@aigne/afs-llm-bench 1.12.0-beta.5

package/dist/runner.d.mts

@@ -0,0 +1,345 @@
+import { BenchPrompt, DimensionScores } from "./scoring.mjs";
+import { OutlierEntry } from "./outlier.mjs";
+import { Prompt } from "./prompts.mjs";
+import { ValidationError } from "./errors.mjs";
+import { AgentRunResult } from "@aigne/afs-agent";
+
+//#region src/runner.d.ts
+declare const ALLOWED_JUDGE_DIMENSIONS: readonly ["instruction_following"];
+interface ToolEntry {
+  path: string;
+  ops: string[];
+  maxDepth?: number;
+}
+/** Permissive read-only AFS sandbox so benchmark prompts can explore mounts. */
+declare function defaultBenchTools(): ToolEntry[];
+declare function defaultBenchSystem(): string;
+interface ExecLike {
+  exec(path: string, args: Record<string, unknown>): Promise<{
+    success: boolean;
+    data?: Record<string, unknown> | unknown;
+    error?: {
+      code?: string;
+      message?: string;
+    };
+  }>;
+}
+interface RunOneInput {
+  deps: ExecLike;
+  model: string;
+  /** Hub used to expand short `model` ids; ignored when `model` is a full
+   *  AFS path. Defaults to `"vertex"` (see DEFAULT_HUB). */
+  hub?: string;
+  prompt: Prompt;
+  benchPrompt: BenchPrompt;
+}
+interface RunOneResult {
+  model: string;
+  prompt: string;
+  scores: DimensionScores;
+  raw: AgentRunResult;
+  /**
+   * Per-judge raw scores from a multi-judge dispatch (Phase 0 of Run 4).
+   * Keyed by judge model id; value is `null` when that judge failed for this
+   * sample. Absent when no judging was requested or only single-judge mode
+   * was used (existing single-judge behaviour stays in `scores.instruction_following`).
+   */
+  judges_raw?: Record<string, number | null>;
+}
+declare function runOne(input: RunOneInput): Promise<RunOneResult>;
+declare function promptToBenchPrompt(prompt: Prompt): BenchPrompt;
+/**
+ * Extract the raw `# Expected outcomes` section text. Returns the entire
+ * section body (including any markdown table) so the LLM judge has the full
+ * rubric. Empty string when the section is missing — the judge still runs
+ * but has no rubric to score against; in practice this means a low score.
+ */
+declare function extractExpectedOutcomes(prompt: Prompt): string;
+/**
+ * Extract expected_paths from the `# Expected paths` body section's first
+ * fenced code block. Strips `// …` line comments. Lines that don't start with
+ * `/` are ignored (so prose inside the block doesn't pollute the path set).
+ */
+declare function extractExpectedPaths(prompt: Prompt): string[];
+interface ValidatedRunArgs {
+  models: string[];
+  prompts: "all" | string[];
+  samples: number;
+  parallel: number;
+  /** Hub for short-model-id expansion. See `RunOneInput.hub`. */
+  hub?: string;
+}
+declare function validateRunArgs(args: Record<string, unknown>): ValidatedRunArgs;
+interface ValidatedRunFullArgs {
+  models: string[];
+  prompts: "all" | string[];
+  samples: number;
+  parallel: number;
+  /**
+   * Hub used to expand short model ids — passed through to runOne's
+   * `resolveModelPath`. Falls back to `DEFAULT_HUB` when omitted. Models
+   * given as a full `/dev/ai/hubs/<hub>/models/<id>` path bypass this.
+   */
+  hub?: string;
+  judgeFor?: ReadonlyArray<(typeof ALLOWED_JUDGE_DIMENSIONS)[number]>;
+  /**
+   * Multi-judge ensemble (Phase 0 of Run 4). When present, takes precedence
+   * over `judgeModel` — each judge dispatches its own chat call per sample
+   * and the per-sample scores are aggregated by `aggregateJudgeScores`.
+   * Shares the same `judgeHub` for every judge (default `"vertex"`).
+   * Length 1 is allowed and reduces to single-judge passthrough.
+   */
+  judges?: string[];
+  /**
+   * When true, write a sidecar `<stamp>-raw.json` next to the main results
+   * file containing `trace[]` for every successful sample. Default false to
+   * keep the main results file small and avoid leaking trace details into
+   * the dir that maintainers commit.
+   */
+  includeRaw: boolean;
+  /**
+   * Per-dimension stddev cap for outlier flagging (Phase 3 of Run 4 /
+   * plan.md L1070-1072). When `undefined`, the runner uses
+   * `DEFAULT_OUTLIER_THRESHOLD` (0.15). Clamped to `[0.01, 1.0]` so a
+   * misconfig can't disable outlier detection (`0`) or flag every dim (`>1`).
+   */
+  outlierThreshold?: number;
+  /**
+   * When true, after the main matrix completes the runner schedules one
+   * extra batch of `min(N, 3)` samples for each (model, prompt) flagged by
+   * `findOutliers` and recomputes the aggregate + outliers from the merged
+   * samples. Bounded so total samples per (model, prompt) ≤ `2 × N` (no
+   * runaway budget).
+   */
+  outlierRerun: boolean;
+}
+declare function validateRunFullArgs(args: Record<string, unknown>): ValidatedRunFullArgs;
+interface SampleError {
+  /** Coarse-grained error class for triage. */
+  kind: "rate_limit" | "auth" | "other";
+  /** Path that failed (almost always /dev/agent/.actions/run). */
+  path?: string;
+  message: string;
+  /** Number of attempts made before giving up (1 for non-retryable errors). */
+  attempts: number;
+}
+interface ModelAggregate {
+  n_samples: number;
+  scores: DimensionScores;
+  stddev: Record<string, number>;
+  /**
+   * Per-prompt mean scores: `{ <prompt_id>: { <dim>: mean across that prompt's
+   * samples } }`. Lets a maintainer audit which prompts dragged the model
+   * average down without re-running the matrix. Keys are sorted alphabetically
+   * for stable diffing across runs. Empty object when there are no samples.
+   */
+  per_prompt: Record<string, DimensionScores>;
+  /**
+   * Per-domain aggregate score (Phase 1 of Run 4 / plan.md L1054-1061).
+   * Single number per domain — the mean of every sample's per-dimension
+   * average for prompts that belong to that domain. Domain is read from the
+   * prompt's `frontmatter.domain` (default `"english"` per INTENT). Absent
+   * when the caller doesn't supply the prompt list, so unit tests that only
+   * pass `samples` keep their existing shape.
+   */
+  domains?: Record<string, number>;
+  /**
+   * Multi-judge audit trail (Phase 0 of Run 4). Per-judge mean across all the
+   * model's samples that received that judge's verdict; `null` when no sample
+   * succeeded for that judge. Absent when no multi-judge dispatch happened.
+   * Lets a maintainer detect a single rogue judge dragging the aggregate.
+   */
+  judges_raw?: Record<string, number | null>;
+  /** Per-sample failures; absent when all samples succeed. */
+  errors?: SampleError[];
+}
+declare function aggregateAcrossPrompts(samples: RunOneResult[], prompts?: Prompt[]): ModelAggregate;
+interface RunMatrixInput {
+  deps: ExecLike;
+  prompts: Prompt[];
+  models: string[];
+  samples: number;
+  parallel: number;
+  resultsDir: string;
+  /** Hub for short-model-id expansion. See `RunOneInput.hub`. */
+  hub?: string;
+}
+interface ResultsFile {
+  version: string;
+  /**
+   * Judge model id(s) used to score `instruction_following`. Single-judge
+   * runs serialise as a string (back-compat with patcher.ts); multi-judge
+   * (Phase 0 of Run 4) serialises as an array. `null` when no judging happened.
+   */
+  judge_model?: string | string[] | null;
+  config: {
+    samples_per_pair: number;
+    models_tested: string[];
+    prompts_used: string[];
+  };
+  results: Record<string, ModelAggregate>;
+  /**
+   * Per-(model, prompt, dim) tuples whose stddev exceeded the configured
+   * threshold (Phase 3 of Run 4). Always present — empty array when nothing
+   * was flagged. Audit-only field; **not** propagated to the catalog by
+   * `patcher.ts`. See `outlier.ts` for the algorithm.
+   */
+  outliers?: OutlierEntry[];
+}
+interface RunMatrixResult {
+  results: Record<string, ModelAggregate>;
+  file: string;
+  ranBatch: {
+    models: string[];
+    prompts: string[];
+    samples: number;
+  };
+}
+declare function runMatrix(input: RunMatrixInput): Promise<RunMatrixResult>;
+interface ProgressEvent {
+  model: string;
+  prompt: string;
+  sampleIndex: number;
+  totalSamples: number;
+  /** "ok" if scoring succeeded, "error" if all retries exhausted. */
+  status: "ok" | "error";
+  /** Five-dim summary on success; only `error` on failure. */
+  scores?: DimensionScores;
+  error?: SampleError;
+}
+interface RunFullInput {
+  deps: ExecLike;
+  prompts: Prompt[];
+  models: string[];
+  samples: number;
+  parallel: number;
+  resultsDir: string;
+  /** Hub for short-model-id expansion. See `RunOneInput.hub`. */
+  hub?: string;
+  /**
+   * Sleep hook for backoff; defaults to setTimeout. Tests inject a no-op so
+   * they don't actually wait 6+ seconds for backoff windows.
+   */
+  sleep?: (ms: number) => Promise<void>;
+  /** Per-sample progress callback (fires after each retry-resolved sample). */
+  onProgress?: (event: ProgressEvent) => void;
+  /** Fires after each (model, prompt) pair flushes to disk. */
+  onPairComplete?: (model: string, promptId: string, partialFile: string) => void | Promise<void>;
+  /** Fires after each model completes (after final pair flush). */
+  onModelComplete?: (model: string, partialFile: string) => void | Promise<void>;
+  /**
+   * Optional dimensions to evaluate via LLM-as-judge. When the array
+   * includes `"instruction_following"`, each successful sample is sent to
+   * `/dev/ai/.actions/chat` for scoring before progress is emitted.
+   */
+  judgeFor?: ReadonlyArray<(typeof ALLOWED_JUDGE_DIMENSIONS)[number]>;
+  /**
+   * Optional override for the judge model id. Validated against
+   * `[A-Za-z0-9._-]+` inside `judgeInstructionFollowing`. Defaults to
+   * `claude-opus-4-7` (routed through ai-device). Ignored when `judges`
+   * is set (multi-judge takes precedence; tasks.md L23).
+   */
+  judgeModel?: string;
+  /**
+   * Optional override for the hub the judge model lives on. Defaults to
+   * `aignehub`. Used together with `judgeModel` to drive ai-device's
+   * bypass-mode dispatch.
+   */
+  judgeHub?: string;
+  /**
+   * Multi-judge ensemble (Phase 0 of Run 4). When set with at least one
+   * judge id, this **overrides** `judgeModel` — every successful sample
+   * dispatches one chat call per judge id and `aggregateJudgeScores`
+   * collapses the N scores into a single `instruction_following`. All
+   * judges share the same `judgeHub`.
+   */
+  judges?: string[];
+  /**
+   * When true, write a sidecar `<stamp>-raw.json` next to the main results
+   * file containing the AgentRunResult (trace[] et al) for every successful
+   * sample. Off by default — raw traces can be large and may contain prompt
+   * fragments the maintainer doesn't want to commit.
+   */
+  includeRaw?: boolean;
+  /**
+   * Per-dimension stddev cap for outlier flagging (Phase 3 of Run 4).
+   * Defaults to `DEFAULT_OUTLIER_THRESHOLD` (0.15). Validated/clamped at
+   * the action boundary (`validateRunFullArgs`).
+   */
+  outlierThreshold?: number;
+  /**
+   * When true, after the main matrix the runner schedules one extra batch
+   * of `min(N, 3)` samples per (model, prompt) flagged by `findOutliers`,
+   * appends them to the existing samples, and recomputes the aggregate +
+   * outliers. Bounded so total samples per (model, prompt) ≤ `2 × N`.
+   */
+  outlierRerun?: boolean;
+}
+/**
+ * Per-sample raw payload kept around when `includeRaw: true`. Indexed
+ * `model → prompt → [{ sampleIndex, raw }]` so a maintainer can audit a
+ * specific cell of the matrix without scanning the whole sidecar.
+ */
+interface RawResultsFile {
+  version: string;
+  config: {
+    samples_per_pair: number;
+    models_tested: string[];
+    prompts_used: string[];
+  };
+  raw: Record<string, Record<string, Array<{
+    sampleIndex: number;
+    raw: AgentRunResult;
+  }>>>;
+}
+interface RunFullResult {
+  results: Record<string, ModelAggregate>;
+  file: string;
+  /** Path to the sidecar raw-trace JSON when `includeRaw: true`; otherwise null. */
+  rawFile: string | null;
+  ranBatch: {
+    models: string[];
+    prompts: string[];
+    samples: number;
+  };
+}
+/**
+ * `runMatrixFull` is the headless full-matrix runner used by `/bench/.actions/runFull`.
+ *
+ * Differences vs `runMatrix`:
+ *  - Allows up to 200 models / 200 prompts (vs 5/5 for the fast `run` action).
+ *  - Flushes the results JSON to disk after every (model, prompt) pair so a
+ *    SIGKILL mid-run preserves the entries already completed.
+ *  - Retries rate-limit errors with exponential backoff (2s/4s/8s, max 3
+ *    retries). Auth errors (`INVALID_API_KEY`, `UNAUTHENTICATED`, etc.) fail
+ *    immediately without retry.
+ *  - Records per-sample failures on the model aggregate's `errors[]` instead
+ *    of throwing — one bad sample doesn't block the rest of the matrix.
+ *  - Emits per-sample progress via `onProgress` so callers can stream live
+ *    output (`▸ <model> × <prompt> sample N/M: ...`).
+ */
+declare function runMatrixFull(input: RunFullInput): Promise<RunFullResult>;
+/**
+ * Write `payload` to `<dir>/<ISO-date>.json` atomically (write to a randomised
+ * `.tmp-<id>.json` then rename). Best-effort redaction runs before
+ * serialisation: both sensitive labels (api_key / Authorization / …) and
+ * common credential VALUE shapes (Bearer tokens, `sk-`/`ya29.` keys, PEM
+ * blocks, AWS/Slack ids) are blanked so credentials that leak into agent-run
+ * traces are scrubbed from the on-disk record. This is a safety net, not a
+ * guarantee — the design keeps credentials out of traces. Aborts (without
+ * writing) when
+ * the serialised payload would exceed `MAX_RESULTS_SIZE_BYTES` — the failure
+ * mode is intentionally noisy so a bug that bloats the results dir never lands
+ * on disk silently.
+ */
+declare function writeResultsAtomic(payload: ResultsFile, dir: string): Promise<string>;
+/**
+ * Write the raw-trace sidecar to `<dir>/<ISO-date>-raw.json` atomically. Uses
+ * the same sensitive-string redaction as the main results file. Has no size
+ * cap — the maintainer opted in via `includeRaw: true` and a hard fail
+ * mid-run would discard the agent traces they explicitly asked to keep.
+ */
+declare function writeRawResultsAtomic(payload: RawResultsFile, dir: string): Promise<string>;
+//#endregion
+export { ExecLike, ModelAggregate, ProgressEvent, RawResultsFile, ResultsFile, RunFullInput, RunFullResult, RunMatrixInput, RunMatrixResult, RunOneInput, RunOneResult, SampleError, ToolEntry, ValidatedRunArgs, ValidatedRunFullArgs, aggregateAcrossPrompts, defaultBenchSystem, defaultBenchTools, extractExpectedOutcomes, extractExpectedPaths, promptToBenchPrompt, runMatrix, runMatrixFull, runOne, validateRunArgs, validateRunFullArgs, writeRawResultsAtomic, writeResultsAtomic };
+//# sourceMappingURL=runner.d.mts.map

package/dist/runner.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.mts","names":[],"sources":["../src/runner.ts"],"mappings":";;;;;;;cAyEM,wBAAA;AAAA,UAwDW,SAAA;EACf,IAAA;EACA,GAAA;EACA,QAAA;AAAA;;iBAIc,iBAAA,CAAA,GAAqB,SAAA;AAAA,iBAIrB,kBAAA,CAAA;AAAA,UAWC,QAAA;EACf,IAAA,CACE,IAAA,UACA,IAAA,EAAM,MAAA,oBACL,OAAA;IACD,OAAA;IACA,IAAA,GAAO,MAAA;IACP,KAAA;MAAU,IAAA;MAAe,OAAA;IAAA;EAAA;AAAA;AAAA,UAIZ,WAAA;EACf,IAAA,EAAM,QAAA;EACN,KAAA;EATU;;EAYV,GAAA;EACA,MAAA,EAAQ,MAAA;EACR,WAAA,EAAa,WAAA;AAAA;AAAA,UAGE,YAAA;EACf,KAAA;EACA,MAAA;EACA,MAAA,EAAQ,eAAA;EACR,GAAA,EAAK,cAAA;EAlBsB;;;AAI7B;;;EAqBE,UAAA,GAAa,MAAA;AAAA;AAAA,iBAGO,MAAA,CAAO,KAAA,EAAO,WAAA,GAAc,OAAA,CAAQ,YAAA;AAAA,iBAkE1C,mBAAA,CAAoB,MAAA,EAAQ,MAAA,GAAS,WAAA;;;;;;;iBAcrC,uBAAA,CAAwB,MAAA,EAAQ,MAAA;;;;;AA9FhD;iBAyGgB,oBAAA,CAAqB,MAAA,EAAQ,MAAA;AAAA,UAgB5B,gBAAA;EACf,MAAA;EACA,OAAA;EACA,OAAA;EACA,QAAA;EAlHmB;EAoHnB,GAAA;AAAA;AAAA,iBAGc,eAAA,CAAgB,IAAA,EAAM,MAAA,oBAA0B,gBAAA;AAAA,UA8D/C,oBAAA;EACf,MAAA;EACA,OAAA;EACA,OAAA;EACA,QAAA;EAzLmB;;AAGrB;;;EA4LE,GAAA;EACA,QAAA,GAAW,aAAA,SAAsB,wBAAA;EA7Le;;;;;;;EAqMhD,MAAA;EArMoE;AAkEtE;;;;;EA0IE,UAAA;EA1ImD;;;AAcrD;;;EAmIE,gBAAA;EAnIoD;AAWtD;;;;;AAgBA;EAgHE,YAAA;AAAA;AAAA,iBAGc,mBAAA,CAAoB,IAAA,EAAM,MAAA,oBAA0B,oBAAA;AAAA,UA2HnD,WAAA;EA5Of;EA8OA,IAAA;EA5OA;EA8OA,IAAA;EACA,OAAA;EA7OG;EA+OH,QAAA;AAAA;AAAA,UAGe,cAAA;EACf,SAAA;EACA,MAAA,EAAQ,eAAA;EACR,MAAA,EAAQ,MAAA;EAlPsD;;;AA8DhE;;;EA2LE,UAAA,EAAY,MAAA,SAAe,eAAA;EA1L3B;;;;;;;;EAmMA,OAAA,GAAU,MAAA;EA1KV;;;;;AAkBF;EA+JE,UAAA,GAAa,MAAA;;EAEb,MAAA,GAAS,WAAA;AAAA;AAAA,iBA+CK,sBAAA,CACd,OAAA,EAAS,YAAA,IACT,OAAA,GAAU,MAAA,KACT,cAAA;AAAA,UA0Gc,cAAA;EACf,IAAA,EAAM,QAAA;EACN,OAAA,EAAS,MAAA;EACT,MAAA;EACA,OAAA;EACA,QAAA;EACA,UAAA;EA5LQ;EA8LR,GAAA;AAAA;AAAA,UAGe,WAAA;EACf,OAAA;EA1Ka;;;;;EAgLb,WAAA;EACA,MAAA;IACE,gBAAA;IACA,aAAA;IACA,YAAA;EAAA;EAEF,OAAA,EAAS,MAAA,SAAe,cAAA;EA7LxB;;;;;;EAoMA,QAAA,GAAW,YAAA;AAAA;AAAA,UAGI,eAAA;EACf,OAAA,EAAS,MAAA,SAAe,cAAA;EACxB,IAAA;EACA,QAAA;IAAY,MAAA;IAAkB,OAAA;IAAmB,OAAA;EAAA;AAAA;AAAA,iBAG7B,SAAA,CAAU,KAAA,EAAO,cAAA,GAAiB,OAAA,CAAQ,eAAA;AAAA,UA2D/C,aAAA;EACf,KAAA;EACA,MAAA;EACA,WAAA;EACA,YAAA;EAvGe;EAyGf,MAAA;;EAEA,MAAA,GAAS,eAAA;EACT,KAAA,GAAQ,WAAA;AAAA;AAAA,UAGO,YAAA;EACf,IAAA,EAAM,QAAA;EACN,OAAA,EAAS,MAAA;EACT,MAAA;EACA,OAAA;EACA,QAAA;EACA,UAAA;EA7GG;EA+GH,GAAA;EA5Ge;;;;EAiHf,KAAA,IAAS,EAAA,aAAe,OAAA;EA7Fb;EA+FX,UAAA,IAAc,KAAA,EAAO,aAAA;EA/FE;EAiGvB,cAAA,IAAkB,KAAA,UAAe,QAAA,UAAkB,WAAA,oBAA+B,OAAA;EA9GlF;EAgHA,eAAA,IAAmB,KAAA,UAAe,WAAA,oBAA+B,OAAA;EA9G/D;;;;;EAoHF,QAAA,GAAW,aAAA,SAAsB,wBAAA;EAzGjC;;;;AAGF;;EA6GE,UAAA;EA5Ge;;;;;EAkHf,QAAA;EAhHY;;;;;AAGd;;EAqHE,MAAA;EArHqC;;;;;;EA4HrC,UAAA;EA5HsD;;;;AA2DxD;EAuEE,gBAAA;;;;;;;EAOA,YAAA;AAAA;;;;;;UAQe,cAAA;EACf,OAAA;EACA,MAAA;IACE,gBAAA;IACA,aAAA;IACA,YAAA;EAAA;EAEF,GAAA,EAAK,MAAA,SAAe,MAAA,SAAe,KAAA;IAAQ,WAAA;IAAqB,GAAA,EAAK,cAAA;EAAA;AAAA;AAAA,UAGtD,aAAA;EACf,OAAA,EAAS,MAAA,SAAe,cAAA;EACxB,IAAA;EApFA;EAsFA,OAAA;EACA,QAAA;IAAY,MAAA;IAAkB,OAAA;IAAmB,OAAA;EAAA;AAAA;;;;;;;;;;;;;;;;iBAkB7B,aAAA,CAAc,KAAA,EAAO,YAAA,GAAe,OAAA,CAAQ,aAAA;;;;;;;;;;AAjClE;;;;iBAsWsB,kBAAA,CAAmB,OAAA,EAAS,WAAA,EAAa,GAAA,WAAc,OAAA;;;;;;;iBAoBvD,qBAAA,CAAsB,OAAA,EAAS,cAAA,EAAgB,GAAA,WAAc,OAAA"}