@zanii/blackbox 0.22.0 → 0.23.0

package/dist/evals/index.d.ts

@@ -0,0 +1,233 @@
+import type { Summary } from "../fleet/index.ts";
+type Obj = {
+    [key: string]: unknown;
+};
+export declare const SCORERS: readonly ["exact", "contains", "regex", "json_schema", "numeric", "tool_called", "tool_not_called", "findings_absent", "judge"];
+export type ScorerKind = (typeof SCORERS)[number];
+export interface Scorer {
+    kind: ScorerKind;
+    pattern?: string;
+    schema?: Obj;
+    tolerance_milli?: number;
+    tool?: string;
+    codes?: string[];
+}
+export interface EvalCase {
+    id: string;
+    input: unknown;
+    expected?: string | number;
+    rubric?: string;
+    tags: string[];
+    scorer: Scorer;
+}
+export declare const CASE_ID: RegExp;
+/** §2: a scorer, checked: the normalised scorer, or what's wrong with it. */
+export declare function checkScorer(v: unknown, at?: string): Scorer | string;
+/**
+ * §1: a dataset, checked: `{name, cases}` with every case normalised (an id, tags, its scorer or
+ * the dataset's), or the first problem. Numbers must be integers: decimals are written as strings.
+ */
+export declare function checkDataset(v: unknown): {
+    name: string;
+    cases: EvalCase[];
+} | string;
+/** §1: what names a dataset's content exactly: `sha256:` over the canonical JSON of {name, cases}. */
+export declare function datasetHash(d: {
+    name: string;
+    cases: readonly EvalCase[];
+}): string;
+/** §1: a JSON Lines file of cases: one object per non-empty line, or the first line that isn't. */
+export declare function parseCasesJsonl(text: string): unknown[] | string;
+/** §1: one case per saved scenario (spec/replay.md §3): a fork of the incident, passing when none
+ * of its findings comes back. */
+export declare function casesFromScenarios(scenarios: readonly unknown[]): Obj[] | string;
+/** §1: one case per recorded session: a fork of it, expected to answer what it answered then. */
+export declare function casesFromSessions(items: ReadonlyArray<{
+    session_id: string;
+    answer: string;
+}>): Obj[];
+/** What one run of the agent left, read from its record: the final answer, the tools it called,
+ * its findings, and the judge's verdict (undefined: not asked; null: an outage). */
+export interface Observation {
+    answer: string | null;
+    tools: {
+        [tool: string]: number;
+    };
+    finding_codes: {
+        [code: string]: number;
+    };
+    judge?: {
+        steps: Array<{
+            rubric: string;
+            pass: boolean;
+            reason?: string;
+        }>;
+    } | null;
+}
+export interface CaseScore {
+    pass: boolean;
+    reason: string;
+    outage?: true;
+}
+/** A decimal as integer milli-units, the digits past the third decimal dropped. */
+export declare function toMilli(s: string): bigint;
+/** §2: one case scored against what its run left. An outage of the judge is never a pass. */
+export declare function scoreCase(c: EvalCase, o: Observation): CaseScore;
+export interface RunSummary {
+    total: number;
+    passed: number;
+    failed: number;
+    outages: number;
+    pass_ppm: number;
+    pass_low_ppm: number;
+    pass_high_ppm: number;
+    cost_micro_usd: number;
+    cost_per_case_micro_usd: number;
+    tags: {
+        [tag: string]: {
+            total: number;
+            passed: number;
+        };
+    };
+}
+/** §3: a run's totals: passes with the 95% range, outages, cost, and per tag. */
+export declare function runSummary(results: ReadonlyArray<{
+    pass: boolean;
+    outage?: boolean;
+    cost_micro_usd: number;
+    tags?: readonly string[];
+}>): RunSummary;
+type Verdict = "better" | "worse" | "unclear";
+interface RunDoc {
+    id: string;
+    dataset: {
+        id: string;
+        hash: string;
+    };
+    results: ReadonlyArray<{
+        case: string;
+        pass: boolean;
+    }>;
+    summary: RunSummary;
+}
+/** §3: run `b` against run `a`: better, worse or unclear, and the cases each fixed or broke. */
+export declare function compareRuns(a: RunDoc, b: RunDoc): {
+    a: string;
+    b: string;
+    same_dataset: boolean;
+    verdict: Verdict;
+    pass_ppm: {
+        a: number;
+        b: number;
+    };
+    pass_range_ppm: {
+        a: number[];
+        b: number[];
+    };
+    fixed: string[];
+    broken: string[];
+    cost_per_case_delta_micro_usd: number;
+};
+export declare const PROMPT_NAME: RegExp;
+/** §4: a prompt text's version hash: the gateway's `system_sha256` of a request whose system
+ * prompt is this text (an Anthropic `system` string, or a Chat Completions system message). */
+export declare function promptSha256(text: string): string;
+/** §5: the arm a session falls in: the first 8 hex of sha256(`<experiment>:<session>`) mod 1000,
+ * walked through the arms' per-mille shares in order. The same session always gets the same arm. */
+export declare function assignArm(experimentId: string, sessionId: string, arms: ReadonlyArray<{
+    name: string;
+    share_permille: number;
+}>): string;
+export interface Arm {
+    name: string;
+    version: number;
+    share_permille: number;
+}
+/** §5: an experiment, checked against its prompt's versions: `{name, prompt, arms}`, or why not. */
+export declare function checkExperiment(v: unknown, versions: readonly number[]): {
+    name: string;
+    prompt: string;
+    arms: Arm[];
+} | string;
+/** §4: how a group of runs did: success with the 95% range, spend per run, findings. */
+export declare function armStats(runs: readonly Summary[]): {
+    sessions: number;
+    success: number;
+    success_ppm: number;
+    success_low_ppm: number;
+    success_high_ppm: number;
+    spend_per_run_micro_usd: number;
+    findings: {
+        advisory: number;
+        caution: number;
+        warning: number;
+    };
+    finding_codes: {
+        [k: string]: number;
+    };
+};
+/**
+ * §4: each version of a prompt, from closed sessions' summaries: the runs that declared it (an
+ * experiment's arm or the SDK's `promptVersion`), and the runs that declared none whose first
+ * request's system prompt hash is the version's. Oldest version first, each against the one before.
+ */
+export declare function promptResults(summaries: readonly Summary[], prompt: {
+    name: string;
+    versions: ReadonlyArray<{
+        version: number;
+        sha256: string;
+    }>;
+}): {
+    name: string;
+    sessions: number;
+    versions: {
+        sessions: number;
+        success: number;
+        success_ppm: number;
+        success_low_ppm: number;
+        success_high_ppm: number;
+        spend_per_run_micro_usd: number;
+        findings: {
+            advisory: number;
+            caution: number;
+            warning: number;
+        };
+        finding_codes: {
+            [k: string]: number;
+        };
+        version: number;
+        sha256: string;
+        declared: number;
+        detected: number;
+        vs_previous: Verdict | null;
+    }[];
+};
+/** §5: each arm of an experiment, from the closed sessions it assigned, against the first arm. */
+export declare function experimentResults(summaries: readonly Summary[], experiment: {
+    id: string;
+    arms: readonly Arm[];
+}): {
+    id: string;
+    sessions: number;
+    arms: {
+        sessions: number;
+        success: number;
+        success_ppm: number;
+        success_low_ppm: number;
+        success_high_ppm: number;
+        spend_per_run_micro_usd: number;
+        findings: {
+            advisory: number;
+            caution: number;
+            warning: number;
+        };
+        finding_codes: {
+            [k: string]: number;
+        };
+        name: string;
+        version: number;
+        share_permille: number;
+        vs_control: Verdict | null;
+    }[];
+};
+export {};