@tangle-network/agent-eval 0.93.0 → 0.95.0

package/dist/benchmarks/index.d.ts

@@ -1,4 +1,23 @@
-export { B as BENCHMARK_SPLIT_SEED, a as BenchmarkAdapter, b as BenchmarkDatasetItem, c as BenchmarkEvaluation, d as deterministicSplit, e as routing } from '../index-Bx3gZ8xl.js';
-import '../run-record-e7vj1uZQ.js';
-import '../errors-CzMUYo7b.js';
-import '../schema-m0gsnbt3.js';
+/**
+ * Reference benchmark wrappers — entry point.
+ *
+ * Core surface (exported here):
+ *   - The `BenchmarkAdapter` contract.
+ *   - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.
+ *   - `routing` — synthetic 16-task router benchmark. The only novel
+ *     benchmark we built; ships in the package.
+ *
+ * Example wrappers (under `examples/benchmarks/`, NOT in the bundle):
+ *   - `gsm8k`         — exact-match math reasoning (HF mirror, dataset
+ *                       not bundled).
+ *   - `swebench-lite` — 30-instance SWE-Bench subset via an external
+ *                       grader command.
+ *
+ * The example wrappers are reference implementations of `BenchmarkAdapter`.
+ * Read them, copy them, adapt them. They're intentionally not in the main
+ * entry — every team will configure them differently.
+ */
+export * as routing from './routing/index';
+export type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation, } from './types';
+export { BENCHMARK_SPLIT_SEED, deterministicSplit } from './types';
+//# sourceMappingURL=index.d.ts.map

package/dist/benchmarks/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/benchmarks/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAA;AAC1C,YAAY,EACV,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,SAAS,CAAA;AAChB,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAA"}

package/dist/benchmarks/routing/dataset.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
+ * deterministic, dependency-free benchmark for any router that maps a
+ * natural-language request to one of a fixed set of route labels.
+ *
+ * Format (see `routing/README.md` for prose):
+ *
+ *   {
+ *     id:           stable per-task ID (matches across processes).
+ *     category:     one of the four route labels.
+ *     prompt:       the user-facing request the router must classify.
+ *     route:        the ground-truth route the router should pick.
+ *     synonyms:     other strings that count as a correct answer.
+ *     hardNegatives:close-but-wrong route labels — used to detect the
+ *                   "always picks the popular route" failure mode.
+ *   }
+ *
+ * The four categories are intentionally cross-domain (file ops,
+ * math, search, conversation) so a router that collapses to one
+ * category is easy to spot.
+ */
+export interface RoutingItem {
+    id: string;
+    category: 'file' | 'math' | 'search' | 'chat';
+    prompt: string;
+    /** Canonical correct route label. */
+    route: string;
+    /** Alternate route labels that also count as correct. */
+    synonyms: string[];
+    /** Wrong-but-tempting route labels (for analysis, not grading). */
+    hardNegatives: string[];
+}
+export declare const ROUTING_DATASET: RoutingItem[];
+//# sourceMappingURL=dataset.d.ts.map

package/dist/benchmarks/routing/dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/routing/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAA;IACV,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAA;IAC7C,MAAM,EAAE,MAAM,CAAA;IACd,qCAAqC;IACrC,KAAK,EAAE,MAAM,CAAA;IACb,yDAAyD;IACzD,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,mEAAmE;IACnE,aAAa,EAAE,MAAM,EAAE,CAAA;CACxB;AAED,eAAO,MAAM,eAAe,EAAE,WAAW,EAiIxC,CAAA"}

package/dist/benchmarks/routing/index.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Routing benchmark — synthetic, dependency-free, ships in the
+ * package. 16 cross-category items in `dataset.ts`. See
+ * `routing/README.md` for the format.
+ *
+ * `evaluate` does case-insensitive exact match against the canonical
+ * route plus declared synonyms. The first valid route token in the
+ * response wins; everything else is ignored. Wrong answers also
+ * report whether they hit a hard negative — useful when triaging
+ * "always picks the popular route" failure modes.
+ */
+import type { RunSplitTag } from '../../run-record';
+import type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from '../types';
+import { ROUTING_DATASET, type RoutingItem } from './dataset';
+export type { RoutingItem };
+export type RoutingPayload = RoutingItem;
+export type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
+declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
+    loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
+    evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
+    assignSplit(itemId: string): RunSplitTag;
+}
+/**
+ * Pull route-shaped tokens out of a model response. Routes look like
+ * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
+ * are not routes, but `category.action` patterns are robust to most
+ * model wrappers (JSON output, prose explanations, code fences).
+ */
+export declare function extractRouteTokens(response: string): string[];
+export declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
+export declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
+export declare const assignSplit: (itemId: string) => RunSplitTag;
+export { ROUTING_DATASET, RoutingAdapter };
+//# sourceMappingURL=index.d.ts.map

package/dist/benchmarks/routing/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/routing/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAA;AAE3F,OAAO,EAAE,eAAe,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAA;AAE7D,YAAY,EAAE,WAAW,EAAE,CAAA;AAC3B,MAAM,MAAM,cAAc,GAAG,WAAW,CAAA;AACxC,MAAM,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;AAErE,cAAM,cAAe,YAAW,gBAAgB,CAAC,kBAAkB,EAAE,cAAc,CAAC;IAC5E,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAM9D,QAAQ,CAAC,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAqBxF,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW;CAGzC;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAG7D;AAID,eAAO,MAAM,WAAW,UAjDG,WAAW,KAAG,OAAO,CAAC,kBAAkB,EAAE,CAiDT,CAAA;AAC5D,eAAO,MAAM,QAAQ,SA5CE,kBAAkB,YAAY,MAAM,KAAG,OAAO,CAAC,mBAAmB,CA4CnC,CAAA;AACtD,eAAO,MAAM,WAAW,WAxBF,MAAM,KAAG,WAwB6B,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,CAAA"}

package/dist/benchmarks/types.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Shared types for the reference benchmark wrappers under
+ * `src/benchmarks/`. Each wrapper exports the three functions in
+ * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.
+ */
+import type { RunSplitTag } from '../run-record';
+export interface BenchmarkDatasetItem<TPayload = unknown> {
+    /** Stable dataset-local item id (used for split assignment + paper
+     *  references). Unique within a benchmark. */
+    id: string;
+    /** Free-form payload. Each benchmark defines its own shape. */
+    payload: TPayload;
+}
+export interface BenchmarkEvaluation {
+    /** [0, 1] score for the response on this item. Exact-match
+     *  benchmarks use 0/1; partial-credit benchmarks may return
+     *  fractional values. */
+    score: number;
+    /** Optional bag of raw scoring signals — e.g. parsed numeric
+     *  answer, regex match, judge sub-scores. */
+    raw: Record<string, unknown>;
+}
+/** Common signature implemented by every adapter under `src/benchmarks/*`. */
+export interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown> {
+    /** Load the dataset for the given split. May hit the network on
+     *  first call but should be cache-friendly. Adapters that don't
+     *  ship the dataset itself MUST throw a clearly-marked error
+     *  pointing the caller at the loader script. */
+    loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
+    /** Score a single response. Pure with respect to the inputs. */
+    evaluate(item: BenchmarkDatasetItem<TPayload>, response: string): Promise<BenchmarkEvaluation>;
+    /** Deterministic split assignment via item id hashing. The
+     *  fraction of items in each split is implementation-defined but
+     *  MUST be stable across processes and platforms. */
+    assignSplit(itemId: string): RunSplitTag;
+}
+/** Split-assignment seed shared across all benchmarks. Bumping this
+ *  value reshuffles every split — do NOT do that lightly. */
+export declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
+/**
+ * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
+ * stable 32-bit hash of `${seed}::${id}`. Default proportions:
+ *
+ *   search: 60%   (optimization-readable)
+ *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
+ *   holdout:20%   (paper-grade held-out, gated reads)
+ */
+export declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
+//# sourceMappingURL=types.d.ts.map

package/dist/benchmarks/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/benchmarks/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAEhD,MAAM,WAAW,oBAAoB,CAAC,QAAQ,GAAG,OAAO;IACtD;kDAC8C;IAC9C,EAAE,EAAE,MAAM,CAAA;IACV,+DAA+D;IAC/D,OAAO,EAAE,QAAQ,CAAA;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC;;6BAEyB;IACzB,KAAK,EAAE,MAAM,CAAA;IACb;iDAC6C;IAC7C,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC7B;AAED,8EAA8E;AAK9E,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO,EAAE,QAAQ,GAAG,OAAO;IACpE;;;oDAGgD;IAChD,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;IAC1E,gEAAgE;IAChE,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAA;IAC9F;;yDAEqD;IACrD,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAA;CACzC;AAkBD;6DAC6D;AAC7D,eAAO,MAAM,oBAAoB,kBAAkB,CAAA;AAEnD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,MAA6B,GAClC,WAAW,CAMb"}

package/dist/bisector.d.ts

@@ -0,0 +1,81 @@
+/**
+ * Bisector — auto-locate the change that introduced an eval regression.
+ *
+ * Two shapes:
+ *   - `commitBisect` — walk an ordered SHA list, binary-search for the
+ *     first commit that fails.
+ *   - `promptBisect` — given a good and bad prompt, progressively port
+ *     paragraphs from good→bad to localize the breaking change.
+ *
+ * Generic `bisect<T>` lets callers drive any ordered state space
+ * (dataset versions, config files, CLI flag combinations).
+ */
+export interface BisectOptions<T> {
+    /** State known to pass. */
+    good: T;
+    /** State known to fail. */
+    bad: T;
+    /** Equality test on state values — default Object.is. */
+    equals?: (a: T, b: T) => boolean;
+    /** Pick the halfway state between good + bad. Return null when no further
+     *  split is possible (e.g. adjacent commits). */
+    halfway: (good: T, bad: T) => T | null;
+    /** Produce a verdict for a state. */
+    runEval: (state: T) => Promise<{
+        score: number;
+        pass: boolean;
+    }>;
+    /** Hard cap on iterations (default 40 — covers ~1T ordered states). */
+    maxIterations?: number;
+}
+export interface BisectStep<T> {
+    state: T;
+    score: number;
+    pass: boolean;
+}
+export interface BisectResult<T> {
+    /** The first bad state — typically `bad` in the final (good, bad) adjacent pair. */
+    culprit: T;
+    /** Ordered trace of all states evaluated. */
+    path: BisectStep<T>[];
+    /** True when we narrowed to an adjacent (good, bad) pair. */
+    converged: boolean;
+    /** True when `good` itself failed or `bad` itself passed — the caller's
+     *  premise was broken. */
+    inputInconsistent: boolean;
+}
+export declare function bisect<T>(options: BisectOptions<T>): Promise<BisectResult<T>>;
+/**
+ * Commit bisect — `commits` is an ordered SHA list, oldest to newest.
+ * `good` and `bad` must both be present in the list.
+ */
+export declare function commitBisect(options: {
+    commits: string[];
+    good: string;
+    bad: string;
+    runEval: (sha: string) => Promise<{
+        score: number;
+        pass: boolean;
+    }>;
+    maxIterations?: number;
+}): Promise<BisectResult<string>>;
+/**
+ * Prompt bisect — splits the good and bad prompts into paragraphs, then
+ * progressively replaces paragraphs in `good` with their counterparts
+ * from `bad` to localize the offending change. Only works when the two
+ * prompts have the same paragraph count (a common editorial workflow
+ * constraint — one paragraph = one change unit).
+ */
+export declare function promptBisect(options: {
+    good: string;
+    bad: string;
+    runEval: (prompt: string) => Promise<{
+        score: number;
+        pass: boolean;
+    }>;
+    maxIterations?: number;
+    paragraphSplitter?: (prompt: string) => string[];
+}): Promise<BisectResult<string> & {
+    offendingParagraphIndex?: number;
+}>;
+//# sourceMappingURL=bisector.d.ts.map

package/dist/bisector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bisector.d.ts","sourceRoot":"","sources":["../src/bisector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,2BAA2B;IAC3B,IAAI,EAAE,CAAC,CAAA;IACP,2BAA2B;IAC3B,GAAG,EAAE,CAAC,CAAA;IACN,yDAAyD;IACzD,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,OAAO,CAAA;IAChC;qDACiD;IACjD,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,CAAA;IACtC,qCAAqC;IACrC,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IAChE,uEAAuE;IACvE,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,KAAK,EAAE,CAAC,CAAA;IACR,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,OAAO,CAAA;CACd;AAED,MAAM,WAAW,YAAY,CAAC,CAAC;IAC7B,oFAAoF;IACpF,OAAO,EAAE,CAAC,CAAA;IACV,6CAA6C;IAC7C,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAA;IACrB,6DAA6D;IAC7D,SAAS,EAAE,OAAO,CAAA;IAClB;8BAC0B;IAC1B,iBAAiB,EAAE,OAAO,CAAA;CAC3B;AAED,wBAAsB,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CA8BnF;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE;IAC1C,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACnE,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAwBhC;AAED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE;IAC1C,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACtE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,EAAE,CAAA;CACjD,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG;IAAE,uBAAuB,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CA6EvE"}

package/dist/budget-guard.d.ts

@@ -0,0 +1,31 @@
+/**
+ * BudgetGuard — enforces token / wall-clock / call / $ caps, records
+ * a ledger entry on every decrement, emits `budget_breach` + throws
+ * `BudgetBreachError` when a cap is hit.
+ *
+ * Wraps a TraceEmitter. The emitter persists ledger entries + breach
+ * events so the classifier, pipelines, and reports can all read
+ * budget state from the trace corpus — no separate accounting.
+ */
+import { AgentEvalError } from './errors';
+import type { TraceEmitter } from './trace/emitter';
+import type { BudgetSpec } from './trace/schema';
+export declare class BudgetBreachError extends AgentEvalError {
+    dimension: keyof BudgetSpec;
+    limit: number;
+    attempted: number;
+    constructor(dimension: keyof BudgetSpec, limit: number, attempted: number);
+}
+export declare class BudgetGuard {
+    private consumed;
+    private emitter;
+    private budget;
+    private startedAt;
+    constructor(emitter: TraceEmitter, budget: BudgetSpec, now?: () => number);
+    /** Record consumption. Throws `BudgetBreachError` if any dimension exceeds its cap. */
+    charge(delta: Partial<Record<keyof BudgetSpec, number>>, spanId?: string): Promise<void>;
+    /** Convenience: advance wall-clock budget based on elapsed wall time. */
+    tickWall(nowMs: number, spanId?: string): Promise<void>;
+    get state(): Record<keyof BudgetSpec, number>;
+}
+//# sourceMappingURL=budget-guard.d.ts.map

package/dist/budget-guard.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"budget-guard.d.ts","sourceRoot":"","sources":["../src/budget-guard.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,qBAAa,iBAAkB,SAAQ,cAAc;IAE1C,SAAS,EAAE,MAAM,UAAU;IAC3B,KAAK,EAAE,MAAM;IACb,SAAS,EAAE,MAAM;gBAFjB,SAAS,EAAE,MAAM,UAAU,EAC3B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM;CAI3B;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAA+E;IAC/F,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,SAAS,CAAQ;gBAEb,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,GAAE,MAAM,MAAyB;IAM3F,uFAAuF;IACjF,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,UAAU,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA0B9F,yEAAyE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7D,IAAI,KAAK,IAAI,MAAM,CAAC,MAAM,UAAU,EAAE,MAAM,CAAC,CAE5C;CACF"}

package/dist/builder-eval/builder-session.d.ts

@@ -0,0 +1,111 @@
+/**
+ * BuilderSession — ties a builder-of-builders workflow together.
+ *
+ * Models agent-builder's shape: Project → Chat → Edit → Ship → App →
+ * AppAgent. Each layer is a Run (linked via parentRunId). The
+ * framework-enforced invariants:
+ *
+ *   - One Project → many Chats; chatId scopes runs within a project.
+ *   - One Chat = one builder Run with `layer='builder'`.
+ *   - One Ship = one child Run with `layer='app-build'` + SandboxHarness.
+ *   - One AppScenario = one grandchild Run with `layer='app-runtime'`.
+ *
+ * Consumers obtain a BuilderSession, call `startChat`, drive the
+ * builder agent (emitting spans), and call `ship` / `runAppScenario`
+ * as the workflow progresses. The session reconstructs itself from
+ * trace data via `resume(store, projectId)`.
+ */
+import type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from '../sandbox-harness';
+import type { TestGradedRunResult, TestGradedScenario } from '../test-graded-scenario';
+import { TraceEmitter } from '../trace/emitter';
+import type { Run } from '../trace/schema';
+import type { TraceStore } from '../trace/store';
+export interface BuilderSessionInit {
+    projectId: string;
+    chatId?: string;
+    /** Free-form: user's task description, project name, etc. Stored on the builder Run. */
+    tags?: Record<string, string>;
+}
+export interface ShipOptions {
+    harness: HarnessConfig;
+    driver?: SandboxDriver;
+    /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */
+    scenarioId?: string;
+}
+export interface RunAppScenarioOptions {
+    scenario: TestGradedScenario;
+    /** Harness driver override; defaults to the one the session was created with. */
+    driver?: SandboxDriver;
+}
+export declare class BuilderSession {
+    private store;
+    private builderEmitter;
+    readonly projectId: string;
+    readonly chatId: string;
+    private builderRunId?;
+    private lastBuildRunId?;
+    private defaultDriver?;
+    constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver);
+    /** Start the builder (L0) run for this chat. Returns the runId. */
+    startChat(scenarioId?: string): Promise<string>;
+    /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */
+    get emitter(): TraceEmitter;
+    /**
+     * Ship the project's generated app: run the sandbox harness as a child
+     * Run (`layer='app-build'`). Returns the build result + runId.
+     */
+    ship(options: ShipOptions): Promise<{
+        runId: string;
+        result: SandboxHarnessResult;
+    }>;
+    /**
+     * Run a domain scenario against the just-built app as a grandchild Run
+     * (`layer='app-runtime'`). The `ship` call must precede this so the
+     * parent is set correctly; if no build exists yet the session attaches
+     * directly to the builder run (useful for prototypes).
+     */
+    runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult>;
+    /** Record an end-of-chat meta score (judge verdict on whether the builder
+     *  served the user's intent). Accepts a numeric score + optional rationale. */
+    recordMetaScore(score: number, rationale?: string): Promise<void>;
+    /** Close the builder Run with a final outcome. */
+    endChat(outcome: {
+        pass: boolean;
+        score?: number;
+        notes?: string;
+    }): Promise<void>;
+    /**
+     * Inline app-runtime run — for cases where the "scenario" isn't a
+     * SWE-bench-style test suite but a live agent interaction (LLM chat,
+     * domain flow). Returns an emitter bound to a fresh Run in the
+     * `app-runtime` layer; caller emits spans inside and calls
+     * `.endRun()` with the final verdict.
+     */
+    startAppRuntime(scenarioId: string): Promise<TraceEmitter>;
+    /**
+     * Lightweight "ship marker" — record an app-build Run with a caller-
+     * provided verdict. Use when there isn't a sandbox harness to run but
+     * you still want to mark the build state at publish time.
+     */
+    recordShipMarker(args: {
+        pass: boolean;
+        score: number;
+        scenarioId?: string;
+        notes?: string;
+    }): Promise<string>;
+    get lastBuildRunIdValue(): string | undefined;
+    get builderRunIdValue(): string | undefined;
+}
+/**
+ * Reconstruct the most recent BuilderSession state for a given project —
+ * returns { builderRunId, lastBuildRunId, chatRuns }. For chat-first UIs
+ * this is how a resumed session finds its place in the edit history.
+ */
+export declare function resumeBuilderSession(store: TraceStore, projectId: string): Promise<{
+    projectId: string;
+    chatRuns: Run[];
+    lastBuilderRun?: Run;
+    lastBuildRun?: Run;
+    lastAppRuntimeRuns: Run[];
+}>;
+//# sourceMappingURL=builder-session.d.ts.map

package/dist/builder-eval/builder-session.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"builder-session.d.ts","sourceRoot":"","sources":["../../src/builder-eval/builder-session.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAE5F,OAAO,KAAK,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAEtF,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAC/C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,wFAAwF;IACxF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,aAAa,CAAA;IACtB,MAAM,CAAC,EAAE,aAAa,CAAA;IACtB,0EAA0E;IAC1E,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,iFAAiF;IACjF,MAAM,CAAC,EAAE,aAAa,CAAA;CACvB;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,cAAc,CAAc;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,YAAY,CAAC,CAAQ;IAC7B,OAAO,CAAC,cAAc,CAAC,CAAQ;IAC/B,OAAO,CAAC,aAAa,CAAC,CAAe;gBAEzB,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,aAAa;IAQ/E,mEAAmE;IAC7D,SAAS,CAAC,UAAU,SAA2B,GAAG,OAAO,CAAC,MAAM,CAAC;IAWvE,gFAAgF;IAChF,IAAI,OAAO,IAAI,YAAY,CAG1B;IAED;;;OAGG;IACG,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,oBAAoB,CAAA;KAAE,CAAC;IAqB1F;;;;;OAKG;IACG,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAmBlF;mFAC+E;IACzE,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAavE,kDAAkD;IAC5C,OAAO,CAAC,OAAO,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAQxF;;;;;;OAMG;IACG,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAiBhE;;;;OAIG;IACG,gBAAgB,CAAC,IAAI,EAAE;QAC3B,IAAI,EAAE,OAAO,CAAA;QACb,KAAK,EAAE,MAAM,CAAA;QACb,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBnB,IAAI,mBAAmB,IAAI,MAAM,GAAG,SAAS,CAE5C;IACD,IAAI,iBAAiB,IAAI,MAAM,GAAG,SAAS,CAE1C;CACF;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,UAAU,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IACT,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,GAAG,EAAE,CAAA;IACf,cAAc,CAAC,EAAE,GAAG,CAAA;IACpB,YAAY,CAAC,EAAE,GAAG,CAAA;IAClB,kBAAkB,EAAE,GAAG,EAAE,CAAA;CAC1B,CAAC,CAkBD"}

package/dist/builder-eval/correlation.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Meta-eval correlation — the highest-leverage signal in the framework.
+ *
+ * Given a corpus of three-layer project reports, compute how well each
+ * pair of layers correlates. The question we care about most:
+ *
+ *   Does `metaScore` (what the builder thinks it did) predict
+ *   `runtimeScore` (what the user actually gets)?
+ *
+ * If r < ~0.4, the builder's self-scoring is broken — it's optimizing
+ * for something other than real-world success. If r > 0.7, meta_score
+ * is a usable proxy and can drive CI gates cheaply.
+ *
+ * Non-parametric rank correlation (Spearman) is also reported because
+ * meta scores are often ordinal-ish.
+ */
+import type { ThreeLayerProjectReport } from './three-layer-eval';
+export interface LayerCorrelation {
+    n: number;
+    pearson: number;
+    spearman: number;
+}
+export interface CorrelationReport {
+    /** Pairs present in the corpus (layers with ≥ 2 matched data points). */
+    metaVsBuild?: LayerCorrelation;
+    metaVsRuntime?: LayerCorrelation;
+    buildVsRuntime?: LayerCorrelation;
+    /** Number of complete projects (all 3 scores present). */
+    completeProjects: number;
+}
+export declare function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport;
+//# sourceMappingURL=correlation.d.ts.map

package/dist/builder-eval/correlation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"correlation.d.ts","sourceRoot":"","sources":["../../src/builder-eval/correlation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAA;AAEjE,MAAM,WAAW,gBAAgB;IAC/B,CAAC,EAAE,MAAM,CAAA;IACT,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,yEAAyE;IACzE,WAAW,CAAC,EAAE,gBAAgB,CAAA;IAC9B,aAAa,CAAC,EAAE,gBAAgB,CAAA;IAChC,cAAc,CAAC,EAAE,gBAAgB,CAAA;IACjC,0DAA0D;IAC1D,gBAAgB,EAAE,MAAM,CAAA;CACzB;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,uBAAuB,EAAE,GAAG,iBAAiB,CAoBrF"}