@tangle-network/agent-eval 0.93.0 → 0.95.0

package/dist/builder-eval/index.d.ts

@@ -1,250 +1,5 @@
-import { S as SandboxDriver, H as HarnessConfig, a as SandboxHarnessResult, T as TestGradedScenario, b as TestGradedRunResult } from '../test-graded-scenario-BdVaPyHT.js';
-import { T as TraceEmitter } from '../emitter-DEZwY14K.js';
-import { R as Run } from '../schema-m0gsnbt3.js';
-import { T as TraceStore } from '../store-CKUAgsJz.js';
-
-/**
- * BuilderSession — ties a builder-of-builders workflow together.
- *
- * Models agent-builder's shape: Project → Chat → Edit → Ship → App →
- * AppAgent. Each layer is a Run (linked via parentRunId). The
- * framework-enforced invariants:
- *
- *   - One Project → many Chats; chatId scopes runs within a project.
- *   - One Chat = one builder Run with `layer='builder'`.
- *   - One Ship = one child Run with `layer='app-build'` + SandboxHarness.
- *   - One AppScenario = one grandchild Run with `layer='app-runtime'`.
- *
- * Consumers obtain a BuilderSession, call `startChat`, drive the
- * builder agent (emitting spans), and call `ship` / `runAppScenario`
- * as the workflow progresses. The session reconstructs itself from
- * trace data via `resume(store, projectId)`.
- */
-
-interface BuilderSessionInit {
-    projectId: string;
-    chatId?: string;
-    /** Free-form: user's task description, project name, etc. Stored on the builder Run. */
-    tags?: Record<string, string>;
-}
-interface ShipOptions {
-    harness: HarnessConfig;
-    driver?: SandboxDriver;
-    /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */
-    scenarioId?: string;
-}
-interface RunAppScenarioOptions {
-    scenario: TestGradedScenario;
-    /** Harness driver override; defaults to the one the session was created with. */
-    driver?: SandboxDriver;
-}
-declare class BuilderSession {
-    private store;
-    private builderEmitter;
-    readonly projectId: string;
-    readonly chatId: string;
-    private builderRunId?;
-    private lastBuildRunId?;
-    private defaultDriver?;
-    constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver);
-    /** Start the builder (L0) run for this chat. Returns the runId. */
-    startChat(scenarioId?: string): Promise<string>;
-    /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */
-    get emitter(): TraceEmitter;
-    /**
-     * Ship the project's generated app: run the sandbox harness as a child
-     * Run (`layer='app-build'`). Returns the build result + runId.
-     */
-    ship(options: ShipOptions): Promise<{
-        runId: string;
-        result: SandboxHarnessResult;
-    }>;
-    /**
-     * Run a domain scenario against the just-built app as a grandchild Run
-     * (`layer='app-runtime'`). The `ship` call must precede this so the
-     * parent is set correctly; if no build exists yet the session attaches
-     * directly to the builder run (useful for prototypes).
-     */
-    runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult>;
-    /** Record an end-of-chat meta score (judge verdict on whether the builder
-     *  served the user's intent). Accepts a numeric score + optional rationale. */
-    recordMetaScore(score: number, rationale?: string): Promise<void>;
-    /** Close the builder Run with a final outcome. */
-    endChat(outcome: {
-        pass: boolean;
-        score?: number;
-        notes?: string;
-    }): Promise<void>;
-    /**
-     * Inline app-runtime run — for cases where the "scenario" isn't a
-     * SWE-bench-style test suite but a live agent interaction (LLM chat,
-     * domain flow). Returns an emitter bound to a fresh Run in the
-     * `app-runtime` layer; caller emits spans inside and calls
-     * `.endRun()` with the final verdict.
-     */
-    startAppRuntime(scenarioId: string): Promise<TraceEmitter>;
-    /**
-     * Lightweight "ship marker" — record an app-build Run with a caller-
-     * provided verdict. Use when there isn't a sandbox harness to run but
-     * you still want to mark the build state at publish time.
-     */
-    recordShipMarker(args: {
-        pass: boolean;
-        score: number;
-        scenarioId?: string;
-        notes?: string;
-    }): Promise<string>;
-    get lastBuildRunIdValue(): string | undefined;
-    get builderRunIdValue(): string | undefined;
-}
-/**
- * Reconstruct the most recent BuilderSession state for a given project —
- * returns { builderRunId, lastBuildRunId, chatRuns }. For chat-first UIs
- * this is how a resumed session finds its place in the edit history.
- */
-declare function resumeBuilderSession(store: TraceStore, projectId: string): Promise<{
-    projectId: string;
-    chatRuns: Run[];
-    lastBuilderRun?: Run;
-    lastBuildRun?: Run;
-    lastAppRuntimeRuns: Run[];
-}>;
-
-/**
- * Three-layer evaluation — the canonical scoring breakdown for
- * builder-of-builders workflows.
- *
- *   meta_score:    did the builder understand + satisfy user intent?
- *                  (judge verdict attached to the builder run)
- *   build_score:   did the generated scaffold build + pass its own tests?
- *                  (outcome.score on the app-build child run)
- *   runtime_score: did the generated agent pass its domain scenarios?
- *                  (mean outcome.score over app-runtime grandchild runs)
- *
- * Returns a structured report per project. The cross-layer correlation
- * is the highest-leverage signal the framework computes — if
- * meta_score doesn't predict runtime_score, the builder's self-scoring
- * is broken.
- *
- * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a
- * scaffold-builder eval that grades compose + build without driving a
- * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures
- * meta + build only. Consumers can tell the two apart without having to
- * interpret null-runtime as either "not yet computed" or "N/A for this
- * project shape".
- */
-
-type ProjectKind = 'full' | 'scaffold-only';
-interface ThreeLayerProjectReport {
-    projectId: string;
-    /**
-     * `'full'` when the project has at least one `app-runtime` run;
-     * `'scaffold-only'` when it only has meta + build layers. Lets
-     * downstream consumers treat a null runtime score as expected
-     * (scaffold-only) vs. missing (full, pipeline broke).
-     */
-    kind: ProjectKind;
-    builderRunId?: string;
-    /** Judge-verdict score on the builder run (0..1 after normalization). */
-    metaScore: number | null;
-    buildRunId?: string;
-    /** 0..1 from the sandbox harness (testsPassed / testsTotal). */
-    buildScore: number | null;
-    appRuntimeRunIds: string[];
-    /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */
-    runtimeScore: number | null;
-    runtimePassRate: number | null;
-    /**
-     * Layer-aware completeness:
-     *   - `kind='full'`: all three layers scored
-     *   - `kind='scaffold-only'`: meta + build scored (runtime not applicable)
-     */
-    complete: boolean;
-}
-declare function scoreProject(store: TraceStore, projectId: string): Promise<ThreeLayerProjectReport>;
-/** Aggregate scoring across every project in a corpus. */
-declare function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]>;
-
-/**
- * Meta-eval correlation — the highest-leverage signal in the framework.
- *
- * Given a corpus of three-layer project reports, compute how well each
- * pair of layers correlates. The question we care about most:
- *
- *   Does `metaScore` (what the builder thinks it did) predict
- *   `runtimeScore` (what the user actually gets)?
- *
- * If r < ~0.4, the builder's self-scoring is broken — it's optimizing
- * for something other than real-world success. If r > 0.7, meta_score
- * is a usable proxy and can drive CI gates cheaply.
- *
- * Non-parametric rank correlation (Spearman) is also reported because
- * meta scores are often ordinal-ish.
- */
-
-interface LayerCorrelation {
-    n: number;
-    pearson: number;
-    spearman: number;
-}
-interface CorrelationReport {
-    /** Pairs present in the corpus (layers with ≥ 2 matched data points). */
-    metaVsBuild?: LayerCorrelation;
-    metaVsRuntime?: LayerCorrelation;
-    buildVsRuntime?: LayerCorrelation;
-    /** Number of complete projects (all 3 scores present). */
-    completeProjects: number;
-}
-declare function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport;
-
-/**
- * ProjectRegistry — project-level aggregation over the trace corpus.
- *
- * Thin reader over TraceStore that answers the questions a chat-first,
- * resumable UI needs:
- *   - listProjects() → project IDs with latest activity
- *   - projectTimeline(id) → chats + builds + runtime runs, chronological
- *   - projectChats(id) → chat-level summaries (turn count, outcome)
- *
- * All queries are pure reads; no state duplication.
- */
-
-interface ProjectSummary {
-    projectId: string;
-    chatCount: number;
-    buildCount: number;
-    appRuntimeCount: number;
-    lastActivityAt: number;
-    latestChatId?: string;
-    latestOutcome?: {
-        pass: boolean;
-        score?: number;
-    };
-}
-interface ChatSummary {
-    chatId: string;
-    projectId: string;
-    builderRunId: string;
-    startedAt: number;
-    endedAt?: number;
-    status: Run['status'];
-    outcome?: Run['outcome'];
-    /** Counts of spans emitted during the chat. */
-    llmTurns?: number;
-    toolCalls?: number;
-    buildRunId?: string;
-    appRuntimeRunIds: string[];
-}
-interface ProjectTimelineEntry {
-    run: Run;
-    layerBucket: 'chat' | 'build' | 'runtime' | 'other';
-}
-declare class ProjectRegistry {
-    private store;
-    constructor(store: TraceStore);
-    listProjects(): Promise<ProjectSummary[]>;
-    projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]>;
-    projectChats(projectId: string): Promise<ChatSummary[]>;
-}
-
-export { BuilderSession, type BuilderSessionInit, type ChatSummary, type CorrelationReport, type LayerCorrelation, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type RunAppScenarioOptions, type ShipOptions, type ThreeLayerProjectReport, correlateLayers, resumeBuilderSession, scoreAllProjects, scoreProject };
+export * from './builder-session';
+export * from './correlation';
+export * from './project-registry';
+export * from './three-layer-eval';
+//# sourceMappingURL=index.d.ts.map

package/dist/builder-eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/builder-eval/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA;AACjC,cAAc,eAAe,CAAA;AAC7B,cAAc,oBAAoB,CAAA;AAClC,cAAc,oBAAoB,CAAA"}

package/dist/builder-eval/index.js

@@ -1,10 +1,10 @@
 import {
   SandboxHarness,
   runTestGradedScenario
-} from "../chunk-T375SUOZ.js";
+} from "../chunk-ZOPLCJSY.js";
 import {
   judgeSpans
-} from "../chunk-47X6LRCE.js";
+} from "../chunk-UFSG7ACU.js";
 import "../chunk-5BKGXME7.js";
 import {
   TraceEmitter

package/dist/builder-eval/project-registry.d.ts

@@ -0,0 +1,51 @@
+/**
+ * ProjectRegistry — project-level aggregation over the trace corpus.
+ *
+ * Thin reader over TraceStore that answers the questions a chat-first,
+ * resumable UI needs:
+ *   - listProjects() → project IDs with latest activity
+ *   - projectTimeline(id) → chats + builds + runtime runs, chronological
+ *   - projectChats(id) → chat-level summaries (turn count, outcome)
+ *
+ * All queries are pure reads; no state duplication.
+ */
+import type { Run } from '../trace/schema';
+import type { TraceStore } from '../trace/store';
+export interface ProjectSummary {
+    projectId: string;
+    chatCount: number;
+    buildCount: number;
+    appRuntimeCount: number;
+    lastActivityAt: number;
+    latestChatId?: string;
+    latestOutcome?: {
+        pass: boolean;
+        score?: number;
+    };
+}
+export interface ChatSummary {
+    chatId: string;
+    projectId: string;
+    builderRunId: string;
+    startedAt: number;
+    endedAt?: number;
+    status: Run['status'];
+    outcome?: Run['outcome'];
+    /** Counts of spans emitted during the chat. */
+    llmTurns?: number;
+    toolCalls?: number;
+    buildRunId?: string;
+    appRuntimeRunIds: string[];
+}
+export interface ProjectTimelineEntry {
+    run: Run;
+    layerBucket: 'chat' | 'build' | 'runtime' | 'other';
+}
+export declare class ProjectRegistry {
+    private store;
+    constructor(store: TraceStore);
+    listProjects(): Promise<ProjectSummary[]>;
+    projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]>;
+    projectChats(projectId: string): Promise<ChatSummary[]>;
+}
+//# sourceMappingURL=project-registry.d.ts.map

package/dist/builder-eval/project-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"project-registry.d.ts","sourceRoot":"","sources":["../../src/builder-eval/project-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,MAAM,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;IACtB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,aAAa,CAAC,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAClD;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAA;IACrB,OAAO,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,CAAA;IACxB,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,gBAAgB,EAAE,MAAM,EAAE,CAAA;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,GAAG,CAAA;IACR,WAAW,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,OAAO,CAAA;CACpD;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,KAAK;gBAAL,KAAK,EAAE,UAAU;IAE/B,YAAY,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAgCzC,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAgBnE,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;CAkC9D"}

package/dist/builder-eval/three-layer-eval.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Three-layer evaluation — the canonical scoring breakdown for
+ * builder-of-builders workflows.
+ *
+ *   meta_score:    did the builder understand + satisfy user intent?
+ *                  (judge verdict attached to the builder run)
+ *   build_score:   did the generated scaffold build + pass its own tests?
+ *                  (outcome.score on the app-build child run)
+ *   runtime_score: did the generated agent pass its domain scenarios?
+ *                  (mean outcome.score over app-runtime grandchild runs)
+ *
+ * Returns a structured report per project. The cross-layer correlation
+ * is the highest-leverage signal the framework computes — if
+ * meta_score doesn't predict runtime_score, the builder's self-scoring
+ * is broken.
+ *
+ * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a
+ * scaffold-builder eval that grades compose + build without driving a
+ * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures
+ * meta + build only. Consumers can tell the two apart without having to
+ * interpret null-runtime as either "not yet computed" or "N/A for this
+ * project shape".
+ */
+import type { TraceStore } from '../trace/store';
+export type ProjectKind = 'full' | 'scaffold-only';
+export interface ThreeLayerProjectReport {
+    projectId: string;
+    /**
+     * `'full'` when the project has at least one `app-runtime` run;
+     * `'scaffold-only'` when it only has meta + build layers. Lets
+     * downstream consumers treat a null runtime score as expected
+     * (scaffold-only) vs. missing (full, pipeline broke).
+     */
+    kind: ProjectKind;
+    builderRunId?: string;
+    /** Judge-verdict score on the builder run (0..1 after normalization). */
+    metaScore: number | null;
+    buildRunId?: string;
+    /** 0..1 from the sandbox harness (testsPassed / testsTotal). */
+    buildScore: number | null;
+    appRuntimeRunIds: string[];
+    /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */
+    runtimeScore: number | null;
+    runtimePassRate: number | null;
+    /**
+     * Layer-aware completeness:
+     *   - `kind='full'`: all three layers scored
+     *   - `kind='scaffold-only'`: meta + build scored (runtime not applicable)
+     */
+    complete: boolean;
+}
+export declare function scoreProject(store: TraceStore, projectId: string): Promise<ThreeLayerProjectReport>;
+/** Aggregate scoring across every project in a corpus. */
+export declare function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]>;
+//# sourceMappingURL=three-layer-eval.d.ts.map

package/dist/builder-eval/three-layer-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"three-layer-eval.d.ts","sourceRoot":"","sources":["../../src/builder-eval/three-layer-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,eAAe,CAAA;AAElD,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAA;IACjB;;;;;OAKG;IACH,IAAI,EAAE,WAAW,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,yEAAyE;IACzE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,gEAAgE;IAChE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,4FAA4F;IAC5F,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B;;;;OAIG;IACH,QAAQ,EAAE,OAAO,CAAA;CAClB;AAED,wBAAsB,YAAY,CAChC,KAAK,EAAE,UAAU,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,uBAAuB,CAAC,CAoClC;AAED,0DAA0D;AAC1D,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAI5F"}

package/dist/campaign/analyst-surface.d.ts

@@ -0,0 +1,108 @@
+/**
+ * Make the trace-analyst's OWN prompt a GEPA-optimizable surface.
+ *
+ * The analyst that drives self-improvement is itself a prompt — and a
+ * hand-tuned one (a hardcoded, hand-versioned `const`). This module lets the
+ * loop optimize it: the analyst `actorDescription` becomes a `MutableSurface`
+ * that `gepaProposer` / `haloProposer` / any `SurfaceProposer` can mutate inside
+ * `runImprovementLoop` or `compareProposers`. That is the second-order loop —
+ * optimizing the optimizer's eyes, not just the agent's prompt.
+ *
+ * Two pieces, both deliberately small (the loop engine already exists — this
+ * only supplies the analyst-shaped dispatch + an objective scorer):
+ *
+ *  - `buildAnalystSurfaceDispatch` — `dispatchWithSurface(surface, scenario)`
+ *    runs `analyzeTraces` with `surface` as the actorDescription over the
+ *    scenario's fixed trace corpus, returning its findings.
+ *  - `failureModeRecallJudge` — a DETERMINISTIC judge (no LLM, no opinion)
+ *    that scores those findings against the scenario's GROUND-TRUTH failure
+ *    modes. This is what keeps optimizing the analyst prompt ungameable: the
+ *    labels come from objective signal (e.g. AppWorld `world.evaluate()` tells
+ *    us which task failed and which API calls were wrong), so we reward an
+ *    analyst for surfacing the failures that really happened — not for
+ *    pleasing a judge that could be talked into anything (Goodhart).
+ *
+ * Wiring (the loop is unchanged; you only pass these in):
+ *
+ *   const dispatchWithSurface = buildAnalystSurfaceDispatch({ analystOptions: { ai } })
+ *   await runImprovementLoop({
+ *     baselineSurface: TRACE_ANALYST_ACTOR_DESCRIPTION,  // the prompt under optimization
+ *     scenarios: trainScenarios,                          // labeled trace corpora
+ *     holdoutScenarios: heldOutScenarios,
+ *     dispatchWithSurface,
+ *     judges: [failureModeRecallJudge()],
+ *     proposer: gepaProposer({ baseUrl, apiKey }),
+ *     gate: heldOutGate({ minDelta: 0.02 }),
+ *     autoOnPromote: 'none',
+ *   })
+ */
+import type { AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult } from '../trace-analyst/analyst';
+import type { DispatchContext, JudgeConfig, MutableSurface, Scenario } from './types';
+/**
+ * A labeled trace scenario: a FIXED trace corpus plus the failure modes a
+ * competent analyst MUST surface from it. The labels are ground truth — the
+ * objective failures that actually occurred — which is what makes optimizing
+ * the analyst prompt against them meaningful rather than circular.
+ */
+export interface AnalystScenario extends Scenario {
+    kind: 'analyst-surface';
+    /** OTLP-JSONL path or an in-memory store of the traces to analyze. */
+    source: AnalyzeTracesOptions['source'];
+    /** The domain question handed to the analyst (framing lives here, not in
+     *  the surface under optimization). */
+    question: string;
+    /**
+     * Ground-truth failure modes a good analyst must identify. A finding "hits"
+     * a mode when it contains ANY of the mode's case-insensitive cues. Derive
+     * these from objective signal (failed task + which step broke), never from
+     * the analyst's own prior output.
+     */
+    expectedFailureModes: Array<{
+        id: string;
+        cues: string[];
+    }>;
+    /**
+     * Cues that mark a finding as HALLUCINATED / out-of-scope for this corpus —
+     * naming a tool, error, or failure that did not occur. Presence penalizes
+     * precision. Optional; omit to score recall only.
+     */
+    forbiddenCues?: string[];
+}
+/** The analyst's output for one scenario — the artifact the judge scores. */
+export interface AnalystArtifact {
+    answer: string;
+    findings: string[];
+    /** The hardcoded-prompt version the analyst reported (provenance only; the
+     *  optimized surface overrides the actual prompt text used). */
+    actorPromptVersion: string;
+}
+export interface BuildAnalystSurfaceDispatchOptions {
+    /**
+     * Everything `analyzeTraces` needs EXCEPT `actorDescription` (supplied by the
+     * surface under optimization) and `source` (supplied by the scenario). `ai`
+     * (the AxAIService) is required for a live run.
+     */
+    analystOptions: Omit<AnalyzeTracesOptions, 'actorDescription' | 'source'>;
+    /** Test seam: defaults to the real `analyzeTraces`. */
+    analyze?: (input: AnalyzeTracesInput, options: AnalyzeTracesOptions) => Promise<AnalyzeTracesResult>;
+}
+/**
+ * Build the `dispatchWithSurface(surface, scenario, ctx)` the improvement loop
+ * calls: run the analyst with `surface` as its actorDescription over the
+ * scenario's trace corpus and return its findings.
+ */
+export declare function buildAnalystSurfaceDispatch(opts: BuildAnalystSurfaceDispatchOptions): (surface: MutableSurface, scenario: AnalystScenario, ctx: DispatchContext) => Promise<AnalystArtifact>;
+export interface FailureModeRecallJudgeOptions {
+    /** Weight on recall when precision is also scored (forbiddenCues present).
+     *  Default 0.5 (equal). Recall-only when no forbiddenCues exist. */
+    recallWeight?: number;
+}
+/**
+ * Deterministic, ground-truth judge for analyst findings. Composite =
+ * recall of the scenario's `expectedFailureModes` (optionally blended with a
+ * precision term that penalizes findings tripping `forbiddenCues`). No LLM —
+ * the score is a function of the labels, so the analyst prompt is optimized
+ * toward surfacing real failures, not toward a judge it can flatter.
+ */
+export declare function failureModeRecallJudge(opts?: FailureModeRecallJudgeOptions): JudgeConfig<AnalystArtifact, AnalystScenario>;
+//# sourceMappingURL=analyst-surface.d.ts.map

package/dist/campaign/analyst-surface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyst-surface.d.ts","sourceRoot":"","sources":["../../src/campaign/analyst-surface.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EACV,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,0BAA0B,CAAA;AAEjC,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAErF;;;;;GAKG;AACH,MAAM,WAAW,eAAgB,SAAQ,QAAQ;IAC/C,IAAI,EAAE,iBAAiB,CAAA;IACvB,sEAAsE;IACtE,MAAM,EAAE,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IACtC;2CACuC;IACvC,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;;OAKG;IACH,oBAAoB,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAA;IAC3D;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB;AAED,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB;oEACgE;IAChE,kBAAkB,EAAE,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,kCAAkC;IACjD;;;;OAIG;IACH,cAAc,EAAE,IAAI,CAAC,oBAAoB,EAAE,kBAAkB,GAAG,QAAQ,CAAC,CAAA;IACzE,uDAAuD;IACvD,OAAO,CAAC,EAAE,CACR,KAAK,EAAE,kBAAkB,EACzB,OAAO,EAAE,oBAAoB,KAC1B,OAAO,CAAC,mBAAmB,CAAC,CAAA;CAClC;AAYD;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,kCAAkC,GACvC,CACD,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,eAAe,EACzB,GAAG,EAAE,eAAe,KACjB,OAAO,CAAC,eAAe,CAAC,CAc5B;AAED,MAAM,WAAW,6BAA6B;IAC5C;wEACoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,GAAE,6BAAkC,GACvC,WAAW,CAAC,eAAe,EAAE,eAAe,CAAC,CA6C/C"}

package/dist/campaign/analyst-surface.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=analyst-surface.test.d.ts.map

package/dist/campaign/analyst-surface.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyst-surface.test.d.ts","sourceRoot":"","sources":["../../src/campaign/analyst-surface.test.ts"],"names":[],"mappings":""}

package/dist/campaign/auto-pr.d.ts

@@ -0,0 +1,46 @@
+/**
+ * `openAutoPr` — thin shell-out helper for the `runImprovementLoop` preset's
+ * `autoOnPromote: 'pr'` mode. Substitutes for the per-product PR-opening
+ * code consumers duplicated 4 times. The PR body includes the campaign's
+ * manifest hash, gate verdict, and scorecard summary so reviewers can see
+ * exactly what was promoted + why.
+ *
+ * NOT a deploy mechanism — this only OPENS a PR. The human reviews + merges.
+ * The Shape B (`autoOnPromote: 'config'`) live-runtime-mutation path is
+ * deferred to Pass B with the full shadow / canary / rollback stack.
+ */
+import type { CampaignResult, GateResult, Scenario } from './types';
+export interface OpenAutoPrOptions<TArtifact, TScenario extends Scenario> {
+    /** Campaign result to attach to the PR. */
+    result: CampaignResult<TArtifact, TScenario>;
+    /** Gate verdict explaining the promotion. Substrate refuses to open a PR
+     *  when `gate.decision !== 'ship'` — fails loud. */
+    gate: GateResult;
+    /** Promoted surface diff — typically the new system prompt addendum or
+     *  full profile diff. Substrate writes it as the PR body. */
+    promotedDiff: string;
+    /** GH owner/repo target (e.g., `tangle-network/gtm-agent`). */
+    ghOwner: string;
+    ghRepo: string;
+    /** Branch name for the PR. Default `auto/<manifestHash[:12]>`. */
+    branch?: string;
+    /** PR title. Default includes manifest hash. */
+    title?: string;
+    /** Whether to actually open the PR or just dry-run. Default reads
+     *  `GH_AUTO_PR_TOKEN` env — present = open, absent = dry-run. */
+    dryRun?: boolean;
+    /** Test seam — substitute `gh pr create` invocation. */
+    ghExec?: (args: string[]) => {
+        stdout: string;
+        stderr: string;
+        status: number;
+    };
+}
+export interface OpenAutoPrResult {
+    opened: boolean;
+    prUrl?: string;
+    dryRun: boolean;
+    reason: string;
+}
+export declare function openAutoPr<TArtifact, TScenario extends Scenario>(options: OpenAutoPrOptions<TArtifact, TScenario>): OpenAutoPrResult;
+//# sourceMappingURL=auto-pr.d.ts.map

package/dist/campaign/auto-pr.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auto-pr.d.ts","sourceRoot":"","sources":["../../src/campaign/auto-pr.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAEnE,MAAM,WAAW,iBAAiB,CAAC,SAAS,EAAE,SAAS,SAAS,QAAQ;IACtE,2CAA2C;IAC3C,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IAC5C;wDACoD;IACpD,IAAI,EAAE,UAAU,CAAA;IAChB;iEAC6D;IAC7D,YAAY,EAAE,MAAM,CAAA;IACpB,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,gDAAgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;qEACiE;IACjE,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,wDAAwD;IACxD,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;CAChF;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,OAAO,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;CACf;AAED,wBAAgB,UAAU,CAAC,SAAS,EAAE,SAAS,SAAS,QAAQ,EAC9D,OAAO,EAAE,iBAAiB,CAAC,SAAS,EAAE,SAAS,CAAC,GAC/C,gBAAgB,CAgDlB"}

package/dist/campaign/distillation/agreement-judge.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Agreement judge for teacher→student distillation. Scores a STUDENT artifact
+ * (the cheap analyst's produced label) against the GoldScenario's gold label
+ * (the teacher's verdict). The score IS the distillation objective: 1.0 means
+ * the student reproduced the teacher exactly, 0.0 means total disagreement.
+ *
+ * The comparison function is INJECTED (`compareLabels`) so the judge is
+ * domain-agnostic — distilling a skill-audit analyst, a triage analyst, or any
+ * other student is a one-line comparator swap. A default `fieldAgreement`
+ * comparator is provided for the common case: a flat verdict object with
+ * categorical and array fields.
+ *
+ * Everything here is PURE + unit-testable — no LLM. (The student spends tokens
+ * producing the artifact; scoring it against frozen gold does not.)
+ */
+import type { JudgeConfig } from '../types';
+import type { GoldScenario } from './gold-scenarios';
+/** What an injected comparator returns: a [0,1] composite plus the per-field
+ *  (per-dimension) agreement breakdown the GEPA proposer reflects on to learn
+ *  WHICH part of the verdict the student is getting wrong. */
+export interface AgreementResult {
+    /** Overall agreement in [0,1]. */
+    score: number;
+    /** Per-dimension agreement in [0,1] — keyed by field/aspect name. The
+     *  reflective proposer surfaces the weakest of these as the lever to fix. */
+    dimensions: Record<string, number>;
+}
+/** Compare a produced label against a gold label → agreement. Injected so the
+ *  judge is domain-agnostic. */
+export type CompareLabels<TProduced = unknown, TLabel = unknown> = (produced: TProduced, gold: TLabel) => AgreementResult;
+export interface BuildAgreementJudgeOptions<TProduced = unknown, TLabel = unknown> {
+    /** Judge name surfaced in `CampaignResult.aggregates.byJudge` + the gate. */
+    name?: string;
+    /** The agreement function — produced student label vs gold teacher label. */
+    compareLabels: CompareLabels<TProduced, TLabel>;
+    /** Dimension keys the judge declares up-front (for `JudgeConfig.dimensions`).
+     *  When omitted, the dimensions present on the first scored result are used
+     *  for display only; the composite is unaffected. */
+    dimensionKeys?: string[];
+    /** Only score `gold`-kind scenarios. Default true — a mixed campaign won't
+     *  mis-apply the agreement judge to non-gold scenarios. */
+    goldOnly?: boolean;
+}
+/** Build a `JudgeConfig` that scores a produced student artifact against the
+ *  scenario's gold label. Conforms to the substrate `JudgeConfig` contract:
+ *  `score({artifact, scenario, signal}) => JudgeScore`. The `composite` is the
+ *  comparator's `score`; `dimensions` carries its per-field breakdown plus the
+ *  scalar `agreement` so a single-dimension consumer still sees the number. */
+export declare function buildAgreementJudge<TProduced, TInput, TLabel>(options: BuildAgreementJudgeOptions<TProduced, TLabel>): JudgeConfig<TProduced, GoldScenario<TInput, TLabel>>;
+export interface FieldAgreementSpec {
+    /** Categorical fields — scored exact-match (1 if equal, else 0). Compared
+     *  with `===` after `JSON`-normalizing so `true`/`'high'`/`3` all work. */
+    categorical?: string[];
+    /** Array fields — scored by Jaccard overlap (|A∩B| / |A∪B|). Two empty
+     *  arrays agree perfectly (1.0). Order-insensitive; elements compared by
+     *  their `JSON.stringify`. */
+    array?: string[];
+}
+/** Default comparator: average per-field agreement over a flat verdict object.
+ *  Categorical fields score exact-match; array fields score set-overlap
+ *  (Jaccard). The composite is the unweighted mean across all declared fields,
+ *  so missing a single boolean (e.g. `public_leak_risk`) costs `1/nFields` of
+ *  the score — the leak-detection lever the audit cares about is a real,
+ *  non-trivial fraction of the objective, not rounding noise.
+ *
+ *  Pure. A field absent from BOTH produced + gold is treated as agreeing
+ *  (both undefined ⇒ 1.0); a field present in only one side disagrees. */
+export declare function fieldAgreement<TProduced extends Record<string, unknown>, TLabel>(spec: FieldAgreementSpec): CompareLabels<TProduced, TLabel>;
+//# sourceMappingURL=agreement-judge.d.ts.map

package/dist/campaign/distillation/agreement-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agreement-judge.d.ts","sourceRoot":"","sources":["../../../src/campaign/distillation/agreement-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAwB,MAAM,UAAU,CAAA;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAEpD;;8DAE8D;AAC9D,MAAM,WAAW,eAAe;IAC9B,kCAAkC;IAClC,KAAK,EAAE,MAAM,CAAA;IACb;iFAC6E;IAC7E,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACnC;AAED;gCACgC;AAChC,MAAM,MAAM,aAAa,CAAC,SAAS,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,IAAI,CACjE,QAAQ,EAAE,SAAS,EACnB,IAAI,EAAE,MAAM,KACT,eAAe,CAAA;AAEpB,MAAM,WAAW,0BAA0B,CAAC,SAAS,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO;IAC/E,6EAA6E;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,6EAA6E;IAC7E,aAAa,EAAE,aAAa,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IAC/C;;yDAEqD;IACrD,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;IACxB;+DAC2D;IAC3D,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAID;;;;+EAI+E;AAC/E,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAC3D,OAAO,EAAE,0BAA0B,CAAC,SAAS,EAAE,MAAM,CAAC,GACrD,WAAW,CAAC,SAAS,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CA6BtD;AAED,MAAM,WAAW,kBAAkB;IACjC;+EAC2E;IAC3E,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB;;kCAE8B;IAC9B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;CACjB;AAED;;;;;;;;0EAQ0E;AAC1E,wBAAgB,cAAc,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAC9E,IAAI,EAAE,kBAAkB,GACvB,aAAa,CAAC,SAAS,EAAE,MAAM,CAAC,CAoBlC"}