@tangle-network/agent-eval 0.129.0 → 0.130.1

package/dist/builder-eval/index.d.ts

@@ -1,546 +1,83 @@
-type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
-interface BudgetSpec {
-    tokens?: number;
-    wallMs?: number;
-    calls?: number;
-    usd?: number;
-}
-interface RunOutcome {
-    score?: number;
-    pass?: boolean;
-    failureClass?: FailureClass;
-    notes?: string;
-}
-/**
- * Layer — optional classification in a nested build workflow.
- * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
- * `app-build`: sandbox harness that compiled + tested the generated scaffold.
- * `app-runtime`: a run of the generated agent against a domain scenario.
- * `meta`: any meta-eval (judge replay, correlation analysis).
- */
-type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
-interface Run {
-    runId: string;
-    /**
-     * Stable identifier of the scenario being executed.
-     *
-     * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts
-     * input WITHOUT this field, substituting a sensible default
-     * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no
-     * curated scenario to anchor to (runtime / operator / meta-eval runs). This
-     * keeps the persisted shape unambiguous for downstream filters + aggregations
-     * while removing the boilerplate of inventing placeholder ids at the call site.
-     */
-    scenarioId: string;
-    variantId?: string;
-    datasetVersion?: string;
-    /** Git SHA of agent code at run time. */
-    codeSha?: string;
-    /** Hash of the prompt template + any system prompt. */
-    promptSha?: string;
-    /** Model id + date + system-prompt hash, concatenated. */
-    modelFingerprint?: string;
-    seed?: number;
-    /** Arbitrary environment markers (shell, docker version, tz). */
-    envFingerprint?: Record<string, string>;
-    /** Version of the redaction rules applied to this run. */
-    redactionVersion?: string;
-    /** Parent run in a nested build workflow. A builder run's children are
-     *  app-build runs; those children are app-runtime runs. */
-    parentRunId?: string;
-    /** Stable project identifier — groups runs across chats + sessions. */
-    projectId?: string;
-    /** Chat/conversation identifier within a project. */
-    chatId?: string;
-    /** Layer classification — hint for aggregation; not enforced. */
-    layer?: RunLayer;
-    startedAt: number;
-    endedAt?: number;
-    status: RunStatus;
-    outcome?: RunOutcome;
-    budget?: BudgetSpec;
-    /** Free-form labels for downstream grouping. */
-    tags?: Record<string, string>;
-}
-type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
-type SpanStatus = 'ok' | 'error';
-interface SpanBase {
-    spanId: string;
-    parentSpanId?: string;
-    runId: string;
-    kind: SpanKind;
-    name: string;
-    startedAt: number;
-    endedAt?: number;
-    status?: SpanStatus;
-    error?: string;
-    /** Anything not covered by typed fields. Kept deliberately free-form. */
-    attributes?: Record<string, unknown>;
-}
-interface Message {
-    role: 'system' | 'user' | 'assistant' | 'tool';
-    content: string;
-    tokens?: number;
-    /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
-    images?: Array<{
-        artifactId?: string;
-        url?: string;
-        mime?: string;
-    }>;
-}
-interface LlmSpan extends SpanBase {
-    kind: 'llm';
-    model: string;
-    messages: Message[];
-    output?: string;
-    inputTokens?: number;
-    /** All generated tokens, including the reasoning subset when present. */
-    outputTokens?: number;
-    cachedTokens?: number;
-    cacheWriteTokens?: number;
-    /** Reasoning-token subset of `outputTokens`. */
-    reasoningTokens?: number;
-    costUsd?: number;
-    finishReason?: string;
-}
-interface ToolSpan extends SpanBase {
-    kind: 'tool';
-    toolName: string;
-    args: unknown;
-    /** False when the source observed the call but did not capture its arguments. */
-    argsCaptured?: boolean;
-    result?: unknown;
-    latencyMs?: number;
-}
-interface RetrievalSpan extends SpanBase {
-    kind: 'retrieval';
-    query: string;
-    hits: Array<{
-        docId: string;
-        score: number;
-        content?: string;
-    }>;
-}
-interface JudgeSpan extends SpanBase {
-    kind: 'judge';
-    judgeId: string;
-    /** Span this judgment applies to. */
-    targetSpanId: string;
-    dimension: string;
-    /** Numeric score (free-range; interpretation up to the judge). */
-    score: number;
-    rationale?: string;
-    evidence?: string;
-}
-interface SandboxSpan extends SpanBase {
-    kind: 'sandbox';
-    image?: string;
-    command?: string;
-    exitCode?: number;
-    testsTotal?: number;
-    testsPassed?: number;
-    stdoutHash?: string;
-    stderrHash?: string;
-    /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
-    wallMs?: number;
-}
-interface GenericSpan extends SpanBase {
-    kind: 'agent' | 'custom';
-}
-type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
-type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
-interface TraceEvent {
-    eventId: string;
-    runId: string;
-    spanId?: string;
-    kind: EventKind;
-    timestamp: number;
-    payload: Record<string, unknown>;
-}
-interface BudgetLedgerEntry {
-    runId: string;
-    dimension: keyof BudgetSpec;
-    limit: number;
-    consumed: number;
-    remaining: number;
-    timestamp: number;
-    breached: boolean;
-    /** Span that triggered this entry, if any. */
-    spanId?: string;
-}
-interface Artifact {
-    artifactId: string;
-    runId: string;
-    spanId?: string;
-    contentType: string;
-    sizeBytes: number;
-    /** sha256 in hex. */
-    hash: string;
-    /** External storage URL (R2, S3, filesystem path). */
-    storageUrl?: string;
-    /** Inline content for small blobs — keep under ~64KB. */
-    inlineContent?: string;
-}
-type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
-
-interface RunFilter {
-    scenarioId?: string;
-    variantId?: string;
-    status?: RunStatus;
-    since?: number;
-    until?: number;
-    tag?: {
-        key: string;
-        value: string;
-    };
-    parentRunId?: string;
-    projectId?: string;
-    chatId?: string;
-    layer?: RunLayer;
-}
-interface SpanFilter {
-    runId?: string;
-    parentSpanId?: string;
-    kind?: SpanKind;
-    name?: string;
-    toolName?: string;
-    judgeId?: string;
-    since?: number;
-    until?: number;
-}
-interface EventFilter {
-    runId?: string;
-    spanId?: string;
-    kind?: EventKind;
-    since?: number;
-    until?: number;
-}
-interface TraceStore {
-    appendRun(run: Run): Promise<void>;
-    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
-    appendSpan(span: Span): Promise<void>;
-    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
-    appendEvent(event: TraceEvent): Promise<void>;
-    appendArtifact(artifact: Artifact): Promise<void>;
-    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
-    getRun(runId: string): Promise<Run | undefined>;
-    listRuns(filter?: RunFilter): Promise<Run[]>;
-    spans(filter?: SpanFilter): Promise<Span[]>;
-    events(filter?: EventFilter): Promise<TraceEvent[]>;
-    budget(runId: string): Promise<BudgetLedgerEntry[]>;
-    artifacts(runId: string): Promise<Artifact[]>;
-}
-
-/**
- * TraceEmitter — hierarchical span builder that auto-parents using an
- * internal stack. One emitter per Run; emitters do NOT share state.
- *
- * Convenience methods (`llm`, `tool`, `retrieval`, `judge`, `sandbox`)
- * return a `SpanHandle` with `.end()` / `.fail()` so callers don't
- * have to thread spanIds manually. For async workflows that can't use
- * the stack (e.g. fan-out parallel calls), pass `parentSpanId`
- * explicitly.
- */
-
-interface SpanHandle<S extends Span = Span> {
-    span: S;
-    end(patch?: Partial<S>): Promise<void>;
-    fail(error: string | Error, patch?: Partial<S>): Promise<void>;
-}
-interface RunCompleteHookContext {
-    runId: string;
-    emitter: TraceEmitter;
-    store: TraceStore;
-    /** Outcome the caller passed to `endRun` (undefined for `abortRun`). */
-    outcome?: RunOutcome;
-    /** Final run status. */
-    status: 'completed' | 'failed' | 'aborted';
-}
-type RunCompleteHook = (ctx: RunCompleteHookContext) => Promise<void> | void;
-interface TraceEmitterOptions {
-    runId?: string;
-    /** Inject a clock for deterministic tests. */
-    now?: () => number;
-    /** Inject an id generator for deterministic tests. */
-    id?: () => string;
-    /**
-     * Hooks fired after `endRun` / `abortRun` writes the final run state.
-     * Designed for trace-analyst auto-execution, integrity assertions, and
-     * outbound notifications. Hooks run sequentially in the order supplied.
-     *
-     * By default a hook that throws is swallowed and logged as a `note` event
-     * on the run — auto-orchestration must not crash the underlying flow.
-     * Set `hookErrors: 'throw'` to propagate.
-     */
-    onRunComplete?: RunCompleteHook[];
-    /** `'swallow'` (default) | `'throw'`. */
-    hookErrors?: 'swallow' | 'throw';
-}
-declare class TraceEmitter {
-    private store;
-    private stack;
-    private _runId;
-    private now;
-    private id;
-    private hooks;
-    private hookErrors;
-    constructor(store: TraceStore, options?: TraceEmitterOptions);
-    get runId(): string;
-    get traceStore(): TraceStore;
-    /** Append a hook after construction (e.g. attach the trace analyst). */
-    addRunCompleteHook(hook: RunCompleteHook): void;
-    /**
-     * Begin a Run.
-     *
-     * `scenarioId` is required on the persisted Run shape — every Run downstream
-     * gets a non-empty scenarioId so filters and aggregations stay simple — but
-     * the INPUT here accepts it as optional. When omitted, startRun substitutes
-     * a sensible default (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) so
-     * runtime / operator / meta-eval runs that have no curated-scenario corpus
-     * to anchor to don't have to invent placeholder strings at the call site.
-     */
-    startRun(run: Omit<Run, 'runId' | 'scenarioId' | 'startedAt' | 'status'> & {
-        scenarioId?: string;
-    }): Promise<Run>;
-    endRun(outcome?: RunOutcome): Promise<void>;
-    abortRun(reason: string): Promise<void>;
-    private runHooks;
-    span<S extends Span = Span>(init: {
-        kind: SpanKind;
-        name: string;
-        parentSpanId?: string;
-        attributes?: Record<string, unknown>;
-    } & Partial<Omit<S, 'spanId' | 'runId' | 'startedAt' | 'kind' | 'name'>>): Promise<SpanHandle<S>>;
-    private handle;
-    private pop;
-    llm(init: Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<LlmSpan>>;
-    tool(init: Omit<ToolSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<ToolSpan>>;
-    retrieval(init: Omit<RetrievalSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<RetrievalSpan>>;
-    recordJudge(verdict: Omit<JudgeSpan, 'spanId' | 'runId' | 'kind' | 'startedAt' | 'endedAt'>): Promise<JudgeSpan>;
-    sandbox(init: Omit<SandboxSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<SandboxSpan>>;
-    emit(event: {
-        kind: EventKind;
-        spanId?: string;
-        payload?: Record<string, unknown>;
-    }): Promise<TraceEvent>;
-    recordBudget(entry: Omit<BudgetLedgerEntry, 'runId' | 'timestamp'> & {
-        timestamp?: number;
-    }): Promise<BudgetLedgerEntry>;
-    recordArtifact(artifact: Omit<Artifact, 'artifactId' | 'runId'>): Promise<Artifact>;
-    /**
-     * Runs `fn` inside a span; auto-ends on success, auto-fails on throw.
-     * Returns the fn's return value. Use this for the 95% case.
-     */
-    within<T>(init: Parameters<TraceEmitter['span']>[0], fn: (handle: SpanHandle) => Promise<T>): Promise<T>;
-}
-
-/**
- * SandboxHarness — executes a scenario in an isolated environment and
- * emits a rich SandboxSpan into the trace.
- *
- * Two built-in drivers:
- *   - `SubprocessSandboxDriver` — spawn in a local cwd with env vars.
- *     Fast, no dependencies, fine for unit tests and most CI gates.
- *   - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;
- *     shells out to `docker run`. Stronger isolation, slower startup.
- *
- * Consumers implement `SandboxDriver` for custom backends (Firecracker,
- * Cloudflare sandbox product, etc.). The harness doesn't care which.
- */
-
-interface HarnessConfig {
-    /** Setup command (e.g. "pnpm install"). Non-zero exit fails the run. */
-    setupCommand?: string;
-    /** Run command (e.g. "pnpm build"). */
-    runCommand?: string;
-    /** Test command (e.g. "pnpm test --run"). Drives the test count + pass count. */
-    testCommand?: string;
-    /** Absolute cwd for the subprocess driver. Ignored by docker driver. */
-    cwd?: string;
-    /** Max wall-clock per phase in ms. Default 10 minutes. */
-    timeoutMs?: number;
-    /**
-     * Cap on captured stdout+stderr bytes per phase. A runaway process can
-     * otherwise grow the in-memory buffer without bound. Once hit, further
-     * output is dropped and `outputTruncated` is set. Default 16 MiB.
-     */
-    maxOutputBytes?: number;
-    /** Image for the docker driver. */
-    image?: string;
-    /** Extra env vars (validated; shell-escaped). */
-    env?: Record<string, string>;
-    /** Parser for the test output — maps stdout/stderr/exit code → pass count. */
-    testParser?: TestOutputParser;
-}
-interface TestOutputParser {
-    id: string;
-    parse(stdout: string, stderr: string, exitCode: number): {
-        testsTotal: number;
-        testsPassed: number;
-    } | undefined;
-}
-interface SandboxResult {
-    phase: 'setup' | 'run' | 'test';
-    exitCode: number;
-    stdout: string;
-    stderr: string;
-    wallMs: number;
-    testsTotal?: number;
-    testsPassed?: number;
-    /**
-     * True when the process was killed because it exceeded `timeoutMs`. A
-     * SIGKILLed child can still close with exit code 0; callers MUST treat
-     * a timed-out phase as a hard failure regardless of `exitCode`, never
-     * as a pass. `undefined`/`false` means the process completed on its own.
-     */
-    killedByTimeout?: boolean;
-    /**
-     * True when captured stdout/stderr hit `maxOutputBytes` and further
-     * output was dropped. The result is still returned (the process was
-     * not killed for this), but downstream parsers see truncated text.
-     */
-    outputTruncated?: boolean;
-}
-interface SandboxDriver {
-    id: string;
-    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
-}
-interface SandboxHarnessResult {
-    passed: boolean;
-    setup?: SandboxResult;
-    run?: SandboxResult;
-    test?: SandboxResult;
-    totalWallMs: number;
-    /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */
-    score: number;
-}
-
-/**
- * TestGradedScenario — a scenario whose score comes from a test suite.
- *
- * This is the SWE-bench pattern generalized. The scenario ships:
- *   - fixture data (setup instructions)
- *   - a test command the harness runs
- *   - optional assertion overrides
- *
- * The runner emits a run, delegates to SandboxHarness, records the
- * outcome, and returns a structured verdict. Consumers bind their own
- * agent execution to this contract.
- */
-
-interface TestGradedScenario {
-    id: string;
-    description?: string;
-    harness: HarnessConfig;
-    /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */
-    passThreshold?: number;
-    /** Provenance for dataset tracking. */
-    datasetVersion?: string;
-    /** Free-form tags (difficulty, category, etc.). */
-    tags?: Record<string, string>;
-}
-interface TestGradedRunResult {
-    runId: string;
-    scenario: TestGradedScenario;
-    harness: SandboxHarnessResult;
-    pass: boolean;
-    score: number;
-    failureClass?: FailureClass;
-}
-
-/**
- * BuilderSession — ties a builder-of-builders workflow together.
- *
- * Models agent-builder's shape: Project → Chat → Edit → Ship → App →
- * AppAgent. Each layer is a Run (linked via parentRunId). The
- * framework-enforced invariants:
- *
- *   - One Project → many Chats; chatId scopes runs within a project.
- *   - One Chat = one builder Run with `layer='builder'`.
- *   - One Ship = one child Run with `layer='app-build'` + SandboxHarness.
- *   - One AppScenario = one grandchild Run with `layer='app-runtime'`.
- *
- * Consumers obtain a BuilderSession, call `startChat`, drive the
- * builder agent (emitting spans), and call `ship` / `runAppScenario`
- * as the workflow progresses. The session reconstructs itself from
- * trace data via `resume(store, projectId)`.
- */
-
+import { f as Run } from "../schema-BtVldJ3T.js";
+import { s as TraceStore } from "../store-CT9YIIve.js";
+import { i as TraceEmitter } from "../emitter-DGQGoLyj.js";
+import { l as SandboxHarnessResult, n as TestGradedRunResult, o as HarnessConfig, r as TestGradedScenario, s as SandboxDriver } from "../test-graded-scenario-D1TaI2va.js";
+//#region src/builder-eval/builder-session.d.ts
 interface BuilderSessionInit {
-    projectId: string;
-    chatId?: string;
-    /** Free-form: user's task description, project name, etc. Stored on the builder Run. */
-    tags?: Record<string, string>;
+  projectId: string;
+  chatId?: string;
+  /** Free-form: user's task description, project name, etc. Stored on the builder Run. */
+  tags?: Record<string, string>;
 }
 interface ShipOptions {
-    harness: HarnessConfig;
-    driver?: SandboxDriver;
-    /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */
-    scenarioId?: string;
+  harness: HarnessConfig;
+  driver?: SandboxDriver;
+  /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */
+  scenarioId?: string;
 }
 interface RunAppScenarioOptions {
-    scenario: TestGradedScenario;
-    /** Harness driver override; defaults to the one the session was created with. */
-    driver?: SandboxDriver;
+  scenario: TestGradedScenario;
+  /** Harness driver override; defaults to the one the session was created with. */
+  driver?: SandboxDriver;
 }
 declare class BuilderSession {
-    private store;
-    private builderEmitter;
-    readonly projectId: string;
-    readonly chatId: string;
-    private builderRunId?;
-    private lastBuildRunId?;
-    private defaultDriver?;
-    constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver);
-    /** Start the builder (L0) run for this chat. Returns the runId. */
-    startChat(scenarioId?: string): Promise<string>;
-    /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */
-    get emitter(): TraceEmitter;
-    /**
-     * Ship the project's generated app: run the sandbox harness as a child
-     * Run (`layer='app-build'`). Returns the build result + runId.
-     */
-    ship(options: ShipOptions): Promise<{
-        runId: string;
-        result: SandboxHarnessResult;
-    }>;
-    /**
-     * Run a domain scenario against the just-built app as a grandchild Run
-     * (`layer='app-runtime'`). The `ship` call must precede this so the
-     * parent is set correctly; if no build exists yet the session attaches
-     * directly to the builder run (useful for prototypes).
-     */
-    runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult>;
-    /** Record an end-of-chat meta score (judge verdict on whether the builder
-     *  served the user's intent). Accepts a numeric score + optional rationale. */
-    recordMetaScore(score: number, rationale?: string): Promise<void>;
-    /** Close the builder Run with a final outcome. */
-    endChat(outcome: {
-        pass: boolean;
-        score?: number;
-        notes?: string;
-    }): Promise<void>;
-    /**
-     * Inline app-runtime run — for cases where the "scenario" isn't a
-     * SWE-bench-style test suite but a live agent interaction (LLM chat,
-     * domain flow). Returns an emitter bound to a fresh Run in the
-     * `app-runtime` layer; caller emits spans inside and calls
-     * `.endRun()` with the final verdict.
-     */
-    startAppRuntime(scenarioId: string): Promise<TraceEmitter>;
-    /**
-     * Lightweight "ship marker" — record an app-build Run with a caller-
-     * provided verdict. Use when there isn't a sandbox harness to run but
-     * you still want to mark the build state at publish time.
-     */
-    recordShipMarker(args: {
-        pass: boolean;
-        score: number;
-        scenarioId?: string;
-        notes?: string;
-    }): Promise<string>;
-    get lastBuildRunIdValue(): string | undefined;
-    get builderRunIdValue(): string | undefined;
+  private store;
+  private builderEmitter;
+  readonly projectId: string;
+  readonly chatId: string;
+  private builderRunId?;
+  private lastBuildRunId?;
+  private defaultDriver?;
+  constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver);
+  /** Start the builder (L0) run for this chat. Returns the runId. */
+  startChat(scenarioId?: string): Promise<string>;
+  /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */
+  get emitter(): TraceEmitter;
+  /**
+   * Ship the project's generated app: run the sandbox harness as a child
+   * Run (`layer='app-build'`). Returns the build result + runId.
+   */
+  ship(options: ShipOptions): Promise<{
+    runId: string;
+    result: SandboxHarnessResult;
+  }>;
+  /**
+   * Run a domain scenario against the just-built app as a grandchild Run
+   * (`layer='app-runtime'`). The `ship` call must precede this so the
+   * parent is set correctly; if no build exists yet the session attaches
+   * directly to the builder run (useful for prototypes).
+   */
+  runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult>;
+  /** Record an end-of-chat meta score (judge verdict on whether the builder
+   *  served the user's intent). Accepts a numeric score + optional rationale. */
+  recordMetaScore(score: number, rationale?: string): Promise<void>;
+  /** Close the builder Run with a final outcome. */
+  endChat(outcome: {
+    pass: boolean;
+    score?: number;
+    notes?: string;
+  }): Promise<void>;
+  /**
+   * Inline app-runtime run — for cases where the "scenario" isn't a
+   * SWE-bench-style test suite but a live agent interaction (LLM chat,
+   * domain flow). Returns an emitter bound to a fresh Run in the
+   * `app-runtime` layer; caller emits spans inside and calls
+   * `.endRun()` with the final verdict.
+   */
+  startAppRuntime(scenarioId: string): Promise<TraceEmitter>;
+  /**
+   * Lightweight "ship marker" — record an app-build Run with a caller-
+   * provided verdict. Use when there isn't a sandbox harness to run but
+   * you still want to mark the build state at publish time.
+   */
+  recordShipMarker(args: {
+    pass: boolean;
+    score: number;
+    scenarioId?: string;
+    notes?: string;
+  }): Promise<string>;
+  get lastBuildRunIdValue(): string | undefined;
+  get builderRunIdValue(): string | undefined;
 }
 /**
  * Reconstruct the most recent BuilderSession state for a given project —
@@ -548,148 +85,99 @@ declare class BuilderSession {
  * this is how a resumed session finds its place in the edit history.
  */
 declare function resumeBuilderSession(store: TraceStore, projectId: string): Promise<{
-    projectId: string;
-    chatRuns: Run[];
-    lastBuilderRun?: Run;
-    lastBuildRun?: Run;
-    lastAppRuntimeRuns: Run[];
+  projectId: string;
+  chatRuns: Run[];
+  lastBuilderRun?: Run;
+  lastBuildRun?: Run;
+  lastAppRuntimeRuns: Run[];
 }>;
-
-/**
- * Three-layer evaluation — the canonical scoring breakdown for
- * builder-of-builders workflows.
- *
- *   meta_score:    did the builder understand + satisfy user intent?
- *                  (judge verdict attached to the builder run)
- *   build_score:   did the generated scaffold build + pass its own tests?
- *                  (outcome.score on the app-build child run)
- *   runtime_score: did the generated agent pass its domain scenarios?
- *                  (mean outcome.score over app-runtime grandchild runs)
- *
- * Returns a structured report per project. The cross-layer correlation
- * is the highest-leverage signal the framework computes — if
- * meta_score doesn't predict runtime_score, the builder's self-scoring
- * is broken.
- *
- * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a
- * scaffold-builder eval that grades compose + build without driving a
- * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures
- * meta + build only. Consumers can tell the two apart without having to
- * interpret null-runtime as either "not yet computed" or "N/A for this
- * project shape".
- */
-
+//#endregion
+//#region src/builder-eval/three-layer-eval.d.ts
 type ProjectKind = 'full' | 'scaffold-only';
 interface ThreeLayerProjectReport {
-    projectId: string;
-    /**
-     * `'full'` when the project has at least one `app-runtime` run;
-     * `'scaffold-only'` when it only has meta + build layers. Lets
-     * downstream consumers treat a null runtime score as expected
-     * (scaffold-only) vs. missing (full, pipeline broke).
-     */
-    kind: ProjectKind;
-    builderRunId?: string;
-    /** Judge-verdict score on the builder run (0..1 after normalization). */
-    metaScore: number | null;
-    buildRunId?: string;
-    /** 0..1 from the sandbox harness (testsPassed / testsTotal). */
-    buildScore: number | null;
-    appRuntimeRunIds: string[];
-    /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */
-    runtimeScore: number | null;
-    runtimePassRate: number | null;
-    /**
-     * Layer-aware completeness:
-     *   - `kind='full'`: all three layers scored
-     *   - `kind='scaffold-only'`: meta + build scored (runtime not applicable)
-     */
-    complete: boolean;
+  projectId: string;
+  /**
+   * `'full'` when the project has at least one `app-runtime` run;
+   * `'scaffold-only'` when it only has meta + build layers. Lets
+   * downstream consumers treat a null runtime score as expected
+   * (scaffold-only) vs. missing (full, pipeline broke).
+   */
+  kind: ProjectKind;
+  builderRunId?: string;
+  /** Judge-verdict score on the builder run (0..1 after normalization). */
+  metaScore: number | null;
+  buildRunId?: string;
+  /** 0..1 from the sandbox harness (testsPassed / testsTotal). */
+  buildScore: number | null;
+  appRuntimeRunIds: string[];
+  /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */
+  runtimeScore: number | null;
+  runtimePassRate: number | null;
+  /**
+   * Layer-aware completeness:
+   *   - `kind='full'`: all three layers scored
+   *   - `kind='scaffold-only'`: meta + build scored (runtime not applicable)
+   */
+  complete: boolean;
 }
 declare function scoreProject(store: TraceStore, projectId: string): Promise<ThreeLayerProjectReport>;
 /** Aggregate scoring across every project in a corpus. */
 declare function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]>;
-
-/**
- * Meta-eval correlation — the highest-leverage signal in the framework.
- *
- * Given a corpus of three-layer project reports, compute how well each
- * pair of layers correlates. The question we care about most:
- *
- *   Does `metaScore` (what the builder thinks it did) predict
- *   `runtimeScore` (what the user actually gets)?
- *
- * If r < ~0.4, the builder's self-scoring is broken — it's optimizing
- * for something other than real-world success. If r > 0.7, meta_score
- * is a usable proxy and can drive CI gates cheaply.
- *
- * Non-parametric rank correlation (Spearman) is also reported because
- * meta scores are often ordinal-ish.
- */
-
+//#endregion
+//#region src/builder-eval/correlation.d.ts
 interface LayerCorrelation {
-    n: number;
-    pearson: number;
-    spearman: number;
+  n: number;
+  pearson: number;
+  spearman: number;
 }
 interface CorrelationReport {
-    /** Pairs present in the corpus (layers with ≥ 2 matched data points). */
-    metaVsBuild?: LayerCorrelation;
-    metaVsRuntime?: LayerCorrelation;
-    buildVsRuntime?: LayerCorrelation;
-    /** Number of complete projects (all 3 scores present). */
-    completeProjects: number;
+  /** Pairs present in the corpus (layers with ≥ 2 matched data points). */
+  metaVsBuild?: LayerCorrelation;
+  metaVsRuntime?: LayerCorrelation;
+  buildVsRuntime?: LayerCorrelation;
+  /** Number of complete projects (all 3 scores present). */
+  completeProjects: number;
 }
 declare function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport;
-
-/**
- * ProjectRegistry — project-level aggregation over the trace corpus.
- *
- * Thin reader over TraceStore that answers the questions a chat-first,
- * resumable UI needs:
- *   - listProjects() → project IDs with latest activity
- *   - projectTimeline(id) → chats + builds + runtime runs, chronological
- *   - projectChats(id) → chat-level summaries (turn count, outcome)
- *
- * All queries are pure reads; no state duplication.
- */
-
+//#endregion
+//#region src/builder-eval/project-registry.d.ts
 interface ProjectSummary {
-    projectId: string;
-    chatCount: number;
-    buildCount: number;
-    appRuntimeCount: number;
-    lastActivityAt: number;
-    latestChatId?: string;
-    latestOutcome?: {
-        pass: boolean;
-        score?: number;
-    };
+  projectId: string;
+  chatCount: number;
+  buildCount: number;
+  appRuntimeCount: number;
+  lastActivityAt: number;
+  latestChatId?: string;
+  latestOutcome?: {
+    pass: boolean;
+    score?: number;
+  };
 }
 interface ChatSummary {
-    chatId: string;
-    projectId: string;
-    builderRunId: string;
-    startedAt: number;
-    endedAt?: number;
-    status: Run['status'];
-    outcome?: Run['outcome'];
-    /** Counts of spans emitted during the chat. */
-    llmTurns?: number;
-    toolCalls?: number;
-    buildRunId?: string;
-    appRuntimeRunIds: string[];
+  chatId: string;
+  projectId: string;
+  builderRunId: string;
+  startedAt: number;
+  endedAt?: number;
+  status: Run['status'];
+  outcome?: Run['outcome'];
+  /** Counts of spans emitted during the chat. */
+  llmTurns?: number;
+  toolCalls?: number;
+  buildRunId?: string;
+  appRuntimeRunIds: string[];
 }
 interface ProjectTimelineEntry {
-    run: Run;
-    layerBucket: 'chat' | 'build' | 'runtime' | 'other';
+  run: Run;
+  layerBucket: 'chat' | 'build' | 'runtime' | 'other';
 }
 declare class ProjectRegistry {
-    private store;
-    constructor(store: TraceStore);
-    listProjects(): Promise<ProjectSummary[]>;
-    projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]>;
-    projectChats(projectId: string): Promise<ChatSummary[]>;
-}
-
-export { BuilderSession, type BuilderSessionInit, type ChatSummary, type CorrelationReport, type LayerCorrelation, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type RunAppScenarioOptions, type ShipOptions, type ThreeLayerProjectReport, correlateLayers, resumeBuilderSession, scoreAllProjects, scoreProject };
+  private store;
+  constructor(store: TraceStore);
+  listProjects(): Promise<ProjectSummary[]>;
+  projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]>;
+  projectChats(projectId: string): Promise<ChatSummary[]>;
+}
+//#endregion
+export { BuilderSession, BuilderSessionInit, ChatSummary, CorrelationReport, LayerCorrelation, ProjectKind, ProjectRegistry, ProjectSummary, ProjectTimelineEntry, RunAppScenarioOptions, ShipOptions, ThreeLayerProjectReport, correlateLayers, resumeBuilderSession, scoreAllProjects, scoreProject };
+//# sourceMappingURL=index.d.ts.map