@tangle-network/agent-eval 0.95.0 → 0.95.1

package/dist/builder-eval/index.d.ts

@@ -1,5 +1,250 @@
-export * from './builder-session';
-export * from './correlation';
-export * from './project-registry';
-export * from './three-layer-eval';
-//# sourceMappingURL=index.d.ts.map
+import { S as SandboxDriver, H as HarnessConfig, a as SandboxHarnessResult, T as TestGradedScenario, b as TestGradedRunResult } from '../test-graded-scenario-DeODGLra.js';
+import { T as TraceEmitter } from '../emitter-C2rqGH_l.js';
+import { R as Run } from '../schema-m0gsnbt3.js';
+import { T as TraceStore } from '../store-BcFXE6LG.js';
+
+/**
+ * BuilderSession — ties a builder-of-builders workflow together.
+ *
+ * Models agent-builder's shape: Project → Chat → Edit → Ship → App →
+ * AppAgent. Each layer is a Run (linked via parentRunId). The
+ * framework-enforced invariants:
+ *
+ *   - One Project → many Chats; chatId scopes runs within a project.
+ *   - One Chat = one builder Run with `layer='builder'`.
+ *   - One Ship = one child Run with `layer='app-build'` + SandboxHarness.
+ *   - One AppScenario = one grandchild Run with `layer='app-runtime'`.
+ *
+ * Consumers obtain a BuilderSession, call `startChat`, drive the
+ * builder agent (emitting spans), and call `ship` / `runAppScenario`
+ * as the workflow progresses. The session reconstructs itself from
+ * trace data via `resume(store, projectId)`.
+ */
+
+interface BuilderSessionInit {
+    projectId: string;
+    chatId?: string;
+    /** Free-form: user's task description, project name, etc. Stored on the builder Run. */
+    tags?: Record<string, string>;
+}
+interface ShipOptions {
+    harness: HarnessConfig;
+    driver?: SandboxDriver;
+    /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */
+    scenarioId?: string;
+}
+interface RunAppScenarioOptions {
+    scenario: TestGradedScenario;
+    /** Harness driver override; defaults to the one the session was created with. */
+    driver?: SandboxDriver;
+}
+declare class BuilderSession {
+    private store;
+    private builderEmitter;
+    readonly projectId: string;
+    readonly chatId: string;
+    private builderRunId?;
+    private lastBuildRunId?;
+    private defaultDriver?;
+    constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver);
+    /** Start the builder (L0) run for this chat. Returns the runId. */
+    startChat(scenarioId?: string): Promise<string>;
+    /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */
+    get emitter(): TraceEmitter;
+    /**
+     * Ship the project's generated app: run the sandbox harness as a child
+     * Run (`layer='app-build'`). Returns the build result + runId.
+     */
+    ship(options: ShipOptions): Promise<{
+        runId: string;
+        result: SandboxHarnessResult;
+    }>;
+    /**
+     * Run a domain scenario against the just-built app as a grandchild Run
+     * (`layer='app-runtime'`). The `ship` call must precede this so the
+     * parent is set correctly; if no build exists yet the session attaches
+     * directly to the builder run (useful for prototypes).
+     */
+    runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult>;
+    /** Record an end-of-chat meta score (judge verdict on whether the builder
+     *  served the user's intent). Accepts a numeric score + optional rationale. */
+    recordMetaScore(score: number, rationale?: string): Promise<void>;
+    /** Close the builder Run with a final outcome. */
+    endChat(outcome: {
+        pass: boolean;
+        score?: number;
+        notes?: string;
+    }): Promise<void>;
+    /**
+     * Inline app-runtime run — for cases where the "scenario" isn't a
+     * SWE-bench-style test suite but a live agent interaction (LLM chat,
+     * domain flow). Returns an emitter bound to a fresh Run in the
+     * `app-runtime` layer; caller emits spans inside and calls
+     * `.endRun()` with the final verdict.
+     */
+    startAppRuntime(scenarioId: string): Promise<TraceEmitter>;
+    /**
+     * Lightweight "ship marker" — record an app-build Run with a caller-
+     * provided verdict. Use when there isn't a sandbox harness to run but
+     * you still want to mark the build state at publish time.
+     */
+    recordShipMarker(args: {
+        pass: boolean;
+        score: number;
+        scenarioId?: string;
+        notes?: string;
+    }): Promise<string>;
+    get lastBuildRunIdValue(): string | undefined;
+    get builderRunIdValue(): string | undefined;
+}
+/**
+ * Reconstruct the most recent BuilderSession state for a given project —
+ * returns { builderRunId, lastBuildRunId, chatRuns }. For chat-first UIs
+ * this is how a resumed session finds its place in the edit history.
+ */
+declare function resumeBuilderSession(store: TraceStore, projectId: string): Promise<{
+    projectId: string;
+    chatRuns: Run[];
+    lastBuilderRun?: Run;
+    lastBuildRun?: Run;
+    lastAppRuntimeRuns: Run[];
+}>;
+
+/**
+ * Three-layer evaluation — the canonical scoring breakdown for
+ * builder-of-builders workflows.
+ *
+ *   meta_score:    did the builder understand + satisfy user intent?
+ *                  (judge verdict attached to the builder run)
+ *   build_score:   did the generated scaffold build + pass its own tests?
+ *                  (outcome.score on the app-build child run)
+ *   runtime_score: did the generated agent pass its domain scenarios?
+ *                  (mean outcome.score over app-runtime grandchild runs)
+ *
+ * Returns a structured report per project. The cross-layer correlation
+ * is the highest-leverage signal the framework computes — if
+ * meta_score doesn't predict runtime_score, the builder's self-scoring
+ * is broken.
+ *
+ * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a
+ * scaffold-builder eval that grades compose + build without driving a
+ * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures
+ * meta + build only. Consumers can tell the two apart without having to
+ * interpret null-runtime as either "not yet computed" or "N/A for this
+ * project shape".
+ */
+
+type ProjectKind = 'full' | 'scaffold-only';
+interface ThreeLayerProjectReport {
+    projectId: string;
+    /**
+     * `'full'` when the project has at least one `app-runtime` run;
+     * `'scaffold-only'` when it only has meta + build layers. Lets
+     * downstream consumers treat a null runtime score as expected
+     * (scaffold-only) vs. missing (full, pipeline broke).
+     */
+    kind: ProjectKind;
+    builderRunId?: string;
+    /** Judge-verdict score on the builder run (0..1 after normalization). */
+    metaScore: number | null;
+    buildRunId?: string;
+    /** 0..1 from the sandbox harness (testsPassed / testsTotal). */
+    buildScore: number | null;
+    appRuntimeRunIds: string[];
+    /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */
+    runtimeScore: number | null;
+    runtimePassRate: number | null;
+    /**
+     * Layer-aware completeness:
+     *   - `kind='full'`: all three layers scored
+     *   - `kind='scaffold-only'`: meta + build scored (runtime not applicable)
+     */
+    complete: boolean;
+}
+declare function scoreProject(store: TraceStore, projectId: string): Promise<ThreeLayerProjectReport>;
+/** Aggregate scoring across every project in a corpus. */
+declare function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]>;
+
+/**
+ * Meta-eval correlation — the highest-leverage signal in the framework.
+ *
+ * Given a corpus of three-layer project reports, compute how well each
+ * pair of layers correlates. The question we care about most:
+ *
+ *   Does `metaScore` (what the builder thinks it did) predict
+ *   `runtimeScore` (what the user actually gets)?
+ *
+ * If r < ~0.4, the builder's self-scoring is broken — it's optimizing
+ * for something other than real-world success. If r > 0.7, meta_score
+ * is a usable proxy and can drive CI gates cheaply.
+ *
+ * Non-parametric rank correlation (Spearman) is also reported because
+ * meta scores are often ordinal-ish.
+ */
+
+interface LayerCorrelation {
+    n: number;
+    pearson: number;
+    spearman: number;
+}
+interface CorrelationReport {
+    /** Pairs present in the corpus (layers with ≥ 2 matched data points). */
+    metaVsBuild?: LayerCorrelation;
+    metaVsRuntime?: LayerCorrelation;
+    buildVsRuntime?: LayerCorrelation;
+    /** Number of complete projects (all 3 scores present). */
+    completeProjects: number;
+}
+declare function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport;
+
+/**
+ * ProjectRegistry — project-level aggregation over the trace corpus.
+ *
+ * Thin reader over TraceStore that answers the questions a chat-first,
+ * resumable UI needs:
+ *   - listProjects() → project IDs with latest activity
+ *   - projectTimeline(id) → chats + builds + runtime runs, chronological
+ *   - projectChats(id) → chat-level summaries (turn count, outcome)
+ *
+ * All queries are pure reads; no state duplication.
+ */
+
+interface ProjectSummary {
+    projectId: string;
+    chatCount: number;
+    buildCount: number;
+    appRuntimeCount: number;
+    lastActivityAt: number;
+    latestChatId?: string;
+    latestOutcome?: {
+        pass: boolean;
+        score?: number;
+    };
+}
+interface ChatSummary {
+    chatId: string;
+    projectId: string;
+    builderRunId: string;
+    startedAt: number;
+    endedAt?: number;
+    status: Run['status'];
+    outcome?: Run['outcome'];
+    /** Counts of spans emitted during the chat. */
+    llmTurns?: number;
+    toolCalls?: number;
+    buildRunId?: string;
+    appRuntimeRunIds: string[];
+}
+interface ProjectTimelineEntry {
+    run: Run;
+    layerBucket: 'chat' | 'build' | 'runtime' | 'other';
+}
+declare class ProjectRegistry {
+    private store;
+    constructor(store: TraceStore);
+    listProjects(): Promise<ProjectSummary[]>;
+    projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]>;
+    projectChats(projectId: string): Promise<ChatSummary[]>;
+}
+
+export { BuilderSession, type BuilderSessionInit, type ChatSummary, type CorrelationReport, type LayerCorrelation, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type RunAppScenarioOptions, type ShipOptions, type ThreeLayerProjectReport, correlateLayers, resumeBuilderSession, scoreAllProjects, scoreProject };

package/dist/calibration-BPmzuVPk.d.ts

@@ -0,0 +1,101 @@
+import { T as TraceStore } from './store-BcFXE6LG.js';
+import { R as Run } from './schema-m0gsnbt3.js';
+import { O as OutcomeFilter, b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
+
+/**
+ * Correlation study — "does our eval score predict real-world outcomes?"
+ *
+ * This is the load-bearing signal. Takes a TraceStore + OutcomeStore,
+ * joins on runId, computes Pearson + Spearman + bootstrap CI for every
+ * (evalMetric, outcomeMetric) pair the caller declares.
+ *
+ * Without this number the framework is ornamental. With it and r > 0.6
+ * the framework is a moat — no other agent-eval tool publishes one.
+ */
+
+interface EvalMetricSpec {
+    id: string;
+    /** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface OutcomePair {
+    evalMetric: string;
+    outcomeMetric: string;
+}
+interface CorrelationResult {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    /** 95% bootstrap CI for Pearson. */
+    pearsonCi95: {
+        lower: number;
+        upper: number;
+    };
+    /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
+    verdict: 'strong' | 'moderate' | 'weak';
+}
+interface CorrelationStudyResult {
+    pairs: CorrelationResult[];
+    joinedSamples: number;
+    skippedRuns: number;
+}
+interface CorrelationStudyOptions {
+    /** Only join outcomes captured within this window after run.startedAt. */
+    maxCaptureLagMs?: number;
+    /** Restrict to a subset of outcomes (cohort, region, source). */
+    outcomeFilter?: OutcomeFilter;
+    /** Which outcome per run to use when multiple exist. Default 'latest'. */
+    reduction?: 'latest' | 'mean' | 'max';
+    /** Bootstrap iterations for the CI. Default 500. */
+    bootstrapIterations?: number;
+}
+declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;
+
+/**
+ * Calibration curve — binned "if eval says X, what does reality show?"
+ *
+ * Companion to correlationStudy. Raw correlation is a single number;
+ * the calibration curve shows *where* the eval is well-calibrated vs
+ * overconfident / underconfident. Buckets the eval metric, computes
+ * mean outcome per bucket, reports expected-calibration-error (ECE).
+ */
+
+interface CalibrationBin {
+    lower: number;
+    upper: number;
+    n: number;
+    evalMean: number;
+    outcomeMean: number;
+    /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
+    gap: number;
+}
+interface CalibrationReport {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    bins: CalibrationBin[];
+    /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
+    ece: number;
+    /** Max bin gap — upper bound on miscalibration. */
+    maxGap: number;
+}
+interface CalibrationOptions {
+    bins?: number;
+    /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
+    binning?: 'equal-width' | 'equal-frequency';
+    /** Clip eval values to [lo, hi] before binning. */
+    range?: {
+        lo: number;
+        hi: number;
+    };
+}
+interface CalibrationPair {
+    evalScore: number;
+    outcome: number;
+}
+declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
+declare function calibrationFromPairs(inputPairs: CalibrationPair[], evalMetric: string, outcomeMetric: string, options?: CalibrationOptions): CalibrationReport | null;
+
+export { type CalibrationBin as C, type EvalMetricSpec as E, type OutcomePair as O, type CalibrationOptions as a, type CalibrationPair as b, type CalibrationReport as c, type CorrelationResult as d, type CorrelationStudyOptions as e, type CorrelationStudyResult as f, calibrationCurve as g, calibrationFromPairs as h, correlationStudy as i };