@tangle-network/agent-eval 0.83.0 → 0.85.0

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.83.0",
+    "version": "0.85.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/runtime-trajectory-BLRiaifm.d.ts

@@ -0,0 +1,49 @@
+import { a as RunSplitTag } from './run-record-De9VarXR.js';
+
+interface RuntimeTrajectoryHookEvent {
+    id: string;
+    runId: string;
+    scenarioId?: string;
+    target: string;
+    phase: string;
+    timestamp: number;
+    stepIndex?: number;
+    parentId?: string;
+    payload?: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface RuntimeTrajectoryRecord {
+    id?: string;
+    scenarioId?: string;
+    splitTag?: RunSplitTag;
+    runtimeEvents?: unknown;
+    [key: string]: unknown;
+}
+interface RuntimeTrajectoryRunRecord {
+    runId: string;
+    scenarioId?: string;
+    splitTag: RunSplitTag;
+}
+interface RuntimeTrajectoryEvidenceSummary {
+    recordCount: number;
+    recordWithRuntimeEventsCount: number;
+    runtimeRunCount: number;
+    lifecycleEventCount: number;
+    defaultedSplitCount: number;
+}
+interface RuntimeTrajectoryEvidenceProjection {
+    runs: RuntimeTrajectoryRunRecord[];
+    events: RuntimeTrajectoryHookEvent[];
+    summary: RuntimeTrajectoryEvidenceSummary;
+    diagnostics: string[];
+}
+interface ProjectRuntimeTrajectoryEvidenceOptions<TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord> {
+    records: TRecord[];
+    defaultSplitTag?: RunSplitTag;
+    recordIdOf?: (record: TRecord, index: number) => string | undefined;
+    scenarioIdOf?: (record: TRecord, index: number) => string | undefined;
+}
+declare function projectRuntimeTrajectoryEvidence<TRecord extends RuntimeTrajectoryRecord>(options: ProjectRuntimeTrajectoryEvidenceOptions<TRecord>): RuntimeTrajectoryEvidenceProjection;
+declare function parseRuntimeTrajectoryHookEvent(input: unknown): RuntimeTrajectoryHookEvent | null;
+
+export { type ProjectRuntimeTrajectoryEvidenceOptions as P, type RuntimeTrajectoryRecord as R, type RuntimeTrajectoryEvidenceProjection as a, type RuntimeTrajectoryEvidenceSummary as b, type RuntimeTrajectoryHookEvent as c, type RuntimeTrajectoryRunRecord as d, projectRuntimeTrajectoryEvidence as e, parseRuntimeTrajectoryHookEvent as p };

package/docs/building-doctrine.md

@@ -0,0 +1,42 @@
+# Building doctrine
+
+How every fleet agent that consumes `agent-eval` is built. Each rule is mechanical: a primitive or test makes the rule enforceable rather than aspirational. For the mental model and primitives this references, see [`concepts.md`](./concepts.md) and the [`/contract`](../src/contract/index.ts) surface.
+
+## 1. Defaults must be provably reachable
+
+Every hard-coded model id or endpoint default is verifiable against the live router. Membership in `{baseUrl}/models` is the free check; an optional 1-token probe per model confirms the router will actually serve it. A default the router cannot serve is a config bug caught before the run, not a runtime surprise that silently degrades into a stub. Backend ids are namespaced by binding: cli-bridge ids (`claude-code/*`, `kimi-code/*`, `opencode/*`) never appear as defaults in code reachable from production — bridge use is an explicit env opt-in, never an implicit fallback.
+
+Enforced by: `preflightModels` (membership + optional probe) and `assertModelsServed` (gate that names every unreachable id with status + detail).
+
+## 2. Probe the platform before peeling client layers
+
+When a request fails, one direct call against the live endpoint bisects platform-versus-client before any code-level debugging begins. A 401 from the router on a `model_not_found` is the platform telling you the default is dead; a connection refused is the platform being unreachable. Establish which side is at fault with a probe first, then debug only the side that is actually broken.
+
+Enforced by: `preflightModels({ probe: true })` — the probe is the platform-side bisection, carrying the router's own `error.message` back to the caller.
+
+## 3. Agent-produced findings are hypotheses
+
+Enumeration of candidate problems may fan out to agents, but agent output is not evidence. Truth comes from probes against ground truth, not from an agent's assertion. Every classification carries quoted evidence, and nothing unverified is merged or reported as fact. A confident-sounding agent claim with no probe behind it is a hypothesis awaiting falsification.
+
+Enforced by: `assertRealBackend` over the resulting `RunRecord[]` — an agent that claims success while the backend was never called reads as a stub, not a pass.
+
+## 4. Experiment integrity checklist
+
+Any lift or benchmark claim satisfies all of the following before it is reported:
+
+- A frozen, disjoint held-out set, spent exactly once, after candidate selection.
+- The propose and selection steps never see held-out data.
+- The paired bootstrap confidence interval excludes zero for a "ship" or "match" verdict.
+- The same scorer and the same items on both sides of any comparison.
+- A leakage check from builder inputs into the evaluation set.
+- Cross-family judge panels, with inter-rater reliability reported and gated.
+- Missing evidence is never scored as zero — fail loud over fabricate.
+- No optional stopping: the stopping rule is fixed before the run.
+
+Enforced by: `pairedBootstrap` (CI), `assertCrossFamily` (panel diversity), `interRaterReliability` (agreement), and `assertRealBackend` (no stub run masquerading as a result).
+
+## 5. Fix the class, not the instance
+
+A drifted default is the symptom of a missing convention. The fix ships the convention and its guard alongside the one-line correction, so the same drift cannot recur silently. Patching the single dead id without adding the preflight gate leaves the class open; the next default rots the same way.
+
+Enforced by: `assertModelsServed` wired into the campaign preflight — the guard that turns "this one model was dead" into "no campaign spends tokens against an unreachable default."

package/docs/research/belief-state-agent-eval-roadmap.md

@@ -238,6 +238,8 @@ Follow-up local smoke on 2026-06-05 over 50 recent Codex JSONL sessions under 20
 
 Runtime hook bridge on 2026-06-05: `src/belief-state/runtime-hooks.ts` now converts `agent-runtime` decision hooks into outcome-blind shadow-probe inputs, attaches matching lifecycle hook events as probe evidence, and only converts them into full `BeliefDecisionPoint` rows when the observed action is supplied. This keeps `agent-eval` trace/analysis-only while letting the runtime emit producer-backed decision boundaries and context for the next experiment.
 
+Phase 0 measurement helper on 2026-06-06: `src/belief-state/phase0-measurement.ts` now joins runtime producer decisions, lifecycle evidence, observed action/outcome labels, and `RunRecord` split metadata into completed `BeliefDecisionPoint` rows plus a `BeliefDecisionResearchEvidencePacket`. This closes the local join/packet path; it does not clear the empirical gate until the corpus has >= 200 labeled producer-backed decisions with integrity checks and a recorded baseline.
+
 Taxonomy tightening on 2026-06-05: `src/belief-state/types.ts` now exports stable decision kinds, evidence sources, evidence quality labels, evaluation criteria, and reason codes. The intent is to make future dashboards and paper artifacts aggregate by stable IDs (`calibration`, `ope-support`, `memory-health`, `surface-attribution`, `promotion`, etc.) instead of parsing prose diagnostics.
 
 ### Q4 2026 - Phase 1: Selective Prediction and Abstention
@@ -567,6 +569,7 @@ Promotion:
 | `src/belief-state/report.test.ts` | full report status: `ship`, `hold`, `need_more_data`; recommendation cannot ship on OPE alone. |
 | `src/belief-state/code-agent-corpus.test.ts` | extracts code-agent decision corpora across Codex, Claude Code, OpenCode, Kimi Code, and Pi/PiGraph-shaped traces; inventories targets; picks failure recovery first; holds when OPE propensities are absent. |
 | `src/belief-state/runtime-hooks.test.ts` | converts runtime decision hooks to outcome-blind shadow probes; attaches matching lifecycle hook events as evidence; requires observed action for full belief rows; collector stays structurally compatible with runtime hooks. |
+| `tests/belief-state/phase0-measurement.test.ts` | joins runtime producer decisions, lifecycle events, observed labels, and run split metadata into completed rows and a research evidence packet without fabricating missing joins or propensities. |
 | `src/belief-state/types.test.ts` | guards the stable decision kinds, evidence sources/qualities, evaluation criteria, and reason-code taxonomy. |
 | `src/meta-eval/calibration.test.ts` | existing `calibrationCurve()` still works after extracting pure helper. |
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.83.0",
+  "version": "0.85.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -169,19 +169,6 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {
-    "build": "tsup && pnpm openapi",
-    "dev": "tsup --watch",
-    "prepare": "husky",
-    "prepublishOnly": "pnpm build",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit",
-    "lint": "biome check src",
-    "format": "biome format --write src",
-    "openapi": "node dist/cli.js openapi --out dist/openapi.json",
-    "verify:package": "node scripts/verify-package-exports.mjs"
-  },
   "dependencies": {
     "@asteasolutions/zod-to-openapi": "^8.5.0",
     "@ax-llm/ax": "^19.0.25",
@@ -209,16 +196,6 @@
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
   },
-  "pnpm": {
-    "minimumReleaseAge": 4320,
-    "minimumReleaseAgeExclude": [
-      "@tangle-network/sandbox"
-    ],
-    "overrides": {
-      "postcss@<8.5.10": "^8.5.10",
-      "ws@>=8.0.0 <8.20.1": "^8.20.1"
-    }
-  },
   "engines": {
     "node": ">=20"
   },
@@ -231,5 +208,15 @@
     ]
   },
   "license": "MIT",
-  "packageManager": "pnpm@10.22.0"
-}
+  "scripts": {
+    "build": "tsup && pnpm openapi",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check src",
+    "format": "biome format --write src",
+    "openapi": "node dist/cli.js openapi --out dist/openapi.json",
+    "verify:package": "node scripts/verify-package-exports.mjs"
+  }
+}