@tangle-network/agent-eval 0.144.6 → 0.144.7

package/CHANGELOG.md

@@ -4,6 +4,30 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [Unreleased]
+
+### Added
+
+- `AnalystDefinition` — the declarative unit behind an analyst arm: a profile fragment, an `EvidenceProjection` (`inline` | `chunked` | `repl-variable` | `agent-tools`), a `ReplyContract` (generalized from `PrimeReplyContract`, which is now a type alias of it), and budget plus repair-turn declarations.
+- `bindAnalyst(definition, transports)` compiles a definition into a runnable `AnalystBenchmarkRunner`; a projection × transport pair with no strategy fails loud with `AnalystExpressivenessError`.
+- The three benchmark arms are re-expressed as exported definitions (`publicDirectAnalystDefinition`, `publicRlmAnalystDefinition`, `primeCodeTraceAnalystDefinition`); `createPublicBenchmarkDirectRunner`, `createPublicBenchmarkRlmRunner`, and `createPrimeBenchmarkRunner` are thin shells over them, so no consumer changes.
+- `analystDefinitionProtocolSha256` (equals `primeAnalystProtocolSha256()` for the inline arm) and `analystDefinitionAsymmetries` (refuses unequal repair turns, renders declared differences).
+- `ExecutionProbe` — the optional live-execution port on `AnalystContext` (`context.probe`); a runtime fills it later, this package defines only the port.
+- Definition parity kill test (`src/analyst/definition-parity.test.ts`): each compiled arm and its entry point must send byte-identical request bodies and record equal protocol digests, so expression loss fails CI by construction.
+- Verification-strategy family (`VerificationStrategySource`, `VERIFICATION_STRATEGIES`): the reward-source union opens with `proof-kernel`, `invariant`, `replication`, and `agreement`, each carrying its documented failure mode (the formalization gap, uncalibrated invariants, method self-replication, the shared blind spot). The answer-key members and the deterministic/probabilistic axis are unchanged.
+- Verdict epistemics: `DefaultVerdict` gains an optional `certification` (`VerdictCertification`) — strategy, exact checker identity with content pins, assumption list, evidence digest. Additive: consumers that do not read the field are unaffected.
+- Checker port (`StrategyChecker`, `CheckerIdentity`, `CheckerOutcome`): a checker is an injected executable boundary returning a typed outcome; the package ships no checker implementation, so a consumer binds its own kernel.
+- Blind statement-equivalence protocol (`defineEquivalenceCheck`, `buildEquivalenceRecord`, `runEquivalenceCheck`): the two-arm design as a typed primitive with fail-loud refusals (`EquivalenceProtocolError`) — a non-blind arm, a wrong arm count, a refutation without its separating witness, or a mismatched checker strategy throws instead of recording.
+- `docs/verification-strategies.md`: the family, each member's failure mode, and the BCWW (4.6) formalization pilot as the worked example.
+
+## [0.144.7] - 2026-08-10 - Runtime journal alignment
+
+### Fixed
+
+- Runtime recursive supervisor journals now retain every event envelope at the Eval reader boundary.
+  Transport events such as `materialized`, `execution-bound`, and `trace-unpropagated` remain readable as unavailable evidence instead of becoming malformed rows.
+  The reader also projects Runtime's nested profile identity and structural tree roles, so recursive trees retain their nodes, settlements, token spend, and unknown-cost fields.
+
 ## [0.144.6] - 2026-08-09 - interaction binding cohort alignment
 
 ### Changed

package/README.md

@@ -440,6 +440,8 @@ See [concepts](./docs/concepts.md), [customer paths](./docs/customer-journeys.md
 | `@tangle-network/agent-eval/analyst` | Built-in and custom trace analysts, labeled comparison, costs, and reports. |
 | `@tangle-network/agent-eval/traces` | Store, replay, and inspect structured traces. |
 | `@tangle-network/agent-eval/supervisor-run` | Read loops or agent-runtime recursive run directories without collapsing missing measurements to zero. |
+| `@tangle-network/agent-eval/trajectory-replay` | Re-execute a recorded shell trajectory in its own image and check whether the recorded failure reproduces. |
+| `@tangle-network/agent-eval/trace-repair` | Grade one analyst finding by executing the repair it proposes, and measure Delta-repair against a no-fix control. |
 | `@tangle-network/agent-eval/benchmarks` | Benchmark adapters and retrieval metrics. |
 | `@tangle-network/agent-eval/rl` | Export rewards, preferences, and training rows. |
 | `@tangle-network/agent-eval/wire` | HTTP and RPC schemas for other languages. |

package/dist/{benchmark-J9Qe6j2_.d.ts → agent-profile-CgDTo40f.d.ts}

@@ -1,5 +1,6 @@
-import { c as AnalystRunInputs, f as AnalystUsageReceipt, i as AnalystFinding, p as EvidenceRef, x as TraceAnalysisStore } from "./types-BhP9q0Fq.js";
-import { E as AnalystRegistry, M as RegistryRunOpts } from "./completion-verifier-CBiee74w.js";
+import { c as AnalystRunInputs, f as AnalystUsageReceipt, i as AnalystFinding, p as EvidenceRef, w as TraceAnalysisStore } from "./types-DF_Udrp-.js";
+import { E as AnalystRegistry, M as RegistryRunOpts } from "./completion-verifier-foUCLif_.js";
+import { AgentProfile, AgentProfile as AgentProfile$1, HarnessType, HarnessType as HarnessType$1 } from "@tangle-network/agent-interface";
 //#region src/analyst/benchmark-scoring.d.ts
 declare function scoreAnalystFindings(testCase: Pick<AnalystBenchmarkCase, 'id' | 'expectedIssues' | 'labeledEvidence'>, findings: readonly AnalystFinding[]): AnalystFindingScore;
 //#endregion
@@ -232,5 +233,91 @@ declare function registryBenchmarkRunner(options: {
   failOnAnalystFailure?: boolean;
 }): AnalystBenchmarkRunner<AnalystRunInputs>;
 //#endregion
-export { scoreAnalystFindings as C, traceStoreEvidenceResolver as S, AnalystIssueExpectation as _, AnalystBenchmarkLabelState as a, registryBenchmarkRunner as b, AnalystBenchmarkProvenance as c, AnalystBenchmarkSummary as d, AnalystEvidenceExpectation as f, AnalystFindingScore as g, AnalystEvidenceResolver as h, AnalystBenchmarkError as i, AnalystBenchmarkResult as l, AnalystEvidenceResolutionError as m, AnalystBenchmarkDatasetRef as n, AnalystBenchmarkObservation as o, AnalystEvidenceResolution as p, AnalystBenchmarkDescriptor as r, AnalystBenchmarkOutput as s, AnalystBenchmarkCase as t, AnalystBenchmarkRunner as u, AnalystLatencyDistribution as v, runAnalystBenchmark as x, RunAnalystBenchmarkOptions as y };
-//# sourceMappingURL=benchmark-J9Qe6j2_.d.ts.map
+//#region src/agent-profile.d.ts
+/**
+ * The agentic coding harnesses an eval sweeps by default — the ones we care about
+ * ranking. This is the SINGLE source of that list; consumers import it instead of
+ * re-declaring their own (a re-declared list is how the fleet drifts). Pass an
+ * explicit `harnesses` (e.g. `harnessTypeSchema.options` for literally every known
+ * harness) to widen beyond these.
+ */
+declare const CODING_HARNESSES: readonly HarnessType[];
+interface ProfileAxisSpec {
+  /** The domain profile to sweep. Its prompt/tools/skills are held fixed; only the
+   *  harness and model vary. `model.default` is the fallback model. */
+  base: AgentProfile;
+  /** Harnesses to cross. Default: {@link CODING_HARNESSES}. */
+  harnesses?: readonly HarnessType[];
+  /** Models to cross. Default: `[base.model.default]` — one model, i.e. today's
+   *  single-model behaviour, so omitting this never changes an existing run. */
+  models?: readonly string[];
+  /** Force every (harness, model) pair verbatim, even ones the harness can't run —
+   *  for deliberately testing failure modes. Default (false): SNAP instead — a
+   *  vendor-locked harness runs only the swept models in its family, or its native
+   *  default when it supports none, so no harness is dropped and none gets a
+   *  guaranteed-failing foreign-model cell. */
+  keepIncompatible?: boolean;
+}
+/** Model sentinel for a vendor-locked harness that supports none of the swept models:
+ *  it carries no provider prefix, so `harnessSupportsModel` accepts it and the harness
+ *  resolves it to its own native default model at runtime (e.g. kimi-code → its Kimi
+ *  model). Lets `expandProfileAxes` snap-instead-of-drop without a per-harness flagship
+ *  table that would rot as router catalogs change. */
+declare const HARNESS_NATIVE_MODEL = "default";
+/**
+ * Expand a base profile across the harness × model matrix into the `AgentProfile[]`
+ * that `runProfileMatrix` / `selfImprove` score — the ONE place "which harnesses ×
+ * which models do we evaluate" lives, so no product hand-rolls its own harness list
+ * or column→profile mapping (the pattern that let those copies drift and silently
+ * break the harness pivot).
+ *
+ * Each cell clones `base`, sets the canonical top-level `harness` and `model.default`,
+ * and stamps `metadata.harness` + `metadata.harnessModel` for matrix grouping. Both
+ * metadata fields are hash-bearing, so every cell gets a distinct `agentProfileId` row
+ * and results join back by harness/model via {@link harnessAxisOf} with no
+ * hand-recomputed key. A vendor-locked harness snaps to its family's swept models — or
+ * its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none — so every
+ * requested harness runs; `keepIncompatible` forces every pair verbatim.
+ *
+ * Omit `harnesses`/`models` to sweep the full default set — the "turn it on for
+ * everything we care about" switch, identical in shape whether one harness or all.
+ */
+declare function expandProfileAxes(spec: ProfileAxisSpec): AgentProfile[];
+/**
+ * Read the (harness, model) a matrix cell ran under, off a profile or a result row's
+ * profile — the join-back for a `byHarness` pivot. Returns undefined when the profile
+ * wasn't produced by {@link expandProfileAxes}. Callers group `result.byProfile` by
+ * this instead of recomputing an id (recomputing the wrong key is what broke the pivot
+ * in the hand-rolled copies).
+ */
+declare function harnessAxisOf(profile: Pick<AgentProfile, 'metadata'>): {
+  harness: HarnessType;
+  model: string;
+} | undefined;
+/**
+ * Collision-resistant, path-safe, human-readable profile id for eval artifacts.
+ * Scorecard joins still use `agentProfileHash`; this id is for run ids, matrix
+ * keys, and directory names where two profiles must not collapse onto one row.
+ * The suffix is the first 64 bits of the behaviour hash, enough for ordinary
+ * eval matrices while keeping filenames readable.
+ */
+declare function agentProfileId(profile: AgentProfile): string;
+/**
+ * Model snapshot used for `RunRecord.model`. Eval surfaces require a concrete
+ * model id because run records reject bare/missing model aliases.
+ */
+declare function agentProfileModelId(profile: AgentProfile): string;
+/**
+ * Deterministic behaviour identity for the canonical
+ * `@tangle-network/agent-interface` AgentProfile.
+ *
+ * `name` and `description` are labels and do not affect the hash. Profile
+ * `version`, prompt, model hints, tools, resources, hooks, modes, permissions,
+ * and extensions do affect the hash. Resource array order is hash-bearing
+ * because mount order can change agent behaviour. Undefined fields are treated
+ * as absent; explicit `null` fields remain hash-bearing.
+ */
+declare function agentProfileHash(profile: AgentProfile): string;
+//#endregion
+export { registryBenchmarkRunner as A, AnalystEvidenceResolution as C, AnalystIssueExpectation as D, AnalystFindingScore as E, traceStoreEvidenceResolver as M, scoreAnalystFindings as N, AnalystLatencyDistribution as O, AnalystEvidenceExpectation as S, AnalystEvidenceResolver as T, AnalystBenchmarkOutput as _, ProfileAxisSpec as a, AnalystBenchmarkRunner as b, agentProfileModelId as c, AnalystBenchmarkCase as d, AnalystBenchmarkDatasetRef as f, AnalystBenchmarkObservation as g, AnalystBenchmarkLabelState as h, HarnessType$1 as i, runAnalystBenchmark as j, RunAnalystBenchmarkOptions as k, expandProfileAxes as l, AnalystBenchmarkError as m, CODING_HARNESSES as n, agentProfileHash as o, AnalystBenchmarkDescriptor as p, HARNESS_NATIVE_MODEL as r, agentProfileId as s, AgentProfile$1 as t, harnessAxisOf as u, AnalystBenchmarkProvenance as v, AnalystEvidenceResolutionError as w, AnalystBenchmarkSummary as x, AnalystBenchmarkResult as y };
+//# sourceMappingURL=agent-profile-CgDTo40f.d.ts.map

package/dist/agent-profile-CgDTo40f.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-profile-CgDTo40f.d.ts","names":[],"sources":["../src/analyst/benchmark-scoring.ts","../src/analyst/benchmark.ts","../src/agent-profile.ts"],"mappings":";;;;iBASgB,qBACd,UAAU,KAAK,oEACf,mBAAmB,mBAClB;;;UCIc;EACf;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA,oBAAoB;EACpB;;EAEA,4BAA4B;;KAGlB;UAEK,qBAAqB;EACpC;;EAEA;;EAEA,YAAY;EACZ,OAAO;EACP,yBAAyB;;EAEzB,2BAA2B;EAC3B;EACA,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA,mBAAmB;EACnB;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;UAGe;EACf;EACA;EACA,oBAAoB;EACpB,QAAQ;;EAER;;KAGU,wBAAwB,qBAAqB;EACvD;EACA,WAAW;EACX,UAAU;EACV,SAAS;gBACK;;;;;iBAMA,2BAA2B,QACzC,WAAW,OAAO,WAAW,qBAC5B,wBAAwB;UAoBV;EACf,mBAAmB;EACnB,QAAQ;EACR,WAAW;;;;;EAKX;;EAEA,QAAQ;;UAGO;EACf;EACA;EACA;EACA;;UAGe,uBAAuB;EACtC;EACA,QACE,OAAO,QACP;IAAW;IAAgB;IAAoB,SAAS;MACvD,yBAAyB,QAAQ;;UAGrB;EACf;EACA;EACA;EACA,YAAY;EACZ;EACA;EACA;EACA;EACA,mBAAmB;EACnB,OAAO;EACP,qBAAqB;EACrB;EACA,eAAe;EACf,QAAQ;EACR,iBAAiB;EACjB,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;EACA,WAAW;EACX;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA,UAAU;EACV;EACA,cAAc;EACd,WAAW;;UAGI,mCAAmC;EAClD;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,YAAY;EACZ,cAAc;EACd,WAAW;;UAGI,2BAA2B;EAC1C,gBAAgB,qBAAqB;EACrC,kBAAkB,uBAAuB;EACzC;EACA;EACA;EACA,kBAAkB,wBAAwB;EAC1C,YAAY;;EAEZ,+BAA+B;EAC/B,iBAAiB,aAAa,uCAAuC;EACrE,SAAS;;iBAGW,oBAAoB,QACxC,SAAS,2BAA2B,UACnC,QAAQ;iBAyDK,wBAAwB;EACtC;EACA,UAAU;EACV,aAAa,KAAK;;EAElB;IACE,uBAAuB;;;;;;;;;;cCpUd,2BAA2B;UAOvB;;;EAGf,MAAM;;EAEN,qBAAqB;;;EAGrB;;;;;;EAMA;;;;;;;cAQW;;;;;;;;;;;;;;;;;;;iBAoBG,kBAAkB,MAAM,kBAAkB;;;;;;;;iBAwD1C,cACd,SAAS,KAAK;EACX,SAAS;EAAa;;;;;;;;;iBAiBX,eAAe,SAAS;;;;;iBASxB,oBAAoB,SAAS;;;;;;;;;;;iBA0C7B,iBAAiB,SAAS"}