@tangle-network/agent-eval 0.128.2 → 0.129.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +265 -0
  2. package/README.md +18 -0
  3. package/dist/analyst/index.d.ts +107 -165
  4. package/dist/analyst/index.js +5 -9
  5. package/dist/analyst/index.js.map +1 -1
  6. package/dist/belief-state/index.d.ts +2 -19
  7. package/dist/belief-state/index.js +30 -31
  8. package/dist/belief-state/index.js.map +1 -1
  9. package/dist/benchmarks/index.d.ts +5 -8
  10. package/dist/benchmarks/index.js +12 -11
  11. package/dist/builder-eval/index.js +1 -1
  12. package/dist/campaign/index.d.ts +30 -39
  13. package/dist/campaign/index.js +11 -10
  14. package/dist/{chunk-NKAGIDE2.js → chunk-2QU3YOPR.js} +15 -274
  15. package/dist/chunk-2QU3YOPR.js.map +1 -0
  16. package/dist/{chunk-EJGRPCO3.js → chunk-3OCR4R5I.js} +245 -134
  17. package/dist/chunk-3OCR4R5I.js.map +1 -0
  18. package/dist/{chunk-2JX3CFMB.js → chunk-56TAVBOK.js} +5 -2
  19. package/dist/chunk-56TAVBOK.js.map +1 -0
  20. package/dist/{chunk-DPUHNQLN.js → chunk-7FO3TNPI.js} +2 -2
  21. package/dist/{chunk-DJKY2TSY.js → chunk-BSO5JDQH.js} +27 -120
  22. package/dist/chunk-BSO5JDQH.js.map +1 -0
  23. package/dist/{chunk-EZJEIH2R.js → chunk-C6LXANRU.js} +11 -20
  24. package/dist/chunk-C6LXANRU.js.map +1 -0
  25. package/dist/{chunk-ZUUWPZCV.js → chunk-DODXQREJ.js} +4 -4
  26. package/dist/{chunk-2MKQIFS4.js → chunk-E7QXT7SX.js} +2 -2
  27. package/dist/{chunk-NYLOYM6N.js → chunk-EG66UGL4.js} +37 -28
  28. package/dist/chunk-EG66UGL4.js.map +1 -0
  29. package/dist/{chunk-P5W7RQKK.js → chunk-FXTVJPYD.js} +2 -2
  30. package/dist/{chunk-VZSRQ272.js → chunk-G7MGMCZD.js} +6 -2
  31. package/dist/chunk-G7MGMCZD.js.map +1 -0
  32. package/dist/{chunk-IHQDPH7D.js → chunk-H23X7XKK.js} +85 -75
  33. package/dist/chunk-H23X7XKK.js.map +1 -0
  34. package/dist/{chunk-VBQ3CRKH.js → chunk-HPWUNB47.js} +4 -6
  35. package/dist/chunk-HPWUNB47.js.map +1 -0
  36. package/dist/{chunk-XPRT64IE.js → chunk-IYCLP2N2.js} +3 -3
  37. package/dist/chunk-IYCLP2N2.js.map +1 -0
  38. package/dist/{chunk-XDWDC2MP.js → chunk-JQSF5DQT.js} +11 -5
  39. package/dist/chunk-JQSF5DQT.js.map +1 -0
  40. package/dist/{chunk-NACAGYSY.js → chunk-M4YBQKIJ.js} +11 -11
  41. package/dist/chunk-M4YBQKIJ.js.map +1 -0
  42. package/dist/{chunk-YJBNWCAA.js → chunk-NY44NC4A.js} +3 -3
  43. package/dist/chunk-OIUOT4QD.js +44 -0
  44. package/dist/chunk-OIUOT4QD.js.map +1 -0
  45. package/dist/chunk-OWN5NPMC.js +152 -0
  46. package/dist/chunk-OWN5NPMC.js.map +1 -0
  47. package/dist/chunk-PC5DOSM7.js +579 -0
  48. package/dist/chunk-PC5DOSM7.js.map +1 -0
  49. package/dist/{chunk-UB2LOJ6Q.js → chunk-QB6BDBP2.js} +23 -20
  50. package/dist/chunk-QB6BDBP2.js.map +1 -0
  51. package/dist/chunk-RXHCETDZ.js +536 -0
  52. package/dist/chunk-RXHCETDZ.js.map +1 -0
  53. package/dist/{chunk-PBE2LOSS.js → chunk-SFLLL76A.js} +7 -7
  54. package/dist/chunk-SFLLL76A.js.map +1 -0
  55. package/dist/{chunk-VGRCHJON.js → chunk-T6RLYGAD.js} +3 -8
  56. package/dist/chunk-T6RLYGAD.js.map +1 -0
  57. package/dist/{chunk-VLOATJQ2.js → chunk-TJVT4QFF.js} +21 -18
  58. package/dist/chunk-TJVT4QFF.js.map +1 -0
  59. package/dist/{chunk-S5YLIBFX.js → chunk-TQ7LNKZ3.js} +2 -2
  60. package/dist/{chunk-EOSZT7PL.js → chunk-U4L7JRPZ.js} +2 -297
  61. package/dist/chunk-U4L7JRPZ.js.map +1 -0
  62. package/dist/chunk-U4PHLT2N.js +419 -0
  63. package/dist/chunk-U4PHLT2N.js.map +1 -0
  64. package/dist/{chunk-WS3NZZQQ.js → chunk-VCZ5FQYW.js} +3 -4
  65. package/dist/chunk-VCZ5FQYW.js.map +1 -0
  66. package/dist/{chunk-BYT7ELPS.js → chunk-WVATSFCP.js} +2 -2
  67. package/dist/{chunk-TSN7JT6D.js → chunk-X4YIBDER.js} +21 -5
  68. package/dist/{chunk-TSN7JT6D.js.map → chunk-X4YIBDER.js.map} +1 -1
  69. package/dist/{chunk-TBL77AUT.js → chunk-YQN4ICPP.js} +5 -5
  70. package/dist/{chunk-MHELPNRP.js → chunk-ZHTZ4EYI.js} +1 -1
  71. package/dist/chunk-ZHTZ4EYI.js.map +1 -0
  72. package/dist/cli.js +6 -5
  73. package/dist/cli.js.map +1 -1
  74. package/dist/contract/index.d.ts +47 -87
  75. package/dist/contract/index.js +14 -13
  76. package/dist/contract/index.js.map +1 -1
  77. package/dist/control.js +3 -2
  78. package/dist/fuzz.js +3 -2
  79. package/dist/fuzz.js.map +1 -1
  80. package/dist/index.d.ts +659 -203
  81. package/dist/index.js +145 -117
  82. package/dist/index.js.map +1 -1
  83. package/dist/meta-eval/index.js +2 -2
  84. package/dist/multishot/index.d.ts +3 -4
  85. package/dist/multishot/index.js.map +1 -1
  86. package/dist/openapi.json +1 -1
  87. package/dist/pipelines/index.js +5 -5
  88. package/dist/reporting.d.ts +14 -0
  89. package/dist/reporting.js +7 -6
  90. package/dist/rl.d.ts +652 -82
  91. package/dist/rl.js +415 -171
  92. package/dist/rl.js.map +1 -1
  93. package/dist/rollout/index.d.ts +1071 -32
  94. package/dist/rollout/index.js +68 -10
  95. package/dist/run-campaign-OJJ7CZF4.js +18 -0
  96. package/dist/supervisor-run/index.d.ts +114 -4
  97. package/dist/supervisor-run/index.js +4 -3
  98. package/dist/traces.d.ts +1 -1
  99. package/dist/traces.js +6 -5
  100. package/dist/wire/index.d.ts +10 -11
  101. package/dist/wire/index.js +3 -3
  102. package/docs/feature-guide.md +1 -1
  103. package/docs/rollout.md +116 -2
  104. package/package.json +4 -4
  105. package/dist/chunk-2JX3CFMB.js.map +0 -1
  106. package/dist/chunk-DJKY2TSY.js.map +0 -1
  107. package/dist/chunk-EJGRPCO3.js.map +0 -1
  108. package/dist/chunk-EOSZT7PL.js.map +0 -1
  109. package/dist/chunk-EZJEIH2R.js.map +0 -1
  110. package/dist/chunk-IHQDPH7D.js.map +0 -1
  111. package/dist/chunk-MHELPNRP.js.map +0 -1
  112. package/dist/chunk-NACAGYSY.js.map +0 -1
  113. package/dist/chunk-NKAGIDE2.js.map +0 -1
  114. package/dist/chunk-NYLOYM6N.js.map +0 -1
  115. package/dist/chunk-PBE2LOSS.js.map +0 -1
  116. package/dist/chunk-TT4KNT67.js +0 -124
  117. package/dist/chunk-TT4KNT67.js.map +0 -1
  118. package/dist/chunk-UB2LOJ6Q.js.map +0 -1
  119. package/dist/chunk-UWZZKKU7.js +0 -237
  120. package/dist/chunk-UWZZKKU7.js.map +0 -1
  121. package/dist/chunk-VBQ3CRKH.js.map +0 -1
  122. package/dist/chunk-VGRCHJON.js.map +0 -1
  123. package/dist/chunk-VLOATJQ2.js.map +0 -1
  124. package/dist/chunk-VZSRQ272.js.map +0 -1
  125. package/dist/chunk-WS3NZZQQ.js.map +0 -1
  126. package/dist/chunk-XDWDC2MP.js.map +0 -1
  127. package/dist/chunk-XPRT64IE.js.map +0 -1
  128. package/dist/run-campaign-ISHFZ7FJ.js +0 -17
  129. /package/dist/{chunk-DPUHNQLN.js.map → chunk-7FO3TNPI.js.map} +0 -0
  130. /package/dist/{chunk-ZUUWPZCV.js.map → chunk-DODXQREJ.js.map} +0 -0
  131. /package/dist/{chunk-2MKQIFS4.js.map → chunk-E7QXT7SX.js.map} +0 -0
  132. /package/dist/{chunk-P5W7RQKK.js.map → chunk-FXTVJPYD.js.map} +0 -0
  133. /package/dist/{chunk-YJBNWCAA.js.map → chunk-NY44NC4A.js.map} +0 -0
  134. /package/dist/{chunk-S5YLIBFX.js.map → chunk-TQ7LNKZ3.js.map} +0 -0
  135. /package/dist/{chunk-BYT7ELPS.js.map → chunk-WVATSFCP.js.map} +0 -0
  136. /package/dist/{chunk-TBL77AUT.js.map → chunk-YQN4ICPP.js.map} +0 -0
  137. /package/dist/{run-campaign-ISHFZ7FJ.js.map → run-campaign-OJJ7CZF4.js.map} +0 -0
package/docs/rollout.md CHANGED
@@ -4,15 +4,63 @@
4
4
  One JSONL line per agent invocation, with the full transcript inline, a single scalar reward, and fail-closed split labels.
5
5
  No other package may write rollout rows; domain repos (e.g. the bench swe-arena backfill) construct lines from their own joins and serialize them exclusively through this module's `writeRolloutLedger` / `appendRolloutLines`.
6
6
 
7
+ ## The hourglass: sources → waist → sinks
8
+
9
+ Many producers, one waist, many consumers.
10
+ Every invariant is enforced once, at the waist, so no sink can be reached by a path that skips it.
11
+
12
+ ```
13
+ SOURCES WAIST SINKS
14
+ RunRecord × trace ─┐ ┌─→ SFT chat JSONL (toSftRows)
15
+ opencode sqlite ─┤ ├─→ reward rows (toRewardRows)
16
+ Claude Code jsonl ─┼─→ tangle.rollout.v1 ────┼─→ Prime Intellect (toVerifiersRolloutOutputs)
17
+ domain backfills ─┤ (RolloutLine) ├─→ OpenAI RFT (toRftItems)
18
+ Harbor ATIF import ─┘ ├─→ DPO / GRPO / PRM (rl exporters)
19
+ ├─→ HF dataset release (buildHfDataset)
20
+ └─→ Harbor ATIF export (toHarborTrajectory)
21
+ ```
22
+
23
+ A source's job is to produce a `RolloutLine` and nothing else.
24
+ A sink's job is to consume `MintedRolloutLine[]` and nothing else — no sink reads a `RunRecord` or a `TraceStore`, because that would be a second door into training data with no gate on it.
25
+
26
+ `MintedRolloutLine` is `RolloutLine` plus a nominal brand (a phantom `unique symbol`, absent at runtime, so the JSON is unchanged).
27
+ The distinction matters because `RolloutLine` is structural: any object literal of the right shape is one, so declaring `{reward: 0.95, realness_gated: true}` was enough to be accepted by every exporter that "only takes a line".
28
+ Three producers mint the brand, and nothing else can:
29
+
30
+ | producer | how the invariant is established |
31
+ | --- | --- |
32
+ | `mintRolloutRows` | applies the gate (`trainingReward`) and validates the line it built |
33
+ | `readRolloutLedger` | validates every line off disk, then normalizes `realness_gated` to an explicit boolean |
34
+ | `assertMinted` / `assertMintedLines` | the explicit escape hatch for a hand-built or reconstructed line — grep it to enumerate every such site |
35
+
36
+ Readers, analysis, the ledger writer and the Harbor interchange keep taking the plain `RolloutLine`.
37
+
38
+ Where each invariant lives:
39
+
40
+ | invariant | enforced at | mechanism |
41
+ | --- | --- | --- |
42
+ | realness gate (anti-Goodhart) | `validateRolloutLine` (runtime) + the `MintedRolloutLine` brand (compile time) | `reward > 0` together with `realness_gated: true` is an INVALID line: it cannot be written to a ledger, read from one, minted, or handed to an exporter. Mint forces the reward to 0; `toSftRows` additionally refuses gated lines |
43
+ | every guard applies every check | `src/rollout/gate-checks.ts` | `GATE_CHECKS` is the one list of gate checks, and each of the four entry points (`validateRolloutLine`, `assertMinted`, `assertRewardGate`, `assertGateReport`) declares a TOTAL `GatePolicy` over it. Adding a check makes every policy a type error until it is wired, an entry point that skips one must write the reason in `omittedBecause(...)`, and each check's `tripwire` is fed to every entry point that claims to enforce it. Four rounds of leaks were all the same defect — a guard composing its checks by hand |
44
+ | never-screened ≠ screened-clean | `assertMinted` + `RealnessLabels` on every emitted row | `realness_gated: false` is the screen's VERDICT, so a producer with no screen was claiming "we looked and nothing fired". `realness_screened` separates the claims (`true` ran / `false` declared absent / `null` not stated) and travels beside `realness_gated` on every exported row shape. `assertMinted` refuses a positive reward carrying `realness_screened: false`; supervision-journal rows stay writable and readable, only promotion into training is closed |
45
+ | a line-less artifact names ONE invocation | `admitUngatedByInvocation` | `PreferenceTriple` / `PrmTrainingTriple` / `StepReward` carry run ids, and many invocations share a `run_id`. References resolve against `rollout_id` first and fall back to `run_id` only when that run holds exactly one invocation; an ambiguous reference DROPS the item and announces the count, a missing one throws. Keying on `run_id` alone was last-wins, which made the gate depend on array order |
46
+ | no hand-rolled score derivation | `src/rollout/score-derivation-guard.ts`, asserted by `reward-invariant.test.ts` | an AST walk over `src/**` flags every READ of `outcome.holdoutScore` / `outcome.searchScore` outside a counted allowlist. It replaced a line regex that seven ordinary reformattings walked past |
47
+ | split fail-closed | `isTrainableSplit` at every training sink | only `search` ships by default; held-out data requires explicit consent, while `dev` and `canary` never train |
48
+ | unlabeled ≠ zero | schema | `outcome.reward: null` is a labeled gap; reward-row and SFT exporters drop null, they never coerce it to 0 |
49
+ | capture gap is a finding | mint | records without spans become `messages: []` lines with `provenance.gap` AND are listed in `missingTraces` |
50
+ | scrub | release pipeline | `scrubLines` applies the 9 deterministic rules to every string before publication |
51
+
7
52
  ## Module map
8
53
 
9
54
  | concern | file | entry points |
10
55
  | --- | --- | --- |
11
56
  | schema + validation | `src/rollout/schema.ts` | `RolloutLine`, `validateRolloutLine`, `assertRolloutLine`, `isTrainableSplit` |
57
+ | the gate checks themselves | `src/rollout/gate-checks.ts` | `GATE_CHECK_IDS`, `GATE_CHECKS`, `GATE_POLICIES`, `gateErrors(subject, policy)` — the one list every entry point draws from. The subject is the LINE (`{outcome, steps}`), not the outcome alone: a per-step reward is training signal and lives outside `outcome`. `readReward` / `payloadIsPopulated` are the total readers every check narrows through, so a value the gate cannot classify is refused rather than read as clean |
12
58
  | ledger file API | `src/rollout/ledger.ts` | `writeRolloutLedger`, `appendRolloutLines`, `readRolloutLedger` |
13
59
  | minting from records | `src/rollout/mint.ts` | `mintRolloutRows(records, traceStore)`: RunRecord joined to trace via shared `runId` |
14
60
  | harness-store intake | `src/rollout/readers/` | `openOpencodeDb` + `readOpencodeSessionMessages` (opencode sqlite), `findClaudeTranscripts` + `readClaudeTranscript` (Claude Code project jsonl) |
61
+ | interchange | `src/rollout/interchange/harbor.ts` | `toHarborTrajectory` / `toHarborTrajectories` / `fromHarborTrajectory` / `relabelImportedSplit` (Harbor ATIF-v1.7); all root-exported as well as on the `/rollout` subpath |
15
62
  | exporters | `src/rollout/exporters.ts` | `toSftRows`, `toRewardRows`, `toVerifiersRolloutOutputs` (Prime Intellect), `toRftItems` (OpenAI RFT), `toJsonl` |
63
+ | RL exporters | `src/rl/exporters.ts` | `toDpoRows`, `toGrpoRows`, `toSftRows`, `toPrmRows`; all training inputs are `MintedRolloutLine[]` or line-referenced artifacts with explicit line context |
16
64
  | release pipeline | `src/rollout/release/` | `scrubLines` (9 deterministic rules), `buildDatasetCard`, `buildHfDataset`, CLI `agent-eval rollout-release … [--push org/name]` |
17
65
 
18
66
  ## Schema decision table
@@ -33,8 +81,8 @@ Rule applied: where fields conflicted, RunRecord-derived semantics won; the ledg
33
81
  | `policy.*` | ledger | + `prompt_hash`, `config_hash`, `agent_profile_cell_id` from PR #410's RunRecord provenance |
34
82
  | `messages` | ledger | canonical OpenAI chat-with-tools incl. `reasoning_content`; minted lines inline the final llm span's conversation |
35
83
  | `steps` | PR #410 | optional trace-span projections (llm/tool), absent on harness-store-derived lines |
36
- | `outcome.reward` | conflict | **merged**: `number \| null`; `null` means no verdict exists (a labeled gap, never 0); minting requires an explicit RunRecord task score and rejects execution-only records instead of labeling them 0; realness-gated scores are forced to 0 |
37
- | `outcome.realness_gated` | PR #410 | required boolean; the anti-Goodhart decision travels with the row and every training export refuses gated lines |
84
+ | `outcome.reward` | conflict | **merged**: `number \| null`; `null` means no verdict exists (a labeled gap, never 0 — interchange imports and unverdicted ledger lines carry it); minting requires an explicit RunRecord task score and rejects execution-only records instead of labeling them 0; realness-gated scores are forced to 0 |
85
+ | `outcome.realness_gated` | PR #410 | required boolean; the anti-Goodhart gate travels into the data, the validator refuses it beside a positive reward, and SFT export refuses gated lines outright. `outcome.realness_screened` rides beside it (optional tri-state) so a screened-clean reward is distinguishable from a never-screened one |
38
86
  | `outcome.reward_source` / `verdict` / `metrics` | ledger | unchanged; mint fills `metrics` from `RunRecord.outcome.raw` |
39
87
  | `outcome.is_completed` / `is_truncated` / `error` | conflict | mint derives terminal fields from the required `RunRecord.terminalOutcome`; producers use the explicit `unknown` value when terminal evidence is unavailable |
40
88
  | `cost.*` | ledger | superset of PR #410's costUsd/totalTokens; `cost.usd` is `null` when `costProvenance.kind === 'uncaptured'` (never a fake 0) |
@@ -50,4 +98,70 @@ Rule applied: where fields conflicted, RunRecord-derived semantics won; the ledg
50
98
  - Reward rows require a non-empty user prompt.
51
99
  - Verifiers requires both prompt and completion turns.
52
100
  - RFT requires prompt turns before the first assistant turn.
101
+ - Verifiers / RFT additionally accept `gatedLines: 'zero-and-flag'` — the dataset release's declared disposition — which keeps realness-gated lines and honest failures in the output at their non-positive reward with the gate stated on the row (`info.realness_gated`, `reference.realness_gated`): the reward in those formats is a signed learning signal, and dropping the gamed population would bias the negatives toward honest failures while hiding the gaming from the buyer. The split policy is NOT relaxed by the disposition.
102
+ Reward rows carry the same flag at `metadata.realness_gated`, and SFT rows carry it too even though it is always `false` there.
103
+ Every one of those four shapes also carries `realness_screened` beside it (`RealnessLabels`): the verdict alone cannot say whether a screen ever ran, and a row stating one claim without the other is what let never-screened rewards read as clean.
104
+ - Harbor ATIF export: no filter, because the format carries no reward at all — it is an interchange/audit sink, never a training-data door. A realness-gated line is exported with the gate stated in `notes` and in `extra.tangle.outcome.realness_gated`.
53
105
  - Release (`rollout-release`): trainable split only; proposer lines dropped unless `--include-proposers`; every string value scrubbed by the 9 deterministic rules (idempotent, so a second pass counts zero); `--push` requires `huggingface-cli` + `HF_TOKEN` and never prints the token.
106
+ The per-format gate disposition is stated once as data in `src/rollout/release/gate-report.ts` (`sft: exclude`, everything else `zero-and-flag`), the build measures the rows it is about to write against it, and it writes nothing at all if any config would ship a gated row above reward 0, any positive number that reward was derived from, or a positive reward whose producer declared no screen ran on it.
107
+ Those refusals are not listed in that file: it iterates `GATE_CHECK_IDS` under `GATE_POLICIES.assertGateReport`, so a check added anywhere in the package reaches the last door before a public dataset by default.
108
+ - Property tests (`src/rollout/gate-properties.test.ts`) hold the whole exporter surface by GENERATION rather than by example: containment (no gated run's number reaches any exporter, through the minted door AND the raw object-literal door), order independence (leak 1 stated as a law — reordering the input changes no output), and idempotence (the gating transform and the Harbor round-trip). 1000 cases per property by default, `GATE_PROPERTY_RUNS` to raise it; the exporter registry there is a total map, so a new exporter with no probe does not compile.
109
+ The dataset card does not assert anything about the gate — it renders the measured counts (`BuildSummary.gate`) and throws when they disagree with the lines it describes, so a README cannot drift from the data files beside it.
110
+
111
+ ## Harbor ATIF-v1.7 interchange
112
+
113
+ [Agent Trajectory Interchange Format](https://www.harborframework.com/docs/agents/trajectory-format) (normative RFC: `harbor-framework/harbor` `rfcs/0001-trajectory-format.md`) is the portability format for agent trajectories.
114
+ It is both a source and a sink of the hourglass, and it sits below the waist in both directions: `toHarborTrajectory` reads `RolloutLine[]`, `fromHarborTrajectory` writes `RolloutLine[]`.
115
+
116
+ **ATIF models no reward, no judge verdict, and no task/split coordinates.**
117
+ Everything else follows from that one fact.
118
+
119
+ | our field | ATIF | note |
120
+ | --- | --- | --- |
121
+ | `rollout_id` | `trajectory_id` | required by the spec on every embedded subagent |
122
+ | `parent_rollout_id` | tree placement in `subagent_trajectories` | our flat edge becomes ATIF's embedding; import flattens it back |
123
+ | `run_id` | `session_id` | ATIF's session is RUN-scoped, and `run_id` is our run-scoped coordinate; every node of an episode carries it, and two roots of one run therefore group as one session |
124
+ | `policy.harness` / `harness_version` | `agent.name` / `agent.version` | ATIF requires both as strings; a null becomes `unknown` / `0.0.0` and is restored exactly from escrow, never guessed back |
125
+ | `policy.model` | `agent.model_name` + `step.model_name` | |
126
+ | `tool_defs` | `agent.tool_definitions` | byte-identical OpenAI function schema, zero transform |
127
+ | `messages[]` | `steps[]` | coalescing fold: ATIF has no `tool` source, so a tool result becomes an `observation.results[]` entry on the step that DECLARED its `tool_call_id` (RFC MUST rule 2), never on whichever step happened to precede it |
128
+ | `ChatMessage.is_copied_context` | `step.is_copied_context` | RFC rule 7: a turn copied in from another trajectory. Native field both ways, and `toSftRows` drops those turns |
129
+ | `ChatToolCall.function.arguments` (string) | `tool_calls[].arguments` (object) | the raw string is kept in the call's `extra` so a malformed argument blob round-trips verbatim instead of being normalized |
130
+ | `cost.usd` / `tokens_in` / `tokens_out` / `cache_read` | `final_metrics.total_*` | omitted, never 0, when not captured |
131
+ | `cost.tokens_reasoning` / `cache_write` / `wall_s` / `llm_call_count` | `final_metrics.extra.*` | no ATIF field; `reasoning_tokens` is the key the RFC's own example uses |
132
+ | `steps[]` (span projections) | `extra.tangle.spans` + `step.metrics` / `step.llm_call_count` | the whole span stays escrowed (ATIF steps are conversation turns, and folding spans in would double-count the run), but the four RL fields ALSO ride ATIF's native per-step channel — a field only our own reader can see is not an interchange field. Escrow wins on import; the native channel is what a foreign document is read through |
133
+ | `task.*`, `policy.*`, `artifacts.*`, `provenance.*`, `role`, ids | `extra.tangle.*` | namespaced escrow; a foreign reader ignores it, our round-trip is exact |
134
+ | `outcome.realness_gated` | `extra.tangle.outcome.realness_gated` + `notes` | stated in prose too, because a consumer that ignores `extra` would otherwise see a gamed trajectory with nothing marking it |
135
+ | `outcome.reward` / `reward_source` / `verdict` | — **dropped** | not escrowed either: a third party must not be able to mistake an agent-eval judge score for something ATIF sanctioned |
136
+
137
+ Import consequences, all deliberate:
138
+
139
+ - Every imported line is **unlabeled**: `reward`, `reward_source` and `verdict` are `null` and `provenance.gap` says `imported from Harbor ATIF; no verdict`. That is the existing "null reward is a labeled gap, never 0" rule — an imported trajectory becomes a training example only once a judge scores it.
140
+ - **Every** imported trajectory lands on **`holdout`**, whatever the document says. `extra.tangle` is namespaced, not authenticated: a hand-written or third-party file can set `task.split: 'search'` as easily as our exporter can, and honouring it made "this file says so" enough to reach a training export. Promoting an import to a trainable split is a separate, greppable act — `relabelImportedSplit(lines, split)` — so `grep relabelImportedSplit` enumerates every place foreign data was declared trainable. The claim itself is not destroyed; it stays readable in the source document.
141
+ - Round-tripping is **idempotent**: `provenance.gap` is composed as a de-duplicated ordered set (it used to accrete one copy of the import note per pass), and every imported `ChatMessage` is emitted with keys in the canonical schema order, so a ledger hashed on serialized bytes sees no diff on further passes. The first import may re-order a producer's keys — that is the canonicalization.
142
+ - We do not synthesize `observation.subagent_trajectory_ref` on export. Our ledger records which *invocation* spawned a worker, not which *step* did; attaching the ref to a guessed step would fabricate a causal claim. The edge lives in the child's escrowed `parent_rollout_id`.
143
+ - Multimodal `ContentPart[]` flattens to text (our chat content is `string | null`); image parts become a `[image <media_type> <path>]` marker rather than being dropped silently.
144
+ - A cycle in `parent_rollout_id` or `subagent_trajectories` throws instead of emitting zero documents or recursing forever.
145
+
146
+ ### Fields adopted from ATIF
147
+
148
+ Four optional fields were added to our schema because ATIF carries real RL signal we were not capturing.
149
+ All are optional and never back-filled, so every ledger written before they existed still validates: absent means "not captured", which is a different claim from `0` or `[]`.
150
+
151
+ | field | on | why |
152
+ | --- | --- | --- |
153
+ | `logprobs` | `RolloutStep` | per-completion-token log probabilities — required for off-policy correction when the rollout came from a different policy than the one being trained |
154
+ | `completion_token_ids` | `RolloutStep` | exact completion tokenization; removes the ambiguity of re-tokenizing text at train time |
155
+ | `prompt_token_ids` | `RolloutStep` | exact prompt tokenization |
156
+ | `llm_call_count` | `RolloutStep` and `RolloutCostBlock` | inferences per span and per invocation; `0` means deterministic dispatch with no model call |
157
+ | `is_copied_context` | `ChatMessage` | RFC 0001 rule 7 makes excluding these turns from SFT a MUST — they were authored by another agent, so training on them credits this run for another's work |
158
+
159
+ The per-step four (`logprobs`, `completion_token_ids`, `prompt_token_ids`, `llm_call_count`) are wired through the interchange in BOTH directions on ATIF's own `step.metrics` / `step.llm_call_count`, not only in our escrow.
160
+ Export matches the k-th `llm` span to the k-th agent step; import prefers the escrow when it exists and falls back to the native channel, which is the only thing a Harbor-native producer writes.
161
+ A step carrying none of the four produces no span — absent stays absent.
162
+
163
+ ### Why there is no Letta converter
164
+
165
+ Letta's trajectory-v1 is a strict subset of what we need from ATIF for this purpose: no per-step or aggregate cost, no multi-agent/subagent structure, and no token-id or logprob channel.
166
+ A Letta sink would therefore carry strictly less than the ATIF one while adding a second format to keep correct against schema drift.
167
+ Do not add one without a concrete consumer that reads Letta and cannot read ATIF.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tangle-network/agent-eval",
3
- "version": "0.128.2",
3
+ "version": "0.129.0",
4
4
  "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
5
5
  "homepage": "https://github.com/tangle-network/agent-eval#readme",
6
6
  "repository": {
@@ -163,9 +163,8 @@
163
163
  "@asteasolutions/zod-to-openapi": "^9.1.0",
164
164
  "@ax-llm/ax": "^23.0.5",
165
165
  "@hono/node-server": "^2.0.11",
166
- "@tangle-network/agent-core": "0.4.21",
167
- "@tangle-network/agent-interface": "0.33.0",
168
- "@tangle-network/tcloud": "^0.4.14",
166
+ "@tangle-network/agent-core": "0.4.22",
167
+ "@tangle-network/agent-interface": "0.34.0",
169
168
  "hono": "^4.12.32",
170
169
  "zod": "^4.4.3"
171
170
  },
@@ -173,6 +172,7 @@
173
172
  "@biomejs/biome": "^2.5.5",
174
173
  "@types/node": "^26.1.1",
175
174
  "esbuild": "^0.28.1",
175
+ "fast-check": "^4.9.0",
176
176
  "husky": "^9.1.7",
177
177
  "lint-staged": "^17.2.0",
178
178
  "openapi3-ts": "^4.6.0",
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/pre-registration.ts","../src/agent-profile-cell.ts","../src/run-record.ts"],"sourcesContent":["/**\n * Pre-registered hypotheses — declare what you're testing BEFORE the\n * run, check it AFTER. Prevents p-hacking, optional stopping, and the\n * \"we ran until it looked good\" failure mode.\n *\n * Manifest is a plain JSON-friendly object. Sign it with a content hash\n * + timestamp; the registered record becomes immutable. Post-run,\n * evaluate the manifest against observed results — the library refuses\n * to let you re-interpret a different metric as the declared one.\n */\n\nexport interface HypothesisManifest {\n id: string\n /** Human prose — goes into the audit trail. */\n hypothesis: string\n /** Metric the hypothesis claims to move. */\n metric: string\n /** 'increase' = candidate should score higher than baseline; 'decrease' = lower. */\n direction: 'increase' | 'decrease'\n /** Minimum effect size to count (same units as the metric). */\n minEffect: number\n /** Alpha threshold. */\n alpha: number\n /** Target statistical power at which sample size was pre-computed. */\n power: number\n /** Declared N per arm before running. */\n preRegisteredN: number\n /** ISO8601 timestamp the manifest was registered. */\n registeredAt: string\n /** Optional identifiers to tie into the trace corpus. */\n baselineLabel?: string\n candidateLabel?: string\n}\n\n/**\n * Identifier for the hashing scheme used to produce `contentHash`.\n *\n * `'sha256-content'` — sha256 hex over the canonicalized manifest with\n * the `contentHash` and `algo` fields stripped. Held as a string union\n * so future schemes can be added without breaking parsers; SignedManifest\n * values without `algo` deserialize cleanly because the field is optional.\n */\nexport type SignedManifestAlgo = 'sha256-content'\n\nexport interface SignedManifest extends HypothesisManifest {\n /** sha256 hex of canonicalized manifest (everything except contentHash and algo). */\n contentHash: string\n /**\n * Algorithm string describing how `contentHash` was produced.\n *\n * Optional on the type so serialized manifests without it still parse,\n * but ALWAYS populated by {@link signManifest}. Consumers that want to\n * enforce a known algorithm should reject manifests where this field\n * is missing or unrecognized.\n */\n algo?: SignedManifestAlgo\n}\n\nexport interface HypothesisResult {\n manifest: SignedManifest\n observedN: number\n observedEffect: number\n observedPValue: number\n /** True iff the observed effect hits the pre-declared direction with\n * magnitude ≥ minEffect AND p < alpha. */\n confirmed: boolean\n /** Enumerated reasons the hypothesis was rejected (each a machine-tag). */\n rejectionReasons: Array<\n 'wrong_direction' | 'effect_too_small' | 'not_significant' | 'undersampled'\n >\n notes?: string\n}\n\n/**\n * Deterministic JSON canonicalization — sort object keys recursively.\n *\n * Two semantically-equal objects produce byte-identical canonicalized output;\n * this is what makes a content-hash stable across encoders, key insertion\n * orders, and runtime versions. Exported for any consumer that needs the same\n * canonicalization guarantee outside the manifest-signing path (e.g., signing\n * an artifact bundle, hashing a dataset version, etc.).\n */\nexport function canonicalize(v: unknown): unknown {\n if (v === null || typeof v !== 'object') return v\n if (Array.isArray(v)) return v.map(canonicalize)\n const keys = Object.keys(v as Record<string, unknown>).sort()\n const out: Record<string, unknown> = {}\n for (const k of keys) out[k] = canonicalize((v as Record<string, unknown>)[k])\n return out\n}\n\n/**\n * SHA-256 hex (full 64 chars) over the canonicalized JSON encoding of `obj`.\n *\n * The same primitive `signManifest` and `verifyManifest` are built on, exposed\n * directly so consumers signing arbitrary structured content (artifact bundles,\n * production packets, dataset manifests, etc.) don't have to re-derive\n * canonicalize+sha256 from scratch.\n *\n * Stable across:\n * - object key insertion order (canonicalization sorts keys recursively)\n * - encoder choice (UTF-8 via TextEncoder, fixed)\n * - runtime (uses the Web Crypto subtle digest, present in Node ≥18 and browsers)\n *\n * Named `hashJson` to disambiguate from `prompt-registry.ts`'s `hashContent`,\n * which takes a string input and returns a truncated 12-char prompt id.\n * Use `hashJson` when you mean \"canonicalize then hash.\"\n *\n * @example\n * const hash = await hashJson({ id: '1', kind: 'spec' })\n * // 'a3f1...' (64 hex chars)\n */\nexport async function hashJson<T>(obj: T): Promise<string> {\n const canonical = canonicalize(obj)\n const bytes = new TextEncoder().encode(JSON.stringify(canonical))\n const digest = await globalThis.crypto.subtle.digest('SHA-256', bytes)\n return Array.from(new Uint8Array(digest))\n .map((b) => b.toString(16).padStart(2, '0'))\n .join('')\n}\n\n/**\n * Sign a manifest with a SHA-256 content hash.\n *\n * The hash covers the canonicalized manifest with the `contentHash`\n * and `algo` fields stripped; this lets verifiers re-sign the rest and\n * compare. Returned manifest always carries `algo: 'sha256-content'`\n * so downstream consumers can identify the scheme; manifests without\n * `algo` still verify because it is stripped before hashing on both sides.\n */\nexport async function signManifest(m: HypothesisManifest): Promise<SignedManifest> {\n const hash = await hashJson(m)\n return { ...m, contentHash: hash, algo: 'sha256-content' }\n}\n\n/**\n * Verify that a signed manifest has not been tampered with.\n *\n * Strips `contentHash` and `algo` before re-signing so manifests without\n * `algo` verify identically to ones that carry it.\n */\nexport async function verifyManifest(m: SignedManifest): Promise<boolean> {\n const { contentHash, algo: _algo, ...rest } = m\n void _algo\n const resigned = await signManifest(rest)\n return resigned.contentHash === contentHash\n}\n\n/**\n * Evaluate a pre-registered hypothesis against observed results.\n * Mechanical — no re-interpretation permitted.\n */\nexport async function evaluateHypothesis(\n manifest: SignedManifest,\n observed: { n: number; effect: number; pValue: number },\n): Promise<HypothesisResult> {\n if (!(await verifyManifest(manifest))) {\n throw new Error('evaluateHypothesis: manifest content hash mismatch (tampered)')\n }\n const reasons: HypothesisResult['rejectionReasons'] = []\n const directionOk = manifest.direction === 'increase' ? observed.effect > 0 : observed.effect < 0\n if (!directionOk) reasons.push('wrong_direction')\n if (Math.abs(observed.effect) < manifest.minEffect) reasons.push('effect_too_small')\n if (observed.pValue >= manifest.alpha) reasons.push('not_significant')\n if (observed.n < manifest.preRegisteredN) reasons.push('undersampled')\n return {\n manifest,\n observedN: observed.n,\n observedEffect: observed.effect,\n observedPValue: observed.pValue,\n confirmed: reasons.length === 0,\n rejectionReasons: reasons,\n }\n}\n","import type { AgentProfile } from '@tangle-network/agent-interface'\nimport { ValidationError } from './errors'\nimport { hashJson } from './pre-registration'\n\nexport type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1'\n\nexport type AgentProfileJsonObject = { [key: string]: AgentProfileJson }\n\nexport type AgentProfileJson =\n | string\n | number\n | boolean\n | null\n | AgentProfileJson[]\n | AgentProfileJsonObject\n\nexport type AgentProfileDimensionValue = string | number | boolean | null\n\nexport interface AgentProfileSource {\n /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */\n kind: string\n /** sha256 over the canonical source profile object. */\n hash: string\n}\n\nexport interface AgentProfileSourceInput {\n kind: string\n /** Precomputed sha256 for callers that already sign their profile artifact. */\n hash?: string\n /** Full canonical runtime profile; hashed and then discarded from the cell. */\n profile?: AgentProfileJson\n}\n\nexport interface AgentProfileHarness {\n id: string\n version?: string\n hash?: string\n}\n\nexport interface AgentProfileCellInput {\n profileId: string\n sourceProfile: AgentProfileSourceInput\n harness?: AgentProfileHarness\n model?: string\n promptHash?: string\n dimensions?: Record<string, AgentProfileDimensionValue>\n}\n\nexport interface AgentProfileCell {\n schemaVersion: AgentProfileCellSchemaVersion\n cellId: string\n profileId: string\n sourceProfile: AgentProfileSource\n harness?: AgentProfileHarness\n model?: string\n promptHash?: string\n dimensions?: Record<string, AgentProfileDimensionValue>\n}\n\nexport class AgentProfileCellValidationError extends ValidationError {\n readonly path: string\n constructor(message: string, path = '') {\n super(path ? `${message} (at ${path})` : message)\n this.path = path\n }\n}\n\nconst SHA256_HEX = /^[0-9a-f]{64}$/\nconst CELL_ID = /^agent-profile-cell:sha256:[0-9a-f]{64}$/\n\nexport async function buildAgentProfileCell(\n input: AgentProfileCellInput,\n): Promise<AgentProfileCell> {\n const material = await normalizeAgentProfileCellInput(input)\n const cellId = `agent-profile-cell:sha256:${await hashJson(material)}`\n return { ...material, cellId }\n}\n\nexport function agentProfileCellHashMaterial(\n cell: AgentProfileCell,\n): Omit<AgentProfileCell, 'cellId'> {\n const { cellId: _cellId, ...material } = cell\n void _cellId\n return normalizeAgentProfileCell(material)\n}\n\n/**\n * Verify an `AgentProfileCell`'s `cellId` matches the sha256 of its hash-material fields, confirming the record has not been tampered with.\n */\nexport async function verifyAgentProfileCell(cell: AgentProfileCell): Promise<boolean> {\n validateAgentProfileCell(cell)\n return (\n cell.cellId ===\n `agent-profile-cell:sha256:${await hashJson(agentProfileCellHashMaterial(cell))}`\n )\n}\n\nexport function validateAgentProfileCell(input: unknown): AgentProfileCell {\n if (input === null || typeof input !== 'object') {\n throw new AgentProfileCellValidationError('expected object')\n }\n const obj = input as Record<string, unknown>\n expectLiteral(obj.schemaVersion, 'agent-profile-cell/v1', 'schemaVersion')\n if (typeof obj.cellId !== 'string' || !CELL_ID.test(obj.cellId)) {\n throw new AgentProfileCellValidationError(\n 'cellId must match agent-profile-cell:sha256:<64 lowercase hex chars>',\n 'cellId',\n )\n }\n expectString(obj.profileId, 'profileId')\n validateSource(obj.sourceProfile, 'sourceProfile')\n if (obj.harness !== undefined) validateHarness(obj.harness, 'harness')\n if (obj.model !== undefined) expectString(obj.model, 'model')\n if (obj.promptHash !== undefined) expectString(obj.promptHash, 'promptHash')\n if (obj.dimensions !== undefined) validateDimensions(obj.dimensions, 'dimensions')\n return input as AgentProfileCell\n}\n\nexport function requireAgentProfileCell(record: {\n runId: string\n agentProfile?: AgentProfileCell\n}): AgentProfileCell {\n if (!record.agentProfile) {\n throw new AgentProfileCellValidationError(\n `run \"${record.runId}\" is missing agentProfile; profile-cell grouping requires explicit profile identity`,\n 'agentProfile',\n )\n }\n return validateAgentProfileCell(record.agentProfile)\n}\n\nexport function agentProfileCellKey(record: {\n runId: string\n agentProfile?: AgentProfileCell\n}): string {\n return requireAgentProfileCell(record).cellId\n}\n\nexport async function assertRunAgentProfileCell(record: {\n runId: string\n model: string\n promptHash: string\n agentProfile?: AgentProfileCell\n}): Promise<AgentProfileCell> {\n const profile = requireAgentProfileCell(record)\n if (!(await verifyAgentProfileCell(profile))) {\n throw new AgentProfileCellValidationError(\n `run \"${record.runId}\" has an agentProfile.cellId that does not match its content`,\n 'agentProfile.cellId',\n )\n }\n if (profile.model !== undefined && profile.model !== record.model) {\n throw new AgentProfileCellValidationError(\n `run \"${record.runId}\" agentProfile.model \"${profile.model}\" does not match model \"${record.model}\"`,\n 'agentProfile.model',\n )\n }\n if (profile.promptHash !== undefined && profile.promptHash !== record.promptHash) {\n throw new AgentProfileCellValidationError(\n `run \"${record.runId}\" agentProfile.promptHash \"${profile.promptHash}\" does not match promptHash \"${record.promptHash}\"`,\n 'agentProfile.promptHash',\n )\n }\n return profile\n}\n\nexport function groupRunsByAgentProfileCell<\n T extends { runId: string; agentProfile?: AgentProfileCell },\n>(records: readonly T[]): Map<string, T[]> {\n const groups = new Map<string, T[]>()\n for (const record of records) {\n const key = agentProfileCellKey(record)\n const bucket = groups.get(key)\n if (bucket) bucket.push(record)\n else groups.set(key, [record])\n }\n return groups\n}\n\nasync function normalizeAgentProfileCellInput(\n input: AgentProfileCellInput,\n): Promise<Omit<AgentProfileCell, 'cellId'>> {\n return normalizeAgentProfileCell({\n schemaVersion: 'agent-profile-cell/v1',\n profileId: input.profileId,\n sourceProfile: await normalizeSourceInput(input.sourceProfile),\n harness: input.harness,\n model: input.model,\n promptHash: input.promptHash,\n dimensions: input.dimensions,\n })\n}\n\nfunction normalizeAgentProfileCell(\n input: Omit<AgentProfileCell, 'cellId'>,\n): Omit<AgentProfileCell, 'cellId'> {\n return compactObject({\n schemaVersion: 'agent-profile-cell/v1' as const,\n profileId: requireNonEmpty(input.profileId, 'profileId'),\n sourceProfile: normalizeSource(input.sourceProfile),\n harness: input.harness ? normalizeHarness(input.harness, 'harness') : undefined,\n model: optionalNonEmpty(input.model, 'model'),\n promptHash: optionalNonEmpty(input.promptHash, 'promptHash'),\n dimensions: input.dimensions\n ? nonEmptyRecord(normalizeDimensions(input.dimensions))\n : undefined,\n })\n}\n\nasync function normalizeSourceInput(input: AgentProfileSourceInput): Promise<AgentProfileSource> {\n const kind = requireNonEmpty(input.kind, 'sourceProfile.kind')\n if (input.hash !== undefined && input.profile !== undefined) {\n throw new AgentProfileCellValidationError(\n 'sourceProfile must provide either hash or profile, not both',\n 'sourceProfile',\n )\n }\n if (input.hash !== undefined) {\n return { kind, hash: requireSha256Hex(input.hash, 'sourceProfile.hash') }\n }\n if (input.profile === undefined) {\n throw new AgentProfileCellValidationError(\n 'sourceProfile must provide hash or profile',\n 'sourceProfile',\n )\n }\n assertJson(input.profile, 'sourceProfile.profile')\n return { kind, hash: await hashJson(input.profile) }\n}\n\nfunction normalizeSource(input: AgentProfileSource): AgentProfileSource {\n return {\n kind: requireNonEmpty(input.kind, 'sourceProfile.kind'),\n hash: requireSha256Hex(input.hash, 'sourceProfile.hash'),\n }\n}\n\nfunction normalizeHarness(input: AgentProfileHarness, path: string): AgentProfileHarness {\n return compactObject({\n id: requireNonEmpty(input.id, `${path}.id`),\n version: optionalNonEmpty(input.version, `${path}.version`),\n hash: optionalNonEmpty(input.hash, `${path}.hash`),\n })\n}\n\nfunction normalizeDimensions(\n input: Record<string, AgentProfileDimensionValue>,\n): Record<string, AgentProfileDimensionValue> {\n const out: Record<string, AgentProfileDimensionValue> = {}\n for (const key of Object.keys(input).sort()) {\n const value = input[key]\n requireNonEmpty(key, 'dimensions.<key>')\n if (\n value !== null &&\n typeof value !== 'string' &&\n typeof value !== 'number' &&\n typeof value !== 'boolean'\n ) {\n throw new AgentProfileCellValidationError(\n 'expected primitive dimension value',\n `dimensions.${key}`,\n )\n }\n if (typeof value === 'number' && !Number.isFinite(value)) {\n throw new AgentProfileCellValidationError('expected finite number', `dimensions.${key}`)\n }\n out[key] = value\n }\n return out\n}\n\nfunction compactObject<T extends Record<string, unknown>>(input: T): T {\n const out: Record<string, unknown> = {}\n for (const [key, value] of Object.entries(input)) {\n if (value !== undefined) out[key] = value\n }\n return out as T\n}\n\nfunction nonEmptyRecord<T extends Record<string, unknown>>(input: T): T | undefined {\n return Object.keys(input).length > 0 ? input : undefined\n}\n\nfunction validateSource(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object' || Array.isArray(value)) {\n throw new AgentProfileCellValidationError('expected object', path)\n }\n const rec = value as Record<string, unknown>\n expectString(rec.kind, `${path}.kind`)\n requireSha256Hex(rec.hash, `${path}.hash`)\n}\n\nfunction validateHarness(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object' || Array.isArray(value)) {\n throw new AgentProfileCellValidationError('expected object', path)\n }\n const rec = value as Record<string, unknown>\n expectString(rec.id, `${path}.id`)\n if (rec.version !== undefined) expectString(rec.version, `${path}.version`)\n if (rec.hash !== undefined) expectString(rec.hash, `${path}.hash`)\n}\n\nfunction validateDimensions(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object' || Array.isArray(value)) {\n throw new AgentProfileCellValidationError('expected object', path)\n }\n normalizeDimensions(value as Record<string, AgentProfileDimensionValue>)\n}\n\nfunction assertJson(value: AgentProfileJson, path: string): void {\n if (value === null) return\n const type = typeof value\n if (type === 'string' || type === 'boolean') return\n if (type === 'number') {\n if (!Number.isFinite(value)) {\n throw new AgentProfileCellValidationError('expected finite number', path)\n }\n return\n }\n if (Array.isArray(value)) {\n value.forEach((item, index) => {\n assertJson(item, `${path}[${index}]`)\n })\n return\n }\n if (type === 'object') {\n for (const [key, nested] of Object.entries(value)) {\n requireNonEmpty(key, `${path}.<key>`)\n assertJson(nested, `${path}.${key}`)\n }\n return\n }\n throw new AgentProfileCellValidationError('expected JSON-compatible value', path)\n}\n\nfunction expectLiteral(value: unknown, expected: string, path: string): void {\n if (value !== expected) {\n throw new AgentProfileCellValidationError(`expected ${expected}`, path)\n }\n}\n\nfunction expectString(value: unknown, path: string): void {\n if (typeof value !== 'string' || value.length === 0) {\n throw new AgentProfileCellValidationError('expected non-empty string', path)\n }\n}\n\nfunction requireNonEmpty(value: string, path: string): string {\n if (typeof value !== 'string' || value.length === 0) {\n throw new AgentProfileCellValidationError('expected non-empty string', path)\n }\n return value\n}\n\nfunction optionalNonEmpty(value: string | undefined, path: string): string | undefined {\n if (value === undefined) return undefined\n return requireNonEmpty(value, path)\n}\n\nfunction requireSha256Hex(value: unknown, path: string): string {\n if (typeof value !== 'string' || !SHA256_HEX.test(value)) {\n throw new AgentProfileCellValidationError('expected 64 lowercase sha256 hex chars', path)\n }\n return value\n}\n\n// ── Consumer helpers ─────────────────────────────────────────────────\n//\n// Boilerplate every product consuming `buildAgentProfileCell` used to duplicate:\n//\n// 1. A `JSON.parse(JSON.stringify(value))` helper that canonicalizes an\n// arbitrary `@tangle-network/agent-interface` `AgentProfile` into the recursive\n// `AgentProfileJson` shape, with a fail-loud error when the profile\n// is not JSON-serializable.\n//\n// 2. The magic string `'agent-interface-profile'` for `sourceProfile.kind`.\n//\n// Both belong here so the cross-product cell join (same canonical profile\n// hashes to the same `sourceProfile.hash` across products) is enforced by\n// the type system, not by every consumer remembering to do it right.\n// See blueprint-agent issue tangle-network/agent-eval#82.\n\n/** Canonical `sourceProfile.kind` values. Two products fingerprinting the\n * same canonical profile MUST use the same kind for their cells to share\n * `sourceProfile.hash`. Extend rather than create new strings — adding a\n * new kind is a deliberate cross-product schema change. */\nexport const AGENT_PROFILE_KINDS = {\n /** A profile declared via `defineAgentProfile(...)` from\n * `@tangle-network/agent-interface`. The default kind for router-backed\n * and sandbox-backed products. */\n AGENT_INTERFACE_PROFILE: 'agent-interface-profile',\n} as const\n\nexport type AgentProfileKind = (typeof AGENT_PROFILE_KINDS)[keyof typeof AGENT_PROFILE_KINDS]\n\n/** Canonicalize an arbitrary value into `AgentProfileJson` by JSON\n * round-trip. Throws when the value contains anything not representable\n * as JSON (functions, BigInt, cycles) — non-portable profiles fail loud\n * rather than silently dropping fields. */\nexport function toAgentProfileJson(value: unknown): AgentProfileJson {\n let serialized: string | undefined\n try {\n serialized = JSON.stringify(value)\n } catch (err) {\n throw new AgentProfileCellValidationError(\n `agent profile must be JSON-serializable: ${err instanceof Error ? err.message : String(err)}`,\n 'sourceProfile.profile',\n )\n }\n if (serialized === undefined) {\n throw new AgentProfileCellValidationError(\n 'agent profile must be JSON-serializable (got undefined after JSON.stringify)',\n 'sourceProfile.profile',\n )\n }\n return JSON.parse(serialized) as AgentProfileJson\n}\n\n/** Canonical AgentProfile shape required when deriving a stable cell id. */\nexport type AgentInterfaceProfileLike = AgentProfile & { name: string; version: string }\n\n/** Higher-level helper that hard-codes the canonical\n * `agent-interface-profile` kind plus the JSON canonicalization. Equivalent\n * to calling `buildAgentProfileCell` with `profileId = \\`${name}@${version}\\``\n * and `sourceProfile = { kind: AGENT_INTERFACE_PROFILE, profile: <round-tripped> }`.\n *\n * Use this from any product consuming an agent-interface `AgentProfile`; the\n * manual `buildAgentProfileCell` call is reserved for advanced cases\n * (custom kinds, pre-computed source hashes, alternate profileId\n * conventions). */\nexport async function buildAgentInterfaceProfileCell(\n profile: AgentInterfaceProfileLike,\n input: Omit<AgentProfileCellInput, 'profileId' | 'sourceProfile'>,\n): Promise<AgentProfileCell> {\n if (!profile || typeof profile !== 'object') {\n throw new AgentProfileCellValidationError('AgentProfile must be an object', 'profile')\n }\n if (typeof profile.name !== 'string' || profile.name.length === 0) {\n throw new AgentProfileCellValidationError(\n 'AgentProfile must have a non-empty `name`',\n 'profile.name',\n )\n }\n if (typeof profile.version !== 'string' || profile.version.length === 0) {\n throw new AgentProfileCellValidationError(\n 'AgentProfile must have a non-empty `version`',\n 'profile.version',\n )\n }\n return buildAgentProfileCell({\n ...input,\n profileId: `${profile.name}@${profile.version}`,\n sourceProfile: {\n kind: AGENT_PROFILE_KINDS.AGENT_INTERFACE_PROFILE,\n profile: toAgentProfileJson(profile),\n },\n })\n}\n","/**\n * Paper-grade RunRecord schema + runtime validator.\n *\n * Every run that participates in a promotion gate, paper table, or\n * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory\n * fields are exactly those the paper \"Two Loops, Three Roles\" requires\n * for reproducibility: who/what/when/cost/seed/hash, plus the search vs\n * holdout split tag. A task score is optional because execution-only records\n * must preserve missing labels instead of converting errors into zero quality.\n *\n * This is intentionally NOT a replacement for the rich `Run` /\n * `ProposeReviewReport` / `ScenarioResult` types already in the\n * package. Those are runtime structures with full provenance. A\n * `RunRecord` is the analysis-time projection — the JSON-friendly\n * row you'd put in a parquet file or paste into a notebook.\n *\n * Validate at the boundary:\n *\n * const rec = validateRunRecord(rawJson) // throws on missing\n * const ok = isRunRecord(rawJson) // boolean check\n * const rec = parseRunRecordSafe(rawJson) // { ok, value | error }\n *\n * The validator runs in pure TS — zod is intentionally NOT a\n * dependency. Round-trip tested in `tests/run-record.test.ts`.\n */\n\nimport type { AgentProfileCell } from './agent-profile-cell'\nimport { validateAgentProfileCell } from './agent-profile-cell'\nimport { ValidationError } from './errors'\nimport { FAILURE_CLASSES, type FailureClass } from './trace/schema'\n\n/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the\n * combined train+test pool that the optimizer is allowed to read. */\nexport type RunSplitTag = 'search' | 'dev' | 'holdout'\n\n/**\n * Explicit execution-lifecycle result for a run.\n *\n * This is separate from task quality (`outcome`) and failure classification.\n * Producers set it only from root-run or process evidence.\n */\nexport type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown'\n\nexport interface RunTokenUsage {\n input: number\n /** All generated tokens charged as output, including reasoning tokens. */\n output: number\n /** Reasoning-token subset of `output`, when the provider reports it. */\n reasoning?: number\n /** Prompt tokens served from a provider cache. */\n cached?: number\n /** Prompt tokens written into a provider cache. */\n cacheWrite?: number\n}\n\n/**\n * How a run's USD amount was obtained.\n */\nexport type RunCostProvenance =\n | { kind: 'observed'; usd: number }\n | { kind: 'estimated'; usd: number }\n | { kind: 'uncaptured'; usd: null }\n\nexport interface RunJudgeMetadata {\n model: string\n promptVersion: string\n /** [0,1] confidence the judge declared. Constant judge confidence\n * across many runs is a fallback signal (see `canary.ts`). */\n confidence: number\n /** True if the judge degraded to a fallback path (rules-only,\n * prior-call cache, etc.). The canary uses this to alert. */\n fallback: boolean\n}\n\n/**\n * Per-judge / per-dimension breakdown for runs scored by an ensemble of\n * judges over a multi-dimensional rubric.\n *\n * The collapsed `outcome.searchScore` / `holdoutScore` carries the\n * composite the gate uses. The full breakdown belongs here so consumers\n * can answer \"which judge disagreed?\", \"which dimension dragged the\n * composite down?\", and \"did half the panel fail?\" without re-running.\n *\n * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and\n * `composite` are convenience projections — derivable but precomputed so\n * downstream IRR primitives (`interRaterReliability`,\n * `corpusInterRaterAgreement`) and reporters don't pay the same\n * aggregation twice.\n *\n * Fail-loud discipline: judges that errored out land in `failedJudges`\n * by id. A missing key in `perJudge` is ambiguous (silent zero vs not\n * run); the explicit list makes a partial-failure recorded as such.\n */\nexport interface JudgeScoresRecord {\n /** Per-judge per-dimension scores. `{ \"kimi-k2.6\": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */\n perJudge: Record<string, Record<string, number>>\n /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */\n perDimMean: Record<string, number>\n /** Composite mean across successful judges. Mirrors the task score only\n * when `failedJudges` is empty. */\n composite: number\n /** Judges that errored or returned an unparseable verdict. Recorded\n * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,\n * not inferred from missing keys in `perJudge`. */\n failedJudges?: string[]\n /** Free-form notes the judges emitted (joined across judges or\n * first-judge only — consumer's choice). */\n notes?: string\n}\n\nexport interface RunOutcome {\n /** Score on the search/optimization split. Optional for holdout-only and\n * execution-only records. */\n searchScore?: number\n /** Score on the held-out split. Optional for search-only and execution-only\n * records. When both scores are absent, the run is explicitly unlabeled. */\n holdoutScore?: number\n /** Bag of any other metric the run produced — judge dimensions,\n * pass/fail counters, latency stats, etc. Numeric only — keeps\n * reporters honest. */\n raw: Record<string, number>\n /** Per-judge / per-dim breakdown. Consumers writing ensemble\n * judgements populate this; substrate primitives like\n * `interRaterReliability` and `corpusInterRaterAgreement` accept\n * these records as input. Optional — single-judge or scalar-only\n * runs leave it unset. */\n judgeScores?: JudgeScoresRecord\n /** Authenticity / realness verdict — did the run build the REAL thing on the\n * intended infra, or fake it (see `./authenticity`)? Optional: only domains\n * with an authenticity config populate it. Carried in the corpus so the\n * flywheel / off-policy learning can optimize for real completion, not gamed\n * pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run\n * must not count as a real success regardless of `score`. */\n realness?: { score: number; gated: boolean; reason?: string }\n}\n\n/**\n * Mandatory paper-grade fields for a single evaluation run. Optional\n * fields are extension points; mandatory fields throw if missing.\n *\n * Hash discipline:\n * - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the\n * model (after any steering bundle merge).\n * - `configHash` is the sha256 of the effective run config (model,\n * temperature, tools, judges, splits). The pair (promptHash,\n * configHash) uniquely identifies an experiment cell.\n *\n * Model snapshot discipline:\n * - `model` MUST encode a snapshot version. Bare aliases like\n * `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.\n * Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.\n */\nexport interface RunRecord {\n /** UUID for the run. */\n runId: string\n /** Logical experiment grouping (a treatment vs a baseline within\n * the same sweep should share `experimentId`). */\n experimentId: string\n /** Stable identifier for the candidate (variant) being run. The\n * promotion gate compares two `candidateId`s on matched items. */\n candidateId: string\n /** RNG seed for the run. Always recorded — silent re-seeding is\n * the most common cause of non-reproducible numbers. */\n seed: number\n /** Model identifier WITH snapshot version. */\n model: string\n /** sha256 of the effective prompt (post-steering). */\n promptHash: string\n /** sha256 of the effective config. */\n configHash: string\n /** Git SHA the harness was run from. */\n commitSha: string\n /** End-to-end wall-clock duration in milliseconds. */\n wallMs: number\n /** Time spent queued before execution started, if known. */\n queueMs?: number\n /** Total USD cost, or null when the producer could not capture one. */\n costUsd: number | null\n /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */\n costProvenance: RunCostProvenance\n /** Token usage breakdown. */\n tokenUsage: RunTokenUsage\n /** Root-run or process terminal result. Never inferred from a child span. */\n terminalOutcome: RunTerminalOutcome\n /** Root-run or process failure reason. Valid only for a failed, cancelled,\n * or incomplete terminal result; never populated from a child span. */\n terminalFailureReason?: string\n /** Judge-side metadata, if a judge was used. */\n judgeMetadata?: RunJudgeMetadata\n /** Per-split scores + raw bag. */\n outcome: RunOutcome\n /** Canonical task-failure class drawn from the shared\n * `FAILURE_CLASSES` taxonomy. Producers set it only from task-result\n * evidence. Execution errors belong in\n * `outcome.raw.execution_error_count`. */\n failureClass?: FailureClass\n /** Free-form task-failure detail scoped under a non-success\n * `failureClass`. It is invalid without that class. */\n failureMode?: string\n /** Which split this run was drawn from. */\n splitTag: RunSplitTag\n /**\n * Stable scenario identifier the run observed or was scored against.\n * Comparison primitives match this identity rather than input order.\n */\n scenarioId: string\n /**\n * Canonical identity for the agent profile cell that produced this row:\n * profile artifact hash plus optional harness/model/prompt/reporting\n * dimensions. Use `agentProfile.cellId` to group persona sweeps and\n * longitudinal reports by the complete source profile, not by a loose\n * candidate label or opaque config hash.\n */\n agentProfile?: AgentProfileCell\n}\n\n/**\n * Canonical task-result classification.\n *\n * A producer may omit classification, record explicit success, or attach\n * domain-specific detail to a non-success class. Detail can never stand alone.\n * Execution errors belong in `outcome.raw.execution_error_count`.\n */\nexport type RunTaskFailure =\n | { failureClass?: undefined; failureMode?: undefined }\n | { failureClass: 'success'; failureMode?: undefined }\n | {\n failureClass: Exclude<FailureClass, 'success'>\n failureMode?: string\n }\n\n/** Return task quality, preferring held-out evidence when both scores exist. */\nexport function runTaskScore(record: RunRecord): number | undefined {\n const score = record.outcome.holdoutScore ?? record.outcome.searchScore\n return typeof score === 'number' && Number.isFinite(score) ? score : undefined\n}\n\n// ── Validation ───────────────────────────────────────────────────────\n\nconst MANDATORY_TOP_LEVEL = [\n 'runId',\n 'experimentId',\n 'candidateId',\n 'seed',\n 'model',\n 'promptHash',\n 'configHash',\n 'commitSha',\n 'wallMs',\n 'costUsd',\n 'costProvenance',\n 'tokenUsage',\n 'terminalOutcome',\n 'outcome',\n 'splitTag',\n 'scenarioId',\n] as const\n\nconst SPLIT_TAGS: ReadonlyArray<RunSplitTag> = ['search', 'dev', 'holdout']\nconst TERMINAL_OUTCOMES: ReadonlyArray<RunTerminalOutcome> = [\n 'succeeded',\n 'failed',\n 'cancelled',\n 'incomplete',\n 'unknown',\n]\n\nexport class RunRecordValidationError extends ValidationError {\n readonly path: string\n constructor(message: string, path = '') {\n super(path ? `${message} (at ${path})` : message)\n this.path = path\n }\n}\n\n/**\n * Strict validator. Throws `RunRecordValidationError` on the first\n * missing or wrongly-typed field. Returns the input cast to\n * `RunRecord` on success — the validator does not coerce.\n */\nexport function validateRunRecord(input: unknown): RunRecord {\n if (input === null || typeof input !== 'object') {\n throw new RunRecordValidationError('expected object')\n }\n const obj = input as Record<string, unknown>\n\n for (const key of MANDATORY_TOP_LEVEL) {\n if (!(key in obj)) {\n throw new RunRecordValidationError(`missing mandatory field \"${key}\"`)\n }\n }\n\n expectString(obj.runId, 'runId')\n expectString(obj.experimentId, 'experimentId')\n expectString(obj.candidateId, 'candidateId')\n expectFiniteNumber(obj.seed, 'seed')\n expectString(obj.model, 'model')\n expectString(obj.promptHash, 'promptHash')\n expectString(obj.configHash, 'configHash')\n expectString(obj.commitSha, 'commitSha')\n expectNonNegativeNumber(obj.wallMs, 'wallMs')\n if (obj.queueMs !== undefined) expectNonNegativeNumber(obj.queueMs, 'queueMs')\n validateCost(obj.costUsd, obj.costProvenance)\n\n // Snapshot discipline: bare model aliases are not paper-grade.\n if (!modelHasSnapshot(obj.model as string)) {\n throw new RunRecordValidationError(\n `model \"${obj.model}\" lacks a snapshot version (use 'name@YYYY-MM-DD' or 'name-YYYYMMDD')`,\n 'model',\n )\n }\n\n // Token usage.\n const tu = obj.tokenUsage\n if (tu === null || typeof tu !== 'object') {\n throw new RunRecordValidationError('tokenUsage must be an object', 'tokenUsage')\n }\n const tuRec = tu as Record<string, unknown>\n expectNonNegativeNumber(tuRec.input, 'tokenUsage.input')\n expectNonNegativeNumber(tuRec.output, 'tokenUsage.output')\n if (tuRec.reasoning !== undefined) {\n expectNonNegativeNumber(tuRec.reasoning, 'tokenUsage.reasoning')\n if ((tuRec.reasoning as number) > (tuRec.output as number)) {\n throw new RunRecordValidationError(\n 'reasoning tokens must be a subset of output tokens',\n 'tokenUsage.reasoning',\n )\n }\n }\n if (tuRec.cached !== undefined) expectNonNegativeNumber(tuRec.cached, 'tokenUsage.cached')\n if (tuRec.cacheWrite !== undefined) {\n expectNonNegativeNumber(tuRec.cacheWrite, 'tokenUsage.cacheWrite')\n }\n\n // Judge metadata, optional.\n if (obj.judgeMetadata !== undefined) {\n const jm = obj.judgeMetadata\n if (jm === null || typeof jm !== 'object') {\n throw new RunRecordValidationError('judgeMetadata must be an object', 'judgeMetadata')\n }\n const jmRec = jm as Record<string, unknown>\n expectString(jmRec.model, 'judgeMetadata.model')\n expectString(jmRec.promptVersion, 'judgeMetadata.promptVersion')\n expectFiniteNumber(jmRec.confidence, 'judgeMetadata.confidence')\n if (typeof jmRec.fallback !== 'boolean') {\n throw new RunRecordValidationError(\n 'judgeMetadata.fallback must be boolean',\n 'judgeMetadata.fallback',\n )\n }\n }\n\n // Outcome.\n const out = obj.outcome\n if (out === null || typeof out !== 'object') {\n throw new RunRecordValidationError('outcome must be an object', 'outcome')\n }\n const outRec = out as Record<string, unknown>\n if (outRec.searchScore !== undefined)\n expectFiniteNumber(outRec.searchScore, 'outcome.searchScore')\n if (outRec.holdoutScore !== undefined)\n expectFiniteNumber(outRec.holdoutScore, 'outcome.holdoutScore')\n const raw = outRec.raw\n if (raw === null || typeof raw !== 'object') {\n throw new RunRecordValidationError('outcome.raw must be an object', 'outcome.raw')\n }\n for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {\n expectFiniteNumber(v, `outcome.raw.${k}`)\n }\n // Realness verdict, optional.\n if (outRec.realness !== undefined) {\n const r = outRec.realness\n if (r === null || typeof r !== 'object') {\n throw new RunRecordValidationError('outcome.realness must be an object', 'outcome.realness')\n }\n const rr = r as Record<string, unknown>\n expectFiniteNumber(rr.score, 'outcome.realness.score')\n if (typeof rr.gated !== 'boolean') {\n throw new RunRecordValidationError(\n 'outcome.realness.gated must be a boolean',\n 'outcome.realness.gated',\n )\n }\n }\n\n // Per-judge / per-dim breakdown, optional.\n if (outRec.judgeScores !== undefined) {\n validateJudgeScores(outRec.judgeScores, 'outcome.judgeScores')\n }\n\n // Failure mode optional.\n if (\n obj.failureClass !== undefined &&\n (typeof obj.failureClass !== 'string' ||\n !FAILURE_CLASSES.includes(obj.failureClass as FailureClass))\n ) {\n throw new RunRecordValidationError(\n `failureClass must be one of ${FAILURE_CLASSES.join(', ')}`,\n 'failureClass',\n )\n }\n if (obj.failureMode !== undefined) {\n expectString(obj.failureMode, 'failureMode')\n if (obj.failureClass === undefined || obj.failureClass === 'success') {\n throw new RunRecordValidationError(\n 'failureMode requires a non-success failureClass',\n 'failureMode',\n )\n }\n }\n\n if (\n typeof obj.terminalOutcome !== 'string' ||\n !TERMINAL_OUTCOMES.includes(obj.terminalOutcome as RunTerminalOutcome)\n ) {\n throw new RunRecordValidationError(\n `terminalOutcome must be one of ${TERMINAL_OUTCOMES.join(', ')}`,\n 'terminalOutcome',\n )\n }\n if (obj.terminalFailureReason !== undefined) {\n expectString(obj.terminalFailureReason, 'terminalFailureReason')\n if (\n obj.terminalOutcome !== 'failed' &&\n obj.terminalOutcome !== 'cancelled' &&\n obj.terminalOutcome !== 'incomplete'\n ) {\n throw new RunRecordValidationError(\n 'terminalFailureReason requires terminalOutcome failed, cancelled, or incomplete',\n 'terminalFailureReason',\n )\n }\n }\n\n if (obj.agentProfile !== undefined) {\n try {\n const profile = validateAgentProfileCell(obj.agentProfile)\n if (profile.model !== undefined && profile.model !== obj.model) {\n throw new RunRecordValidationError(\n `agentProfile.model \"${profile.model}\" does not match model \"${obj.model}\"`,\n 'agentProfile.model',\n )\n }\n if (profile.promptHash !== undefined && profile.promptHash !== obj.promptHash) {\n throw new RunRecordValidationError(\n `agentProfile.promptHash \"${profile.promptHash}\" does not match promptHash \"${obj.promptHash}\"`,\n 'agentProfile.promptHash',\n )\n }\n } catch (error) {\n if (error instanceof RunRecordValidationError) throw error\n if (error instanceof Error) {\n throw new RunRecordValidationError(error.message, 'agentProfile')\n }\n throw error\n }\n }\n\n expectString(obj.scenarioId, 'scenarioId')\n\n // Split tag.\n if (typeof obj.splitTag !== 'string' || !SPLIT_TAGS.includes(obj.splitTag as RunSplitTag)) {\n throw new RunRecordValidationError(\n `splitTag must be one of ${SPLIT_TAGS.join(', ')}, got ${String(obj.splitTag)}`,\n 'splitTag',\n )\n }\n\n return input as RunRecord\n}\n\nfunction validateCost(costUsd: unknown, provenance: unknown): void {\n if (provenance === null || typeof provenance !== 'object') {\n throw new RunRecordValidationError('costProvenance must be an object', 'costProvenance')\n }\n const value = provenance as Record<string, unknown>\n if (value.kind !== 'observed' && value.kind !== 'estimated' && value.kind !== 'uncaptured') {\n throw new RunRecordValidationError(\n 'costProvenance.kind must be observed, estimated, or uncaptured',\n 'costProvenance.kind',\n )\n }\n if (value.kind === 'uncaptured') {\n if (value.usd !== null) {\n throw new RunRecordValidationError(\n 'uncaptured costProvenance.usd must be null',\n 'costProvenance.usd',\n )\n }\n if (costUsd !== null) {\n throw new RunRecordValidationError('uncaptured cost requires costUsd to be null', 'costUsd')\n }\n return\n }\n expectNonNegativeNumber(costUsd, 'costUsd')\n expectNonNegativeNumber(value.usd, 'costProvenance.usd')\n if (value.usd !== costUsd) {\n throw new RunRecordValidationError(\n 'costProvenance.usd must equal costUsd',\n 'costProvenance.usd',\n )\n }\n}\n\n/** Boolean validator — convenience for filtering arrays. */\nexport function isRunRecord(input: unknown): input is RunRecord {\n try {\n validateRunRecord(input)\n return true\n } catch {\n return false\n }\n}\n\n/** Non-throwing validator — returns a discriminated union. */\nexport function parseRunRecordSafe(\n input: unknown,\n): { ok: true; value: RunRecord } | { ok: false; error: RunRecordValidationError } {\n try {\n return { ok: true, value: validateRunRecord(input) }\n } catch (e) {\n if (e instanceof RunRecordValidationError) return { ok: false, error: e }\n throw e\n }\n}\n\n/** Round-trip helper — `JSON.parse(JSON.stringify(record))` then validate. */\nexport function roundTripRunRecord(record: RunRecord): RunRecord {\n const json = JSON.stringify(record)\n return validateRunRecord(JSON.parse(json))\n}\n\n// ── Internals ────────────────────────────────────────────────────────\n\nfunction expectString(value: unknown, path: string): void {\n if (typeof value !== 'string' || value.length === 0) {\n throw new RunRecordValidationError(`expected non-empty string`, path)\n }\n}\n\nfunction expectFiniteNumber(value: unknown, path: string): void {\n if (typeof value !== 'number' || !Number.isFinite(value)) {\n throw new RunRecordValidationError(`expected finite number`, path)\n }\n}\n\nfunction expectNonNegativeNumber(value: unknown, path: string): void {\n expectFiniteNumber(value, path)\n if ((value as number) < 0) {\n throw new RunRecordValidationError('expected non-negative number', path)\n }\n}\n\nfunction validateJudgeScores(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object') {\n throw new RunRecordValidationError('judgeScores must be an object', path)\n }\n const rec = value as Record<string, unknown>\n\n const perJudge = rec.perJudge\n if (perJudge === null || typeof perJudge !== 'object') {\n throw new RunRecordValidationError('perJudge must be an object', `${path}.perJudge`)\n }\n for (const [judgeId, dims] of Object.entries(perJudge as Record<string, unknown>)) {\n if (dims === null || typeof dims !== 'object') {\n throw new RunRecordValidationError(\n 'per-judge entry must be an object of dimension scores',\n `${path}.perJudge.${judgeId}`,\n )\n }\n for (const [dim, score] of Object.entries(dims as Record<string, unknown>)) {\n expectFiniteNumber(score, `${path}.perJudge.${judgeId}.${dim}`)\n }\n }\n\n const perDimMean = rec.perDimMean\n if (perDimMean === null || typeof perDimMean !== 'object') {\n throw new RunRecordValidationError('perDimMean must be an object', `${path}.perDimMean`)\n }\n for (const [dim, mean] of Object.entries(perDimMean as Record<string, unknown>)) {\n expectFiniteNumber(mean, `${path}.perDimMean.${dim}`)\n }\n\n expectFiniteNumber(rec.composite, `${path}.composite`)\n\n if (rec.failedJudges !== undefined) {\n if (!Array.isArray(rec.failedJudges)) {\n throw new RunRecordValidationError(\n 'failedJudges must be an array of strings',\n `${path}.failedJudges`,\n )\n }\n for (let i = 0; i < rec.failedJudges.length; i++) {\n const id = rec.failedJudges[i]\n if (typeof id !== 'string' || id.length === 0) {\n throw new RunRecordValidationError(\n 'failedJudges entry must be a non-empty string',\n `${path}.failedJudges[${i}]`,\n )\n }\n }\n }\n\n if (rec.notes !== undefined && typeof rec.notes !== 'string') {\n throw new RunRecordValidationError('notes must be a string', `${path}.notes`)\n }\n}\n\n/**\n * Heuristic snapshot check. Accepts:\n * - `name@YYYY-MM-DD` (Anthropic style: `claude-sonnet-4-6@2025-04-15`)\n * - `name-YYYYMMDD` (OpenAI style: `gpt-4o-2024-11-20`)\n * - `name@<arbitrary-token>` (allow opaque snapshots like `@v3`)\n * - explicit `:date-...` Vertex-style tags\n *\n * Rejects bare aliases like `claude-sonnet-4` or `gpt-4o` that remap\n * silently as providers ship new snapshots.\n */\nexport function modelHasSnapshot(model: string): boolean {\n if (model.includes('@')) return true\n if (/-\\d{8}$/.test(model)) return true\n if (/-\\d{4}-\\d{2}-\\d{2}$/.test(model)) return true\n if (/:date-/.test(model)) return true\n return false\n}\n"],"mappings":";;;;;;;;AAkFO,SAAS,aAAa,GAAqB;AAChD,MAAI,MAAM,QAAQ,OAAO,MAAM,SAAU,QAAO;AAChD,MAAI,MAAM,QAAQ,CAAC,EAAG,QAAO,EAAE,IAAI,YAAY;AAC/C,QAAM,OAAO,OAAO,KAAK,CAA4B,EAAE,KAAK;AAC5D,QAAM,MAA+B,CAAC;AACtC,aAAW,KAAK,KAAM,KAAI,CAAC,IAAI,aAAc,EAA8B,CAAC,CAAC;AAC7E,SAAO;AACT;AAuBA,eAAsB,SAAY,KAAyB;AACzD,QAAM,YAAY,aAAa,GAAG;AAClC,QAAM,QAAQ,IAAI,YAAY,EAAE,OAAO,KAAK,UAAU,SAAS,CAAC;AAChE,QAAM,SAAS,MAAM,WAAW,OAAO,OAAO,OAAO,WAAW,KAAK;AACrE,SAAO,MAAM,KAAK,IAAI,WAAW,MAAM,CAAC,EACrC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG,CAAC,EAC1C,KAAK,EAAE;AACZ;AAWA,eAAsB,aAAa,GAAgD;AACjF,QAAM,OAAO,MAAM,SAAS,CAAC;AAC7B,SAAO,EAAE,GAAG,GAAG,aAAa,MAAM,MAAM,iBAAiB;AAC3D;AAQA,eAAsB,eAAe,GAAqC;AACxE,QAAM,EAAE,aAAa,MAAM,OAAO,GAAG,KAAK,IAAI;AAC9C,OAAK;AACL,QAAM,WAAW,MAAM,aAAa,IAAI;AACxC,SAAO,SAAS,gBAAgB;AAClC;AAMA,eAAsB,mBACpB,UACA,UAC2B;AAC3B,MAAI,CAAE,MAAM,eAAe,QAAQ,GAAI;AACrC,UAAM,IAAI,MAAM,+DAA+D;AAAA,EACjF;AACA,QAAM,UAAgD,CAAC;AACvD,QAAM,cAAc,SAAS,cAAc,aAAa,SAAS,SAAS,IAAI,SAAS,SAAS;AAChG,MAAI,CAAC,YAAa,SAAQ,KAAK,iBAAiB;AAChD,MAAI,KAAK,IAAI,SAAS,MAAM,IAAI,SAAS,UAAW,SAAQ,KAAK,kBAAkB;AACnF,MAAI,SAAS,UAAU,SAAS,MAAO,SAAQ,KAAK,iBAAiB;AACrE,MAAI,SAAS,IAAI,SAAS,eAAgB,SAAQ,KAAK,cAAc;AACrE,SAAO;AAAA,IACL;AAAA,IACA,WAAW,SAAS;AAAA,IACpB,gBAAgB,SAAS;AAAA,IACzB,gBAAgB,SAAS;AAAA,IACzB,WAAW,QAAQ,WAAW;AAAA,IAC9B,kBAAkB;AAAA,EACpB;AACF;;;AClHO,IAAM,kCAAN,cAA8C,gBAAgB;AAAA,EAC1D;AAAA,EACT,YAAY,SAAiB,OAAO,IAAI;AACtC,UAAM,OAAO,GAAG,OAAO,QAAQ,IAAI,MAAM,OAAO;AAChD,SAAK,OAAO;AAAA,EACd;AACF;AAEA,IAAM,aAAa;AACnB,IAAM,UAAU;AAEhB,eAAsB,sBACpB,OAC2B;AAC3B,QAAM,WAAW,MAAM,+BAA+B,KAAK;AAC3D,QAAM,SAAS,6BAA6B,MAAM,SAAS,QAAQ,CAAC;AACpE,SAAO,EAAE,GAAG,UAAU,OAAO;AAC/B;AAEO,SAAS,6BACd,MACkC;AAClC,QAAM,EAAE,QAAQ,SAAS,GAAG,SAAS,IAAI;AACzC,OAAK;AACL,SAAO,0BAA0B,QAAQ;AAC3C;AAKA,eAAsB,uBAAuB,MAA0C;AACrF,2BAAyB,IAAI;AAC7B,SACE,KAAK,WACL,6BAA6B,MAAM,SAAS,6BAA6B,IAAI,CAAC,CAAC;AAEnF;AAEO,SAAS,yBAAyB,OAAkC;AACzE,MAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;AAC/C,UAAM,IAAI,gCAAgC,iBAAiB;AAAA,EAC7D;AACA,QAAM,MAAM;AACZ,gBAAc,IAAI,eAAe,yBAAyB,eAAe;AACzE,MAAI,OAAO,IAAI,WAAW,YAAY,CAAC,QAAQ,KAAK,IAAI,MAAM,GAAG;AAC/D,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,eAAa,IAAI,WAAW,WAAW;AACvC,iBAAe,IAAI,eAAe,eAAe;AACjD,MAAI,IAAI,YAAY,OAAW,iBAAgB,IAAI,SAAS,SAAS;AACrE,MAAI,IAAI,UAAU,OAAW,cAAa,IAAI,OAAO,OAAO;AAC5D,MAAI,IAAI,eAAe,OAAW,cAAa,IAAI,YAAY,YAAY;AAC3E,MAAI,IAAI,eAAe,OAAW,oBAAmB,IAAI,YAAY,YAAY;AACjF,SAAO;AACT;AAEO,SAAS,wBAAwB,QAGnB;AACnB,MAAI,CAAC,OAAO,cAAc;AACxB,UAAM,IAAI;AAAA,MACR,QAAQ,OAAO,KAAK;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACA,SAAO,yBAAyB,OAAO,YAAY;AACrD;AAEO,SAAS,oBAAoB,QAGzB;AACT,SAAO,wBAAwB,MAAM,EAAE;AACzC;AAEA,eAAsB,0BAA0B,QAKlB;AAC5B,QAAM,UAAU,wBAAwB,MAAM;AAC9C,MAAI,CAAE,MAAM,uBAAuB,OAAO,GAAI;AAC5C,UAAM,IAAI;AAAA,MACR,QAAQ,OAAO,KAAK;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACA,MAAI,QAAQ,UAAU,UAAa,QAAQ,UAAU,OAAO,OAAO;AACjE,UAAM,IAAI;AAAA,MACR,QAAQ,OAAO,KAAK,yBAAyB,QAAQ,KAAK,2BAA2B,OAAO,KAAK;AAAA,MACjG;AAAA,IACF;AAAA,EACF;AACA,MAAI,QAAQ,eAAe,UAAa,QAAQ,eAAe,OAAO,YAAY;AAChF,UAAM,IAAI;AAAA,MACR,QAAQ,OAAO,KAAK,8BAA8B,QAAQ,UAAU,gCAAgC,OAAO,UAAU;AAAA,MACrH;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,4BAEd,SAAyC;AACzC,QAAM,SAAS,oBAAI,IAAiB;AACpC,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,oBAAoB,MAAM;AACtC,UAAM,SAAS,OAAO,IAAI,GAAG;AAC7B,QAAI,OAAQ,QAAO,KAAK,MAAM;AAAA,QACzB,QAAO,IAAI,KAAK,CAAC,MAAM,CAAC;AAAA,EAC/B;AACA,SAAO;AACT;AAEA,eAAe,+BACb,OAC2C;AAC3C,SAAO,0BAA0B;AAAA,IAC/B,eAAe;AAAA,IACf,WAAW,MAAM;AAAA,IACjB,eAAe,MAAM,qBAAqB,MAAM,aAAa;AAAA,IAC7D,SAAS,MAAM;AAAA,IACf,OAAO,MAAM;AAAA,IACb,YAAY,MAAM;AAAA,IAClB,YAAY,MAAM;AAAA,EACpB,CAAC;AACH;AAEA,SAAS,0BACP,OACkC;AAClC,SAAO,cAAc;AAAA,IACnB,eAAe;AAAA,IACf,WAAW,gBAAgB,MAAM,WAAW,WAAW;AAAA,IACvD,eAAe,gBAAgB,MAAM,aAAa;AAAA,IAClD,SAAS,MAAM,UAAU,iBAAiB,MAAM,SAAS,SAAS,IAAI;AAAA,IACtE,OAAO,iBAAiB,MAAM,OAAO,OAAO;AAAA,IAC5C,YAAY,iBAAiB,MAAM,YAAY,YAAY;AAAA,IAC3D,YAAY,MAAM,aACd,eAAe,oBAAoB,MAAM,UAAU,CAAC,IACpD;AAAA,EACN,CAAC;AACH;AAEA,eAAe,qBAAqB,OAA6D;AAC/F,QAAM,OAAO,gBAAgB,MAAM,MAAM,oBAAoB;AAC7D,MAAI,MAAM,SAAS,UAAa,MAAM,YAAY,QAAW;AAC3D,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,MAAI,MAAM,SAAS,QAAW;AAC5B,WAAO,EAAE,MAAM,MAAM,iBAAiB,MAAM,MAAM,oBAAoB,EAAE;AAAA,EAC1E;AACA,MAAI,MAAM,YAAY,QAAW;AAC/B,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,aAAW,MAAM,SAAS,uBAAuB;AACjD,SAAO,EAAE,MAAM,MAAM,MAAM,SAAS,MAAM,OAAO,EAAE;AACrD;AAEA,SAAS,gBAAgB,OAA+C;AACtE,SAAO;AAAA,IACL,MAAM,gBAAgB,MAAM,MAAM,oBAAoB;AAAA,IACtD,MAAM,iBAAiB,MAAM,MAAM,oBAAoB;AAAA,EACzD;AACF;AAEA,SAAS,iBAAiB,OAA4B,MAAmC;AACvF,SAAO,cAAc;AAAA,IACnB,IAAI,gBAAgB,MAAM,IAAI,GAAG,IAAI,KAAK;AAAA,IAC1C,SAAS,iBAAiB,MAAM,SAAS,GAAG,IAAI,UAAU;AAAA,IAC1D,MAAM,iBAAiB,MAAM,MAAM,GAAG,IAAI,OAAO;AAAA,EACnD,CAAC;AACH;AAEA,SAAS,oBACP,OAC4C;AAC5C,QAAM,MAAkD,CAAC;AACzD,aAAW,OAAO,OAAO,KAAK,KAAK,EAAE,KAAK,GAAG;AAC3C,UAAM,QAAQ,MAAM,GAAG;AACvB,oBAAgB,KAAK,kBAAkB;AACvC,QACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU,WACjB;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA,cAAc,GAAG;AAAA,MACnB;AAAA,IACF;AACA,QAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;AACxD,YAAM,IAAI,gCAAgC,0BAA0B,cAAc,GAAG,EAAE;AAAA,IACzF;AACA,QAAI,GAAG,IAAI;AAAA,EACb;AACA,SAAO;AACT;AAEA,SAAS,cAAiD,OAAa;AACrE,QAAM,MAA+B,CAAC;AACtC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,QAAI,UAAU,OAAW,KAAI,GAAG,IAAI;AAAA,EACtC;AACA,SAAO;AACT;AAEA,SAAS,eAAkD,OAAyB;AAClF,SAAO,OAAO,KAAK,KAAK,EAAE,SAAS,IAAI,QAAQ;AACjD;AAEA,SAAS,eAAe,OAAgB,MAAoB;AAC1D,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACvE,UAAM,IAAI,gCAAgC,mBAAmB,IAAI;AAAA,EACnE;AACA,QAAM,MAAM;AACZ,eAAa,IAAI,MAAM,GAAG,IAAI,OAAO;AACrC,mBAAiB,IAAI,MAAM,GAAG,IAAI,OAAO;AAC3C;AAEA,SAAS,gBAAgB,OAAgB,MAAoB;AAC3D,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACvE,UAAM,IAAI,gCAAgC,mBAAmB,IAAI;AAAA,EACnE;AACA,QAAM,MAAM;AACZ,eAAa,IAAI,IAAI,GAAG,IAAI,KAAK;AACjC,MAAI,IAAI,YAAY,OAAW,cAAa,IAAI,SAAS,GAAG,IAAI,UAAU;AAC1E,MAAI,IAAI,SAAS,OAAW,cAAa,IAAI,MAAM,GAAG,IAAI,OAAO;AACnE;AAEA,SAAS,mBAAmB,OAAgB,MAAoB;AAC9D,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AACvE,UAAM,IAAI,gCAAgC,mBAAmB,IAAI;AAAA,EACnE;AACA,sBAAoB,KAAmD;AACzE;AAEA,SAAS,WAAW,OAAyB,MAAoB;AAC/D,MAAI,UAAU,KAAM;AACpB,QAAM,OAAO,OAAO;AACpB,MAAI,SAAS,YAAY,SAAS,UAAW;AAC7C,MAAI,SAAS,UAAU;AACrB,QAAI,CAAC,OAAO,SAAS,KAAK,GAAG;AAC3B,YAAM,IAAI,gCAAgC,0BAA0B,IAAI;AAAA,IAC1E;AACA;AAAA,EACF;AACA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,UAAM,QAAQ,CAAC,MAAM,UAAU;AAC7B,iBAAW,MAAM,GAAG,IAAI,IAAI,KAAK,GAAG;AAAA,IACtC,CAAC;AACD;AAAA,EACF;AACA,MAAI,SAAS,UAAU;AACrB,eAAW,CAAC,KAAK,MAAM,KAAK,OAAO,QAAQ,KAAK,GAAG;AACjD,sBAAgB,KAAK,GAAG,IAAI,QAAQ;AACpC,iBAAW,QAAQ,GAAG,IAAI,IAAI,GAAG,EAAE;AAAA,IACrC;AACA;AAAA,EACF;AACA,QAAM,IAAI,gCAAgC,kCAAkC,IAAI;AAClF;AAEA,SAAS,cAAc,OAAgB,UAAkB,MAAoB;AAC3E,MAAI,UAAU,UAAU;AACtB,UAAM,IAAI,gCAAgC,YAAY,QAAQ,IAAI,IAAI;AAAA,EACxE;AACF;AAEA,SAAS,aAAa,OAAgB,MAAoB;AACxD,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACnD,UAAM,IAAI,gCAAgC,6BAA6B,IAAI;AAAA,EAC7E;AACF;AAEA,SAAS,gBAAgB,OAAe,MAAsB;AAC5D,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACnD,UAAM,IAAI,gCAAgC,6BAA6B,IAAI;AAAA,EAC7E;AACA,SAAO;AACT;AAEA,SAAS,iBAAiB,OAA2B,MAAkC;AACrF,MAAI,UAAU,OAAW,QAAO;AAChC,SAAO,gBAAgB,OAAO,IAAI;AACpC;AAEA,SAAS,iBAAiB,OAAgB,MAAsB;AAC9D,MAAI,OAAO,UAAU,YAAY,CAAC,WAAW,KAAK,KAAK,GAAG;AACxD,UAAM,IAAI,gCAAgC,0CAA0C,IAAI;AAAA,EAC1F;AACA,SAAO;AACT;AAsBO,IAAM,sBAAsB;AAAA;AAAA;AAAA;AAAA,EAIjC,yBAAyB;AAC3B;AAQO,SAAS,mBAAmB,OAAkC;AACnE,MAAI;AACJ,MAAI;AACF,iBAAa,KAAK,UAAU,KAAK;AAAA,EACnC,SAAS,KAAK;AACZ,UAAM,IAAI;AAAA,MACR,4CAA4C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MAC5F;AAAA,IACF;AAAA,EACF;AACA,MAAI,eAAe,QAAW;AAC5B,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,SAAO,KAAK,MAAM,UAAU;AAC9B;AAcA,eAAsB,+BACpB,SACA,OAC2B;AAC3B,MAAI,CAAC,WAAW,OAAO,YAAY,UAAU;AAC3C,UAAM,IAAI,gCAAgC,kCAAkC,SAAS;AAAA,EACvF;AACA,MAAI,OAAO,QAAQ,SAAS,YAAY,QAAQ,KAAK,WAAW,GAAG;AACjE,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,MAAI,OAAO,QAAQ,YAAY,YAAY,QAAQ,QAAQ,WAAW,GAAG;AACvE,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,SAAO,sBAAsB;AAAA,IAC3B,GAAG;AAAA,IACH,WAAW,GAAG,QAAQ,IAAI,IAAI,QAAQ,OAAO;AAAA,IAC7C,eAAe;AAAA,MACb,MAAM,oBAAoB;AAAA,MAC1B,SAAS,mBAAmB,OAAO;AAAA,IACrC;AAAA,EACF,CAAC;AACH;;;ACjOO,SAAS,aAAa,QAAuC;AAClE,QAAM,QAAQ,OAAO,QAAQ,gBAAgB,OAAO,QAAQ;AAC5D,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE;AAIA,IAAM,sBAAsB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,IAAM,aAAyC,CAAC,UAAU,OAAO,SAAS;AAC1E,IAAM,oBAAuD;AAAA,EAC3D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,IAAM,2BAAN,cAAuC,gBAAgB;AAAA,EACnD;AAAA,EACT,YAAY,SAAiB,OAAO,IAAI;AACtC,UAAM,OAAO,GAAG,OAAO,QAAQ,IAAI,MAAM,OAAO;AAChD,SAAK,OAAO;AAAA,EACd;AACF;AAOO,SAAS,kBAAkB,OAA2B;AAC3D,MAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;AAC/C,UAAM,IAAI,yBAAyB,iBAAiB;AAAA,EACtD;AACA,QAAM,MAAM;AAEZ,aAAW,OAAO,qBAAqB;AACrC,QAAI,EAAE,OAAO,MAAM;AACjB,YAAM,IAAI,yBAAyB,4BAA4B,GAAG,GAAG;AAAA,IACvE;AAAA,EACF;AAEA,EAAAA,cAAa,IAAI,OAAO,OAAO;AAC/B,EAAAA,cAAa,IAAI,cAAc,cAAc;AAC7C,EAAAA,cAAa,IAAI,aAAa,aAAa;AAC3C,qBAAmB,IAAI,MAAM,MAAM;AACnC,EAAAA,cAAa,IAAI,OAAO,OAAO;AAC/B,EAAAA,cAAa,IAAI,YAAY,YAAY;AACzC,EAAAA,cAAa,IAAI,YAAY,YAAY;AACzC,EAAAA,cAAa,IAAI,WAAW,WAAW;AACvC,0BAAwB,IAAI,QAAQ,QAAQ;AAC5C,MAAI,IAAI,YAAY,OAAW,yBAAwB,IAAI,SAAS,SAAS;AAC7E,eAAa,IAAI,SAAS,IAAI,cAAc;AAG5C,MAAI,CAAC,iBAAiB,IAAI,KAAe,GAAG;AAC1C,UAAM,IAAI;AAAA,MACR,UAAU,IAAI,KAAK;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AAGA,QAAM,KAAK,IAAI;AACf,MAAI,OAAO,QAAQ,OAAO,OAAO,UAAU;AACzC,UAAM,IAAI,yBAAyB,gCAAgC,YAAY;AAAA,EACjF;AACA,QAAM,QAAQ;AACd,0BAAwB,MAAM,OAAO,kBAAkB;AACvD,0BAAwB,MAAM,QAAQ,mBAAmB;AACzD,MAAI,MAAM,cAAc,QAAW;AACjC,4BAAwB,MAAM,WAAW,sBAAsB;AAC/D,QAAK,MAAM,YAAwB,MAAM,QAAmB;AAC1D,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,MAAM,WAAW,OAAW,yBAAwB,MAAM,QAAQ,mBAAmB;AACzF,MAAI,MAAM,eAAe,QAAW;AAClC,4BAAwB,MAAM,YAAY,uBAAuB;AAAA,EACnE;AAGA,MAAI,IAAI,kBAAkB,QAAW;AACnC,UAAM,KAAK,IAAI;AACf,QAAI,OAAO,QAAQ,OAAO,OAAO,UAAU;AACzC,YAAM,IAAI,yBAAyB,mCAAmC,eAAe;AAAA,IACvF;AACA,UAAM,QAAQ;AACd,IAAAA,cAAa,MAAM,OAAO,qBAAqB;AAC/C,IAAAA,cAAa,MAAM,eAAe,6BAA6B;AAC/D,uBAAmB,MAAM,YAAY,0BAA0B;AAC/D,QAAI,OAAO,MAAM,aAAa,WAAW;AACvC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,MAAM,IAAI;AAChB,MAAI,QAAQ,QAAQ,OAAO,QAAQ,UAAU;AAC3C,UAAM,IAAI,yBAAyB,6BAA6B,SAAS;AAAA,EAC3E;AACA,QAAM,SAAS;AACf,MAAI,OAAO,gBAAgB;AACzB,uBAAmB,OAAO,aAAa,qBAAqB;AAC9D,MAAI,OAAO,iBAAiB;AAC1B,uBAAmB,OAAO,cAAc,sBAAsB;AAChE,QAAM,MAAM,OAAO;AACnB,MAAI,QAAQ,QAAQ,OAAO,QAAQ,UAAU;AAC3C,UAAM,IAAI,yBAAyB,iCAAiC,aAAa;AAAA,EACnF;AACA,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,GAA8B,GAAG;AACnE,uBAAmB,GAAG,eAAe,CAAC,EAAE;AAAA,EAC1C;AAEA,MAAI,OAAO,aAAa,QAAW;AACjC,UAAM,IAAI,OAAO;AACjB,QAAI,MAAM,QAAQ,OAAO,MAAM,UAAU;AACvC,YAAM,IAAI,yBAAyB,sCAAsC,kBAAkB;AAAA,IAC7F;AACA,UAAM,KAAK;AACX,uBAAmB,GAAG,OAAO,wBAAwB;AACrD,QAAI,OAAO,GAAG,UAAU,WAAW;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,MAAI,OAAO,gBAAgB,QAAW;AACpC,wBAAoB,OAAO,aAAa,qBAAqB;AAAA,EAC/D;AAGA,MACE,IAAI,iBAAiB,WACpB,OAAO,IAAI,iBAAiB,YAC3B,CAAC,gBAAgB,SAAS,IAAI,YAA4B,IAC5D;AACA,UAAM,IAAI;AAAA,MACR,+BAA+B,gBAAgB,KAAK,IAAI,CAAC;AAAA,MACzD;AAAA,IACF;AAAA,EACF;AACA,MAAI,IAAI,gBAAgB,QAAW;AACjC,IAAAA,cAAa,IAAI,aAAa,aAAa;AAC3C,QAAI,IAAI,iBAAiB,UAAa,IAAI,iBAAiB,WAAW;AACpE,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MACE,OAAO,IAAI,oBAAoB,YAC/B,CAAC,kBAAkB,SAAS,IAAI,eAAqC,GACrE;AACA,UAAM,IAAI;AAAA,MACR,kCAAkC,kBAAkB,KAAK,IAAI,CAAC;AAAA,MAC9D;AAAA,IACF;AAAA,EACF;AACA,MAAI,IAAI,0BAA0B,QAAW;AAC3C,IAAAA,cAAa,IAAI,uBAAuB,uBAAuB;AAC/D,QACE,IAAI,oBAAoB,YACxB,IAAI,oBAAoB,eACxB,IAAI,oBAAoB,cACxB;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,iBAAiB,QAAW;AAClC,QAAI;AACF,YAAM,UAAU,yBAAyB,IAAI,YAAY;AACzD,UAAI,QAAQ,UAAU,UAAa,QAAQ,UAAU,IAAI,OAAO;AAC9D,cAAM,IAAI;AAAA,UACR,uBAAuB,QAAQ,KAAK,2BAA2B,IAAI,KAAK;AAAA,UACxE;AAAA,QACF;AAAA,MACF;AACA,UAAI,QAAQ,eAAe,UAAa,QAAQ,eAAe,IAAI,YAAY;AAC7E,cAAM,IAAI;AAAA,UACR,4BAA4B,QAAQ,UAAU,gCAAgC,IAAI,UAAU;AAAA,UAC5F;AAAA,QACF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,yBAA0B,OAAM;AACrD,UAAI,iBAAiB,OAAO;AAC1B,cAAM,IAAI,yBAAyB,MAAM,SAAS,cAAc;AAAA,MAClE;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,EAAAA,cAAa,IAAI,YAAY,YAAY;AAGzC,MAAI,OAAO,IAAI,aAAa,YAAY,CAAC,WAAW,SAAS,IAAI,QAAuB,GAAG;AACzF,UAAM,IAAI;AAAA,MACR,2BAA2B,WAAW,KAAK,IAAI,CAAC,SAAS,OAAO,IAAI,QAAQ,CAAC;AAAA,MAC7E;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,aAAa,SAAkB,YAA2B;AACjE,MAAI,eAAe,QAAQ,OAAO,eAAe,UAAU;AACzD,UAAM,IAAI,yBAAyB,oCAAoC,gBAAgB;AAAA,EACzF;AACA,QAAM,QAAQ;AACd,MAAI,MAAM,SAAS,cAAc,MAAM,SAAS,eAAe,MAAM,SAAS,cAAc;AAC1F,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,MAAI,MAAM,SAAS,cAAc;AAC/B,QAAI,MAAM,QAAQ,MAAM;AACtB,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY,MAAM;AACpB,YAAM,IAAI,yBAAyB,+CAA+C,SAAS;AAAA,IAC7F;AACA;AAAA,EACF;AACA,0BAAwB,SAAS,SAAS;AAC1C,0BAAwB,MAAM,KAAK,oBAAoB;AACvD,MAAI,MAAM,QAAQ,SAAS;AACzB,UAAM,IAAI;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAGO,SAAS,YAAY,OAAoC;AAC9D,MAAI;AACF,sBAAkB,KAAK;AACvB,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,mBACd,OACiF;AACjF,MAAI;AACF,WAAO,EAAE,IAAI,MAAM,OAAO,kBAAkB,KAAK,EAAE;AAAA,EACrD,SAAS,GAAG;AACV,QAAI,aAAa,yBAA0B,QAAO,EAAE,IAAI,OAAO,OAAO,EAAE;AACxE,UAAM;AAAA,EACR;AACF;AAGO,SAAS,mBAAmB,QAA8B;AAC/D,QAAM,OAAO,KAAK,UAAU,MAAM;AAClC,SAAO,kBAAkB,KAAK,MAAM,IAAI,CAAC;AAC3C;AAIA,SAASA,cAAa,OAAgB,MAAoB;AACxD,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACnD,UAAM,IAAI,yBAAyB,6BAA6B,IAAI;AAAA,EACtE;AACF;AAEA,SAAS,mBAAmB,OAAgB,MAAoB;AAC9D,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;AACxD,UAAM,IAAI,yBAAyB,0BAA0B,IAAI;AAAA,EACnE;AACF;AAEA,SAAS,wBAAwB,OAAgB,MAAoB;AACnE,qBAAmB,OAAO,IAAI;AAC9B,MAAK,QAAmB,GAAG;AACzB,UAAM,IAAI,yBAAyB,gCAAgC,IAAI;AAAA,EACzE;AACF;AAEA,SAAS,oBAAoB,OAAgB,MAAoB;AAC/D,MAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;AAC/C,UAAM,IAAI,yBAAyB,iCAAiC,IAAI;AAAA,EAC1E;AACA,QAAM,MAAM;AAEZ,QAAM,WAAW,IAAI;AACrB,MAAI,aAAa,QAAQ,OAAO,aAAa,UAAU;AACrD,UAAM,IAAI,yBAAyB,8BAA8B,GAAG,IAAI,WAAW;AAAA,EACrF;AACA,aAAW,CAAC,SAAS,IAAI,KAAK,OAAO,QAAQ,QAAmC,GAAG;AACjF,QAAI,SAAS,QAAQ,OAAO,SAAS,UAAU;AAC7C,YAAM,IAAI;AAAA,QACR;AAAA,QACA,GAAG,IAAI,aAAa,OAAO;AAAA,MAC7B;AAAA,IACF;AACA,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,IAA+B,GAAG;AAC1E,yBAAmB,OAAO,GAAG,IAAI,aAAa,OAAO,IAAI,GAAG,EAAE;AAAA,IAChE;AAAA,EACF;AAEA,QAAM,aAAa,IAAI;AACvB,MAAI,eAAe,QAAQ,OAAO,eAAe,UAAU;AACzD,UAAM,IAAI,yBAAyB,gCAAgC,GAAG,IAAI,aAAa;AAAA,EACzF;AACA,aAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC/E,uBAAmB,MAAM,GAAG,IAAI,eAAe,GAAG,EAAE;AAAA,EACtD;AAEA,qBAAmB,IAAI,WAAW,GAAG,IAAI,YAAY;AAErD,MAAI,IAAI,iBAAiB,QAAW;AAClC,QAAI,CAAC,MAAM,QAAQ,IAAI,YAAY,GAAG;AACpC,YAAM,IAAI;AAAA,QACR;AAAA,QACA,GAAG,IAAI;AAAA,MACT;AAAA,IACF;AACA,aAAS,IAAI,GAAG,IAAI,IAAI,aAAa,QAAQ,KAAK;AAChD,YAAM,KAAK,IAAI,aAAa,CAAC;AAC7B,UAAI,OAAO,OAAO,YAAY,GAAG,WAAW,GAAG;AAC7C,cAAM,IAAI;AAAA,UACR;AAAA,UACA,GAAG,IAAI,iBAAiB,CAAC;AAAA,QAC3B;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,IAAI,UAAU,UAAa,OAAO,IAAI,UAAU,UAAU;AAC5D,UAAM,IAAI,yBAAyB,0BAA0B,GAAG,IAAI,QAAQ;AAAA,EAC9E;AACF;AAYO,SAAS,iBAAiB,OAAwB;AACvD,MAAI,MAAM,SAAS,GAAG,EAAG,QAAO;AAChC,MAAI,UAAU,KAAK,KAAK,EAAG,QAAO;AAClC,MAAI,sBAAsB,KAAK,KAAK,EAAG,QAAO;AAC9C,MAAI,SAAS,KAAK,KAAK,EAAG,QAAO;AACjC,SAAO;AACT;","names":["expectString"]}