@tangle-network/agent-eval 0.144.6 → 0.144.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +2 -0
- package/dist/{benchmark-J9Qe6j2_.d.ts → agent-profile-CgDTo40f.d.ts} +91 -4
- package/dist/agent-profile-CgDTo40f.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +470 -88
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +24 -5
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DWIvOAGk.js → analyze-runs-BNNK7irB.js} +3 -3
- package/dist/{analyze-runs-DWIvOAGk.js.map → analyze-runs-BNNK7irB.js.map} +1 -1
- package/dist/{baseline-D_fT6277.d.ts → baseline-CavEbRyH.d.ts} +2 -23
- package/dist/baseline-CavEbRyH.d.ts.map +1 -0
- package/dist/{benchmark-command-CQd78YHt.js → benchmark-command-BKENp2s5.js} +662 -605
- package/dist/benchmark-command-BKENp2s5.js.map +1 -0
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-BEOkuvIg.js → benchmarks-BlPmjd88.js} +4 -4
- package/dist/{benchmarks-BEOkuvIg.js.map → benchmarks-BlPmjd88.js.map} +1 -1
- package/dist/campaign/index.d.ts +7 -5
- package/dist/campaign/index.js +5 -3
- package/dist/{campaign-CXsdyym7.js → campaign--HVSuvV0.js} +17 -301
- package/dist/campaign--HVSuvV0.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-0JI64ovJ.d.ts → client-DjXROWpx.d.ts} +3 -3
- package/dist/{client-0JI64ovJ.d.ts.map → client-DjXROWpx.d.ts.map} +1 -1
- package/dist/{completion-verifier-CBiee74w.d.ts → completion-verifier-foUCLif_.d.ts} +5 -5
- package/dist/{completion-verifier-CBiee74w.d.ts.map → completion-verifier-foUCLif_.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +9 -8
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +7 -6
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/control.js +1 -1
- package/dist/counterfactual-CWPTrMH7.js +126 -0
- package/dist/counterfactual-CWPTrMH7.js.map +1 -0
- package/dist/counterfactual-CxmxAONP.d.ts +72 -0
- package/dist/counterfactual-CxmxAONP.d.ts.map +1 -0
- package/dist/{default-registry-J9m-_tya.d.ts → default-registry-BZhStdjl.d.ts} +6 -5
- package/dist/default-registry-BZhStdjl.d.ts.map +1 -0
- package/dist/{default-registry-Dta70shL.js → default-registry-BaQXW1Ow.js} +2 -2
- package/dist/{default-registry-Dta70shL.js.map → default-registry-BaQXW1Ow.js.map} +1 -1
- package/dist/{dspy-rlm-engine-19FQEMBK.js → dspy-rlm-engine-BiN49gK6.js} +4 -3
- package/dist/{dspy-rlm-engine-19FQEMBK.js.map → dspy-rlm-engine-BiN49gK6.js.map} +1 -1
- package/dist/{tool-groups-CK0JCkqO.d.ts → engine-nB64f48I.d.ts} +18 -31
- package/dist/engine-nB64f48I.d.ts.map +1 -0
- package/dist/{eval-campaign-CfLQQs9B.js → eval-campaign-DNjCvAm-.js} +7 -6
- package/dist/{eval-campaign-CfLQQs9B.js.map → eval-campaign-DNjCvAm-.js.map} +1 -1
- package/dist/{exact-types-CBYF5MGd.d.ts → exact-types-Djvzosly.d.ts} +2 -2
- package/dist/{exact-types-CBYF5MGd.d.ts.map → exact-types-Djvzosly.d.ts.map} +1 -1
- package/dist/exec-BLtYZdWo.js +49 -0
- package/dist/exec-BLtYZdWo.js.map +1 -0
- package/dist/experiment/index.d.ts +802 -0
- package/dist/experiment/index.d.ts.map +1 -0
- package/dist/experiment/index.js +1108 -0
- package/dist/experiment/index.js.map +1 -0
- package/dist/experiment-tracker-CnRICnMl.js +500 -0
- package/dist/experiment-tracker-CnRICnMl.js.map +1 -0
- package/dist/experiment-tracker-IMntXr6J.d.ts +299 -0
- package/dist/experiment-tracker-IMntXr6J.d.ts.map +1 -0
- package/dist/{external-optimizer-contracts-Q9c0L3eY.d.ts → external-optimizer-contracts-CZuJNcT5.d.ts} +2 -2
- package/dist/{external-optimizer-contracts-Q9c0L3eY.d.ts.map → external-optimizer-contracts-CZuJNcT5.d.ts.map} +1 -1
- package/dist/{feedback-trajectory-GgoS0-MK.d.ts → feedback-trajectory-Rh280oXo.d.ts} +3 -3
- package/dist/{feedback-trajectory-GgoS0-MK.d.ts.map → feedback-trajectory-Rh280oXo.d.ts.map} +1 -1
- package/dist/fuzz.d.ts +1 -1
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-BZUe-ODI.d.ts → index-BZ3-y4YL.d.ts} +3 -3
- package/dist/{index-BZUe-ODI.d.ts.map → index-BZ3-y4YL.d.ts.map} +1 -1
- package/dist/{index-DSC51roc.d.ts → index-C3ssXVLv.d.ts} +2 -2
- package/dist/{index-DSC51roc.d.ts.map → index-C3ssXVLv.d.ts.map} +1 -1
- package/dist/{index-4XwggC10.d.ts → index-C5HOo4ZF2.d.ts} +4 -4
- package/dist/index-C5HOo4ZF2.d.ts.map +1 -0
- package/dist/{index-B6-B0zTB.d.ts → index-CvXXlyz7.d.ts} +2 -2
- package/dist/{index-B6-B0zTB.d.ts.map → index-CvXXlyz7.d.ts.map} +1 -1
- package/dist/{index-Dx1kF3Ez.d.ts → index-CwDrUMe0.d.ts} +2 -2
- package/dist/{index-Dx1kF3Ez.d.ts.map → index-CwDrUMe0.d.ts.map} +1 -1
- package/dist/{index-BIL5vxxt.d.ts → index-Sh2I0DRc.d.ts} +11 -645
- package/dist/index-Sh2I0DRc.d.ts.map +1 -0
- package/dist/index.d.ts +214 -404
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +233 -649
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-DqEsugpr.d.ts → insight-report-C6h6F_4L.d.ts} +3 -3
- package/dist/{insight-report-DqEsugpr.d.ts.map → insight-report-C6h6F_4L.d.ts.map} +1 -1
- package/dist/{integrity-CNGUaGBY.d.ts → integrity-BuqEKu-x.d.ts} +2 -2
- package/dist/{integrity-CNGUaGBY.d.ts.map → integrity-BuqEKu-x.d.ts.map} +1 -1
- package/dist/integrity-MLzHOfV9.js +141 -0
- package/dist/integrity-MLzHOfV9.js.map +1 -0
- package/dist/kind-factory-BHIgPmzS.js.map +1 -1
- package/dist/{llm-client-Dv5BiKLE.js → llm-client-DzvMUsS_.js} +24 -6
- package/dist/llm-client-DzvMUsS_.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/meta-eval/index.d.ts +1 -1
- package/dist/{mint-DD-0oQTA.js → mint-CGEkzPLf.js} +2 -2
- package/dist/{mint-DD-0oQTA.js.map → mint-CGEkzPLf.js.map} +1 -1
- package/dist/{multi-layer-verifier-BHY1gWAc.d.ts → multi-layer-verifier-DnAqwl0h.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BHY1gWAc.d.ts.map → multi-layer-verifier-DnAqwl0h.d.ts.map} +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts +114 -0
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +2 -1
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/prime-bridge-transport-6feEglLf.d.ts +49 -0
- package/dist/prime-bridge-transport-6feEglLf.d.ts.map +1 -0
- package/dist/prime-protocol-BfSalTfR.js +453 -0
- package/dist/prime-protocol-BfSalTfR.js.map +1 -0
- package/dist/profile-cell.js +242 -1
- package/dist/profile-cell.js.map +1 -0
- package/dist/promotion-policy-ChWhTDBH.d.ts +289 -0
- package/dist/promotion-policy-ChWhTDBH.d.ts.map +1 -0
- package/dist/promotion-policy-CrLrmys8.js +682 -0
- package/dist/promotion-policy-CrLrmys8.js.map +1 -0
- package/dist/{propose-review-control-BciUCZoh.js → propose-review-control-qgWLA6E9.js} +2 -2
- package/dist/{propose-review-control-BciUCZoh.js.map → propose-review-control-qgWLA6E9.js.map} +1 -1
- package/dist/{release-report-ChOgpIoQ.d.ts → release-report-CI8uisI1.d.ts} +2 -2
- package/dist/{release-report-ChOgpIoQ.d.ts.map → release-report-CI8uisI1.d.ts.map} +1 -1
- package/dist/{release-report-Sl0xfkFv.js → release-report-Dy39cbFF.js} +3 -3
- package/dist/{release-report-Sl0xfkFv.js.map → release-report-Dy39cbFF.js.map} +1 -1
- package/dist/{replay-Krvb114g.d.ts → replay-DFf-teiC.d.ts} +5 -4
- package/dist/replay-DFf-teiC.d.ts.map +1 -0
- package/dist/{replay-GW61ezMW.js → replay-DyBLaKFc.js} +3 -3
- package/dist/{replay-GW61ezMW.js.map → replay-DyBLaKFc.js.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +2 -2
- package/dist/{researcher-xLeNcpKX.d.ts → researcher-BoaxeCzP.d.ts} +4 -4
- package/dist/{researcher-xLeNcpKX.d.ts.map → researcher-BoaxeCzP.d.ts.map} +1 -1
- package/dist/{reward-hacking-CyuzxKly.js → reward-hacking-B2tPL9a3.js} +12 -3
- package/dist/reward-hacking-B2tPL9a3.js.map +1 -0
- package/dist/{reward-hacking-RZgnGWlx.d.ts → reward-hacking-Cf1PtEOz.d.ts} +33 -3
- package/dist/reward-hacking-Cf1PtEOz.d.ts.map +1 -0
- package/dist/rl.d.ts +17 -7
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +16 -7
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-C2fD1cf4.js → rollout-2ECTXb2N.js} +2 -2
- package/dist/{rollout-C2fD1cf4.js.map → rollout-2ECTXb2N.js.map} +1 -1
- package/dist/{run-evidence-C6G41MSI.d.ts → run-evidence-BDFFai9R.d.ts} +2 -2
- package/dist/{run-evidence-C6G41MSI.d.ts.map → run-evidence-BDFFai9R.d.ts.map} +1 -1
- package/dist/{run-record-CWN8-VsV.js → run-record-DqOw5X6_.js} +2 -2
- package/dist/{run-record-CWN8-VsV.js.map → run-record-DqOw5X6_.js.map} +1 -1
- package/dist/{schema-Cef2cFmb.d.ts → schema-Cef2cFmb2.d.ts} +1 -1
- package/dist/schema-Cef2cFmb2.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-DwF6n05O.js → semantic-concept-judge-Bmrq6yqU.js} +2 -2
- package/dist/{semantic-concept-judge-DwF6n05O.js.map → semantic-concept-judge-Bmrq6yqU.js.map} +1 -1
- package/dist/sequential-D-BLJBKU.js +299 -0
- package/dist/sequential-D-BLJBKU.js.map +1 -0
- package/dist/{server-D6XJQHw7.js → server-iu0ede49.js} +2 -2
- package/dist/{server-D6XJQHw7.js.map → server-iu0ede49.js.map} +1 -1
- package/dist/{single-run-lock-D5iN0Xzb.js → single-run-lock-BMQEv1wG.js} +7 -161
- package/dist/single-run-lock-BMQEv1wG.js.map +1 -0
- package/dist/{skill-usage-GlOphAhX.d.ts → skill-usage-CJlWEUFt.d.ts} +10 -10
- package/dist/{skill-usage-GlOphAhX.d.ts.map → skill-usage-CJlWEUFt.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-7S43rbDB.d.ts → skillopt-optimization-method-BO7NAl3b.d.ts} +8 -294
- package/dist/skillopt-optimization-method-BO7NAl3b.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-Bfb-vBKe.js → skillopt-optimization-method-CkaI2ly4.js} +19 -654
- package/dist/skillopt-optimization-method-CkaI2ly4.js.map +1 -0
- package/dist/statistical-heldout-Dn9ruizm.d.ts +555 -0
- package/dist/statistical-heldout-Dn9ruizm.d.ts.map +1 -0
- package/dist/{statistics-C-dm-J6H.d.ts → statistics-D6Uebe_4.d.ts} +2 -2
- package/dist/{statistics-C-dm-J6H.d.ts.map → statistics-D6Uebe_4.d.ts.map} +1 -1
- package/dist/steps-BArUxhna.d.ts +51 -0
- package/dist/steps-BArUxhna.d.ts.map +1 -0
- package/dist/{integrity-fdt8XPAv.js → store-DNe_Uv1Q.js} +2 -140
- package/dist/store-DNe_Uv1Q.js.map +1 -0
- package/dist/{summary-report-B0cAyA7N.d.ts → summary-report-DuUS_i7W.d.ts} +3 -114
- package/dist/summary-report-DuUS_i7W.d.ts.map +1 -0
- package/dist/{summary-report-9A5y7EsK.js → summary-report-Lf-5I7xh.js} +2 -2
- package/dist/{summary-report-9A5y7EsK.js.map → summary-report-Lf-5I7xh.js.map} +1 -1
- package/dist/supervisor-run/index.d.ts +2 -2
- package/dist/supervisor-run/index.js +1 -1
- package/dist/{supervisor-run-DiyQVczd.js → supervisor-run-D_sokXcO.js} +32 -8
- package/dist/{supervisor-run-DiyQVczd.js.map → supervisor-run-D_sokXcO.js.map} +1 -1
- package/dist/tool-groups-DIVBnJyl.d.ts +28 -0
- package/dist/tool-groups-DIVBnJyl.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +2102 -0
- package/dist/trace-repair/index.d.ts.map +1 -0
- package/dist/trace-repair/index.js +3878 -0
- package/dist/trace-repair/index.js.map +1 -0
- package/dist/traces.d.ts +6 -6
- package/dist/traces.js +3 -2
- package/dist/trajectory-YC15QDYQ.d.ts +24 -0
- package/dist/trajectory-YC15QDYQ.d.ts.map +1 -0
- package/dist/trajectory-replay/index.d.ts +781 -0
- package/dist/trajectory-replay/index.d.ts.map +1 -0
- package/dist/trajectory-replay/index.js +2103 -0
- package/dist/trajectory-replay/index.js.map +1 -0
- package/dist/{types-XMVEdrE_.d.ts → types-D216SgwM.d.ts} +24 -6
- package/dist/types-D216SgwM.d.ts.map +1 -0
- package/dist/{types-yLK8gXE9.d.ts → types-D4mog56g.d.ts} +2 -2
- package/dist/{types-yLK8gXE9.d.ts.map → types-D4mog56g.d.ts.map} +1 -1
- package/dist/{types-BhP9q0Fq.d.ts → types-DF_Udrp-.d.ts} +52 -3
- package/dist/{types-BhP9q0Fq.d.ts.map → types-DF_Udrp-.d.ts.map} +1 -1
- package/dist/{types-DOZyvsFU.d.ts → types-DYuNHo9R.d.ts} +3 -3
- package/dist/{types-DOZyvsFU.d.ts.map → types-DYuNHo9R.d.ts.map} +1 -1
- package/dist/usage-receipt-EVI8B8Xu.js.map +1 -1
- package/dist/verdict-DExhxfgR.d.ts +201 -0
- package/dist/verdict-DExhxfgR.d.ts.map +1 -0
- package/dist/verdict-cache-BCcOh0kF.js +159 -0
- package/dist/verdict-cache-BCcOh0kF.js.map +1 -0
- package/dist/wire/index.d.ts +2 -2
- package/dist/wire/index.js +1 -1
- package/docs/campaign-proposers.md +5 -0
- package/docs/charter.md +112 -0
- package/docs/experiment.md +104 -0
- package/docs/prime-analyst.md +1 -0
- package/docs/trace-analysis.md +26 -0
- package/docs/trace-repair-admission.md +194 -0
- package/docs/trace-repair-analyst-arms.md +121 -0
- package/docs/trace-repair-continuation.md +107 -0
- package/docs/trace-repair-grader.md +163 -0
- package/docs/trajectory-replay.md +110 -0
- package/docs/verification-strategies.md +103 -0
- package/package.json +19 -2
- package/dist/agent-profile-cell-CbfBm2g6.js +0 -335
- package/dist/agent-profile-cell-CbfBm2g6.js.map +0 -1
- package/dist/baseline-D_fT6277.d.ts.map +0 -1
- package/dist/benchmark-J9Qe6j2_.d.ts.map +0 -1
- package/dist/benchmark-command-CQd78YHt.js.map +0 -1
- package/dist/campaign-CXsdyym7.js.map +0 -1
- package/dist/default-registry-J9m-_tya.d.ts.map +0 -1
- package/dist/index-4XwggC10.d.ts.map +0 -1
- package/dist/index-BIL5vxxt.d.ts.map +0 -1
- package/dist/integrity-fdt8XPAv.js.map +0 -1
- package/dist/llm-client-Dv5BiKLE.js.map +0 -1
- package/dist/replay-Krvb114g.d.ts.map +0 -1
- package/dist/reward-hacking-CyuzxKly.js.map +0 -1
- package/dist/reward-hacking-RZgnGWlx.d.ts.map +0 -1
- package/dist/schema-Cef2cFmb.d.ts.map +0 -1
- package/dist/single-run-lock-D5iN0Xzb.js.map +0 -1
- package/dist/skillopt-optimization-method-7S43rbDB.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-Bfb-vBKe.js.map +0 -1
- package/dist/summary-report-B0cAyA7N.d.ts.map +0 -1
- package/dist/tool-groups-CK0JCkqO.d.ts.map +0 -1
- package/dist/types-XMVEdrE_.d.ts.map +0 -1
- package/dist/verdict-Dps8_okt.d.ts +0 -37
- package/dist/verdict-Dps8_okt.d.ts.map +0 -1
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
//#region src/pre-registration.ts
|
|
2
|
+
/**
|
|
3
|
+
* Deterministic JSON canonicalization — sort object keys recursively.
|
|
4
|
+
*
|
|
5
|
+
* Two semantically-equal objects produce byte-identical canonicalized output;
|
|
6
|
+
* this is what makes a content-hash stable across encoders, key insertion
|
|
7
|
+
* orders, and runtime versions. Exported for any consumer that needs the same
|
|
8
|
+
* canonicalization guarantee outside the manifest-signing path (e.g., signing
|
|
9
|
+
* an artifact bundle, hashing a dataset version, etc.).
|
|
10
|
+
*/
|
|
11
|
+
function canonicalize(v) {
|
|
12
|
+
if (v === null || typeof v !== "object") return v;
|
|
13
|
+
if (Array.isArray(v)) return v.map(canonicalize);
|
|
14
|
+
const keys = Object.keys(v).sort();
|
|
15
|
+
const out = {};
|
|
16
|
+
for (const k of keys) out[k] = canonicalize(v[k]);
|
|
17
|
+
return out;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* SHA-256 hex (full 64 chars) over the canonicalized JSON encoding of `obj`.
|
|
21
|
+
*
|
|
22
|
+
* The same primitive `signManifest` and `verifyManifest` are built on, exposed
|
|
23
|
+
* directly so consumers signing arbitrary structured content (artifact bundles,
|
|
24
|
+
* production packets, dataset manifests, etc.) don't have to re-derive
|
|
25
|
+
* canonicalize+sha256 from scratch.
|
|
26
|
+
*
|
|
27
|
+
* Stable across:
|
|
28
|
+
* - object key insertion order (canonicalization sorts keys recursively)
|
|
29
|
+
* - encoder choice (UTF-8 via TextEncoder, fixed)
|
|
30
|
+
* - runtime (uses the Web Crypto subtle digest, present in Node ≥18 and browsers)
|
|
31
|
+
*
|
|
32
|
+
* Named `hashJson` to disambiguate from `prompt-registry.ts`'s `hashContent`,
|
|
33
|
+
* which takes a string input and returns a truncated 12-char prompt id.
|
|
34
|
+
* Use `hashJson` when you mean "canonicalize then hash."
|
|
35
|
+
*
|
|
36
|
+
* @example
|
|
37
|
+
* const hash = await hashJson({ id: '1', kind: 'spec' })
|
|
38
|
+
* // 'a3f1...' (64 hex chars)
|
|
39
|
+
*/
|
|
40
|
+
async function hashJson(obj) {
|
|
41
|
+
const canonical = canonicalize(obj);
|
|
42
|
+
const bytes = new TextEncoder().encode(JSON.stringify(canonical));
|
|
43
|
+
const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
|
|
44
|
+
return Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Sign a manifest with a SHA-256 content hash.
|
|
48
|
+
*
|
|
49
|
+
* The hash covers the canonicalized manifest with the `contentHash`
|
|
50
|
+
* and `algo` fields stripped; this lets verifiers re-sign the rest and
|
|
51
|
+
* compare. Returned manifest always carries `algo: 'sha256-content'`
|
|
52
|
+
* so downstream consumers can identify the scheme; manifests without
|
|
53
|
+
* `algo` still verify because it is stripped before hashing on both sides.
|
|
54
|
+
*/
|
|
55
|
+
async function signManifest(m) {
|
|
56
|
+
const hash = await hashJson(m);
|
|
57
|
+
return {
|
|
58
|
+
...m,
|
|
59
|
+
contentHash: hash,
|
|
60
|
+
algo: "sha256-content"
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Verify that a signed manifest has not been tampered with.
|
|
65
|
+
*
|
|
66
|
+
* Strips `contentHash` and `algo` before re-signing so manifests without
|
|
67
|
+
* `algo` verify identically to ones that carry it.
|
|
68
|
+
*/
|
|
69
|
+
async function verifyManifest(m) {
|
|
70
|
+
const { contentHash, algo: _algo, ...rest } = m;
|
|
71
|
+
return (await signManifest(rest)).contentHash === contentHash;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Evaluate a pre-registered hypothesis against observed results.
|
|
75
|
+
* Mechanical — no re-interpretation permitted.
|
|
76
|
+
*/
|
|
77
|
+
async function evaluateHypothesis(manifest, observed) {
|
|
78
|
+
if (!await verifyManifest(manifest)) throw new Error("evaluateHypothesis: manifest content hash mismatch (tampered)");
|
|
79
|
+
const reasons = [];
|
|
80
|
+
if (!(manifest.direction === "increase" ? observed.effect > 0 : observed.effect < 0)) reasons.push("wrong_direction");
|
|
81
|
+
if (Math.abs(observed.effect) < manifest.minEffect) reasons.push("effect_too_small");
|
|
82
|
+
if (observed.pValue >= manifest.alpha) reasons.push("not_significant");
|
|
83
|
+
if (observed.n < manifest.preRegisteredN) reasons.push("undersampled");
|
|
84
|
+
return {
|
|
85
|
+
manifest,
|
|
86
|
+
observedN: observed.n,
|
|
87
|
+
observedEffect: observed.effect,
|
|
88
|
+
observedPValue: observed.pValue,
|
|
89
|
+
confirmed: reasons.length === 0,
|
|
90
|
+
rejectionReasons: reasons
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
//#endregion
|
|
94
|
+
export { verifyManifest as a, signManifest as i, evaluateHypothesis as n, hashJson as r, canonicalize as t };
|
|
95
|
+
|
|
96
|
+
//# sourceMappingURL=pre-registration-DakwTRXk.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pre-registration-DakwTRXk.js","names":[],"sources":["../src/pre-registration.ts"],"sourcesContent":["/**\n * Pre-registered hypotheses — declare what you're testing BEFORE the\n * run, check it AFTER. Prevents p-hacking, optional stopping, and the\n * \"we ran until it looked good\" failure mode.\n *\n * Manifest is a plain JSON-friendly object. Sign it with a content hash\n * + timestamp; the registered record becomes immutable. Post-run,\n * evaluate the manifest against observed results — the library refuses\n * to let you re-interpret a different metric as the declared one.\n */\n\nexport interface HypothesisManifest {\n id: string\n /** Human prose — goes into the audit trail. */\n hypothesis: string\n /** Metric the hypothesis claims to move. */\n metric: string\n /** 'increase' = candidate should score higher than baseline; 'decrease' = lower. */\n direction: 'increase' | 'decrease'\n /** Minimum effect size to count (same units as the metric). */\n minEffect: number\n /** Alpha threshold. */\n alpha: number\n /** Target statistical power at which sample size was pre-computed. */\n power: number\n /** Declared N per arm before running. */\n preRegisteredN: number\n /** ISO8601 timestamp the manifest was registered. */\n registeredAt: string\n /** Optional identifiers to tie into the trace corpus. */\n baselineLabel?: string\n candidateLabel?: string\n}\n\n/**\n * Identifier for the hashing scheme used to produce `contentHash`.\n *\n * `'sha256-content'` — sha256 hex over the canonicalized manifest with\n * the `contentHash` and `algo` fields stripped. Held as a string union\n * so future schemes can be added without breaking parsers; SignedManifest\n * values without `algo` deserialize cleanly because the field is optional.\n */\nexport type SignedManifestAlgo = 'sha256-content'\n\nexport interface SignedManifest extends HypothesisManifest {\n /** sha256 hex of canonicalized manifest (everything except contentHash and algo). */\n contentHash: string\n /**\n * Algorithm string describing how `contentHash` was produced.\n *\n * Optional on the type so serialized manifests without it still parse,\n * but ALWAYS populated by {@link signManifest}. Consumers that want to\n * enforce a known algorithm should reject manifests where this field\n * is missing or unrecognized.\n */\n algo?: SignedManifestAlgo\n}\n\nexport interface HypothesisResult {\n manifest: SignedManifest\n observedN: number\n observedEffect: number\n observedPValue: number\n /** True iff the observed effect hits the pre-declared direction with\n * magnitude ≥ minEffect AND p < alpha. */\n confirmed: boolean\n /** Enumerated reasons the hypothesis was rejected (each a machine-tag). */\n rejectionReasons: Array<\n 'wrong_direction' | 'effect_too_small' | 'not_significant' | 'undersampled'\n >\n notes?: string\n}\n\n/**\n * Deterministic JSON canonicalization — sort object keys recursively.\n *\n * Two semantically-equal objects produce byte-identical canonicalized output;\n * this is what makes a content-hash stable across encoders, key insertion\n * orders, and runtime versions. Exported for any consumer that needs the same\n * canonicalization guarantee outside the manifest-signing path (e.g., signing\n * an artifact bundle, hashing a dataset version, etc.).\n */\nexport function canonicalize(v: unknown): unknown {\n if (v === null || typeof v !== 'object') return v\n if (Array.isArray(v)) return v.map(canonicalize)\n const keys = Object.keys(v as Record<string, unknown>).sort()\n const out: Record<string, unknown> = {}\n for (const k of keys) out[k] = canonicalize((v as Record<string, unknown>)[k])\n return out\n}\n\n/**\n * SHA-256 hex (full 64 chars) over the canonicalized JSON encoding of `obj`.\n *\n * The same primitive `signManifest` and `verifyManifest` are built on, exposed\n * directly so consumers signing arbitrary structured content (artifact bundles,\n * production packets, dataset manifests, etc.) don't have to re-derive\n * canonicalize+sha256 from scratch.\n *\n * Stable across:\n * - object key insertion order (canonicalization sorts keys recursively)\n * - encoder choice (UTF-8 via TextEncoder, fixed)\n * - runtime (uses the Web Crypto subtle digest, present in Node ≥18 and browsers)\n *\n * Named `hashJson` to disambiguate from `prompt-registry.ts`'s `hashContent`,\n * which takes a string input and returns a truncated 12-char prompt id.\n * Use `hashJson` when you mean \"canonicalize then hash.\"\n *\n * @example\n * const hash = await hashJson({ id: '1', kind: 'spec' })\n * // 'a3f1...' (64 hex chars)\n */\nexport async function hashJson<T>(obj: T): Promise<string> {\n const canonical = canonicalize(obj)\n const bytes = new TextEncoder().encode(JSON.stringify(canonical))\n const digest = await globalThis.crypto.subtle.digest('SHA-256', bytes)\n return Array.from(new Uint8Array(digest))\n .map((b) => b.toString(16).padStart(2, '0'))\n .join('')\n}\n\n/**\n * Sign a manifest with a SHA-256 content hash.\n *\n * The hash covers the canonicalized manifest with the `contentHash`\n * and `algo` fields stripped; this lets verifiers re-sign the rest and\n * compare. Returned manifest always carries `algo: 'sha256-content'`\n * so downstream consumers can identify the scheme; manifests without\n * `algo` still verify because it is stripped before hashing on both sides.\n */\nexport async function signManifest(m: HypothesisManifest): Promise<SignedManifest> {\n const hash = await hashJson(m)\n return { ...m, contentHash: hash, algo: 'sha256-content' }\n}\n\n/**\n * Verify that a signed manifest has not been tampered with.\n *\n * Strips `contentHash` and `algo` before re-signing so manifests without\n * `algo` verify identically to ones that carry it.\n */\nexport async function verifyManifest(m: SignedManifest): Promise<boolean> {\n const { contentHash, algo: _algo, ...rest } = m\n void _algo\n const resigned = await signManifest(rest)\n return resigned.contentHash === contentHash\n}\n\n/**\n * Evaluate a pre-registered hypothesis against observed results.\n * Mechanical — no re-interpretation permitted.\n */\nexport async function evaluateHypothesis(\n manifest: SignedManifest,\n observed: { n: number; effect: number; pValue: number },\n): Promise<HypothesisResult> {\n if (!(await verifyManifest(manifest))) {\n throw new Error('evaluateHypothesis: manifest content hash mismatch (tampered)')\n }\n const reasons: HypothesisResult['rejectionReasons'] = []\n const directionOk = manifest.direction === 'increase' ? observed.effect > 0 : observed.effect < 0\n if (!directionOk) reasons.push('wrong_direction')\n if (Math.abs(observed.effect) < manifest.minEffect) reasons.push('effect_too_small')\n if (observed.pValue >= manifest.alpha) reasons.push('not_significant')\n if (observed.n < manifest.preRegisteredN) reasons.push('undersampled')\n return {\n manifest,\n observedN: observed.n,\n observedEffect: observed.effect,\n observedPValue: observed.pValue,\n confirmed: reasons.length === 0,\n rejectionReasons: reasons,\n }\n}\n"],"mappings":";;;;;;;;;;AAkFA,SAAgB,aAAa,GAAqB;CAChD,IAAI,MAAM,QAAQ,OAAO,MAAM,UAAU,OAAO;CAChD,IAAI,MAAM,QAAQ,CAAC,GAAG,OAAO,EAAE,IAAI,YAAY;CAC/C,MAAM,OAAO,OAAO,KAAK,CAA4B,CAAC,CAAC,KAAK;CAC5D,MAAM,MAA+B,CAAC;CACtC,KAAK,MAAM,KAAK,MAAM,IAAI,KAAK,aAAc,EAA8B,EAAE;CAC7E,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;AAuBA,eAAsB,SAAY,KAAyB;CACzD,MAAM,YAAY,aAAa,GAAG;CAClC,MAAM,QAAQ,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK,UAAU,SAAS,CAAC;CAChE,MAAM,SAAS,MAAM,WAAW,OAAO,OAAO,OAAO,WAAW,KAAK;CACrE,OAAO,MAAM,KAAK,IAAI,WAAW,MAAM,CAAC,CAAC,CACtC,KAAK,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAC3C,KAAK,EAAE;AACZ;;;;;;;;;;AAWA,eAAsB,aAAa,GAAgD;CACjF,MAAM,OAAO,MAAM,SAAS,CAAC;CAC7B,OAAO;EAAE,GAAG;EAAG,aAAa;EAAM,MAAM;CAAiB;AAC3D;;;;;;;AAQA,eAAsB,eAAe,GAAqC;CACxE,MAAM,EAAE,aAAa,MAAM,OAAO,GAAG,SAAS;CAG9C,QAAO,MADgB,aAAa,IAAI,EAAA,CACxB,gBAAgB;AAClC;;;;;AAMA,eAAsB,mBACpB,UACA,UAC2B;CAC3B,IAAI,CAAE,MAAM,eAAe,QAAQ,GACjC,MAAM,IAAI,MAAM,+DAA+D;CAEjF,MAAM,UAAgD,CAAC;CAEvD,IAAI,EADgB,SAAS,cAAc,aAAa,SAAS,SAAS,IAAI,SAAS,SAAS,IAC9E,QAAQ,KAAK,iBAAiB;CAChD,IAAI,KAAK,IAAI,SAAS,MAAM,IAAI,SAAS,WAAW,QAAQ,KAAK,kBAAkB;CACnF,IAAI,SAAS,UAAU,SAAS,OAAO,QAAQ,KAAK,iBAAiB;CACrE,IAAI,SAAS,IAAI,SAAS,gBAAgB,QAAQ,KAAK,cAAc;CACrE,OAAO;EACL;EACA,WAAW,SAAS;EACpB,gBAAgB,SAAS;EACzB,gBAAgB,SAAS;EACzB,WAAW,QAAQ,WAAW;EAC9B,kBAAkB;CACpB;AACF"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
//#region src/analyst/prime-bridge-transport.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* The bridge seam for the prime analyst protocol: one POST to an
|
|
4
|
+
* OpenAI-compatible `/v1/chat/completions` endpoint, returning the raw status
|
|
5
|
+
* and body text.
|
|
6
|
+
*
|
|
7
|
+
* Deliberately NOT `callLlm` from `../llm-client`, which serves a different
|
|
8
|
+
* contract:
|
|
9
|
+
* - it retries transient failures, and a prime turn holds a single model seat
|
|
10
|
+
* for minutes — a silent second attempt doubles the seat time and the spend;
|
|
11
|
+
* - it normalizes usage, while the protocol's receipt depends on the bridge's
|
|
12
|
+
* non-standard `model_requests` and `estimated` fields;
|
|
13
|
+
* - it composes sampling and response-format options the bridge's CLI backends
|
|
14
|
+
* reject;
|
|
15
|
+
* - it collapses HTTP status, unparseable body, and empty content into two
|
|
16
|
+
* error classes, while the protocol classifies them as three distinct
|
|
17
|
+
* terminal reasons.
|
|
18
|
+
*/
|
|
19
|
+
interface PrimeBridgeTransportRequest {
|
|
20
|
+
url: string;
|
|
21
|
+
body: {
|
|
22
|
+
model: string;
|
|
23
|
+
messages: Array<{
|
|
24
|
+
role: 'user';
|
|
25
|
+
content: string;
|
|
26
|
+
}>;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* The call's only deadline. The protocol aborts it on the per-call timeout
|
|
30
|
+
* and on the caller's cancellation, so a transport that imposes a second
|
|
31
|
+
* deadline of its own competes with this one.
|
|
32
|
+
*/
|
|
33
|
+
signal: AbortSignal;
|
|
34
|
+
}
|
|
35
|
+
interface PrimeBridgeTransportResult {
|
|
36
|
+
status: number;
|
|
37
|
+
text: string;
|
|
38
|
+
}
|
|
39
|
+
/** One POST to the bridge. Injectable so tests run against a fake bridge. */
|
|
40
|
+
type PrimeBridgeTransport = (request: PrimeBridgeTransportRequest) => Promise<PrimeBridgeTransportResult>;
|
|
41
|
+
/**
|
|
42
|
+
* Default transport on node:http/node:https rather than fetch: undici's fixed
|
|
43
|
+
* response-header timeout kills prime calls that legitimately run past five
|
|
44
|
+
* minutes, so the request's AbortSignal is the only deadline.
|
|
45
|
+
*/
|
|
46
|
+
declare function nodeHttpPrimeBridgeTransport(): PrimeBridgeTransport;
|
|
47
|
+
//#endregion
|
|
48
|
+
export { nodeHttpPrimeBridgeTransport as i, PrimeBridgeTransportRequest as n, PrimeBridgeTransportResult as r, PrimeBridgeTransport as t };
|
|
49
|
+
//# sourceMappingURL=prime-bridge-transport-6feEglLf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prime-bridge-transport-6feEglLf.d.ts","names":[],"sources":["../src/analyst/prime-bridge-transport.ts"],"mappings":";;;;;;;;;;;;;;;;;;UAoBiB;EACf;EACA;IACE;IACA,UAAU;MAAQ;MAAc;;;;;;;;EAOlC,QAAQ;;UAGO;EACf;EACA;;;KAIU,wBACV,SAAS,gCACN,QAAQ;;;;;;iBAOG,gCAAgC"}
|
|
@@ -0,0 +1,453 @@
|
|
|
1
|
+
import { c as ValidationError } from "./errors-D-LKuDhb.js";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
//#region src/analyst/equal-terms.ts
|
|
4
|
+
/**
|
|
5
|
+
* Refuses a comparison whose arms are not on equal terms: an empty set, a
|
|
6
|
+
* duplicated id, or unequal repair turns — a retry is a second sample the
|
|
7
|
+
* other arms never got. `noun` names the arm kind in every message so a
|
|
8
|
+
* refusal reads in the caller's vocabulary.
|
|
9
|
+
*/
|
|
10
|
+
function assertEqualDeclarativeTerms(noun, terms) {
|
|
11
|
+
if (terms.length === 0) throw new ValidationError(`a ${noun} comparison needs at least one ${noun}`);
|
|
12
|
+
const ids = terms.map((term) => term.id);
|
|
13
|
+
const duplicate = ids.find((id, index) => ids.indexOf(id) !== index);
|
|
14
|
+
if (duplicate !== void 0) throw new ValidationError(`${noun} '${duplicate}' is declared twice`);
|
|
15
|
+
const repairTurns = terms[0].repairTurns;
|
|
16
|
+
const unequal = terms.find((term) => term.repairTurns !== repairTurns);
|
|
17
|
+
if (unequal) throw new ValidationError(`${noun} '${unequal.id}' gets ${unequal.repairTurns} bounded repair turns and ${noun} '${terms[0].id}' gets ${repairTurns}; a retry is a second sample the other arms never got`);
|
|
18
|
+
return {
|
|
19
|
+
ids,
|
|
20
|
+
repairTurns
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
//#endregion
|
|
24
|
+
//#region src/analyst/prime-protocol.ts
|
|
25
|
+
function buildPrimePrompt(spec) {
|
|
26
|
+
return [
|
|
27
|
+
`QUESTION: ${spec.question}`,
|
|
28
|
+
"",
|
|
29
|
+
...spec.taskDefinition === void 0 ? [] : [
|
|
30
|
+
"TASK DEFINITION:",
|
|
31
|
+
spec.taskDefinition,
|
|
32
|
+
""
|
|
33
|
+
],
|
|
34
|
+
...spec.contractLines,
|
|
35
|
+
"",
|
|
36
|
+
spec.trajectoryHeader,
|
|
37
|
+
spec.renderedTrajectory,
|
|
38
|
+
...spec.trailer === void 0 ? [] : ["", spec.trailer]
|
|
39
|
+
].join("\n");
|
|
40
|
+
}
|
|
41
|
+
/** Carries the malformed reply and the contract — never the trajectory. */
|
|
42
|
+
function buildPrimeRepairPrompt(spec) {
|
|
43
|
+
return [
|
|
44
|
+
"Your previous reply to a trace-analysis task was structurally malformed and could not be parsed",
|
|
45
|
+
`(${spec.defect}). Below is your previous reply verbatim. Re-emit ONLY the corrected JSON — one`,
|
|
46
|
+
"fenced ```json block, no other text, no tools. The JSON object has exactly two fields:",
|
|
47
|
+
...spec.repairContractLines,
|
|
48
|
+
"",
|
|
49
|
+
"PREVIOUS REPLY:",
|
|
50
|
+
spec.previousReply
|
|
51
|
+
].join("\n");
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Recover the reply's JSON object.
|
|
55
|
+
*
|
|
56
|
+
* Distinct from `extractJsonPayload` in ../llm-client, which serves a response
|
|
57
|
+
* that DECLARES a JSON root and therefore must not scan onward. A prime reply
|
|
58
|
+
* is prose plus a fenced block, and when the model emits several fences the
|
|
59
|
+
* last one is its answer — so fences are scanned in reverse, and only then is a
|
|
60
|
+
* brace-to-brace slice tried.
|
|
61
|
+
*/
|
|
62
|
+
function extractPrimeJsonObject(text) {
|
|
63
|
+
const direct = parsePrimeJsonObject(text);
|
|
64
|
+
if (direct) return direct;
|
|
65
|
+
const fenced = [...text.matchAll(/```(?:json)?\s*\n?([\s\S]*?)```/g)];
|
|
66
|
+
for (let index = fenced.length - 1; index >= 0; index -= 1) {
|
|
67
|
+
const candidate = parsePrimeJsonObject(fenced[index][1]);
|
|
68
|
+
if (candidate) return candidate;
|
|
69
|
+
}
|
|
70
|
+
const start = text.indexOf("{");
|
|
71
|
+
const end = text.lastIndexOf("}");
|
|
72
|
+
if (start >= 0 && end > start) {
|
|
73
|
+
const candidate = parsePrimeJsonObject(text.slice(start, end + 1));
|
|
74
|
+
if (candidate) return candidate;
|
|
75
|
+
}
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
/** Why the reply cannot be read as a prime answer, or null when it can. */
|
|
79
|
+
function primeReplyDefect(parsed, rowsField) {
|
|
80
|
+
if (parsed === null) return "no parseable JSON object";
|
|
81
|
+
if (!Array.isArray(parsed[rowsField])) return `JSON has no "${rowsField}" array`;
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
function parsePrimeJsonObject(text) {
|
|
85
|
+
try {
|
|
86
|
+
const value = JSON.parse(text.trim());
|
|
87
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
|
|
88
|
+
} catch {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function emptyPrimeRawUsage() {
|
|
93
|
+
return {
|
|
94
|
+
calls: null,
|
|
95
|
+
inputTokens: null,
|
|
96
|
+
outputTokens: null,
|
|
97
|
+
bridgeEstimated: false
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
/** Read the bridge's OpenAI-shaped `usage` object. */
|
|
101
|
+
function normalizePrimeUsage(raw) {
|
|
102
|
+
if (typeof raw !== "object" || raw === null) return emptyPrimeRawUsage();
|
|
103
|
+
const record = raw;
|
|
104
|
+
return {
|
|
105
|
+
calls: tokenCountOrNull(record.model_requests),
|
|
106
|
+
inputTokens: tokenCountOrNull(record.prompt_tokens),
|
|
107
|
+
outputTokens: tokenCountOrNull(record.completion_tokens),
|
|
108
|
+
bridgeEstimated: record.estimated === true
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Sum two turns. Each side poisons independently: two turns that both report
|
|
113
|
+
* input and neither report output yield a real input total beside a null
|
|
114
|
+
* output, because discarding a measured count is as wrong as inventing one.
|
|
115
|
+
*/
|
|
116
|
+
function mergePrimeRawUsage(a, b) {
|
|
117
|
+
return {
|
|
118
|
+
calls: sumOrNull(a.calls, b.calls),
|
|
119
|
+
inputTokens: sumOrNull(a.inputTokens, b.inputTokens),
|
|
120
|
+
outputTokens: sumOrNull(a.outputTokens, b.outputTokens),
|
|
121
|
+
bridgeEstimated: a.bridgeEstimated || b.bridgeEstimated
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
function sumOrNull(a, b) {
|
|
125
|
+
return a !== null && b !== null ? a + b : null;
|
|
126
|
+
}
|
|
127
|
+
function tokenCountOrNull(value) {
|
|
128
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Bind raw prime usage to agent-eval's typed receipt.
|
|
132
|
+
*
|
|
133
|
+
* `RunTokenUsage.input` and `.output` are non-nullable, so a one-sided count
|
|
134
|
+
* cannot round-trip through `tokens` without writing a zero nobody measured.
|
|
135
|
+
* The complete-accounting field therefore stays null, the reported side is
|
|
136
|
+
* carried verbatim in `partialTokens`, and its price becomes the receipt's
|
|
137
|
+
* `knownCostUsd` lower bound.
|
|
138
|
+
*
|
|
139
|
+
* Only agent-eval calls this; consumers with no pricing table read
|
|
140
|
+
* `PrimeRawUsage` directly.
|
|
141
|
+
*/
|
|
142
|
+
function analystUsageReceiptFromPrimeUsage(usage, pricing) {
|
|
143
|
+
const { calls, inputTokens, outputTokens, bridgeEstimated } = usage;
|
|
144
|
+
const estimatedTokens = bridgeEstimated ? { tokensEstimated: true } : {};
|
|
145
|
+
if (inputTokens !== null && outputTokens !== null) return {
|
|
146
|
+
calls,
|
|
147
|
+
tokens: {
|
|
148
|
+
input: inputTokens,
|
|
149
|
+
output: outputTokens
|
|
150
|
+
},
|
|
151
|
+
cost: {
|
|
152
|
+
kind: "estimated",
|
|
153
|
+
usd: priceTokens(inputTokens, outputTokens, pricing)
|
|
154
|
+
},
|
|
155
|
+
...estimatedTokens
|
|
156
|
+
};
|
|
157
|
+
if (inputTokens === null && outputTokens === null) return {
|
|
158
|
+
calls,
|
|
159
|
+
tokens: null,
|
|
160
|
+
cost: {
|
|
161
|
+
kind: "uncaptured",
|
|
162
|
+
usd: null
|
|
163
|
+
},
|
|
164
|
+
...estimatedTokens
|
|
165
|
+
};
|
|
166
|
+
return {
|
|
167
|
+
calls,
|
|
168
|
+
tokens: null,
|
|
169
|
+
partialTokens: {
|
|
170
|
+
input: inputTokens,
|
|
171
|
+
output: outputTokens
|
|
172
|
+
},
|
|
173
|
+
cost: {
|
|
174
|
+
kind: "uncaptured",
|
|
175
|
+
usd: null
|
|
176
|
+
},
|
|
177
|
+
knownCostUsd: priceTokens(inputTokens ?? 0, outputTokens ?? 0, pricing),
|
|
178
|
+
...estimatedTokens
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
function priceTokens(input, output, pricing) {
|
|
182
|
+
return (input * pricing.inputUsdPerMillion + output * pricing.outputUsdPerMillion) / 1e6;
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Run the protocol: one call, one bounded repair turn on a structurally
|
|
186
|
+
* malformed reply, then decode. Zero valid rows from a well-formed reply is an
|
|
187
|
+
* honest null, not a failure.
|
|
188
|
+
*/
|
|
189
|
+
async function runPrimeExchange(options) {
|
|
190
|
+
const { contract } = options;
|
|
191
|
+
const turns = [];
|
|
192
|
+
const repair = {
|
|
193
|
+
attempted: false,
|
|
194
|
+
succeeded: null
|
|
195
|
+
};
|
|
196
|
+
const first = await callPrimeTurn(options, options.prompt);
|
|
197
|
+
if (!first.ok) return {
|
|
198
|
+
ok: false,
|
|
199
|
+
failure: first.failure,
|
|
200
|
+
usage: mergeTurns(turns),
|
|
201
|
+
turns,
|
|
202
|
+
repair
|
|
203
|
+
};
|
|
204
|
+
turns.push({
|
|
205
|
+
turn: "first",
|
|
206
|
+
usage: first.usage,
|
|
207
|
+
rawUsage: first.rawUsage
|
|
208
|
+
});
|
|
209
|
+
let reply = first.content;
|
|
210
|
+
let parsed = extractPrimeJsonObject(reply);
|
|
211
|
+
let defect = primeReplyDefect(parsed, contract.rowsField);
|
|
212
|
+
if (defect !== null && options.repair) {
|
|
213
|
+
repair.attempted = true;
|
|
214
|
+
const second = await callPrimeTurn(options, buildPrimeRepairPrompt({
|
|
215
|
+
defect,
|
|
216
|
+
previousReply: reply,
|
|
217
|
+
repairContractLines: contract.repairContractLines
|
|
218
|
+
}));
|
|
219
|
+
if (!second.ok) return {
|
|
220
|
+
ok: false,
|
|
221
|
+
failure: second.failure,
|
|
222
|
+
usage: mergeTurns(turns),
|
|
223
|
+
turns,
|
|
224
|
+
repair,
|
|
225
|
+
reply
|
|
226
|
+
};
|
|
227
|
+
turns.push({
|
|
228
|
+
turn: "repair",
|
|
229
|
+
usage: second.usage,
|
|
230
|
+
rawUsage: second.rawUsage
|
|
231
|
+
});
|
|
232
|
+
reply = second.content;
|
|
233
|
+
parsed = extractPrimeJsonObject(reply);
|
|
234
|
+
defect = primeReplyDefect(parsed, contract.rowsField);
|
|
235
|
+
repair.succeeded = defect === null;
|
|
236
|
+
}
|
|
237
|
+
const usage = mergeTurns(turns);
|
|
238
|
+
if (defect !== null) return {
|
|
239
|
+
ok: false,
|
|
240
|
+
failure: {
|
|
241
|
+
kind: "malformed-reply",
|
|
242
|
+
message: `${defect} in prime reply${repair.attempted ? " even after the bounded repair turn" : ""}`
|
|
243
|
+
},
|
|
244
|
+
usage,
|
|
245
|
+
turns,
|
|
246
|
+
repair,
|
|
247
|
+
reply
|
|
248
|
+
};
|
|
249
|
+
const rawRows = parsed[contract.rowsField];
|
|
250
|
+
const rows = [];
|
|
251
|
+
const rejected = [];
|
|
252
|
+
let overflow = 0;
|
|
253
|
+
rawRows.forEach((row, index) => {
|
|
254
|
+
const decoded = contract.decodeRow(row, index);
|
|
255
|
+
if (!decoded.ok) {
|
|
256
|
+
rejected.push({
|
|
257
|
+
index,
|
|
258
|
+
reason: decoded.reason
|
|
259
|
+
});
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
if (contract.maxRows !== void 0 && rows.length >= contract.maxRows) {
|
|
263
|
+
overflow += 1;
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
rows.push(decoded.row);
|
|
267
|
+
});
|
|
268
|
+
const answer = parsed.answer;
|
|
269
|
+
return {
|
|
270
|
+
ok: true,
|
|
271
|
+
answer: typeof answer === "string" ? answer : null,
|
|
272
|
+
rows,
|
|
273
|
+
rejected,
|
|
274
|
+
reportedRows: rawRows.length,
|
|
275
|
+
overflow,
|
|
276
|
+
usage,
|
|
277
|
+
turns,
|
|
278
|
+
repair,
|
|
279
|
+
reply
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
async function callPrimeTurn(options, content) {
|
|
283
|
+
const { transport, url, model, timeoutMs, signal } = options;
|
|
284
|
+
const controller = new AbortController();
|
|
285
|
+
const forwardAbort = () => controller.abort(signal?.reason);
|
|
286
|
+
if (signal?.aborted) controller.abort(signal.reason);
|
|
287
|
+
else signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
288
|
+
const deadline = setTimeout(() => controller.abort(), timeoutMs);
|
|
289
|
+
let result;
|
|
290
|
+
try {
|
|
291
|
+
result = await transport({
|
|
292
|
+
url,
|
|
293
|
+
body: {
|
|
294
|
+
model,
|
|
295
|
+
messages: [{
|
|
296
|
+
role: "user",
|
|
297
|
+
content
|
|
298
|
+
}]
|
|
299
|
+
},
|
|
300
|
+
signal: controller.signal
|
|
301
|
+
});
|
|
302
|
+
} catch (error) {
|
|
303
|
+
if (signal?.aborted) return {
|
|
304
|
+
ok: false,
|
|
305
|
+
failure: {
|
|
306
|
+
kind: "aborted",
|
|
307
|
+
message: "prime exchange cancelled by the caller",
|
|
308
|
+
cause: error
|
|
309
|
+
}
|
|
310
|
+
};
|
|
311
|
+
if (controller.signal.aborted) return {
|
|
312
|
+
ok: false,
|
|
313
|
+
failure: {
|
|
314
|
+
kind: "deadline",
|
|
315
|
+
message: `bridge call exceeded ${timeoutMs}ms`
|
|
316
|
+
}
|
|
317
|
+
};
|
|
318
|
+
return {
|
|
319
|
+
ok: false,
|
|
320
|
+
failure: {
|
|
321
|
+
kind: "transport",
|
|
322
|
+
message: `bridge transport failure: ${error instanceof Error ? error.message : String(error)}`
|
|
323
|
+
}
|
|
324
|
+
};
|
|
325
|
+
} finally {
|
|
326
|
+
clearTimeout(deadline);
|
|
327
|
+
signal?.removeEventListener("abort", forwardAbort);
|
|
328
|
+
}
|
|
329
|
+
if (result.status !== 200) {
|
|
330
|
+
const bodySnippet = result.text.slice(0, 500);
|
|
331
|
+
return {
|
|
332
|
+
ok: false,
|
|
333
|
+
failure: {
|
|
334
|
+
kind: "http-status",
|
|
335
|
+
message: `bridge HTTP ${result.status}: ${bodySnippet}`,
|
|
336
|
+
status: result.status,
|
|
337
|
+
bodySnippet
|
|
338
|
+
}
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
let response;
|
|
342
|
+
try {
|
|
343
|
+
response = JSON.parse(result.text);
|
|
344
|
+
} catch {
|
|
345
|
+
return {
|
|
346
|
+
ok: false,
|
|
347
|
+
failure: {
|
|
348
|
+
kind: "unparseable-json",
|
|
349
|
+
message: `bridge returned unparseable JSON (${result.text.length} bytes)`
|
|
350
|
+
}
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
const replyContent = primeReplyContent(response);
|
|
354
|
+
if (replyContent === null) return {
|
|
355
|
+
ok: false,
|
|
356
|
+
failure: {
|
|
357
|
+
kind: "no-content",
|
|
358
|
+
message: "bridge reply carries no message content"
|
|
359
|
+
}
|
|
360
|
+
};
|
|
361
|
+
const rawUsage = primeReplyUsage(response);
|
|
362
|
+
return {
|
|
363
|
+
ok: true,
|
|
364
|
+
content: replyContent,
|
|
365
|
+
usage: normalizePrimeUsage(rawUsage),
|
|
366
|
+
rawUsage
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* Fold from the FIRST turn, never from an empty receipt: an all-null identity
|
|
371
|
+
* would poison every side it merged with and erase counts the bridge reported.
|
|
372
|
+
*/
|
|
373
|
+
function mergeTurns(turns) {
|
|
374
|
+
if (turns.length === 0) return emptyPrimeRawUsage();
|
|
375
|
+
return turns.slice(1).reduce((total, turn) => mergePrimeRawUsage(total, turn.usage), turns[0].usage);
|
|
376
|
+
}
|
|
377
|
+
function primeReplyContent(response) {
|
|
378
|
+
if (typeof response !== "object" || response === null) return null;
|
|
379
|
+
const choices = response.choices;
|
|
380
|
+
if (!Array.isArray(choices) || choices.length === 0) return null;
|
|
381
|
+
const message = choices[0]?.message;
|
|
382
|
+
if (typeof message !== "object" || message === null) return null;
|
|
383
|
+
const content = message.content;
|
|
384
|
+
return typeof content === "string" && content.length > 0 ? content : null;
|
|
385
|
+
}
|
|
386
|
+
function primeReplyUsage(response) {
|
|
387
|
+
if (typeof response !== "object" || response === null) return null;
|
|
388
|
+
return response.usage ?? null;
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Render, measure, fall back to the capped projection, re-measure, fail loud.
|
|
392
|
+
*
|
|
393
|
+
* Inline is the only delivery prime has, so an oversized trajectory is a
|
|
394
|
+
* refusal rather than a silent truncation: dropping spans would understate the
|
|
395
|
+
* trajectory and the analyst would answer a question about a different run.
|
|
396
|
+
*/
|
|
397
|
+
async function projectPrimeTrajectory(source, limits) {
|
|
398
|
+
let fetch = "full";
|
|
399
|
+
let items = await source.full();
|
|
400
|
+
if (items === null) {
|
|
401
|
+
fetch = "capped";
|
|
402
|
+
items = await source.capped();
|
|
403
|
+
}
|
|
404
|
+
let rendered = JSON.stringify(items);
|
|
405
|
+
if (rendered.length > limits.maxInlineChars && fetch === "full") {
|
|
406
|
+
fetch = "capped";
|
|
407
|
+
items = await source.capped();
|
|
408
|
+
rendered = JSON.stringify(items);
|
|
409
|
+
}
|
|
410
|
+
if (rendered.length > limits.maxInlineChars) return {
|
|
411
|
+
ok: false,
|
|
412
|
+
reason: `trajectory renders to ${rendered.length} chars even at ${source.cappedDescription}; inline delivery impossible`,
|
|
413
|
+
renderedChars: rendered.length
|
|
414
|
+
};
|
|
415
|
+
return {
|
|
416
|
+
ok: true,
|
|
417
|
+
items,
|
|
418
|
+
rendered,
|
|
419
|
+
delivery: {
|
|
420
|
+
mode: "inline-json",
|
|
421
|
+
fetch,
|
|
422
|
+
renderedChars: rendered.length
|
|
423
|
+
}
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Digest of everything a consumer can send to the bridge under the prime
|
|
428
|
+
* protocol, recorded per observation so a prime result names the exact contract
|
|
429
|
+
* that produced it.
|
|
430
|
+
*
|
|
431
|
+
* Computed over the ACTUALLY composed contract, so two consumers that both
|
|
432
|
+
* stamp `analyst_id: 'prime'` while asking materially different questions get
|
|
433
|
+
* different digests by construction. That is what makes 'prime' a reproducible
|
|
434
|
+
* claim rather than a label.
|
|
435
|
+
*/
|
|
436
|
+
function primeProtocolSha256(identity) {
|
|
437
|
+
return createHash("sha256").update(JSON.stringify({
|
|
438
|
+
kind: "prime-analyst-protocol",
|
|
439
|
+
question: identity.question,
|
|
440
|
+
taskPrompt: identity.taskDefinition ?? null,
|
|
441
|
+
outputContract: identity.contractLines,
|
|
442
|
+
repairContract: buildPrimeRepairPrompt({
|
|
443
|
+
defect: "<defect>",
|
|
444
|
+
previousReply: "<previous-reply>",
|
|
445
|
+
repairContractLines: identity.repairContractLines
|
|
446
|
+
}),
|
|
447
|
+
limits: identity.limits
|
|
448
|
+
})).digest("hex");
|
|
449
|
+
}
|
|
450
|
+
//#endregion
|
|
451
|
+
export { extractPrimeJsonObject as a, primeProtocolSha256 as c, runPrimeExchange as d, assertEqualDeclarativeTerms as f, emptyPrimeRawUsage as i, primeReplyDefect as l, buildPrimePrompt as n, mergePrimeRawUsage as o, buildPrimeRepairPrompt as r, normalizePrimeUsage as s, analystUsageReceiptFromPrimeUsage as t, projectPrimeTrajectory as u };
|
|
452
|
+
|
|
453
|
+
//# sourceMappingURL=prime-protocol-BfSalTfR.js.map
|