@tangle-network/agent-eval 0.183.0 → 0.184.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-cell-s__adRnK.d.ts → agent-profile-cell-BHcB-mZX.d.ts} +8 -9
- package/dist/{agent-profile-cell-s__adRnK.d.ts.map → agent-profile-cell-BHcB-mZX.d.ts.map} +1 -1
- package/dist/{agent-profile-cell-Cv6UA-W_.js → agent-profile-cell-DcFW4vOc.js} +16 -15
- package/dist/agent-profile-cell-DcFW4vOc.js.map +1 -0
- package/dist/{agent-profile-CivaSsSy.d.ts → agent-profile-uAIQK3Nr.d.ts} +3 -3
- package/dist/{agent-profile-CivaSsSy.d.ts.map → agent-profile-uAIQK3Nr.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +10 -10
- package/dist/analyst/index.js +8 -7
- package/dist/analyst/index.js.map +1 -1
- package/dist/{benchmark-DQKzykkO.js → benchmark-BeQJyrRC.js} +3 -3
- package/dist/{benchmark-DQKzykkO.js.map → benchmark-BeQJyrRC.js.map} +1 -1
- package/dist/{benchmark-command-NBrqNwgy.js → benchmark-command-DlSePljH.js} +13 -12
- package/dist/{benchmark-command-NBrqNwgy.js.map → benchmark-command-DlSePljH.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +6 -5
- package/dist/campaign/index.js +9 -9
- package/dist/{campaign-BwcEBC91.js → campaign-D6puiGFO.js} +15 -15
- package/dist/{campaign-BwcEBC91.js.map → campaign-D6puiGFO.js.map} +1 -1
- package/dist/{campaign-evidence-PYPUftRy.js → campaign-evidence-aWEmfROB.js} +5 -5
- package/dist/{campaign-evidence-PYPUftRy.js.map → campaign-evidence-aWEmfROB.js.map} +1 -1
- package/dist/{canonical-DPyQ_rpt.js → canonical-BsZVgBhW.js} +2 -4
- package/dist/canonical-BsZVgBhW.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-kh2jOjTK.d.ts → client-Bc8LANCG.d.ts} +4 -4
- package/dist/{client-kh2jOjTK.d.ts.map → client-Bc8LANCG.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +14 -14
- package/dist/{default-registry-C0-N4PkG.js → default-registry-CpLko4jB.js} +5 -4
- package/dist/{default-registry-C0-N4PkG.js.map → default-registry-CpLko4jB.js.map} +1 -1
- package/dist/{default-registry-BwDSWVzg.d.ts → default-registry-DYB4aHVn.d.ts} +6 -6
- package/dist/{default-registry-BwDSWVzg.d.ts.map → default-registry-DYB4aHVn.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CwOWWQt_.d.ts → define-agent-eval-DE67ECH1.d.ts} +7 -7
- package/dist/{define-agent-eval-CwOWWQt_.d.ts.map → define-agent-eval-DE67ECH1.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CneiWyuA.js → define-agent-eval-RTekxhUE.js} +8 -8
- package/dist/{define-agent-eval-CneiWyuA.js.map → define-agent-eval-RTekxhUE.js.map} +1 -1
- package/dist/{dspy-rlm-engine-8pgBSiy7.js → dspy-rlm-engine-Da-l3CpD.js} +3 -3
- package/dist/{dspy-rlm-engine-8pgBSiy7.js.map → dspy-rlm-engine-Da-l3CpD.js.map} +1 -1
- package/dist/{engine-DS1cysJy.d.ts → engine-D1ykrAKR.d.ts} +7 -7
- package/dist/{engine-DS1cysJy.d.ts.map → engine-D1ykrAKR.d.ts.map} +1 -1
- package/dist/{eval-campaign-CCEkwzoH.js → eval-campaign-XFHEpN86.js} +4 -4
- package/dist/{eval-campaign-CCEkwzoH.js.map → eval-campaign-XFHEpN86.js.map} +1 -1
- package/dist/evaluation.d.ts +75 -0
- package/dist/evaluation.d.ts.map +1 -0
- package/dist/evaluation.js +137 -0
- package/dist/evaluation.js.map +1 -0
- package/dist/{exact-types-BZDe0W2D.d.ts → exact-types-BEHN7tnq.d.ts} +2 -2
- package/dist/{exact-types-BZDe0W2D.d.ts.map → exact-types-BEHN7tnq.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/experiment/index.js +5 -5
- package/dist/{experiment-tracker-C7PfnF4b.d.ts → experiment-tracker-URRGP9ek.d.ts} +2 -2
- package/dist/{experiment-tracker-C7PfnF4b.d.ts.map → experiment-tracker-URRGP9ek.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-DIB4YWEe.js → external-optimizer-process-BbhvXTFx.js} +4 -4
- package/dist/{external-optimizer-process-DIB4YWEe.js.map → external-optimizer-process-BbhvXTFx.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-CjguXN2e.js → external-optimizer-subprocess-DMgmqTB6.js} +3 -3
- package/dist/{external-optimizer-subprocess-CjguXN2e.js.map → external-optimizer-subprocess-DMgmqTB6.js.map} +1 -1
- package/dist/{feedback-trajectory-CXmtITBo.d.ts → feedback-trajectory-CytTR7u4.d.ts} +3 -3
- package/dist/{feedback-trajectory-CXmtITBo.d.ts.map → feedback-trajectory-CytTR7u4.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/{index-DoykkxW0.d.ts → index-58wex7KV.d.ts} +11 -11
- package/dist/{index-DoykkxW0.d.ts.map → index-58wex7KV.d.ts.map} +1 -1
- package/dist/{index--_8_Nlpn.d.ts → index-CKZa_rnt.d.ts} +4 -121
- package/dist/index-CKZa_rnt.d.ts.map +1 -0
- package/dist/{index-DbG8QbH9.d.ts → index-DVrtLO5j.d.ts} +6 -6
- package/dist/{index-DbG8QbH9.d.ts.map → index-DVrtLO5j.d.ts.map} +1 -1
- package/dist/{index-Bp_6sj3x.d.ts → index-ww8736Am.d.ts} +20 -315
- package/dist/index-ww8736Am.d.ts.map +1 -0
- package/dist/index.d.ts +23 -22
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +24 -24
- package/dist/{insight-report-D1qa0HWs.d.ts → insight-report-C3Bfl762.d.ts} +3 -3
- package/dist/{insight-report-D1qa0HWs.d.ts.map → insight-report-C3Bfl762.d.ts.map} +1 -1
- package/dist/jev-protocol.d.ts +74 -0
- package/dist/jev-protocol.d.ts.map +1 -0
- package/dist/jev-protocol.js +91 -0
- package/dist/jev-protocol.js.map +1 -0
- package/dist/jev.d.ts +131 -0
- package/dist/jev.d.ts.map +1 -0
- package/dist/jev.js +312 -0
- package/dist/jev.js.map +1 -0
- package/dist/{journal-Cs9f7385.js → journal-BuRLm6ho.js} +2 -2
- package/dist/{journal-Cs9f7385.js.map → journal-BuRLm6ho.js.map} +1 -1
- package/dist/{kind-factory-DxTzg32u.js → kind-factory-DE088Zhv.js} +4 -3
- package/dist/{kind-factory-DxTzg32u.js.map → kind-factory-DE088Zhv.js.map} +1 -1
- package/dist/ledger-core/index.js +2 -2
- package/dist/{llm-judge-FfaRgz9X.js → llm-judge-C5wbADA5.js} +9 -9
- package/dist/{llm-judge-FfaRgz9X.js.map → llm-judge-C5wbADA5.js.map} +1 -1
- package/dist/{matrix-CyhW-vgJ.d.ts → matrix-CPKsRY8Y.d.ts} +2 -2
- package/dist/{matrix-CyhW-vgJ.d.ts.map → matrix-CPKsRY8Y.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +54 -2
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +52 -3
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{mint-ySIIkKlV.js → mint-B4zgSAJW.js} +63 -14
- package/dist/mint-B4zgSAJW.js.map +1 -0
- package/dist/mint-BI14xehJ.d.ts +285 -0
- package/dist/mint-BI14xehJ.d.ts.map +1 -0
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/{produced-state-CGoVInrl.js → produced-state-D3uWsY8t.js} +23 -19
- package/dist/{produced-state-CGoVInrl.js.map → produced-state-D3uWsY8t.js.map} +1 -1
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/{promotion-policy-BKFrl7CC.js → promotion-policy-CD-bUeyR.js} +3 -3
- package/dist/{promotion-policy-BKFrl7CC.js.map → promotion-policy-CD-bUeyR.js.map} +1 -1
- package/dist/{query-D1nLIKt7.js → query-DlV0pdx4.js} +2 -2
- package/dist/{query-D1nLIKt7.js.map → query-DlV0pdx4.js.map} +1 -1
- package/dist/{registry-BRbB6Y0v.d.ts → registry-D5Ly3GRV.d.ts} +3 -3
- package/dist/{registry-BRbB6Y0v.d.ts.map → registry-D5Ly3GRV.d.ts.map} +1 -1
- package/dist/{release-confidence-BcqeQTHW.d.ts → release-confidence-DsETiO4f.d.ts} +3 -3
- package/dist/{release-confidence-BcqeQTHW.d.ts.map → release-confidence-DsETiO4f.d.ts.map} +1 -1
- package/dist/{release-confidence-DMg8n18l.js → release-confidence-RanRUXRW.js} +2 -2
- package/dist/{release-confidence-DMg8n18l.js.map → release-confidence-RanRUXRW.js.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +2 -2
- package/dist/{researcher-64T49THL.d.ts → researcher-CJkH-6Ty.d.ts} +4 -4
- package/dist/{researcher-64T49THL.d.ts.map → researcher-CJkH-6Ty.d.ts.map} +1 -1
- package/dist/{reward-hacking-uzO_ihep.d.ts → reward-hacking-BIQ2BWLG.d.ts} +2 -2
- package/dist/{reward-hacking-uzO_ihep.d.ts.map → reward-hacking-BIQ2BWLG.d.ts.map} +1 -1
- package/dist/rl.d.ts +10 -6
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +21 -7
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-D_54FZVu.js → rollout-C0Ifk3vK.js} +2 -2
- package/dist/{rollout-D_54FZVu.js.map → rollout-C0Ifk3vK.js.map} +1 -1
- package/dist/{rubric-predictive-validity-Bmj2_cll.d.ts → rubric-predictive-validity-CTU8y8Cv.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-Bmj2_cll.d.ts.map → rubric-predictive-validity-CTU8y8Cv.d.ts.map} +1 -1
- package/dist/{run-record-DPa2PeHx.js → run-record-BhMVsHR1.js} +2 -2
- package/dist/{run-record-DPa2PeHx.js.map → run-record-BhMVsHR1.js.map} +1 -1
- package/dist/{run-record-BiTWauyO.d.ts → run-record-D9wS35fK.d.ts} +2 -2
- package/dist/{run-record-BiTWauyO.d.ts.map → run-record-D9wS35fK.d.ts.map} +1 -1
- package/dist/{run-record-DualPTn2.js → run-record-tg_5H2IK.js} +2 -2
- package/dist/{run-record-DualPTn2.js.map → run-record-tg_5H2IK.js.map} +1 -1
- package/dist/search-ledger-types-D7KDVg-o.d.ts +302 -0
- package/dist/search-ledger-types-D7KDVg-o.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-_3Fp6Q_3.js → semantic-concept-judge-5r9hDEBA.js} +4 -4
- package/dist/{semantic-concept-judge-_3Fp6Q_3.js.map → semantic-concept-judge-5r9hDEBA.js.map} +1 -1
- package/dist/{sequential-D_xHrBZe.js → sequential-BfDkCBGo.js} +3 -3
- package/dist/{sequential-D_xHrBZe.js.map → sequential-BfDkCBGo.js.map} +1 -1
- package/dist/{server-BZQR1f0C.js → server-DmpcFiVK.js} +2 -2
- package/dist/{server-BZQR1f0C.js.map → server-DmpcFiVK.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DY1aQ-aL.js → skillopt-optimization-method-B2zxoDk0.js} +7 -7
- package/dist/{skillopt-optimization-method-DY1aQ-aL.js.map → skillopt-optimization-method-B2zxoDk0.js.map} +1 -1
- package/dist/{statistical-heldout-CpVd6FmY.d.ts → statistical-heldout-Bu4oNOJh.d.ts} +3 -3
- package/dist/{statistical-heldout-CpVd6FmY.d.ts.map → statistical-heldout-Bu4oNOJh.d.ts.map} +1 -1
- package/dist/{store-otlp-DAaEbMLf.js → store-otlp-rU5PFlOB.js} +2 -2
- package/dist/{store-otlp-DAaEbMLf.js.map → store-otlp-rU5PFlOB.js.map} +1 -1
- package/dist/{store-tool-spans-Dt-YdAuE.d.ts → store-tool-spans-DwdokbHg.d.ts} +54 -6
- package/dist/store-tool-spans-DwdokbHg.d.ts.map +1 -0
- package/dist/{store-tool-spans-BZ8MPKFe.js → store-tool-spans-zrtftcRe.js} +68 -6
- package/dist/store-tool-spans-zrtftcRe.js.map +1 -0
- package/dist/{summary-report-IGAOtgPd.js → summary-report-BMvou2oZ.js} +2 -2
- package/dist/{summary-report-IGAOtgPd.js.map → summary-report-BMvou2oZ.js.map} +1 -1
- package/dist/{summary-report-D1h4dlrK.d.ts → summary-report-BzFS5UhZ.d.ts} +2 -2
- package/dist/{summary-report-D1h4dlrK.d.ts.map → summary-report-BzFS5UhZ.d.ts.map} +1 -1
- package/dist/{task-failure-attributes-B_JjrOVo.js → task-failure-attributes-BOxjfbhr.js} +2 -2
- package/dist/{task-failure-attributes-B_JjrOVo.js.map → task-failure-attributes-BOxjfbhr.js.map} +1 -1
- package/dist/{tool-groups-B2bSNaJB.d.ts → tool-groups-qHMydbAH.d.ts} +3 -3
- package/dist/tool-groups-qHMydbAH.d.ts.map +1 -0
- package/dist/{tool-waste-C7MU9u1e.js → tool-waste-nbHPAxei.js} +2 -2
- package/dist/{tool-waste-C7MU9u1e.js.map → tool-waste-nbHPAxei.js.map} +1 -1
- package/dist/trace-repair/index.d.ts +2 -2
- package/dist/trace-repair/index.js +3 -3
- package/dist/traces.d.ts +5 -5
- package/dist/traces.js +7 -7
- package/dist/trajectory-replay/index.js +1 -1
- package/dist/{types-CS0qk_Yp.d.ts → types-BTRcIqFr.d.ts} +3 -3
- package/dist/{types-CS0qk_Yp.d.ts.map → types-BTRcIqFr.d.ts.map} +1 -1
- package/dist/{types-D7gEdPoQ.d.ts → types-BjB2wZM9.d.ts} +2 -2
- package/dist/{types-D7gEdPoQ.d.ts.map → types-BjB2wZM9.d.ts.map} +1 -1
- package/dist/types-C8VBbiw6.js +91 -0
- package/dist/types-C8VBbiw6.js.map +1 -0
- package/dist/{types-BvZoPTGa.d.ts → types-c3ZRTbw9.d.ts} +3 -3
- package/dist/{types-BvZoPTGa.d.ts.map → types-c3ZRTbw9.d.ts.map} +1 -1
- package/dist/{types-DQ0e2E7y.js → usage-receipt-Cx3xNeyB.js} +2 -89
- package/dist/usage-receipt-Cx3xNeyB.js.map +1 -0
- package/dist/{verdict-BQ3pCFf8.js → verdict-DhPoXApk.js} +2 -2
- package/dist/{verdict-BQ3pCFf8.js.map → verdict-DhPoXApk.js.map} +1 -1
- package/dist/{verdict-cache-B3eCVQtY.js → verdict-cache-BQEsVy9n.js} +2 -2
- package/dist/{verdict-cache-B3eCVQtY.js.map → verdict-cache-BQEsVy9n.js.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/changes/evaluation-definition-identity.md +27 -0
- package/docs/changes/jev-evidence-review.md +18 -0
- package/docs/changes/jev-product-adoption.md +18 -0
- package/docs/changes/probability-policy-audit.md +81 -0
- package/docs/jev-behavior-review.md +174 -0
- package/docs/jev-integrations.md +50 -0
- package/docs/jev.md +140 -0
- package/docs/produced-state.md +13 -0
- package/docs/profile-json-identity.md +33 -0
- package/package.json +16 -1
- package/dist/agent-profile-cell-Cv6UA-W_.js.map +0 -1
- package/dist/canonical-DPyQ_rpt.js.map +0 -1
- package/dist/exporters-D98GPi1n.d.ts +0 -161
- package/dist/exporters-D98GPi1n.d.ts.map +0 -1
- package/dist/index--_8_Nlpn.d.ts.map +0 -1
- package/dist/index-Bp_6sj3x.d.ts.map +0 -1
- package/dist/mint-ySIIkKlV.js.map +0 -1
- package/dist/store-tool-spans-BZ8MPKFe.js.map +0 -1
- package/dist/store-tool-spans-Dt-YdAuE.d.ts.map +0 -1
- package/dist/tool-groups-B2bSNaJB.d.ts.map +0 -1
- package/dist/types-DQ0e2E7y.js.map +0 -1
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { i as CostLedger } from "./cost-ledger-_IbkejCF.js";
|
|
2
|
+
//#region src/evaluation.ts
|
|
3
|
+
/** A metered function, independent of providers, question formats, and application policy. */
|
|
4
|
+
function createEvaluator(options) {
|
|
5
|
+
const defaults = { ...options };
|
|
6
|
+
const ledger = defaults.costLedger ?? new CostLedger();
|
|
7
|
+
return async (input, context = {}) => {
|
|
8
|
+
context.signal?.throwIfAborted();
|
|
9
|
+
const started = performance.now();
|
|
10
|
+
const paid = await (context.costLedger ?? ledger).runPaidCall({
|
|
11
|
+
callId: context.callId,
|
|
12
|
+
actor: context.actor ?? "evaluation",
|
|
13
|
+
channel: context.channel ?? "evaluation",
|
|
14
|
+
phase: context.costPhase ?? "evaluation",
|
|
15
|
+
tags: context.costTags,
|
|
16
|
+
model: typeof defaults.model === "function" ? defaults.model(input) : defaults.model,
|
|
17
|
+
signal: context.signal,
|
|
18
|
+
maximumCharge: context.maximumCharge ?? defaults.maximumCharge,
|
|
19
|
+
execute: (signal, idempotencyKey) => defaults.execute(input, {
|
|
20
|
+
signal,
|
|
21
|
+
idempotencyKey
|
|
22
|
+
}),
|
|
23
|
+
receipt: defaults.receipt,
|
|
24
|
+
receiptFromError: defaults.receiptFromError
|
|
25
|
+
});
|
|
26
|
+
if (paid.receipt) await context.onReceipt?.(paid.receipt);
|
|
27
|
+
if (!paid.succeeded) throw paid.error;
|
|
28
|
+
context.signal?.throwIfAborted();
|
|
29
|
+
await defaults.validate?.(paid.value, input);
|
|
30
|
+
context.signal?.throwIfAborted();
|
|
31
|
+
return {
|
|
32
|
+
value: paid.value,
|
|
33
|
+
receipt: paid.receipt,
|
|
34
|
+
durationMs: performance.now() - started
|
|
35
|
+
};
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/** Return the existing contract. Mapping never performs another inference call. */
|
|
39
|
+
function asJudge(options) {
|
|
40
|
+
const config = {
|
|
41
|
+
...options,
|
|
42
|
+
dimensions: structuredClone(options.dimensions)
|
|
43
|
+
};
|
|
44
|
+
return {
|
|
45
|
+
name: config.name,
|
|
46
|
+
judgeVersion: config.version,
|
|
47
|
+
dimensions: config.dimensions,
|
|
48
|
+
appliesTo: config.appliesTo,
|
|
49
|
+
async score({ artifact, scenario, signal, costLedger, costPhase, costTags }) {
|
|
50
|
+
signal.throwIfAborted();
|
|
51
|
+
const input = {
|
|
52
|
+
artifact,
|
|
53
|
+
scenario
|
|
54
|
+
};
|
|
55
|
+
const result = await config.evaluate(input, {
|
|
56
|
+
signal,
|
|
57
|
+
costLedger,
|
|
58
|
+
actor: config.name,
|
|
59
|
+
channel: "judge",
|
|
60
|
+
costPhase,
|
|
61
|
+
costTags: {
|
|
62
|
+
...costTags,
|
|
63
|
+
scenarioId: scenario.id
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
await config.record?.(result, input);
|
|
67
|
+
signal.throwIfAborted();
|
|
68
|
+
const score = await config.map(result.value, input);
|
|
69
|
+
signal.throwIfAborted();
|
|
70
|
+
return score;
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/** Registry, graph, and trace consumers keep the same Analyst interface. */
|
|
75
|
+
function asAnalyst(options) {
|
|
76
|
+
const config = {
|
|
77
|
+
...options,
|
|
78
|
+
cost: structuredClone(options.cost)
|
|
79
|
+
};
|
|
80
|
+
return {
|
|
81
|
+
id: config.id,
|
|
82
|
+
version: config.version,
|
|
83
|
+
description: config.description,
|
|
84
|
+
inputKind: config.inputKind,
|
|
85
|
+
cost: config.cost,
|
|
86
|
+
async analyze(input, context) {
|
|
87
|
+
const signals = context.signal ? [context.signal] : [];
|
|
88
|
+
if (context.deadlineMs !== void 0) {
|
|
89
|
+
const remaining = context.deadlineMs - Date.now();
|
|
90
|
+
if (!Number.isFinite(remaining)) throw new TypeError("Analyst deadline must be finite");
|
|
91
|
+
if (remaining <= 0) throw new DOMException("Analyst deadline exceeded", "TimeoutError");
|
|
92
|
+
signals.push(AbortSignal.timeout(Math.ceil(remaining)));
|
|
93
|
+
}
|
|
94
|
+
const signal = AbortSignal.any(signals);
|
|
95
|
+
signal.throwIfAborted();
|
|
96
|
+
const analystContext = {
|
|
97
|
+
...context,
|
|
98
|
+
signal
|
|
99
|
+
};
|
|
100
|
+
const result = await config.evaluate(input, {
|
|
101
|
+
signal,
|
|
102
|
+
costLedger: context.costLedger ?? (context.budgetUsd === void 0 ? void 0 : new CostLedger({ costCeilingUsd: context.budgetUsd })),
|
|
103
|
+
actor: config.id,
|
|
104
|
+
channel: "analyst",
|
|
105
|
+
costPhase: context.costPhase,
|
|
106
|
+
costTags: {
|
|
107
|
+
...context.tags,
|
|
108
|
+
runId: context.runId,
|
|
109
|
+
correlationId: context.correlationId
|
|
110
|
+
},
|
|
111
|
+
onReceipt: (receipt) => context.recordUsage?.({
|
|
112
|
+
calls: 1,
|
|
113
|
+
tokens: receipt.usageUnknown ? null : {
|
|
114
|
+
input: receipt.inputTokens,
|
|
115
|
+
output: receipt.outputTokens
|
|
116
|
+
},
|
|
117
|
+
cost: receipt.costUnknown ? {
|
|
118
|
+
kind: "uncaptured",
|
|
119
|
+
usd: null
|
|
120
|
+
} : {
|
|
121
|
+
kind: receipt.actualCostUsd === void 0 ? "estimated" : "observed",
|
|
122
|
+
usd: receipt.costUsd
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
}, analystContext);
|
|
126
|
+
await config.record?.(result, input, analystContext);
|
|
127
|
+
signal.throwIfAborted();
|
|
128
|
+
const findings = await config.map(result.value, input, analystContext);
|
|
129
|
+
signal.throwIfAborted();
|
|
130
|
+
return findings;
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
//#endregion
|
|
135
|
+
export { asAnalyst, asJudge, createEvaluator };
|
|
136
|
+
|
|
137
|
+
//# sourceMappingURL=evaluation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluation.js","names":[],"sources":["../src/evaluation.ts"],"sourcesContent":["import type { Analyst, AnalystContext, AnalystFinding } from './analyst/types'\nimport type { JudgeConfig, JudgeScore, Scenario } from './campaign/types'\nimport type {\n CostChannel,\n CostLedgerHandle,\n CostReceipt,\n CostReceiptInput,\n MaximumCharge,\n} from './cost-ledger'\nimport { CostLedger } from './cost-ledger'\n\nexport interface EvaluationResult<T> {\n value: T\n receipt: CostReceipt\n durationMs: number\n}\n\nexport interface EvaluationContext {\n /** Physical paid-call identity, forwarded through the existing ledger and transport.\n * Reusing it in one ledger conflicts; it is not an automatic result-replay policy. */\n callId?: string\n signal?: AbortSignal\n costLedger?: CostLedgerHandle\n actor?: string\n channel?: CostChannel\n costPhase?: string\n costTags?: Record<string, string>\n maximumCharge?: MaximumCharge\n /** Awaited after accounting and before validation or result delivery. */\n onReceipt?: (receipt: CostReceipt) => void | Promise<void>\n}\n\nexport type Evaluator<I, O> = (\n input: I,\n context?: EvaluationContext,\n) => Promise<EvaluationResult<O>>\n\nexport interface EvaluatorOptions<I, O> {\n execute: (input: I, context: { signal: AbortSignal; idempotencyKey: string }) => Promise<O>\n receipt: (value: O) => CostReceiptInput\n receiptFromError?: (error: Error) => CostReceiptInput | undefined\n model?: string | ((input: I) => string)\n costLedger?: CostLedgerHandle\n maximumCharge?: MaximumCharge\n /** Awaited after accounting, so invalid output never erases paid work. */\n validate?: (value: O, input: I) => void | Promise<void>\n}\n\n/** A metered function, independent of providers, question formats, and application policy. */\nexport function createEvaluator<I, O>(options: EvaluatorOptions<I, O>): Evaluator<I, O> {\n const defaults = { ...options }\n const ledger = defaults.costLedger ?? new CostLedger()\n return async (input, context = {}) => {\n context.signal?.throwIfAborted()\n const started = performance.now()\n const paid = await (context.costLedger ?? ledger).runPaidCall({\n callId: context.callId,\n actor: context.actor ?? 'evaluation',\n channel: context.channel ?? 'evaluation',\n phase: context.costPhase ?? 'evaluation',\n tags: context.costTags,\n model: typeof defaults.model === 'function' ? defaults.model(input) : defaults.model,\n signal: context.signal,\n maximumCharge: context.maximumCharge ?? defaults.maximumCharge,\n execute: (signal, idempotencyKey) => defaults.execute(input, { signal, idempotencyKey }),\n receipt: defaults.receipt,\n receiptFromError: defaults.receiptFromError,\n })\n if (paid.receipt) await context.onReceipt?.(paid.receipt)\n if (!paid.succeeded) throw paid.error\n // A transport can complete after cancellation. Keep its paid receipt, not a live decision.\n context.signal?.throwIfAborted()\n await defaults.validate?.(paid.value, input)\n context.signal?.throwIfAborted()\n return { value: paid.value, receipt: paid.receipt, durationMs: performance.now() - started }\n }\n}\n\nexport interface EvaluationJudgeOptions<A, S extends Scenario, O> {\n name: string\n version: string\n dimensions: JudgeConfig<A, S>['dimensions']\n evaluate: Evaluator<{ artifact: A; scenario: S }, O>\n map: (value: O, input: { artifact: A; scenario: S }) => JudgeScore | Promise<JudgeScore>\n /** Persist the complete observation before reducing it to a verdict. */\n record?: (\n result: EvaluationResult<O>,\n input: { artifact: A; scenario: S },\n ) => void | Promise<void>\n appliesTo?: (scenario: S) => boolean\n}\n\n/** Return the existing contract. Mapping never performs another inference call. */\nexport function asJudge<A, S extends Scenario, O>(\n options: EvaluationJudgeOptions<A, S, O>,\n): JudgeConfig<A, S> {\n // The version describes these callbacks, not later mutations of the caller's options.\n const config = { ...options, dimensions: structuredClone(options.dimensions) }\n return {\n name: config.name,\n judgeVersion: config.version,\n dimensions: config.dimensions,\n appliesTo: config.appliesTo,\n async score({ artifact, scenario, signal, costLedger, costPhase, costTags }) {\n signal.throwIfAborted()\n const input = { artifact, scenario }\n const result = await config.evaluate(input, {\n signal,\n costLedger,\n actor: config.name,\n channel: 'judge',\n costPhase,\n costTags: { ...costTags, scenarioId: scenario.id },\n })\n await config.record?.(result, input)\n signal.throwIfAborted()\n const score = await config.map(result.value, input)\n signal.throwIfAborted()\n return score\n },\n }\n}\n\nexport interface EvaluationAnalystOptions<I, O> {\n id: string\n version: string\n description: string\n inputKind: Analyst<I>['inputKind']\n cost: Analyst<I>['cost']\n evaluate: (\n input: I,\n context: EvaluationContext,\n analystContext: AnalystContext,\n ) => Promise<EvaluationResult<O>>\n map: (value: O, input: I, context: AnalystContext) => AnalystFinding[] | Promise<AnalystFinding[]>\n record?: (result: EvaluationResult<O>, input: I, context: AnalystContext) => void | Promise<void>\n}\n\n/** Registry, graph, and trace consumers keep the same Analyst interface. */\nexport function asAnalyst<I, O>(options: EvaluationAnalystOptions<I, O>): Analyst<I> {\n const config = { ...options, cost: structuredClone(options.cost) }\n return {\n id: config.id,\n version: config.version,\n description: config.description,\n inputKind: config.inputKind,\n cost: config.cost,\n async analyze(input, context) {\n const signals = context.signal ? [context.signal] : []\n if (context.deadlineMs !== undefined) {\n const remaining = context.deadlineMs - Date.now()\n if (!Number.isFinite(remaining)) throw new TypeError('Analyst deadline must be finite')\n if (remaining <= 0) throw new DOMException('Analyst deadline exceeded', 'TimeoutError')\n signals.push(AbortSignal.timeout(Math.ceil(remaining)))\n }\n const signal = AbortSignal.any(signals)\n signal.throwIfAborted()\n const analystContext = { ...context, signal }\n const result = await config.evaluate(\n input,\n {\n signal,\n // No implicit account may override the evaluator's configured spending authority.\n costLedger:\n context.costLedger ??\n (context.budgetUsd === undefined\n ? undefined\n : new CostLedger({ costCeilingUsd: context.budgetUsd })),\n actor: config.id,\n channel: 'analyst',\n costPhase: context.costPhase,\n costTags: { ...context.tags, runId: context.runId, correlationId: context.correlationId },\n onReceipt: (receipt) =>\n context.recordUsage?.({\n calls: 1,\n tokens: receipt.usageUnknown\n ? null\n : { input: receipt.inputTokens, output: receipt.outputTokens },\n cost: receipt.costUnknown\n ? { kind: 'uncaptured', usd: null }\n : {\n kind: receipt.actualCostUsd === undefined ? 'estimated' : 'observed',\n usd: receipt.costUsd,\n },\n }),\n },\n analystContext,\n )\n await config.record?.(result, input, analystContext)\n signal.throwIfAborted()\n const findings = await config.map(result.value, input, analystContext)\n signal.throwIfAborted()\n return findings\n },\n }\n}\n"],"mappings":";;;AAiDA,SAAgB,gBAAsB,SAAkD;CACtF,MAAM,WAAW,EAAE,GAAG,QAAQ;CAC9B,MAAM,SAAS,SAAS,cAAc,IAAI,WAAW;CACrD,OAAO,OAAO,OAAO,UAAU,CAAC,MAAM;EACpC,QAAQ,QAAQ,eAAe;EAC/B,MAAM,UAAU,YAAY,IAAI;EAChC,MAAM,OAAO,OAAO,QAAQ,cAAc,OAAA,CAAQ,YAAY;GAC5D,QAAQ,QAAQ;GAChB,OAAO,QAAQ,SAAS;GACxB,SAAS,QAAQ,WAAW;GAC5B,OAAO,QAAQ,aAAa;GAC5B,MAAM,QAAQ;GACd,OAAO,OAAO,SAAS,UAAU,aAAa,SAAS,MAAM,KAAK,IAAI,SAAS;GAC/E,QAAQ,QAAQ;GAChB,eAAe,QAAQ,iBAAiB,SAAS;GACjD,UAAU,QAAQ,mBAAmB,SAAS,QAAQ,OAAO;IAAE;IAAQ;GAAe,CAAC;GACvF,SAAS,SAAS;GAClB,kBAAkB,SAAS;EAC7B,CAAC;EACD,IAAI,KAAK,SAAS,MAAM,QAAQ,YAAY,KAAK,OAAO;EACxD,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK;EAEhC,QAAQ,QAAQ,eAAe;EAC/B,MAAM,SAAS,WAAW,KAAK,OAAO,KAAK;EAC3C,QAAQ,QAAQ,eAAe;EAC/B,OAAO;GAAE,OAAO,KAAK;GAAO,SAAS,KAAK;GAAS,YAAY,YAAY,IAAI,IAAI;EAAQ;CAC7F;AACF;;AAiBA,SAAgB,QACd,SACmB;CAEnB,MAAM,SAAS;EAAE,GAAG;EAAS,YAAY,gBAAgB,QAAQ,UAAU;CAAE;CAC7E,OAAO;EACL,MAAM,OAAO;EACb,cAAc,OAAO;EACrB,YAAY,OAAO;EACnB,WAAW,OAAO;EAClB,MAAM,MAAM,EAAE,UAAU,UAAU,QAAQ,YAAY,WAAW,YAAY;GAC3E,OAAO,eAAe;GACtB,MAAM,QAAQ;IAAE;IAAU;GAAS;GACnC,MAAM,SAAS,MAAM,OAAO,SAAS,OAAO;IAC1C;IACA;IACA,OAAO,OAAO;IACd,SAAS;IACT;IACA,UAAU;KAAE,GAAG;KAAU,YAAY,SAAS;IAAG;GACnD,CAAC;GACD,MAAM,OAAO,SAAS,QAAQ,KAAK;GACnC,OAAO,eAAe;GACtB,MAAM,QAAQ,MAAM,OAAO,IAAI,OAAO,OAAO,KAAK;GAClD,OAAO,eAAe;GACtB,OAAO;EACT;CACF;AACF;;AAkBA,SAAgB,UAAgB,SAAqD;CACnF,MAAM,SAAS;EAAE,GAAG;EAAS,MAAM,gBAAgB,QAAQ,IAAI;CAAE;CACjE,OAAO;EACL,IAAI,OAAO;EACX,SAAS,OAAO;EAChB,aAAa,OAAO;EACpB,WAAW,OAAO;EAClB,MAAM,OAAO;EACb,MAAM,QAAQ,OAAO,SAAS;GAC5B,MAAM,UAAU,QAAQ,SAAS,CAAC,QAAQ,MAAM,IAAI,CAAC;GACrD,IAAI,QAAQ,eAAe,KAAA,GAAW;IACpC,MAAM,YAAY,QAAQ,aAAa,KAAK,IAAI;IAChD,IAAI,CAAC,OAAO,SAAS,SAAS,GAAG,MAAM,IAAI,UAAU,iCAAiC;IACtF,IAAI,aAAa,GAAG,MAAM,IAAI,aAAa,6BAA6B,cAAc;IACtF,QAAQ,KAAK,YAAY,QAAQ,KAAK,KAAK,SAAS,CAAC,CAAC;GACxD;GACA,MAAM,SAAS,YAAY,IAAI,OAAO;GACtC,OAAO,eAAe;GACtB,MAAM,iBAAiB;IAAE,GAAG;IAAS;GAAO;GAC5C,MAAM,SAAS,MAAM,OAAO,SAC1B,OACA;IACE;IAEA,YACE,QAAQ,eACP,QAAQ,cAAc,KAAA,IACnB,KAAA,IACA,IAAI,WAAW,EAAE,gBAAgB,QAAQ,UAAU,CAAC;IAC1D,OAAO,OAAO;IACd,SAAS;IACT,WAAW,QAAQ;IACnB,UAAU;KAAE,GAAG,QAAQ;KAAM,OAAO,QAAQ;KAAO,eAAe,QAAQ;IAAc;IACxF,YAAY,YACV,QAAQ,cAAc;KACpB,OAAO;KACP,QAAQ,QAAQ,eACZ,OACA;MAAE,OAAO,QAAQ;MAAa,QAAQ,QAAQ;KAAa;KAC/D,MAAM,QAAQ,cACV;MAAE,MAAM;MAAc,KAAK;KAAK,IAChC;MACE,MAAM,QAAQ,kBAAkB,KAAA,IAAY,cAAc;MAC1D,KAAK,QAAQ;KACf;IACN,CAAC;GACL,GACA,cACF;GACA,MAAM,OAAO,SAAS,QAAQ,OAAO,cAAc;GACnD,OAAO,eAAe;GACtB,MAAM,WAAW,MAAM,OAAO,IAAI,OAAO,OAAO,OAAO,cAAc;GACrE,OAAO,eAAe;GACtB,OAAO;EACT;CACF;AACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { l as AnalystRunResult, s as AnalystRunEvent, t as Analyst, u as AnalystRunSummary } from "./types-
|
|
1
|
+
import { l as AnalystRunResult, s as AnalystRunEvent, t as Analyst, u as AnalystRunSummary } from "./types-BjB2wZM9.js";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
//#region src/analyst/exact-types.d.ts
|
|
4
4
|
/** Analyst metadata required before the exact registry path will execute it. */
|
|
@@ -231,4 +231,4 @@ declare const exactExecutionPlanSchema: z.ZodObject<{
|
|
|
231
231
|
}, z.core.$strict>;
|
|
232
232
|
//#endregion
|
|
233
233
|
export { ExactAnalystRunPolicySnapshot as a, ExactAnalystSnapshot as c, ExactExecutionComponentSnapshot as d, ExactAnalystRunEvent as i, ExactCapableAnalyst as l, ExactAnalystExecutionPlanSnapshot as n, ExactAnalystRunResult as o, ExactAnalystRunCompletion as r, ExactAnalystRunSummary as s, ExactAnalystBudgetSnapshot as t, ExactExecutionComponentIdentity as u };
|
|
234
|
-
//# sourceMappingURL=exact-types-
|
|
234
|
+
//# sourceMappingURL=exact-types-BEHN7tnq.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exact-types-
|
|
1
|
+
{"version":3,"file":"exact-types-BEHN7tnq.d.ts","names":[],"sources":["../src/analyst/exact-types.ts"],"mappings":";;;;UAMiB,oBAAoB,0BAA0B,QAAQ;;WAE5D,iBAAiB,SAAS;;KAGhC,aAAa,KAAK,0BAA0B,mBACpC,aAAa,UACtB,+BACc,aAAa,IAAI,aAAa,EAAE,WAC5C;KAEM,uBAAuB,aAAa,EAAE,aAAa;KACnD,6BAA6B,aAAa,EAAE,aAAa;;UAGpD;EACf;EACA;EACA,QAAQ,SAAS;;;UAIF;EACf;EACA;EACA;;KAyBU,gCAAgC,aAAa,EAAE,aAAa;KAC5D,oCAAoC,aAC9C,EAAE,aAAa;KAGL,yBAAyB;;EAEnC;;KAGU;EACN;;EAEA;EACA;IAAS;IAAe;;;UAGb,8BAA8B;EAC7C,aAAa;EACb,gBAAgB;EAChB,YAAY;;;KAIF,wBACP,QAAQ;EAAmB;;EAC1B,gBAAgB;MAEjB,KAAK,QAAQ;EAAmB;;EAC/B,SAAS;KAEX,QAAQ;EAAmB;MAC1B,KAAK,QAAQ;EAAmB;;EAC/B,SAAS;MAEV,KAAK,QAAQ;EAAmB;;EAC/B,QAAQ;;cAwCR,uBAAqB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAOzB,EAAA,KAAA;cAII,sBAAoB,EAAA,uBAAA,EAAA;;;;;;;;;;;GAaxB,EAAA,KAAA;cAiBI,sBAAoB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAYxB,EAAA,KAAA;cAEI,0BAAwB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+E1B,EAAA,KAAA"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { c as ValidationError, r as CaptureIntegrityError } from "../errors-DEE6u6ot.js";
|
|
2
2
|
import { n as bonferroni, r as holm, t as benjaminiHochberg } from "../multiplicity-DIWHvysC.js";
|
|
3
|
-
import { $ as paretoSignificanceGate, $t as wilson, A as PairArmsOptions, At as FinalEvidenceReservation, B as PowerPreflight, Bt as eProcess, C as hashJson, Dt as FinalEvidenceMeasurement, E as verifyManifest, Et as FinalEvidenceLedger, Ft as summarizeEvaluationUnits, H as powerPreflight, It as EProcess, K as EvidenceVector, L as comparePairedArms, Lt as EProcessOptions, Mt as EvaluationClaim, N as PairedArmRow, Nt as EvaluationUnitSummary, O as MatchedPair, Ot as FinalEvidenceOutcome, Pt as defineEvaluationClaim, Q as paretoPolicy, R as pairArms, Rt as EProcessState, S as evaluateHypothesis, T as signManifest, Tt as FinalEvidenceError, Ut as ProportionInterval, V as PowerPreflightOptions, X as PromotionPolicy, Xt as pairedRiskDifferenceExact, Yt as pairedRiskDifference, Z as buildEvidenceVector, Zt as pairedRiskDifferenceScore, _ as sequentialPairedGate, _t as verifyEvidenceReceipt, at as createCampaignEvidenceReceipt, b as SignedManifest, c as pairHoldout, ct as EVIDENCE_RECEIPT_VERSION, d as SequentialDecision, dt as EvidenceBinding, f as SequentialObservation, ft as EvidenceReceipt, g as sequentialDecide, gt as isIndependentEvidence, ht as createEvidenceReceipt, i as PairedHoldout, it as CampaignEvidenceContext, j as PairArmsResult, jt as openFinalEvidenceLedger, kt as FinalEvidenceRecord, lt as EvidenceAuthority, m as SequentialPairedGateOptions, mt as INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, n as HeldoutSignificance, ot as CreateEvidenceReceiptInput, p as SequentialPairedGate, pt as EvidenceReceiptVerification, qt as mcnemar, r as HeldoutSignificanceOptions, s as heldoutSignificance, st as EVIDENCE_AUTHORITY_KINDS, ut as EvidenceAuthorityKind, v as HypothesisManifest, w as manifestContentDigest, wt as FinalEvidenceConflictError, x as SignedManifestAlgo, y as HypothesisResult, z as pairRunRecords, zt as EProcessStep } from "../statistical-heldout-
|
|
3
|
+
import { $ as paretoSignificanceGate, $t as wilson, A as PairArmsOptions, At as FinalEvidenceReservation, B as PowerPreflight, Bt as eProcess, C as hashJson, Dt as FinalEvidenceMeasurement, E as verifyManifest, Et as FinalEvidenceLedger, Ft as summarizeEvaluationUnits, H as powerPreflight, It as EProcess, K as EvidenceVector, L as comparePairedArms, Lt as EProcessOptions, Mt as EvaluationClaim, N as PairedArmRow, Nt as EvaluationUnitSummary, O as MatchedPair, Ot as FinalEvidenceOutcome, Pt as defineEvaluationClaim, Q as paretoPolicy, R as pairArms, Rt as EProcessState, S as evaluateHypothesis, T as signManifest, Tt as FinalEvidenceError, Ut as ProportionInterval, V as PowerPreflightOptions, X as PromotionPolicy, Xt as pairedRiskDifferenceExact, Yt as pairedRiskDifference, Z as buildEvidenceVector, Zt as pairedRiskDifferenceScore, _ as sequentialPairedGate, _t as verifyEvidenceReceipt, at as createCampaignEvidenceReceipt, b as SignedManifest, c as pairHoldout, ct as EVIDENCE_RECEIPT_VERSION, d as SequentialDecision, dt as EvidenceBinding, f as SequentialObservation, ft as EvidenceReceipt, g as sequentialDecide, gt as isIndependentEvidence, ht as createEvidenceReceipt, i as PairedHoldout, it as CampaignEvidenceContext, j as PairArmsResult, jt as openFinalEvidenceLedger, kt as FinalEvidenceRecord, lt as EvidenceAuthority, m as SequentialPairedGateOptions, mt as INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, n as HeldoutSignificance, ot as CreateEvidenceReceiptInput, p as SequentialPairedGate, pt as EvidenceReceiptVerification, qt as mcnemar, r as HeldoutSignificanceOptions, s as heldoutSignificance, st as EVIDENCE_AUTHORITY_KINDS, ut as EvidenceAuthorityKind, v as HypothesisManifest, w as manifestContentDigest, wt as FinalEvidenceConflictError, x as SignedManifestAlgo, y as HypothesisResult, z as pairRunRecords, zt as EProcessStep } from "../statistical-heldout-Bu4oNOJh.js";
|
|
4
4
|
import { c as BOOTSTRAP_GATE_MIN_N, d as PairedBootstrapResult, h as pairedBootstrap, u as PairedBootstrapOptions } from "../paired-promotion-decision-DPsMQm-0.js";
|
|
5
|
-
import { _ as requiredSampleSize, a as ExperimentVerdict, d as inMemoryExperimentStore, f as mulberry32, g as requiredPairedSampleSize, h as pairedMde, i as ExperimentTracker, l as fileExperimentStore, m as mcnemarRequiredN, n as ExperimentRep, p as mcnemarPower, r as ExperimentStats, t as Experiment } from "../experiment-tracker-
|
|
5
|
+
import { _ as requiredSampleSize, a as ExperimentVerdict, d as inMemoryExperimentStore, f as mulberry32, g as requiredPairedSampleSize, h as pairedMde, i as ExperimentTracker, l as fileExperimentStore, m as mcnemarRequiredN, n as ExperimentRep, p as mcnemarPower, r as ExperimentStats, t as Experiment } from "../experiment-tracker-URRGP9ek.js";
|
|
6
6
|
import { a as PairedEvalueStep, d as sequentialCrossingHorizon, i as PairedEvalueSequence, o as SequentialCrossingHorizon, r as PairedEvalueOptions, s as SequentialCrossingHorizonOptions, u as pairedEvalueSequence } from "../sequential-BhsrMupG.js";
|
|
7
7
|
import { A as classifyReissue, B as evaluateProvenanceGate, C as ReissuePolicy, D as UniformPassDecision, E as SetExpr, F as evaluateIdentityGate, G as runUniformPassBudget, H as projectNLadderBudget, I as evaluateOracleDeterminismGate, L as evaluatePopulationReproducibilityGate, M as computeInterval, N as evaluateCondition, O as UniformPassSchedule, P as evaluateHaltRule, R as evaluatePowerFloorGate, S as Predicate, T as SelectionRule, U as readField, V as executeDecisionRule, W as runSelectionRule, _ as IntervalSpec, a as ComputedInterval, b as NLadderProjection, c as DecisionOutcome, d as Estimand, f as EstimandResult, g as HaltRule, h as HaltOutcome, i as BudgetRule, j as computeEstimand, k as ValidityGate, l as DecisionRule, m as GateResult, n as AdmissionRule, o as Condition, p as EvidenceRecord, r as AdmissionStage, s as DecisionBranch, t as AdmissionPartition, u as DerivedQuantities, v as JsonValue, w as ReissueVerdict, x as Obligation, y as MatchedBudgetRule, z as evaluatePredicate } from "../ast-hI-vjW6J.js";
|
|
8
8
|
import { z } from "zod";
|
package/dist/experiment/index.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { n as CaptureIntegrityError, s as ValidationError } from "../errors-Dngq5h35.js";
|
|
2
|
-
import { a as hashCanonical } from "../canonical-
|
|
3
|
-
import { _ as verifyManifest, g as signManifest, h as manifestContentDigest, m as hashJson, p as evaluateHypothesis } from "../agent-profile-cell-
|
|
2
|
+
import { a as hashCanonical } from "../canonical-BsZVgBhW.js";
|
|
3
|
+
import { _ as verifyManifest, g as signManifest, h as manifestContentDigest, m as hashJson, p as evaluateHypothesis } from "../agent-profile-cell-DcFW4vOc.js";
|
|
4
4
|
import { t as mulberry32 } from "../random-Dn5fPWkt.js";
|
|
5
5
|
import { a as requiredSampleSize, c as holm, i as requiredPairedSampleSize, n as mcnemarRequiredN, o as benjaminiHochberg, r as pairedMde, s as bonferroni, t as mcnemarPower } from "../power-and-mde-BJOpt6pq.js";
|
|
6
6
|
import { f as mcnemar, g as pairedRiskDifferenceScore, h as pairedRiskDifferenceExact, m as pairedRiskDifference, n as pairArms, r as pairRunRecords, t as comparePairedArms, v as wilson } from "../paired-arms-JSDcs2zx.js";
|
|
7
7
|
import { r as pairedBootstrap, t as BOOTSTRAP_GATE_MIN_N } from "../paired-tests-C8iCsioC.js";
|
|
8
8
|
import { t as eProcess } from "../sequential-eprocess-D1jKoihe.js";
|
|
9
|
-
import { A as FinalEvidenceConflictError, M as openFinalEvidenceLedger, N as defineEvaluationClaim, P as summarizeEvaluationUnits, a as createEvidenceReceipt, at as pairHoldout, i as INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, it as heldoutSignificance, j as FinalEvidenceError, k as powerPreflight, n as EVIDENCE_AUTHORITY_KINDS, o as isIndependentEvidence, r as EVIDENCE_RECEIPT_VERSION, s as verifyEvidenceReceipt, t as createCampaignEvidenceReceipt } from "../campaign-evidence-
|
|
9
|
+
import { A as FinalEvidenceConflictError, M as openFinalEvidenceLedger, N as defineEvaluationClaim, P as summarizeEvaluationUnits, a as createEvidenceReceipt, at as pairHoldout, i as INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, it as heldoutSignificance, j as FinalEvidenceError, k as powerPreflight, n as EVIDENCE_AUTHORITY_KINDS, o as isIndependentEvidence, r as EVIDENCE_RECEIPT_VERSION, s as verifyEvidenceReceipt, t as createCampaignEvidenceReceipt } from "../campaign-evidence-aWEmfROB.js";
|
|
10
10
|
import { _ as runSelectionRule, a as evaluateHaltRule, c as evaluatePopulationReproducibilityGate, d as evaluateProvenanceGate, f as executeDecisionRule, g as readField, h as projectNLadderBudget, i as evaluateCondition, l as evaluatePowerFloorGate, m as powerFloorProblems, n as computeEstimand, o as evaluateIdentityGate, p as intervalSpecProblems, r as computeInterval, s as evaluateOracleDeterminismGate, t as classifyReissue, u as evaluatePredicate, v as runUniformPassBudget } from "../ast-CP9ae9B0.js";
|
|
11
11
|
import { a as inMemoryExperimentStore, r as fileExperimentStore, t as ExperimentTracker } from "../experiment-tracker-BKEumQug.js";
|
|
12
12
|
import { n as pairedEvalueSequence, r as sequentialCrossingHorizon } from "../sequential-CzK5DarL.js";
|
|
13
|
-
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-
|
|
14
|
-
import { n as sequentialPairedGate, t as sequentialDecide } from "../sequential-
|
|
13
|
+
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-CD-bUeyR.js";
|
|
14
|
+
import { n as sequentialPairedGate, t as sequentialDecide } from "../sequential-BfDkCBGo.js";
|
|
15
15
|
import { z } from "zod";
|
|
16
16
|
//#region src/experiment/budget.ts
|
|
17
17
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { p as EvidenceRef } from "./types-
|
|
1
|
+
import { p as EvidenceRef } from "./types-BjB2wZM9.js";
|
|
2
2
|
//#region src/statistics/power-and-mde.d.ts
|
|
3
3
|
/**
|
|
4
4
|
* Required N per arm for a two-sample comparison at target effect size,
|
|
@@ -234,4 +234,4 @@ declare class ExperimentTracker {
|
|
|
234
234
|
}
|
|
235
235
|
//#endregion
|
|
236
236
|
export { requiredSampleSize as _, ExperimentVerdict as a, computeExperimentStats as c, inMemoryExperimentStore as d, mulberry32 as f, requiredPairedSampleSize as g, pairedMde as h, ExperimentTracker as i, fileExperimentStore as l, mcnemarRequiredN as m, ExperimentRep as n, ImprovementThresholds as o, mcnemarPower as p, ExperimentStats as r, ImprovementVerdictResult as s, Experiment as t, improvementVerdict as u };
|
|
237
|
-
//# sourceMappingURL=experiment-tracker-
|
|
237
|
+
//# sourceMappingURL=experiment-tracker-URRGP9ek.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment-tracker-
|
|
1
|
+
{"version":3,"file":"experiment-tracker-URRGP9ek.d.ts","names":[],"sources":["../src/statistics/power-and-mde.ts","../src/statistics/random.ts","../src/experiment-tracker.ts"],"mappings":";;;;;;;;iBAWgB,mBAAmB;EACjC;EACA;EACA;EACA;;;;;;;;;;;iBAsBc,yBAAyB;EACvC;EACA;EACA;EACA;;;;;;;;iBAkBc,UAAU;EACxB;EACA;EACA;EACA;;;;;;;;;;;;;;;;iBAyBc,iBAAiB;EAC/B;EACA;EACA;EACA;EACA;;;;;;;iBAyBc,aAAa;EAC3B;EACA;EACA;EACA;EACA;;;;;;;;iBCrHc,WAAW;;;;;KCyBf;;UAGK;;EAEf;;EAEA;;EAEA;;;;UAKe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA,WAAW;;EAEX;;EAEA,UAAU;;UAGK;EACf;EACA;EACA;EACA;;EAEA;;EAEA;;;EAGA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA,YAAY;;EAEZ;;EAEA;EACA,MAAM;EACN,OAAO;EACP,SAAS;;EAET;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;;EAGA;;UAGe;EACf,SAAS;;EAET;;EAEA;;;;;;;iBAkDc,uBACd,MAAM,iBACN,aAAa,wBACZ;;;;;;;iBAgDa,mBACd,WAAW,iBACX,QAAQ,wBACR,aAAa,wBACZ;;;KA6CS,yBAAyB,uBAAuB,QAAQ;;;UAInD;EACf,QAAQ,QAAQ;EAChB,KAAK,aAAa,eAAe;;;;iBAoBnB,wBAAwB,UAAS,eAAoB;;iBAarD,oBAAoB,eAAe;UAyBlC;EACf,QAAQ;EACR,mBAAmB;EACnB,aAAa;;EAEb;;UAGe;EACf;EACA;EACA;EACA;;EAEA,aAAa;;;;;;;;;cAUF;mBACM;mBACA;mBACA;mBACA;EAEjB,YAAY,UAAS;EAOf,OAAO,OAAO,wBAAwB,QAAQ;;;EA6B9C,OACJ,sBACA,KAAK,KAAK;IAAwC;IAAc;MAC/D,QAAQ;EAqCL,IAAI,uBAAuB,QAAQ;EAMnC,QAAQ,QAAQ;;EAKhB,WAAW,uBAAuB,QAAQ"}
|
package/dist/{external-optimizer-process-DIB4YWEe.js → external-optimizer-process-BbhvXTFx.js}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { n as contentHash } from "./verdict-cache-
|
|
2
|
-
import { h as tryAcquireAtomicFileLock, m as probeAtomicFileLock } from "./journal-
|
|
3
|
-
import { a as closeServer, c as waitForActiveHandlers, g as isRecord, h as isExternalTextCandidate, o as listenLocal, s as sendJsonIfOpen, v as resolveExternalOptimizerCallbackLimits } from "./external-optimizer-subprocess-
|
|
1
|
+
import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
|
|
2
|
+
import { h as tryAcquireAtomicFileLock, m as probeAtomicFileLock } from "./journal-BuRLm6ho.js";
|
|
3
|
+
import { a as closeServer, c as waitForActiveHandlers, g as isRecord, h as isExternalTextCandidate, o as listenLocal, s as sendJsonIfOpen, v as resolveExternalOptimizerCallbackLimits } from "./external-optimizer-subprocess-DMgmqTB6.js";
|
|
4
4
|
import { createServer } from "node:http";
|
|
5
5
|
//#region src/campaign/single-run-lock.ts
|
|
6
6
|
/**
|
|
@@ -291,4 +291,4 @@ function cloneCandidate(candidate) {
|
|
|
291
291
|
//#endregion
|
|
292
292
|
export { acquireSingleRunLock as n, startExternalOptimizerCallback as t };
|
|
293
293
|
|
|
294
|
-
//# sourceMappingURL=external-optimizer-process-
|
|
294
|
+
//# sourceMappingURL=external-optimizer-process-BbhvXTFx.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"external-optimizer-process-DIB4YWEe.js","names":[],"sources":["../src/campaign/single-run-lock.ts","../src/campaign/external-optimizer-callback.ts"],"sourcesContent":["/**\n * Single-run lock for evaluations that share one mutable environment.\n *\n * Two concurrent runs against a shared stateful gym silently corrupt each\n * other: each resets/mutates environment state mid-cell of the other, and\n * every score from both becomes garbage that LOOKS like worker variance\n * (agent-lab R357 burned hours on flip-flopping scores before tracing them\n * to exactly this). The fix is a pid lockfile: refuse to start while a live\n * holder exists, reclaim stale locks whose pid is gone, release only if the\n * lock is still ours.\n *\n * `alsoCheck` exists because independent runners can guard the same shared\n * resource with differently named lockfiles; a runner must respect all of\n * them even though it writes only its own.\n */\n\nimport {\n type AtomicFileLockUnavailable,\n probeAtomicFileLock,\n tryAcquireAtomicFileLock,\n} from '../ledger-core/atomic-file-lock'\n\nexport interface SingleRunLockOptions {\n /** Lockfile this runner writes (and checks). */\n readonly lockPath: string\n /** Other runners' lockfiles guarding the same resource; checked, never written. */\n readonly alsoCheck?: readonly string[]\n /** Install a process 'exit' hook that releases the lock. Default true. */\n readonly releaseOnExit?: boolean\n /** Owner pid recorded in the lockfile metadata. Default process.pid. */\n readonly pid?: number\n}\n\nexport interface SingleRunLock {\n /** Remove the lockfile if this process still owns it. Idempotent. */\n release(): void\n}\n\nfunction assertAvailable(path: string): void {\n const unavailable = probeAtomicFileLock({ lockPath: path })\n if (unavailable) throw unavailableError(path, unavailable)\n}\n\nfunction unavailableError(path: string, unavailable: AtomicFileLockUnavailable): Error {\n if (unavailable.reason === 'recovery') {\n return new Error(\n `single-run lock recovery is already in progress (${path}); refusing a concurrent run`,\n )\n }\n return new Error(\n `single-run lock held by live pid ${unavailable.holder.pid} (${path}); refusing a concurrent run on the shared resource`,\n )\n}\n\n/**\n * Acquire the lock or throw naming the live holder. A stale lock (holder pid\n * no longer running) is reclaimed by one contender. An interrupted reclaim\n * leaves a marker that fails closed instead of admitting overlapping runs.\n */\nexport function acquireSingleRunLock(opts: SingleRunLockOptions): SingleRunLock {\n const pid = opts.pid ?? process.pid\n if (!Number.isSafeInteger(pid) || pid <= 0)\n throw new Error('single-run lock pid must be a positive integer')\n for (const path of opts.alsoCheck ?? []) assertAvailable(path)\n const acquisition = tryAcquireAtomicFileLock({\n lockPath: opts.lockPath,\n pid,\n })\n if (!acquisition.acquired) throw unavailableError(opts.lockPath, acquisition)\n const releaseOnExit = opts.releaseOnExit ?? true\n let released = false\n const release = (): void => {\n if (released) return\n released = true\n if (releaseOnExit) process.off('exit', release)\n try {\n acquisition.lock.release()\n } catch {\n // release is best-effort; a leftover stale lock is reclaimed on next acquire\n }\n }\n try {\n for (const path of opts.alsoCheck ?? []) assertAvailable(path)\n } catch (error) {\n release()\n throw error\n }\n if (releaseOnExit) process.on('exit', release)\n return { release }\n}\n","import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'\nimport { contentHash } from '../verdict-cache'\nimport {\n type ExternalOptimizerCallback,\n type ExternalOptimizerCallbackLimits,\n type ExternalOptimizerEvaluationObservation,\n type ExternalTextCandidate,\n type ExternalTextEvaluationRequest,\n isExternalTextCandidate,\n isRecord,\n resolveExternalOptimizerCallbackLimits,\n} from './external-optimizer-contracts'\nimport {\n closeServer,\n listenLocal,\n sendJsonIfOpen,\n waitForActiveHandlers,\n} from './external-optimizer-http'\n\ntype UnsequencedObservation = ExternalOptimizerEvaluationObservation extends infer T\n ? T extends ExternalOptimizerEvaluationObservation\n ? Omit<T, 'sequence'>\n : never\n : never\n\nexport async function startExternalOptimizerCallback<TResponse>(args: {\n token: string\n maxEvaluations: number\n acceptEvaluation?: () => number | undefined\n evaluate: (request: ExternalTextEvaluationRequest, signal: AbortSignal) => Promise<TResponse>\n observe?: (observation: ExternalOptimizerEvaluationObservation) => void\n /** Loopback JSON byte limits. Omitted fields use finite defaults. */\n limits?: Partial<ExternalOptimizerCallbackLimits>\n signal?: AbortSignal\n}): Promise<ExternalOptimizerCallback> {\n assertCallbackConfig(args)\n const limits = resolveExternalOptimizerCallbackLimits(args.limits)\n args.signal?.throwIfAborted()\n let evaluations = 0\n let accepting = true\n let closePromise: Promise<void> | undefined\n const activeControllers = new Set<AbortController>()\n const activeHandlers = new Set<Promise<void>>()\n const proposedCandidateHashes = new Set<string>()\n let observationSequence = 0\n const server = createServer((request, response) => {\n if (!accepting) {\n sendJsonIfOpen(response, 503, { error: 'external optimizer callback is closing' })\n return\n }\n\n const controller = new AbortController()\n const abortRequest = (): void => {\n request.destroy()\n response.destroy()\n }\n activeControllers.add(controller)\n controller.signal.addEventListener('abort', abortRequest, { once: true })\n\n let handler!: Promise<void>\n handler = handleCallback(\n request,\n response,\n controller.signal,\n args,\n limits,\n () => {\n const accepted = args.acceptEvaluation ? args.acceptEvaluation() : evaluations + 1\n if (accepted === undefined) return undefined\n if (!Number.isSafeInteger(accepted) || accepted <= 0) {\n throw new Error('external optimizer callback: invalid accepted evaluation count')\n }\n if (accepted > args.maxEvaluations) return undefined\n evaluations += 1\n return accepted\n },\n (observation) => {\n if (!args.observe) return\n args.observe({ ...observation, sequence: ++observationSequence })\n },\n proposedCandidateHashes,\n ).finally(() => {\n controller.signal.removeEventListener('abort', abortRequest)\n activeControllers.delete(controller)\n activeHandlers.delete(handler)\n })\n activeHandlers.add(handler)\n void handler.catch(() => undefined)\n })\n const port = await listenLocal(server)\n const close = (): Promise<void> => {\n closePromise ??= closeCallbackServer()\n return closePromise\n }\n const onAbort = (): void => {\n void close().catch(() => undefined)\n }\n args.signal?.addEventListener('abort', onAbort, { once: true })\n if (args.signal?.aborted) onAbort()\n return {\n url: `http://127.0.0.1:${port}/evaluate`,\n token: args.token,\n evaluations: () => evaluations,\n close,\n }\n\n async function closeCallbackServer(): Promise<void> {\n args.signal?.removeEventListener('abort', onAbort)\n accepting = false\n const closingServer = closeServer(server)\n server.closeIdleConnections?.()\n for (const controller of activeControllers) controller.abort()\n const [serverResult] = await Promise.allSettled([\n closingServer,\n waitForActiveHandlers(activeHandlers),\n ])\n if (activeControllers.size !== 0 || activeHandlers.size !== 0) {\n throw new Error('external optimizer callback closed with active request work')\n }\n if (serverResult?.status === 'rejected') throw serverResult.reason\n }\n}\n\nasync function handleCallback<TResponse>(\n request: IncomingMessage,\n response: ServerResponse,\n signal: AbortSignal,\n args: {\n token: string\n maxEvaluations: number\n acceptEvaluation?: () => number | undefined\n evaluate: (request: ExternalTextEvaluationRequest, signal: AbortSignal) => Promise<TResponse>\n observe?: (observation: ExternalOptimizerEvaluationObservation) => void\n },\n limits: ExternalOptimizerCallbackLimits,\n nextEvaluation: () => number | undefined,\n observe: (observation: UnsequencedObservation) => void,\n proposedCandidateHashes: Set<string>,\n): Promise<void> {\n try {\n if (request.method !== 'POST' || request.url !== '/evaluate') {\n sendJsonIfOpen(response, 404, { error: 'not found' })\n return\n }\n if (request.headers.authorization !== `Bearer ${args.token}`) {\n sendJsonIfOpen(response, 401, { error: 'unauthorized' })\n return\n }\n let body: unknown\n try {\n body = await readJson(request, limits.maxRequestBytes)\n } catch {\n observe({ kind: 'refusal', reason: 'invalid-request' })\n sendJsonIfOpen(response, 400, { error: 'request body must be bounded valid JSON' })\n return\n }\n if (\n !isRecord(body) ||\n !isExternalTextCandidate(body.candidate) ||\n typeof body.exampleId !== 'string'\n ) {\n observe({ kind: 'refusal', reason: 'invalid-request' })\n sendJsonIfOpen(response, 400, { error: 'candidate and exampleId are required strings' })\n return\n }\n const candidate = cloneCandidate(body.candidate)\n const candidateHash = contentHash({ kind: 'external-text-candidate', candidate })\n if (!proposedCandidateHashes.has(candidateHash)) {\n observe({ kind: 'proposal', candidate, candidateHash })\n proposedCandidateHashes.add(candidateHash)\n }\n const count = nextEvaluation()\n if (count === undefined) {\n observe({\n kind: 'refusal',\n reason: 'evaluation-limit',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n })\n sendJsonIfOpen(response, 429, { error: 'evaluation limit reached' })\n return\n }\n let result: TResponse\n try {\n result = await args.evaluate({ candidate, exampleId: body.exampleId }, signal)\n } catch (error) {\n observe({\n kind: 'refusal',\n reason: 'evaluation-failed',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n })\n // The thrown detail is the only diagnostic for a failed evaluation —\n // an opaque 500 costs the caller a blind debugging round.\n sendJsonIfOpen(response, 500, {\n error: `evaluation failed: ${String(error).slice(0, 400)}`,\n })\n return\n }\n let encoded: string\n try {\n encoded = encodeJsonResponse(result)\n } catch {\n observe({\n kind: 'refusal',\n reason: 'evaluation-failed',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n })\n sendJsonIfOpen(response, 500, { error: 'evaluation response must be valid JSON' })\n return\n }\n if (Buffer.byteLength(encoded) > limits.maxResponseBytes) {\n observe({\n kind: 'refusal',\n reason: 'evaluation-failed',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n })\n sendJsonIfOpen(response, 413, { error: 'evaluation response too large' })\n return\n }\n observe({\n kind: 'evaluation',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n evaluationNumber: count,\n response: structuredClone(result),\n })\n sendEncodedJsonIfOpen(response, 200, encoded)\n } catch {\n sendJsonIfOpen(response, 500, { error: 'evaluation failed' })\n }\n}\n\nfunction readJson(request: IncomingMessage, maxRequestBytes: number): Promise<unknown> {\n return new Promise((resolvePromise, reject) => {\n let size = 0\n const chunks: Buffer[] = []\n request.on('data', (chunk: Buffer) => {\n size += chunk.length\n if (size > maxRequestBytes) {\n reject(new Error('callback body too large'))\n request.destroy()\n return\n }\n chunks.push(chunk)\n })\n request.on('error', reject)\n request.on('end', () => {\n try {\n resolvePromise(JSON.parse(Buffer.concat(chunks).toString('utf8')))\n } catch (error) {\n reject(error)\n }\n })\n })\n}\n\nfunction encodeJsonResponse(value: unknown): string {\n const encoded = JSON.stringify(value)\n if (encoded === undefined) throw new Error('evaluation response must be JSON-serializable')\n return encoded\n}\n\nfunction sendEncodedJsonIfOpen(response: ServerResponse, status: number, encoded: string): void {\n if (response.destroyed || response.writableEnded) return\n response.writeHead(status, {\n 'content-type': 'application/json; charset=utf-8',\n 'content-length': String(Buffer.byteLength(encoded)),\n })\n response.end(encoded)\n}\n\nfunction assertCallbackConfig(args: {\n token: string\n maxEvaluations: number\n acceptEvaluation?: () => number | undefined\n evaluate: (request: ExternalTextEvaluationRequest, signal: AbortSignal) => Promise<unknown>\n observe?: (observation: ExternalOptimizerEvaluationObservation) => void\n limits?: Partial<ExternalOptimizerCallbackLimits>\n signal?: AbortSignal\n}): void {\n if (typeof args.token !== 'string' || !args.token.trim()) {\n throw new Error('external optimizer callback: token must be non-empty')\n }\n if (!Number.isSafeInteger(args.maxEvaluations) || args.maxEvaluations <= 0) {\n throw new Error('external optimizer callback: maxEvaluations must be a positive safe integer')\n }\n if (args.acceptEvaluation !== undefined && typeof args.acceptEvaluation !== 'function') {\n throw new Error('external optimizer callback: acceptEvaluation must be a function')\n }\n if (typeof args.evaluate !== 'function') {\n throw new Error('external optimizer callback: evaluate must be a function')\n }\n if (args.observe !== undefined && typeof args.observe !== 'function') {\n throw new Error('external optimizer callback: observe must be a function')\n }\n}\n\nfunction cloneCandidate(candidate: ExternalTextCandidate): ExternalTextCandidate {\n return typeof candidate === 'string' ? candidate : { ...candidate }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAsCA,SAAS,gBAAgB,MAAoB;CAC3C,MAAM,cAAc,oBAAoB,EAAE,UAAU,KAAK,CAAC;CAC1D,IAAI,aAAa,MAAM,iBAAiB,MAAM,WAAW;AAC3D;AAEA,SAAS,iBAAiB,MAAc,aAA+C;CACrF,IAAI,YAAY,WAAW,YACzB,uBAAO,IAAI,MACT,oDAAoD,KAAK,6BAC3D;CAEF,uBAAO,IAAI,MACT,oCAAoC,YAAY,OAAO,IAAI,IAAI,KAAK,oDACtE;AACF;;;;;;AAOA,SAAgB,qBAAqB,MAA2C;CAC9E,MAAM,MAAM,KAAK,OAAO,QAAQ;CAChC,IAAI,CAAC,OAAO,cAAc,GAAG,KAAK,OAAO,GACvC,MAAM,IAAI,MAAM,gDAAgD;CAClE,KAAK,MAAM,QAAQ,KAAK,aAAa,CAAC,GAAG,gBAAgB,IAAI;CAC7D,MAAM,cAAc,yBAAyB;EAC3C,UAAU,KAAK;EACf;CACF,CAAC;CACD,IAAI,CAAC,YAAY,UAAU,MAAM,iBAAiB,KAAK,UAAU,WAAW;CAC5E,MAAM,gBAAgB,KAAK,iBAAiB;CAC5C,IAAI,WAAW;CACf,MAAM,gBAAsB;EAC1B,IAAI,UAAU;EACd,WAAW;EACX,IAAI,eAAe,QAAQ,IAAI,QAAQ,OAAO;EAC9C,IAAI;GACF,YAAY,KAAK,QAAQ;EAC3B,QAAQ,CAER;CACF;CACA,IAAI;EACF,KAAK,MAAM,QAAQ,KAAK,aAAa,CAAC,GAAG,gBAAgB,IAAI;CAC/D,SAAS,OAAO;EACd,QAAQ;EACR,MAAM;CACR;CACA,IAAI,eAAe,QAAQ,GAAG,QAAQ,OAAO;CAC7C,OAAO,EAAE,QAAQ;AACnB;;;AChEA,eAAsB,+BAA0C,MASzB;CACrC,qBAAqB,IAAI;CACzB,MAAM,SAAS,uCAAuC,KAAK,MAAM;CACjE,KAAK,QAAQ,eAAe;CAC5B,IAAI,cAAc;CAClB,IAAI,YAAY;CAChB,IAAI;CACJ,MAAM,oCAAoB,IAAI,IAAqB;CACnD,MAAM,iCAAiB,IAAI,IAAmB;CAC9C,MAAM,0CAA0B,IAAI,IAAY;CAChD,IAAI,sBAAsB;CAC1B,MAAM,SAAS,cAAc,SAAS,aAAa;EACjD,IAAI,CAAC,WAAW;GACd,eAAe,UAAU,KAAK,EAAE,OAAO,yCAAyC,CAAC;GACjF;EACF;EAEA,MAAM,aAAa,IAAI,gBAAgB;EACvC,MAAM,qBAA2B;GAC/B,QAAQ,QAAQ;GAChB,SAAS,QAAQ;EACnB;EACA,kBAAkB,IAAI,UAAU;EAChC,WAAW,OAAO,iBAAiB,SAAS,cAAc,EAAE,MAAM,KAAK,CAAC;EAExE,IAAI;EACJ,UAAU,eACR,SACA,UACA,WAAW,QACX,MACA,cACM;GACJ,MAAM,WAAW,KAAK,mBAAmB,KAAK,iBAAiB,IAAI,cAAc;GACjF,IAAI,aAAa,KAAA,GAAW,OAAO,KAAA;GACnC,IAAI,CAAC,OAAO,cAAc,QAAQ,KAAK,YAAY,GACjD,MAAM,IAAI,MAAM,gEAAgE;GAElF,IAAI,WAAW,KAAK,gBAAgB,OAAO,KAAA;GAC3C,eAAe;GACf,OAAO;EACT,IACC,gBAAgB;GACf,IAAI,CAAC,KAAK,SAAS;GACnB,KAAK,QAAQ;IAAE,GAAG;IAAa,UAAU,EAAE;GAAoB,CAAC;EAClE,GACA,uBACF,CAAC,CAAC,cAAc;GACd,WAAW,OAAO,oBAAoB,SAAS,YAAY;GAC3D,kBAAkB,OAAO,UAAU;GACnC,eAAe,OAAO,OAAO;EAC/B,CAAC;EACD,eAAe,IAAI,OAAO;EAC1B,QAAa,YAAY,KAAA,CAAS;CACpC,CAAC;CACD,MAAM,OAAO,MAAM,YAAY,MAAM;CACrC,MAAM,cAA6B;EACjC,iBAAiB,oBAAoB;EACrC,OAAO;CACT;CACA,MAAM,gBAAsB;EAC1B,MAAW,CAAC,CAAC,YAAY,KAAA,CAAS;CACpC;CACA,KAAK,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CAC9D,IAAI,KAAK,QAAQ,SAAS,QAAQ;CAClC,OAAO;EACL,KAAK,oBAAoB,KAAK;EAC9B,OAAO,KAAK;EACZ,mBAAmB;EACnB;CACF;CAEA,eAAe,sBAAqC;EAClD,KAAK,QAAQ,oBAAoB,SAAS,OAAO;EACjD,YAAY;EACZ,MAAM,gBAAgB,YAAY,MAAM;EACxC,OAAO,uBAAuB;EAC9B,KAAK,MAAM,cAAc,mBAAmB,WAAW,MAAM;EAC7D,MAAM,CAAC,gBAAgB,MAAM,QAAQ,WAAW,CAC9C,eACA,sBAAsB,cAAc,CACtC,CAAC;EACD,IAAI,kBAAkB,SAAS,KAAK,eAAe,SAAS,GAC1D,MAAM,IAAI,MAAM,6DAA6D;EAE/E,IAAI,cAAc,WAAW,YAAY,MAAM,aAAa;CAC9D;AACF;AAEA,eAAe,eACb,SACA,UACA,QACA,MAOA,QACA,gBACA,SACA,yBACe;CACf,IAAI;EACF,IAAI,QAAQ,WAAW,UAAU,QAAQ,QAAQ,aAAa;GAC5D,eAAe,UAAU,KAAK,EAAE,OAAO,YAAY,CAAC;GACpD;EACF;EACA,IAAI,QAAQ,QAAQ,kBAAkB,UAAU,KAAK,SAAS;GAC5D,eAAe,UAAU,KAAK,EAAE,OAAO,eAAe,CAAC;GACvD;EACF;EACA,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,SAAS,SAAS,OAAO,eAAe;EACvD,QAAQ;GACN,QAAQ;IAAE,MAAM;IAAW,QAAQ;GAAkB,CAAC;GACtD,eAAe,UAAU,KAAK,EAAE,OAAO,0CAA0C,CAAC;GAClF;EACF;EACA,IACE,CAAC,SAAS,IAAI,KACd,CAAC,wBAAwB,KAAK,SAAS,KACvC,OAAO,KAAK,cAAc,UAC1B;GACA,QAAQ;IAAE,MAAM;IAAW,QAAQ;GAAkB,CAAC;GACtD,eAAe,UAAU,KAAK,EAAE,OAAO,+CAA+C,CAAC;GACvF;EACF;EACA,MAAM,YAAY,eAAe,KAAK,SAAS;EAC/C,MAAM,gBAAgB,YAAY;GAAE,MAAM;GAA2B;EAAU,CAAC;EAChF,IAAI,CAAC,wBAAwB,IAAI,aAAa,GAAG;GAC/C,QAAQ;IAAE,MAAM;IAAY;IAAW;GAAc,CAAC;GACtD,wBAAwB,IAAI,aAAa;EAC3C;EACA,MAAM,QAAQ,eAAe;EAC7B,IAAI,UAAU,KAAA,GAAW;GACvB,QAAQ;IACN,MAAM;IACN,QAAQ;IACR;IACA;IACA,WAAW,KAAK;GAClB,CAAC;GACD,eAAe,UAAU,KAAK,EAAE,OAAO,2BAA2B,CAAC;GACnE;EACF;EACA,IAAI;EACJ,IAAI;GACF,SAAS,MAAM,KAAK,SAAS;IAAE;IAAW,WAAW,KAAK;GAAU,GAAG,MAAM;EAC/E,SAAS,OAAO;GACd,QAAQ;IACN,MAAM;IACN,QAAQ;IACR;IACA;IACA,WAAW,KAAK;GAClB,CAAC;GAGD,eAAe,UAAU,KAAK,EAC5B,OAAO,sBAAsB,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,IACzD,CAAC;GACD;EACF;EACA,IAAI;EACJ,IAAI;GACF,UAAU,mBAAmB,MAAM;EACrC,QAAQ;GACN,QAAQ;IACN,MAAM;IACN,QAAQ;IACR;IACA;IACA,WAAW,KAAK;GAClB,CAAC;GACD,eAAe,UAAU,KAAK,EAAE,OAAO,yCAAyC,CAAC;GACjF;EACF;EACA,IAAI,OAAO,WAAW,OAAO,IAAI,OAAO,kBAAkB;GACxD,QAAQ;IACN,MAAM;IACN,QAAQ;IACR;IACA;IACA,WAAW,KAAK;GAClB,CAAC;GACD,eAAe,UAAU,KAAK,EAAE,OAAO,gCAAgC,CAAC;GACxE;EACF;EACA,QAAQ;GACN,MAAM;GACN;GACA;GACA,WAAW,KAAK;GAChB,kBAAkB;GAClB,UAAU,gBAAgB,MAAM;EAClC,CAAC;EACD,sBAAsB,UAAU,KAAK,OAAO;CAC9C,QAAQ;EACN,eAAe,UAAU,KAAK,EAAE,OAAO,oBAAoB,CAAC;CAC9D;AACF;AAEA,SAAS,SAAS,SAA0B,iBAA2C;CACrF,OAAO,IAAI,SAAS,gBAAgB,WAAW;EAC7C,IAAI,OAAO;EACX,MAAM,SAAmB,CAAC;EAC1B,QAAQ,GAAG,SAAS,UAAkB;GACpC,QAAQ,MAAM;GACd,IAAI,OAAO,iBAAiB;IAC1B,uBAAO,IAAI,MAAM,yBAAyB,CAAC;IAC3C,QAAQ,QAAQ;IAChB;GACF;GACA,OAAO,KAAK,KAAK;EACnB,CAAC;EACD,QAAQ,GAAG,SAAS,MAAM;EAC1B,QAAQ,GAAG,aAAa;GACtB,IAAI;IACF,eAAe,KAAK,MAAM,OAAO,OAAO,MAAM,CAAC,CAAC,SAAS,MAAM,CAAC,CAAC;GACnE,SAAS,OAAO;IACd,OAAO,KAAK;GACd;EACF,CAAC;CACH,CAAC;AACH;AAEA,SAAS,mBAAmB,OAAwB;CAClD,MAAM,UAAU,KAAK,UAAU,KAAK;CACpC,IAAI,YAAY,KAAA,GAAW,MAAM,IAAI,MAAM,+CAA+C;CAC1F,OAAO;AACT;AAEA,SAAS,sBAAsB,UAA0B,QAAgB,SAAuB;CAC9F,IAAI,SAAS,aAAa,SAAS,eAAe;CAClD,SAAS,UAAU,QAAQ;EACzB,gBAAgB;EAChB,kBAAkB,OAAO,OAAO,WAAW,OAAO,CAAC;CACrD,CAAC;CACD,SAAS,IAAI,OAAO;AACtB;AAEA,SAAS,qBAAqB,MAQrB;CACP,IAAI,OAAO,KAAK,UAAU,YAAY,CAAC,KAAK,MAAM,KAAK,GACrD,MAAM,IAAI,MAAM,sDAAsD;CAExE,IAAI,CAAC,OAAO,cAAc,KAAK,cAAc,KAAK,KAAK,kBAAkB,GACvE,MAAM,IAAI,MAAM,6EAA6E;CAE/F,IAAI,KAAK,qBAAqB,KAAA,KAAa,OAAO,KAAK,qBAAqB,YAC1E,MAAM,IAAI,MAAM,kEAAkE;CAEpF,IAAI,OAAO,KAAK,aAAa,YAC3B,MAAM,IAAI,MAAM,0DAA0D;CAE5E,IAAI,KAAK,YAAY,KAAA,KAAa,OAAO,KAAK,YAAY,YACxD,MAAM,IAAI,MAAM,yDAAyD;AAE7E;AAEA,SAAS,eAAe,WAAyD;CAC/E,OAAO,OAAO,cAAc,WAAW,YAAY,EAAE,GAAG,UAAU;AACpE"}
|
|
1
|
+
{"version":3,"file":"external-optimizer-process-BbhvXTFx.js","names":[],"sources":["../src/campaign/single-run-lock.ts","../src/campaign/external-optimizer-callback.ts"],"sourcesContent":["/**\n * Single-run lock for evaluations that share one mutable environment.\n *\n * Two concurrent runs against a shared stateful gym silently corrupt each\n * other: each resets/mutates environment state mid-cell of the other, and\n * every score from both becomes garbage that LOOKS like worker variance\n * (agent-lab R357 burned hours on flip-flopping scores before tracing them\n * to exactly this). The fix is a pid lockfile: refuse to start while a live\n * holder exists, reclaim stale locks whose pid is gone, release only if the\n * lock is still ours.\n *\n * `alsoCheck` exists because independent runners can guard the same shared\n * resource with differently named lockfiles; a runner must respect all of\n * them even though it writes only its own.\n */\n\nimport {\n type AtomicFileLockUnavailable,\n probeAtomicFileLock,\n tryAcquireAtomicFileLock,\n} from '../ledger-core/atomic-file-lock'\n\nexport interface SingleRunLockOptions {\n /** Lockfile this runner writes (and checks). */\n readonly lockPath: string\n /** Other runners' lockfiles guarding the same resource; checked, never written. */\n readonly alsoCheck?: readonly string[]\n /** Install a process 'exit' hook that releases the lock. Default true. */\n readonly releaseOnExit?: boolean\n /** Owner pid recorded in the lockfile metadata. Default process.pid. */\n readonly pid?: number\n}\n\nexport interface SingleRunLock {\n /** Remove the lockfile if this process still owns it. Idempotent. */\n release(): void\n}\n\nfunction assertAvailable(path: string): void {\n const unavailable = probeAtomicFileLock({ lockPath: path })\n if (unavailable) throw unavailableError(path, unavailable)\n}\n\nfunction unavailableError(path: string, unavailable: AtomicFileLockUnavailable): Error {\n if (unavailable.reason === 'recovery') {\n return new Error(\n `single-run lock recovery is already in progress (${path}); refusing a concurrent run`,\n )\n }\n return new Error(\n `single-run lock held by live pid ${unavailable.holder.pid} (${path}); refusing a concurrent run on the shared resource`,\n )\n}\n\n/**\n * Acquire the lock or throw naming the live holder. A stale lock (holder pid\n * no longer running) is reclaimed by one contender. An interrupted reclaim\n * leaves a marker that fails closed instead of admitting overlapping runs.\n */\nexport function acquireSingleRunLock(opts: SingleRunLockOptions): SingleRunLock {\n const pid = opts.pid ?? process.pid\n if (!Number.isSafeInteger(pid) || pid <= 0)\n throw new Error('single-run lock pid must be a positive integer')\n for (const path of opts.alsoCheck ?? []) assertAvailable(path)\n const acquisition = tryAcquireAtomicFileLock({\n lockPath: opts.lockPath,\n pid,\n })\n if (!acquisition.acquired) throw unavailableError(opts.lockPath, acquisition)\n const releaseOnExit = opts.releaseOnExit ?? true\n let released = false\n const release = (): void => {\n if (released) return\n released = true\n if (releaseOnExit) process.off('exit', release)\n try {\n acquisition.lock.release()\n } catch {\n // release is best-effort; a leftover stale lock is reclaimed on next acquire\n }\n }\n try {\n for (const path of opts.alsoCheck ?? []) assertAvailable(path)\n } catch (error) {\n release()\n throw error\n }\n if (releaseOnExit) process.on('exit', release)\n return { release }\n}\n","import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'\nimport { contentHash } from '../verdict-cache'\nimport {\n type ExternalOptimizerCallback,\n type ExternalOptimizerCallbackLimits,\n type ExternalOptimizerEvaluationObservation,\n type ExternalTextCandidate,\n type ExternalTextEvaluationRequest,\n isExternalTextCandidate,\n isRecord,\n resolveExternalOptimizerCallbackLimits,\n} from './external-optimizer-contracts'\nimport {\n closeServer,\n listenLocal,\n sendJsonIfOpen,\n waitForActiveHandlers,\n} from './external-optimizer-http'\n\ntype UnsequencedObservation = ExternalOptimizerEvaluationObservation extends infer T\n ? T extends ExternalOptimizerEvaluationObservation\n ? Omit<T, 'sequence'>\n : never\n : never\n\nexport async function startExternalOptimizerCallback<TResponse>(args: {\n token: string\n maxEvaluations: number\n acceptEvaluation?: () => number | undefined\n evaluate: (request: ExternalTextEvaluationRequest, signal: AbortSignal) => Promise<TResponse>\n observe?: (observation: ExternalOptimizerEvaluationObservation) => void\n /** Loopback JSON byte limits. Omitted fields use finite defaults. */\n limits?: Partial<ExternalOptimizerCallbackLimits>\n signal?: AbortSignal\n}): Promise<ExternalOptimizerCallback> {\n assertCallbackConfig(args)\n const limits = resolveExternalOptimizerCallbackLimits(args.limits)\n args.signal?.throwIfAborted()\n let evaluations = 0\n let accepting = true\n let closePromise: Promise<void> | undefined\n const activeControllers = new Set<AbortController>()\n const activeHandlers = new Set<Promise<void>>()\n const proposedCandidateHashes = new Set<string>()\n let observationSequence = 0\n const server = createServer((request, response) => {\n if (!accepting) {\n sendJsonIfOpen(response, 503, { error: 'external optimizer callback is closing' })\n return\n }\n\n const controller = new AbortController()\n const abortRequest = (): void => {\n request.destroy()\n response.destroy()\n }\n activeControllers.add(controller)\n controller.signal.addEventListener('abort', abortRequest, { once: true })\n\n let handler!: Promise<void>\n handler = handleCallback(\n request,\n response,\n controller.signal,\n args,\n limits,\n () => {\n const accepted = args.acceptEvaluation ? args.acceptEvaluation() : evaluations + 1\n if (accepted === undefined) return undefined\n if (!Number.isSafeInteger(accepted) || accepted <= 0) {\n throw new Error('external optimizer callback: invalid accepted evaluation count')\n }\n if (accepted > args.maxEvaluations) return undefined\n evaluations += 1\n return accepted\n },\n (observation) => {\n if (!args.observe) return\n args.observe({ ...observation, sequence: ++observationSequence })\n },\n proposedCandidateHashes,\n ).finally(() => {\n controller.signal.removeEventListener('abort', abortRequest)\n activeControllers.delete(controller)\n activeHandlers.delete(handler)\n })\n activeHandlers.add(handler)\n void handler.catch(() => undefined)\n })\n const port = await listenLocal(server)\n const close = (): Promise<void> => {\n closePromise ??= closeCallbackServer()\n return closePromise\n }\n const onAbort = (): void => {\n void close().catch(() => undefined)\n }\n args.signal?.addEventListener('abort', onAbort, { once: true })\n if (args.signal?.aborted) onAbort()\n return {\n url: `http://127.0.0.1:${port}/evaluate`,\n token: args.token,\n evaluations: () => evaluations,\n close,\n }\n\n async function closeCallbackServer(): Promise<void> {\n args.signal?.removeEventListener('abort', onAbort)\n accepting = false\n const closingServer = closeServer(server)\n server.closeIdleConnections?.()\n for (const controller of activeControllers) controller.abort()\n const [serverResult] = await Promise.allSettled([\n closingServer,\n waitForActiveHandlers(activeHandlers),\n ])\n if (activeControllers.size !== 0 || activeHandlers.size !== 0) {\n throw new Error('external optimizer callback closed with active request work')\n }\n if (serverResult?.status === 'rejected') throw serverResult.reason\n }\n}\n\nasync function handleCallback<TResponse>(\n request: IncomingMessage,\n response: ServerResponse,\n signal: AbortSignal,\n args: {\n token: string\n maxEvaluations: number\n acceptEvaluation?: () => number | undefined\n evaluate: (request: ExternalTextEvaluationRequest, signal: AbortSignal) => Promise<TResponse>\n observe?: (observation: ExternalOptimizerEvaluationObservation) => void\n },\n limits: ExternalOptimizerCallbackLimits,\n nextEvaluation: () => number | undefined,\n observe: (observation: UnsequencedObservation) => void,\n proposedCandidateHashes: Set<string>,\n): Promise<void> {\n try {\n if (request.method !== 'POST' || request.url !== '/evaluate') {\n sendJsonIfOpen(response, 404, { error: 'not found' })\n return\n }\n if (request.headers.authorization !== `Bearer ${args.token}`) {\n sendJsonIfOpen(response, 401, { error: 'unauthorized' })\n return\n }\n let body: unknown\n try {\n body = await readJson(request, limits.maxRequestBytes)\n } catch {\n observe({ kind: 'refusal', reason: 'invalid-request' })\n sendJsonIfOpen(response, 400, { error: 'request body must be bounded valid JSON' })\n return\n }\n if (\n !isRecord(body) ||\n !isExternalTextCandidate(body.candidate) ||\n typeof body.exampleId !== 'string'\n ) {\n observe({ kind: 'refusal', reason: 'invalid-request' })\n sendJsonIfOpen(response, 400, { error: 'candidate and exampleId are required strings' })\n return\n }\n const candidate = cloneCandidate(body.candidate)\n const candidateHash = contentHash({ kind: 'external-text-candidate', candidate })\n if (!proposedCandidateHashes.has(candidateHash)) {\n observe({ kind: 'proposal', candidate, candidateHash })\n proposedCandidateHashes.add(candidateHash)\n }\n const count = nextEvaluation()\n if (count === undefined) {\n observe({\n kind: 'refusal',\n reason: 'evaluation-limit',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n })\n sendJsonIfOpen(response, 429, { error: 'evaluation limit reached' })\n return\n }\n let result: TResponse\n try {\n result = await args.evaluate({ candidate, exampleId: body.exampleId }, signal)\n } catch (error) {\n observe({\n kind: 'refusal',\n reason: 'evaluation-failed',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n })\n // The thrown detail is the only diagnostic for a failed evaluation —\n // an opaque 500 costs the caller a blind debugging round.\n sendJsonIfOpen(response, 500, {\n error: `evaluation failed: ${String(error).slice(0, 400)}`,\n })\n return\n }\n let encoded: string\n try {\n encoded = encodeJsonResponse(result)\n } catch {\n observe({\n kind: 'refusal',\n reason: 'evaluation-failed',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n })\n sendJsonIfOpen(response, 500, { error: 'evaluation response must be valid JSON' })\n return\n }\n if (Buffer.byteLength(encoded) > limits.maxResponseBytes) {\n observe({\n kind: 'refusal',\n reason: 'evaluation-failed',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n })\n sendJsonIfOpen(response, 413, { error: 'evaluation response too large' })\n return\n }\n observe({\n kind: 'evaluation',\n candidate,\n candidateHash,\n exampleId: body.exampleId,\n evaluationNumber: count,\n response: structuredClone(result),\n })\n sendEncodedJsonIfOpen(response, 200, encoded)\n } catch {\n sendJsonIfOpen(response, 500, { error: 'evaluation failed' })\n }\n}\n\nfunction readJson(request: IncomingMessage, maxRequestBytes: number): Promise<unknown> {\n return new Promise((resolvePromise, reject) => {\n let size = 0\n const chunks: Buffer[] = []\n request.on('data', (chunk: Buffer) => {\n size += chunk.length\n if (size > maxRequestBytes) {\n reject(new Error('callback body too large'))\n request.destroy()\n return\n }\n chunks.push(chunk)\n })\n request.on('error', reject)\n request.on('end', () => {\n try {\n resolvePromise(JSON.parse(Buffer.concat(chunks).toString('utf8')))\n } catch (error) {\n reject(error)\n }\n })\n })\n}\n\nfunction encodeJsonResponse(value: unknown): string {\n const encoded = JSON.stringify(value)\n if (encoded === undefined) throw new Error('evaluation response must be JSON-serializable')\n return encoded\n}\n\nfunction sendEncodedJsonIfOpen(response: ServerResponse, status: number, encoded: string): void {\n if (response.destroyed || response.writableEnded) return\n response.writeHead(status, {\n 'content-type': 'application/json; charset=utf-8',\n 'content-length': String(Buffer.byteLength(encoded)),\n })\n response.end(encoded)\n}\n\nfunction assertCallbackConfig(args: {\n token: string\n maxEvaluations: number\n acceptEvaluation?: () => number | undefined\n evaluate: (request: ExternalTextEvaluationRequest, signal: AbortSignal) => Promise<unknown>\n observe?: (observation: ExternalOptimizerEvaluationObservation) => void\n limits?: Partial<ExternalOptimizerCallbackLimits>\n signal?: AbortSignal\n}): void {\n if (typeof args.token !== 'string' || !args.token.trim()) {\n throw new Error('external optimizer callback: token must be non-empty')\n }\n if (!Number.isSafeInteger(args.maxEvaluations) || args.maxEvaluations <= 0) {\n throw new Error('external optimizer callback: maxEvaluations must be a positive safe integer')\n }\n if (args.acceptEvaluation !== undefined && typeof args.acceptEvaluation !== 'function') {\n throw new Error('external optimizer callback: acceptEvaluation must be a function')\n }\n if (typeof args.evaluate !== 'function') {\n throw new Error('external optimizer callback: evaluate must be a function')\n }\n if (args.observe !== undefined && typeof args.observe !== 'function') {\n throw new Error('external optimizer callback: observe must be a function')\n }\n}\n\nfunction cloneCandidate(candidate: ExternalTextCandidate): ExternalTextCandidate {\n return typeof candidate === 'string' ? candidate : { ...candidate }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAsCA,SAAS,gBAAgB,MAAoB;CAC3C,MAAM,cAAc,oBAAoB,EAAE,UAAU,KAAK,CAAC;CAC1D,IAAI,aAAa,MAAM,iBAAiB,MAAM,WAAW;AAC3D;AAEA,SAAS,iBAAiB,MAAc,aAA+C;CACrF,IAAI,YAAY,WAAW,YACzB,uBAAO,IAAI,MACT,oDAAoD,KAAK,6BAC3D;CAEF,uBAAO,IAAI,MACT,oCAAoC,YAAY,OAAO,IAAI,IAAI,KAAK,oDACtE;AACF;;;;;;AAOA,SAAgB,qBAAqB,MAA2C;CAC9E,MAAM,MAAM,KAAK,OAAO,QAAQ;CAChC,IAAI,CAAC,OAAO,cAAc,GAAG,KAAK,OAAO,GACvC,MAAM,IAAI,MAAM,gDAAgD;CAClE,KAAK,MAAM,QAAQ,KAAK,aAAa,CAAC,GAAG,gBAAgB,IAAI;CAC7D,MAAM,cAAc,yBAAyB;EAC3C,UAAU,KAAK;EACf;CACF,CAAC;CACD,IAAI,CAAC,YAAY,UAAU,MAAM,iBAAiB,KAAK,UAAU,WAAW;CAC5E,MAAM,gBAAgB,KAAK,iBAAiB;CAC5C,IAAI,WAAW;CACf,MAAM,gBAAsB;EAC1B,IAAI,UAAU;EACd,WAAW;EACX,IAAI,eAAe,QAAQ,IAAI,QAAQ,OAAO;EAC9C,IAAI;GACF,YAAY,KAAK,QAAQ;EAC3B,QAAQ,CAER;CACF;CACA,IAAI;EACF,KAAK,MAAM,QAAQ,KAAK,aAAa,CAAC,GAAG,gBAAgB,IAAI;CAC/D,SAAS,OAAO;EACd,QAAQ;EACR,MAAM;CACR;CACA,IAAI,eAAe,QAAQ,GAAG,QAAQ,OAAO;CAC7C,OAAO,EAAE,QAAQ;AACnB;;;AChEA,eAAsB,+BAA0C,MASzB;CACrC,qBAAqB,IAAI;CACzB,MAAM,SAAS,uCAAuC,KAAK,MAAM;CACjE,KAAK,QAAQ,eAAe;CAC5B,IAAI,cAAc;CAClB,IAAI,YAAY;CAChB,IAAI;CACJ,MAAM,oCAAoB,IAAI,IAAqB;CACnD,MAAM,iCAAiB,IAAI,IAAmB;CAC9C,MAAM,0CAA0B,IAAI,IAAY;CAChD,IAAI,sBAAsB;CAC1B,MAAM,SAAS,cAAc,SAAS,aAAa;EACjD,IAAI,CAAC,WAAW;GACd,eAAe,UAAU,KAAK,EAAE,OAAO,yCAAyC,CAAC;GACjF;EACF;EAEA,MAAM,aAAa,IAAI,gBAAgB;EACvC,MAAM,qBAA2B;GAC/B,QAAQ,QAAQ;GAChB,SAAS,QAAQ;EACnB;EACA,kBAAkB,IAAI,UAAU;EAChC,WAAW,OAAO,iBAAiB,SAAS,cAAc,EAAE,MAAM,KAAK,CAAC;EAExE,IAAI;EACJ,UAAU,eACR,SACA,UACA,WAAW,QACX,MACA,cACM;GACJ,MAAM,WAAW,KAAK,mBAAmB,KAAK,iBAAiB,IAAI,cAAc;GACjF,IAAI,aAAa,KAAA,GAAW,OAAO,KAAA;GACnC,IAAI,CAAC,OAAO,cAAc,QAAQ,KAAK,YAAY,GACjD,MAAM,IAAI,MAAM,gEAAgE;GAElF,IAAI,WAAW,KAAK,gBAAgB,OAAO,KAAA;GAC3C,eAAe;GACf,OAAO;EACT,IACC,gBAAgB;GACf,IAAI,CAAC,KAAK,SAAS;GACnB,KAAK,QAAQ;IAAE,GAAG;IAAa,UAAU,EAAE;GAAoB,CAAC;EAClE,GACA,uBACF,CAAC,CAAC,cAAc;GACd,WAAW,OAAO,oBAAoB,SAAS,YAAY;GAC3D,kBAAkB,OAAO,UAAU;GACnC,eAAe,OAAO,OAAO;EAC/B,CAAC;EACD,eAAe,IAAI,OAAO;EAC1B,QAAa,YAAY,KAAA,CAAS;CACpC,CAAC;CACD,MAAM,OAAO,MAAM,YAAY,MAAM;CACrC,MAAM,cAA6B;EACjC,iBAAiB,oBAAoB;EACrC,OAAO;CACT;CACA,MAAM,gBAAsB;EAC1B,MAAW,CAAC,CAAC,YAAY,KAAA,CAAS;CACpC;CACA,KAAK,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CAC9D,IAAI,KAAK,QAAQ,SAAS,QAAQ;CAClC,OAAO;EACL,KAAK,oBAAoB,KAAK;EAC9B,OAAO,KAAK;EACZ,mBAAmB;EACnB;CACF;CAEA,eAAe,sBAAqC;EAClD,KAAK,QAAQ,oBAAoB,SAAS,OAAO;EACjD,YAAY;EACZ,MAAM,gBAAgB,YAAY,MAAM;EACxC,OAAO,uBAAuB;EAC9B,KAAK,MAAM,cAAc,mBAAmB,WAAW,MAAM;EAC7D,MAAM,CAAC,gBAAgB,MAAM,QAAQ,WAAW,CAC9C,eACA,sBAAsB,cAAc,CACtC,CAAC;EACD,IAAI,kBAAkB,SAAS,KAAK,eAAe,SAAS,GAC1D,MAAM,IAAI,MAAM,6DAA6D;EAE/E,IAAI,cAAc,WAAW,YAAY,MAAM,aAAa;CAC9D;AACF;AAEA,eAAe,eACb,SACA,UACA,QACA,MAOA,QACA,gBACA,SACA,yBACe;CACf,IAAI;EACF,IAAI,QAAQ,WAAW,UAAU,QAAQ,QAAQ,aAAa;GAC5D,eAAe,UAAU,KAAK,EAAE,OAAO,YAAY,CAAC;GACpD;EACF;EACA,IAAI,QAAQ,QAAQ,kBAAkB,UAAU,KAAK,SAAS;GAC5D,eAAe,UAAU,KAAK,EAAE,OAAO,eAAe,CAAC;GACvD;EACF;EACA,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,SAAS,SAAS,OAAO,eAAe;EACvD,QAAQ;GACN,QAAQ;IAAE,MAAM;IAAW,QAAQ;GAAkB,CAAC;GACtD,eAAe,UAAU,KAAK,EAAE,OAAO,0CAA0C,CAAC;GAClF;EACF;EACA,IACE,CAAC,SAAS,IAAI,KACd,CAAC,wBAAwB,KAAK,SAAS,KACvC,OAAO,KAAK,cAAc,UAC1B;GACA,QAAQ;IAAE,MAAM;IAAW,QAAQ;GAAkB,CAAC;GACtD,eAAe,UAAU,KAAK,EAAE,OAAO,+CAA+C,CAAC;GACvF;EACF;EACA,MAAM,YAAY,eAAe,KAAK,SAAS;EAC/C,MAAM,gBAAgB,YAAY;GAAE,MAAM;GAA2B;EAAU,CAAC;EAChF,IAAI,CAAC,wBAAwB,IAAI,aAAa,GAAG;GAC/C,QAAQ;IAAE,MAAM;IAAY;IAAW;GAAc,CAAC;GACtD,wBAAwB,IAAI,aAAa;EAC3C;EACA,MAAM,QAAQ,eAAe;EAC7B,IAAI,UAAU,KAAA,GAAW;GACvB,QAAQ;IACN,MAAM;IACN,QAAQ;IACR;IACA;IACA,WAAW,KAAK;GAClB,CAAC;GACD,eAAe,UAAU,KAAK,EAAE,OAAO,2BAA2B,CAAC;GACnE;EACF;EACA,IAAI;EACJ,IAAI;GACF,SAAS,MAAM,KAAK,SAAS;IAAE;IAAW,WAAW,KAAK;GAAU,GAAG,MAAM;EAC/E,SAAS,OAAO;GACd,QAAQ;IACN,MAAM;IACN,QAAQ;IACR;IACA;IACA,WAAW,KAAK;GAClB,CAAC;GAGD,eAAe,UAAU,KAAK,EAC5B,OAAO,sBAAsB,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,IACzD,CAAC;GACD;EACF;EACA,IAAI;EACJ,IAAI;GACF,UAAU,mBAAmB,MAAM;EACrC,QAAQ;GACN,QAAQ;IACN,MAAM;IACN,QAAQ;IACR;IACA;IACA,WAAW,KAAK;GAClB,CAAC;GACD,eAAe,UAAU,KAAK,EAAE,OAAO,yCAAyC,CAAC;GACjF;EACF;EACA,IAAI,OAAO,WAAW,OAAO,IAAI,OAAO,kBAAkB;GACxD,QAAQ;IACN,MAAM;IACN,QAAQ;IACR;IACA;IACA,WAAW,KAAK;GAClB,CAAC;GACD,eAAe,UAAU,KAAK,EAAE,OAAO,gCAAgC,CAAC;GACxE;EACF;EACA,QAAQ;GACN,MAAM;GACN;GACA;GACA,WAAW,KAAK;GAChB,kBAAkB;GAClB,UAAU,gBAAgB,MAAM;EAClC,CAAC;EACD,sBAAsB,UAAU,KAAK,OAAO;CAC9C,QAAQ;EACN,eAAe,UAAU,KAAK,EAAE,OAAO,oBAAoB,CAAC;CAC9D;AACF;AAEA,SAAS,SAAS,SAA0B,iBAA2C;CACrF,OAAO,IAAI,SAAS,gBAAgB,WAAW;EAC7C,IAAI,OAAO;EACX,MAAM,SAAmB,CAAC;EAC1B,QAAQ,GAAG,SAAS,UAAkB;GACpC,QAAQ,MAAM;GACd,IAAI,OAAO,iBAAiB;IAC1B,uBAAO,IAAI,MAAM,yBAAyB,CAAC;IAC3C,QAAQ,QAAQ;IAChB;GACF;GACA,OAAO,KAAK,KAAK;EACnB,CAAC;EACD,QAAQ,GAAG,SAAS,MAAM;EAC1B,QAAQ,GAAG,aAAa;GACtB,IAAI;IACF,eAAe,KAAK,MAAM,OAAO,OAAO,MAAM,CAAC,CAAC,SAAS,MAAM,CAAC,CAAC;GACnE,SAAS,OAAO;IACd,OAAO,KAAK;GACd;EACF,CAAC;CACH,CAAC;AACH;AAEA,SAAS,mBAAmB,OAAwB;CAClD,MAAM,UAAU,KAAK,UAAU,KAAK;CACpC,IAAI,YAAY,KAAA,GAAW,MAAM,IAAI,MAAM,+CAA+C;CAC1F,OAAO;AACT;AAEA,SAAS,sBAAsB,UAA0B,QAAgB,SAAuB;CAC9F,IAAI,SAAS,aAAa,SAAS,eAAe;CAClD,SAAS,UAAU,QAAQ;EACzB,gBAAgB;EAChB,kBAAkB,OAAO,OAAO,WAAW,OAAO,CAAC;CACrD,CAAC;CACD,SAAS,IAAI,OAAO;AACtB;AAEA,SAAS,qBAAqB,MAQrB;CACP,IAAI,OAAO,KAAK,UAAU,YAAY,CAAC,KAAK,MAAM,KAAK,GACrD,MAAM,IAAI,MAAM,sDAAsD;CAExE,IAAI,CAAC,OAAO,cAAc,KAAK,cAAc,KAAK,KAAK,kBAAkB,GACvE,MAAM,IAAI,MAAM,6EAA6E;CAE/F,IAAI,KAAK,qBAAqB,KAAA,KAAa,OAAO,KAAK,qBAAqB,YAC1E,MAAM,IAAI,MAAM,kEAAkE;CAEpF,IAAI,OAAO,KAAK,aAAa,YAC3B,MAAM,IAAI,MAAM,0DAA0D;CAE5E,IAAI,KAAK,YAAY,KAAA,KAAa,OAAO,KAAK,YAAY,YACxD,MAAM,IAAI,MAAM,yDAAyD;AAE7E;AAEA,SAAS,eAAe,WAAyD;CAC/E,OAAO,OAAO,cAAc,WAAW,YAAY,EAAE,GAAG,UAAU;AACpE"}
|
package/dist/{external-optimizer-subprocess-CjguXN2e.js → external-optimizer-subprocess-DMgmqTB6.js}
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { s as ValidationError } from "./errors-Dngq5h35.js";
|
|
2
|
-
import { t as canonicalJson } from "./verdict-cache-
|
|
2
|
+
import { t as canonicalJson } from "./verdict-cache-BQEsVy9n.js";
|
|
3
3
|
import { c as costForTokenPricing, i as CostLedger } from "./cost-ledger-_IbkejCF.js";
|
|
4
|
-
import { o as appendLedgerLine, s as tryWithLedgerFileLock } from "./journal-
|
|
4
|
+
import { o as appendLedgerLine, s as tryWithLedgerFileLock } from "./journal-BuRLm6ho.js";
|
|
5
5
|
import "./ledger-core/index.js";
|
|
6
6
|
import { f as ModelSubstitutionError, g as assertServedModelPolicy, h as assertServedModel } from "./llm-client-DDhZ29Vm.js";
|
|
7
7
|
import { createRequire } from "node:module";
|
|
@@ -1916,4 +1916,4 @@ function summarizeProcessOutput(capture, max = 4e3) {
|
|
|
1916
1916
|
//#endregion
|
|
1917
1917
|
export { inMemoryCampaignStorage as C, SearchLedgerIntegrityError as D, SearchLedgerError as E, fsCampaignStorage as S, SearchLedgerConflictError as T, removeCredentialEnvironment as _, closeServer as a, safePathComponent as b, waitForActiveHandlers as c, assertExternalOptimizerModelBudget as d, assertJsonValue as f, isRecord as g, isExternalTextCandidate as h, startExternalOptimizerModelProxy as i, DEFAULT_EXTERNAL_OPTIMIZER_CALLBACK_LIMITS as l, isCandidateText as m, closeExternalOptimizerResources as n, listenLocal as o, assertNoCredentialValues as p, runWithCleanup as r, sendJsonIfOpen as s, runExternalOptimizerProcess as t, DEFAULT_EXTERNAL_OPTIMIZER_PROCESS_LIMITS as u, resolveExternalOptimizerCallbackLimits as v, SEARCH_LEDGER_FILE_CONTEXT as w, createRunCostLedger as x, resolveExternalOptimizerProcessLimits as y };
|
|
1918
1918
|
|
|
1919
|
-
//# sourceMappingURL=external-optimizer-subprocess-
|
|
1919
|
+
//# sourceMappingURL=external-optimizer-subprocess-DMgmqTB6.js.map
|