@tangle-network/agent-eval 0.217.1 → 0.218.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -1
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +1 -1
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +9 -9
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DyeNcenW.d.ts → analyze-runs-DjlZOVXx.d.ts} +2 -2
- package/dist/{analyze-runs-DyeNcenW.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
- package/dist/{benchmark-command-xQK8nOI6.js → benchmark-command-BupdHRyf.js} +661 -562
- package/dist/benchmark-command-BupdHRyf.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +4 -4
- package/dist/campaign/index.js +10 -10
- package/dist/{campaign-CJGkxAQ6.js → campaign-CGvUTnmc.js} +312 -281
- package/dist/{campaign-CJGkxAQ6.js.map → campaign-CGvUTnmc.js.map} +1 -1
- package/dist/{campaign-evidence-CQYw8_0H.js → campaign-evidence-DunzcJGv.js} +2 -2
- package/dist/{campaign-evidence-CQYw8_0H.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
- package/dist/{chat-client-ldqxYYTN.js → chat-client-B2pXlPdx.js} +2 -2
- package/dist/{chat-client-ldqxYYTN.js.map → chat-client-B2pXlPdx.js.map} +1 -1
- package/dist/{chat-json-call-DavzexSp.js → chat-json-call-Dm9c-b41.js} +2 -2
- package/dist/{chat-json-call-DavzexSp.js.map → chat-json-call-Dm9c-b41.js.map} +1 -1
- package/dist/cli.js +10 -10
- package/dist/{completion-verifier-B_jw6C9p.js → completion-verifier-DbLVf3i8.js} +4 -4
- package/dist/{completion-verifier-B_jw6C9p.js.map → completion-verifier-DbLVf3i8.js.map} +1 -1
- package/dist/contract/index.d.ts +7 -7
- package/dist/contract/index.js +90 -10
- package/dist/contract/index.js.map +1 -1
- package/dist/{default-registry-ClYjq3QE.js → default-registry-BJ-Pq6yL.js} +309 -277
- package/dist/{default-registry-ClYjq3QE.js.map → default-registry-BJ-Pq6yL.js.map} +1 -1
- package/dist/{define-agent-eval-BDxNlP3D.js → define-agent-eval-BGdFKlrc.js} +165 -105
- package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
- package/dist/{define-agent-eval-BIkH_gAR.d.ts → define-agent-eval-BlH10zEw.d.ts} +6 -6
- package/dist/{define-agent-eval-BIkH_gAR.d.ts.map → define-agent-eval-BlH10zEw.d.ts.map} +1 -1
- package/dist/diagnosis/index.d.ts +1 -1
- package/dist/{dspy-rlm-engine-DhdCr1Dc.js → dspy-rlm-engine-CSMwudLO.js} +2 -2
- package/dist/{dspy-rlm-engine-DhdCr1Dc.js.map → dspy-rlm-engine-CSMwudLO.js.map} +1 -1
- package/dist/eval-campaign-BUMIxKTE.js +393 -0
- package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
- package/dist/{evaluation-jTjC_QQD.d.ts → evaluation-BOZ38vOH.d.ts} +2 -2
- package/dist/{evaluation-jTjC_QQD.d.ts.map → evaluation-BOZ38vOH.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +110 -71
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-Cl6IeNAu.js → exporters-DWvM16LE.js} +2 -2
- package/dist/{exporters-Cl6IeNAu.js.map → exporters-DWvM16LE.js.map} +1 -1
- package/dist/{external-optimizer-process-KVFGvYX1.js → external-optimizer-process-C3Wm7Vhn.js} +2 -2
- package/dist/{external-optimizer-process-KVFGvYX1.js.map → external-optimizer-process-C3Wm7Vhn.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-XIHBOSKt.js → external-optimizer-subprocess-B5gdLRZv.js} +3 -3
- package/dist/{external-optimizer-subprocess-XIHBOSKt.js.map → external-optimizer-subprocess-B5gdLRZv.js.map} +1 -1
- package/dist/{findings-store-BMVQtEsm.js → findings-store-BtTzPL0d.js} +5 -5
- package/dist/{findings-store-BMVQtEsm.js.map → findings-store-BtTzPL0d.js.map} +1 -1
- package/dist/{hf-dataset-D9f7-oX7.js → hf-dataset-CKxIJQba.js} +3 -3
- package/dist/{hf-dataset-D9f7-oX7.js.map → hf-dataset-CKxIJQba.js.map} +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-ZuWx52WG.d.ts → index-B16FNdW5.d.ts} +34 -17
- package/dist/index-B16FNdW5.d.ts.map +1 -0
- package/dist/{index-9jJ2fKSX.d.ts → index-CalAsXMR.d.ts} +7 -5
- package/dist/index-CalAsXMR.d.ts.map +1 -0
- package/dist/{index-BiT6RtNH.d.ts → index-DbMOXYnP.d.ts} +6 -6
- package/dist/{index-BiT6RtNH.d.ts.map → index-DbMOXYnP.d.ts.map} +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +611 -688
- package/dist/index.js.map +1 -1
- package/dist/{integrity-DOd9Q86O.js → integrity-C8oUeB0g.js} +322 -197
- package/dist/integrity-C8oUeB0g.js.map +1 -0
- package/dist/{journal-CR4QikYp.js → journal-C7om8Gum.js} +48 -38
- package/dist/journal-C7om8Gum.js.map +1 -0
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/ledger-core/index.js +2 -2
- package/dist/ledger-core/lock-holder.d.ts +2 -0
- package/dist/ledger-core/lock-holder.js +181 -0
- package/dist/ledger-core/lock-holder.js.map +1 -0
- package/dist/{ledger-core-DrJhbms9.js → ledger-core-CAxUdDk0.js} +2 -2
- package/dist/{ledger-core-DrJhbms9.js.map → ledger-core-CAxUdDk0.js.map} +1 -1
- package/dist/{llm-client-BshCREn6.js → llm-client-_Y6ic-tf.js} +238 -248
- package/dist/llm-client-_Y6ic-tf.js.map +1 -0
- package/dist/{llm-judge-C8jMTNNG.js → llm-judge-Djxh-Uhs.js} +646 -458
- package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
- package/dist/lock-holder-DaJbfpci.d.ts +86 -0
- package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
- package/dist/{matrix-DHe-dwVJ.d.ts → matrix-CCSBna4q.d.ts} +2 -2
- package/dist/{matrix-DHe-dwVJ.d.ts.map → matrix-CCSBna4q.d.ts.map} +1 -1
- package/dist/{mint-zszyYlpA.js → mint-CbdwjTPW.js} +2 -2
- package/dist/{mint-zszyYlpA.js.map → mint-CbdwjTPW.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{pareto-BzsxidFD.js → pareto-CTv8vozZ.js} +4 -4
- package/dist/{pareto-BzsxidFD.js.map → pareto-CTv8vozZ.js.map} +1 -1
- package/dist/{promotion-gate-DaY95yU6.d.ts → promotion-gate-CxEHRsZW.d.ts} +2 -2
- package/dist/{promotion-gate-DaY95yU6.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
- package/dist/{proposal-findings-BbWidvUk.js → proposal-findings-n-5WBbEx.js} +2 -2
- package/dist/{proposal-findings-BbWidvUk.js.map → proposal-findings-n-5WBbEx.js.map} +1 -1
- package/dist/{render-C-w2EZ6p.js → render-Cs34y0iX.js} +404 -208
- package/dist/render-Cs34y0iX.js.map +1 -0
- package/dist/reporting.d.ts +2 -2
- package/dist/{researcher-0ABVIdGa.d.ts → researcher-GU1tV4dn.d.ts} +2 -2
- package/dist/{researcher-0ABVIdGa.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
- package/dist/rl.d.ts +2 -2
- package/dist/rl.js +4 -4
- package/dist/rollout/index.js +5 -5
- package/dist/{rollout-D6C50mfA.js → rollout-Dd1ucT_M.js} +5 -5
- package/dist/{rollout-D6C50mfA.js.map → rollout-Dd1ucT_M.js.map} +1 -1
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-DxcKTzbi.js → schema-CvhfLjtZ.js} +77 -47
- package/dist/schema-CvhfLjtZ.js.map +1 -0
- package/dist/search/index.d.ts +2 -2
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +2 -2
- package/dist/{search-kernel-DxqjBxA-.d.ts → search-kernel-p9JB3SmL.d.ts} +3 -3
- package/dist/{search-kernel-DxqjBxA-.d.ts.map → search-kernel-p9JB3SmL.d.ts.map} +1 -1
- package/dist/{search-ledger-TUvB5gWH.js → search-ledger-ClpN4I3e.js} +4 -4
- package/dist/{search-ledger-TUvB5gWH.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
- package/dist/{search-ledger-file-UKCO2r8a.js → search-ledger-file-Bd5Tioq6.js} +3 -3
- package/dist/{search-ledger-file-UKCO2r8a.js.map → search-ledger-file-Bd5Tioq6.js.map} +1 -1
- package/dist/{search-shipper-Cw2z3-td.js → search-shipper-Bg2HW3Uv.js} +3 -3
- package/dist/{search-shipper-Cw2z3-td.js.map → search-shipper-Bg2HW3Uv.js.map} +1 -1
- package/dist/{search-summary-DQIvJsSB.js → search-summary-CrBuvVnf.js} +3 -3
- package/dist/{search-summary-DQIvJsSB.js.map → search-summary-CrBuvVnf.js.map} +1 -1
- package/dist/{server-DoVbqqtJ.js → server-C6M0qR8h.js} +2 -2
- package/dist/{server-DoVbqqtJ.js.map → server-C6M0qR8h.js.map} +1 -1
- package/dist/{single-run-lock-U58yf8N6.js → single-run-lock-Dekd8_Wv.js} +7 -7
- package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
- package/dist/{skillopt-optimization-method-kyL94-ri.js → skillopt-optimization-method-5dlex7MJ.js} +1339 -1059
- package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
- package/dist/{statistical-heldout-C9AbgREN.d.ts → statistical-heldout-BHfLZHrm.d.ts} +2 -2
- package/dist/{statistical-heldout-C9AbgREN.d.ts.map → statistical-heldout-BHfLZHrm.d.ts.map} +1 -1
- package/dist/{summary-report-DM7tlml1.d.ts → summary-report-CkbRcSuu.d.ts} +16 -1
- package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +3 -3
- package/dist/{systemone-Bp_Ac0p1.d.ts → systemone-BMzDJb1i.d.ts} +3 -3
- package/dist/{systemone-Bp_Ac0p1.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
- package/dist/systemone.d.ts +2 -2
- package/dist/{terminal-record-78g6E4vh.js → terminal-record-D4Rv-j1W.js} +241 -193
- package/dist/{terminal-record-78g6E4vh.js.map → terminal-record-D4Rv-j1W.js.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +4 -1
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +834 -824
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-BGRglkKd.js → tree-B5e1uAeL.js} +122 -14
- package/dist/tree-B5e1uAeL.js.map +1 -0
- package/dist/{types-DtMbFw6U.d.ts → types-BKCzzmit.d.ts} +2 -2
- package/dist/{types-DtMbFw6U.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
- package/dist/{types-D8GhJoHk.d.ts → types-DtkO1G9L.d.ts} +2 -2
- package/dist/{types-D8GhJoHk.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/design/statistics-decisions.md +1 -1
- package/docs/eval-surface-map.md +1 -0
- package/docs/profile-json-identity.md +1 -2
- package/docs/public-api.md +27 -16
- package/package.json +7 -2
- package/dist/benchmark-command-xQK8nOI6.js.map +0 -1
- package/dist/define-agent-eval-BDxNlP3D.js.map +0 -1
- package/dist/eval-campaign-BLEfBGbA.js +0 -343
- package/dist/eval-campaign-BLEfBGbA.js.map +0 -1
- package/dist/index-9jJ2fKSX.d.ts.map +0 -1
- package/dist/index-ZuWx52WG.d.ts.map +0 -1
- package/dist/integrity-DOd9Q86O.js.map +0 -1
- package/dist/journal-CR4QikYp.js.map +0 -1
- package/dist/llm-client-BshCREn6.js.map +0 -1
- package/dist/llm-judge-C8jMTNNG.js.map +0 -1
- package/dist/render-C-w2EZ6p.js.map +0 -1
- package/dist/schema-DxcKTzbi.js.map +0 -1
- package/dist/single-run-lock-U58yf8N6.js.map +0 -1
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +0 -1
- package/dist/summary-report-DM7tlml1.d.ts.map +0 -1
- package/dist/tree-BGRglkKd.js.map +0 -1
|
@@ -588,269 +588,259 @@ function costReceiptFromLlmError(error, customTokenPricing) {
|
|
|
588
588
|
async function callLlm(req, opts = {}) {
|
|
589
589
|
return callLlmAttempts(req, opts, false);
|
|
590
590
|
}
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
async function callLlmAttempts(req, opts, allowSchemaFallback) {
|
|
591
|
+
const ENDPOINT = "/chat/completions";
|
|
592
|
+
function startAttemptRun(req, opts, allowSchemaFallback) {
|
|
594
593
|
if (!opts.baseUrl) throw new Error("callLlm: opts.baseUrl is required");
|
|
595
594
|
const baseUrl = opts.baseUrl.replace(/\/+$/, "");
|
|
596
|
-
const url = `${baseUrl}/chat/completions`;
|
|
597
|
-
const endpoint = "/chat/completions";
|
|
598
|
-
const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
599
|
-
const maximumAttempts = resolveMaximumAttempts(opts.maximumAttempts);
|
|
600
|
-
const minIntervalMs = resolveMinIntervalMs(opts.minIntervalMs);
|
|
601
|
-
const pacing = pacingStateFor(opts);
|
|
602
|
-
const fetchFn = opts.fetch ?? globalThis.fetch;
|
|
603
|
-
const headers = buildHeaders(opts);
|
|
604
|
-
const provider = opts.provider ?? providerFromBaseUrl(baseUrl);
|
|
605
|
-
const sink = opts.rawSink;
|
|
606
|
-
const redactor = opts.redactor ?? defaultProviderRedactor;
|
|
607
|
-
const traceContext = opts.traceContext;
|
|
608
|
-
const callerSignal = opts.signal;
|
|
609
595
|
const deadlineMs = opts.deadlineMs;
|
|
610
596
|
const deadlineStart = Date.now();
|
|
597
|
+
const remainingMs = () => deadlineMs === void 0 ? Infinity : Math.max(0, deadlineMs - (Date.now() - deadlineStart));
|
|
598
|
+
const run = {
|
|
599
|
+
req,
|
|
600
|
+
opts,
|
|
601
|
+
allowSchemaFallback,
|
|
602
|
+
baseUrl,
|
|
603
|
+
url: `${baseUrl}${ENDPOINT}`,
|
|
604
|
+
timeoutMs: req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
605
|
+
maximumAttempts: resolveMaximumAttempts(opts.maximumAttempts),
|
|
606
|
+
minIntervalMs: resolveMinIntervalMs(opts.minIntervalMs),
|
|
607
|
+
pacing: pacingStateFor(opts),
|
|
608
|
+
headers: buildHeaders(opts),
|
|
609
|
+
provider: opts.provider ?? providerFromBaseUrl(baseUrl),
|
|
610
|
+
deadlineStart,
|
|
611
|
+
remainingMs,
|
|
612
|
+
retryDelay: (ms) => sleep(Math.min(ms, remainingMs()), opts.signal),
|
|
613
|
+
effectiveRequest: req,
|
|
614
|
+
lastErr: void 0
|
|
615
|
+
};
|
|
611
616
|
if (opts.customTokenPricing) costForTokenPricing(opts.customTokenPricing, {
|
|
612
617
|
inputTokens: 0,
|
|
613
618
|
outputTokens: 0
|
|
614
619
|
});
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
620
|
+
return run;
|
|
621
|
+
}
|
|
622
|
+
function deadlineError(run) {
|
|
623
|
+
return run.lastErr instanceof Error ? run.lastErr : new DOMException("callLlm deadline exceeded", "TimeoutError");
|
|
624
|
+
}
|
|
625
|
+
/** Write one raw provider event for this attempt. Does nothing without a sink. */
|
|
626
|
+
async function recordAttemptEvent(run, attempt, fields) {
|
|
627
|
+
const { opts } = run;
|
|
628
|
+
if (!opts.rawSink) return;
|
|
629
|
+
await recordRaw(opts.rawSink, opts.redactor ?? defaultProviderRedactor, {
|
|
630
|
+
eventId: newRecordId(),
|
|
631
|
+
runId: opts.traceContext?.runId,
|
|
632
|
+
spanId: opts.traceContext?.spanId,
|
|
633
|
+
provider: run.provider,
|
|
634
|
+
model: run.req.model,
|
|
635
|
+
endpoint: ENDPOINT,
|
|
636
|
+
baseUrl: run.baseUrl,
|
|
637
|
+
attemptIndex: attempt,
|
|
638
|
+
...fields,
|
|
639
|
+
redactedFields: []
|
|
640
|
+
});
|
|
641
|
+
}
|
|
642
|
+
/**
|
|
643
|
+
* Wait for this client's pacing slot, and claim it. Every wake rechecks, so concurrent waiters
|
|
644
|
+
* cannot dispatch a catch-up burst; the claim is made in the same synchronous step that finds the
|
|
645
|
+
* slot free, so no other waiter can slip in between the check and the claim.
|
|
646
|
+
*/
|
|
647
|
+
async function awaitPacingSlot(run) {
|
|
648
|
+
const { opts, pacing, minIntervalMs } = run;
|
|
649
|
+
while (true) {
|
|
650
|
+
opts.signal?.throwIfAborted();
|
|
651
|
+
if (deadlineExceeded(run.deadlineStart, opts.deadlineMs)) throw deadlineError(run);
|
|
652
|
+
const waitMs = pacing.lastStartedAt === void 0 ? 0 : pacing.lastStartedAt + minIntervalMs - Date.now();
|
|
653
|
+
if (waitMs <= 0) {
|
|
654
|
+
const started = Date.now();
|
|
640
655
|
pacing.lastStartedAt = started;
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
656
|
+
return started;
|
|
657
|
+
}
|
|
658
|
+
await run.retryDelay(waitMs);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
/** Answer a non-2xx response: retry with an adjusted request or after a wait, give up, or throw its error. */
|
|
662
|
+
async function handleHttpFailure(run, attempt, res, body, responseHeaders, started, state) {
|
|
663
|
+
const { req, opts, maximumAttempts, deadlineStart } = run;
|
|
664
|
+
await recordAttemptEvent(run, attempt, {
|
|
665
|
+
direction: "error",
|
|
666
|
+
timestamp: Date.now(),
|
|
667
|
+
durationMs: Date.now() - started,
|
|
668
|
+
statusCode: res.status,
|
|
669
|
+
responseHeaders,
|
|
670
|
+
responseBody: body,
|
|
671
|
+
errorMessage: `HTTP ${res.status}`
|
|
672
|
+
});
|
|
673
|
+
if (opts.rawSink) state.errorRecorded = true;
|
|
674
|
+
const err = new LlmCallError(`LLM call failed with HTTP ${res.status}`, res.status, body, req.model);
|
|
675
|
+
const canRetry = attempt < maximumAttempts - 1 && !deadlineExceeded(deadlineStart, opts.deadlineMs);
|
|
676
|
+
if (isTemperatureOneRejection(res.status, body) && run.effectiveRequest.temperature !== 1 && canRetry) {
|
|
677
|
+
run.lastErr = err;
|
|
678
|
+
run.effectiveRequest = {
|
|
679
|
+
...run.effectiveRequest,
|
|
680
|
+
temperature: 1
|
|
681
|
+
};
|
|
682
|
+
return { kind: "retry" };
|
|
683
|
+
}
|
|
684
|
+
if (run.allowSchemaFallback && opts.jsonSchemaTransport !== "json-object" && run.effectiveRequest.jsonSchema && isSchemaRejection(res.status, body) && canRetry) {
|
|
685
|
+
run.lastErr = err;
|
|
686
|
+
run.effectiveRequest = {
|
|
687
|
+
...run.effectiveRequest,
|
|
688
|
+
jsonMode: true,
|
|
689
|
+
jsonSchema: void 0
|
|
690
|
+
};
|
|
691
|
+
return { kind: "retry" };
|
|
692
|
+
}
|
|
693
|
+
if (RETRYABLE_STATUS.has(res.status) && canRetry) {
|
|
694
|
+
run.lastErr = err;
|
|
695
|
+
const wait = parseRetryAfter(res.headers) ?? backoffMs(attempt);
|
|
696
|
+
if (wait >= run.remainingMs()) {
|
|
697
|
+
opts.signal?.throwIfAborted();
|
|
698
|
+
return { kind: "exhausted" };
|
|
699
|
+
}
|
|
700
|
+
await run.retryDelay(wait);
|
|
701
|
+
return { kind: "retry" };
|
|
702
|
+
}
|
|
703
|
+
throw err;
|
|
704
|
+
}
|
|
705
|
+
/** Turn a parsed completion into the call result: content, tool calls, usage with its completeness, and cost. */
|
|
706
|
+
function parseCompletion(run, json, started) {
|
|
707
|
+
const { req, opts } = run;
|
|
708
|
+
const choice = json.choices?.[0];
|
|
709
|
+
const toolCalls = parseWireToolCalls(choice?.message?.tool_calls, req.model);
|
|
710
|
+
const usageRaw = json.usage && typeof json.usage === "object" && !Array.isArray(json.usage) ? json.usage : void 0;
|
|
711
|
+
const promptTokens = providerTokenCount(usageRaw?.prompt_tokens);
|
|
712
|
+
const completionTokens = providerTokenCount(usageRaw?.completion_tokens);
|
|
713
|
+
const totalTokens = providerTokenCount(usageRaw?.total_tokens);
|
|
714
|
+
const reasoningRaw = (usageRaw?.completion_tokens_details && typeof usageRaw.completion_tokens_details === "object" && !Array.isArray(usageRaw.completion_tokens_details) ? usageRaw.completion_tokens_details : void 0)?.reasoning_tokens;
|
|
715
|
+
const reasoningTokens = reasoningRaw === void 0 ? void 0 : providerTokenCount(reasoningRaw);
|
|
716
|
+
const cachedRaw = usageRaw?.prompt_tokens_details && typeof usageRaw.prompt_tokens_details === "object" && !Array.isArray(usageRaw.prompt_tokens_details) ? usageRaw.prompt_tokens_details.cached_tokens : void 0;
|
|
717
|
+
const cachedPromptTokens = cachedRaw === void 0 ? void 0 : providerTokenCount(cachedRaw);
|
|
718
|
+
const usageCaptured = promptTokens !== void 0 && completionTokens !== void 0 && (reasoningRaw === void 0 || reasoningTokens !== void 0 && reasoningTokens <= completionTokens) && (cachedRaw === void 0 || cachedPromptTokens !== void 0 && cachedPromptTokens <= promptTokens) && (totalTokens === void 0 || totalTokens === promptTokens + completionTokens);
|
|
719
|
+
const costFromProxy = providerReportedCost(json);
|
|
720
|
+
const content = choice?.message?.content ?? "";
|
|
721
|
+
const configuredCost = typeof costFromProxy !== "number" && usageCaptured && opts.customTokenPricing ? costForTokenPricing(opts.customTokenPricing, {
|
|
722
|
+
inputTokens: promptTokens - (cachedPromptTokens ?? 0),
|
|
723
|
+
...cachedPromptTokens ? { cachedTokens: cachedPromptTokens } : {},
|
|
724
|
+
outputTokens: completionTokens
|
|
725
|
+
}) : void 0;
|
|
726
|
+
const servedModel = typeof json.model === "string" && json.model.trim() !== "" ? json.model : null;
|
|
727
|
+
if (opts.assertServedModel) assertServedModel(req.model, servedModel, opts.assertServedModel === true ? {} : opts.assertServedModel);
|
|
728
|
+
return {
|
|
729
|
+
content,
|
|
730
|
+
...toolCalls === void 0 ? {} : { toolCalls },
|
|
731
|
+
...req.logprobs === void 0 ? {} : { logprobs: parseWireLogprobs(choice?.logprobs?.content, req.model) },
|
|
732
|
+
finishReason: choice?.finish_reason ?? null,
|
|
733
|
+
contentEmpty: content.trim().length === 0,
|
|
734
|
+
usage: {
|
|
735
|
+
promptTokens: promptTokens ?? 0,
|
|
736
|
+
completionTokens: completionTokens ?? 0,
|
|
737
|
+
totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0),
|
|
738
|
+
captured: usageCaptured,
|
|
739
|
+
reasoningTokens,
|
|
740
|
+
cachedPromptTokens
|
|
741
|
+
},
|
|
742
|
+
costUsd: typeof costFromProxy === "number" ? costFromProxy : configuredCost ?? null,
|
|
743
|
+
model: servedModel ?? req.model,
|
|
744
|
+
servedModel,
|
|
745
|
+
durationMs: Date.now() - started,
|
|
746
|
+
raw: json
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
/** One attempt: pace, dispatch, read the answer. A failure either becomes a retry outcome or is thrown. */
|
|
750
|
+
async function runAttempt(run, attempt) {
|
|
751
|
+
const { opts } = run;
|
|
752
|
+
const callerSignal = opts.signal;
|
|
753
|
+
if (callerSignal?.aborted) throw new DOMException("callLlm aborted by caller signal", "AbortError");
|
|
754
|
+
if (deadlineExceeded(run.deadlineStart, opts.deadlineMs)) throw deadlineError(run);
|
|
755
|
+
const requestBody = buildBody(run.effectiveRequest, opts.jsonSchemaTransport === "json-object", opts.thinking);
|
|
756
|
+
const controller = new AbortController();
|
|
757
|
+
const attemptSignal = combineAbortSignals(controller.signal, callerSignal);
|
|
758
|
+
let timeoutHandle;
|
|
759
|
+
let started = Date.now();
|
|
760
|
+
const state = {
|
|
761
|
+
dispatched: false,
|
|
762
|
+
errorRecorded: false
|
|
763
|
+
};
|
|
764
|
+
try {
|
|
765
|
+
if (run.minIntervalMs > 0) started = await awaitPacingSlot(run);
|
|
766
|
+
else run.pacing.lastStartedAt = started;
|
|
767
|
+
state.dispatched = true;
|
|
768
|
+
timeoutHandle = setTimeout(() => controller.abort(), Math.min(run.timeoutMs, run.remainingMs()));
|
|
769
|
+
await recordAttemptEvent(run, attempt, {
|
|
770
|
+
direction: "request",
|
|
771
|
+
timestamp: started,
|
|
772
|
+
requestHeaders: run.headers,
|
|
773
|
+
requestBody
|
|
774
|
+
});
|
|
775
|
+
attemptSignal.throwIfAborted();
|
|
776
|
+
const res = await (opts.fetch ?? globalThis.fetch)(run.url, {
|
|
777
|
+
method: "POST",
|
|
778
|
+
headers: run.headers,
|
|
779
|
+
body: JSON.stringify(requestBody),
|
|
780
|
+
signal: attemptSignal
|
|
781
|
+
});
|
|
782
|
+
const responseHeaders = opts.rawSink ? headersToObject(res.headers) : void 0;
|
|
783
|
+
const text = await res.text();
|
|
784
|
+
attemptSignal.throwIfAborted();
|
|
785
|
+
clearTimeout(timeoutHandle);
|
|
786
|
+
if (!res.ok) return await handleHttpFailure(run, attempt, res, text, responseHeaders, started, state);
|
|
787
|
+
let json;
|
|
788
|
+
try {
|
|
789
|
+
json = JSON.parse(text);
|
|
790
|
+
} catch (parseErr) {
|
|
791
|
+
await recordAttemptEvent(run, attempt, {
|
|
792
|
+
direction: "error",
|
|
759
793
|
timestamp: Date.now(),
|
|
760
794
|
durationMs: Date.now() - started,
|
|
761
795
|
statusCode: res.status,
|
|
762
796
|
responseHeaders,
|
|
763
|
-
responseBody:
|
|
764
|
-
|
|
797
|
+
responseBody: text,
|
|
798
|
+
errorMessage: `non-JSON response: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`
|
|
765
799
|
});
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0),
|
|
796
|
-
captured: usageCaptured,
|
|
797
|
-
reasoningTokens,
|
|
798
|
-
cachedPromptTokens
|
|
799
|
-
},
|
|
800
|
-
costUsd: typeof costFromProxy === "number" ? costFromProxy : configuredCost ?? null,
|
|
801
|
-
model: servedModel ?? req.model,
|
|
802
|
-
servedModel,
|
|
803
|
-
durationMs: Date.now() - started,
|
|
804
|
-
raw: json
|
|
805
|
-
};
|
|
806
|
-
} catch (err) {
|
|
807
|
-
clearTimeout(timeoutHandle);
|
|
808
|
-
lastErr = err;
|
|
809
|
-
if (callerSignal?.aborted) {
|
|
810
|
-
if (sink && dispatched && !attemptErrorRecorded) await recordRaw(sink, redactor, {
|
|
811
|
-
eventId: newRecordId(),
|
|
812
|
-
runId: traceContext?.runId,
|
|
813
|
-
spanId: traceContext?.spanId,
|
|
814
|
-
provider,
|
|
815
|
-
model: req.model,
|
|
816
|
-
endpoint,
|
|
817
|
-
baseUrl,
|
|
818
|
-
attemptIndex: attempt,
|
|
819
|
-
direction: "error",
|
|
820
|
-
timestamp: Date.now(),
|
|
821
|
-
durationMs: Date.now() - started,
|
|
822
|
-
errorMessage: err instanceof Error ? err.message : String(err),
|
|
823
|
-
redactedFields: []
|
|
824
|
-
});
|
|
825
|
-
throw err;
|
|
826
|
-
}
|
|
827
|
-
if (sink && dispatched && !attemptErrorRecorded) await recordRaw(sink, redactor, {
|
|
828
|
-
eventId: newRecordId(),
|
|
829
|
-
runId: traceContext?.runId,
|
|
830
|
-
spanId: traceContext?.spanId,
|
|
831
|
-
provider,
|
|
832
|
-
model: req.model,
|
|
833
|
-
endpoint,
|
|
834
|
-
baseUrl,
|
|
835
|
-
attemptIndex: attempt,
|
|
836
|
-
direction: "error",
|
|
837
|
-
timestamp: Date.now(),
|
|
838
|
-
durationMs: Date.now() - started,
|
|
839
|
-
errorMessage: err instanceof Error ? err.message : String(err),
|
|
840
|
-
redactedFields: []
|
|
841
|
-
});
|
|
842
|
-
const wait = backoffMs(attempt);
|
|
843
|
-
if (attempt < maximumAttempts - 1 && isTransientLlmError(err) && !deadlineExceeded(deadlineStart, deadlineMs) && wait < remainingMs()) {
|
|
844
|
-
await retryDelay(wait);
|
|
845
|
-
continue;
|
|
846
|
-
}
|
|
847
|
-
callerSignal?.throwIfAborted();
|
|
848
|
-
throw err;
|
|
849
|
-
} finally {
|
|
850
|
-
clearTimeout(timeoutHandle);
|
|
800
|
+
if (opts.rawSink) state.errorRecorded = true;
|
|
801
|
+
throw parseErr;
|
|
802
|
+
}
|
|
803
|
+
await recordAttemptEvent(run, attempt, {
|
|
804
|
+
direction: "response",
|
|
805
|
+
timestamp: Date.now(),
|
|
806
|
+
durationMs: Date.now() - started,
|
|
807
|
+
statusCode: res.status,
|
|
808
|
+
responseHeaders,
|
|
809
|
+
responseBody: json
|
|
810
|
+
});
|
|
811
|
+
return {
|
|
812
|
+
kind: "result",
|
|
813
|
+
result: parseCompletion(run, json, started)
|
|
814
|
+
};
|
|
815
|
+
} catch (err) {
|
|
816
|
+
clearTimeout(timeoutHandle);
|
|
817
|
+
run.lastErr = err;
|
|
818
|
+
if (state.dispatched && !state.errorRecorded) await recordAttemptEvent(run, attempt, {
|
|
819
|
+
direction: "error",
|
|
820
|
+
timestamp: Date.now(),
|
|
821
|
+
durationMs: Date.now() - started,
|
|
822
|
+
errorMessage: err instanceof Error ? err.message : String(err)
|
|
823
|
+
});
|
|
824
|
+
if (callerSignal?.aborted) throw err;
|
|
825
|
+
const wait = backoffMs(attempt);
|
|
826
|
+
if (attempt < run.maximumAttempts - 1 && isTransientLlmError(err) && !deadlineExceeded(run.deadlineStart, opts.deadlineMs) && wait < run.remainingMs()) {
|
|
827
|
+
await run.retryDelay(wait);
|
|
828
|
+
return { kind: "retry" };
|
|
851
829
|
}
|
|
830
|
+
callerSignal?.throwIfAborted();
|
|
831
|
+
throw err;
|
|
832
|
+
} finally {
|
|
833
|
+
clearTimeout(timeoutHandle);
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
async function callLlmAttempts(req, opts, allowSchemaFallback) {
|
|
837
|
+
const run = startAttemptRun(req, opts, allowSchemaFallback);
|
|
838
|
+
for (let attempt = 0; attempt < run.maximumAttempts; attempt++) {
|
|
839
|
+
const outcome = await runAttempt(run, attempt);
|
|
840
|
+
if (outcome.kind === "result") return outcome.result;
|
|
841
|
+
if (outcome.kind === "exhausted") break;
|
|
852
842
|
}
|
|
853
|
-
throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
|
|
843
|
+
throw run.lastErr instanceof Error ? run.lastErr : new Error(String(run.lastErr));
|
|
854
844
|
}
|
|
855
845
|
async function recordRaw(sink, redactor, event) {
|
|
856
846
|
try {
|
|
@@ -918,4 +908,4 @@ var LlmClient = class {
|
|
|
918
908
|
//#endregion
|
|
919
909
|
export { combineAbortSignals as _, maximumChargeForLlmRequest as a, ModelSubstitutionError as c, assertServedModelPolicy as d, assertServedModels as f, judgeFamily as g, assertCrossFamily as h, costReceiptFromLlmError as i, assertCrossFamilyServed as l, CrossFamilyError as m, callLlmJson as n, backoffMs as o, checkServedModel as p, costReceiptFromLlm as r, isTransientLlmError as s, LlmClient as t, assertServedModel as u };
|
|
920
910
|
|
|
921
|
-
//# sourceMappingURL=llm-client-
|
|
911
|
+
//# sourceMappingURL=llm-client-_Y6ic-tf.js.map
|