@tangle-network/agent-eval 0.137.0 → 0.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +78 -0
- package/README.md +34 -0
- package/dist/analyst/index.d.ts +485 -104
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +10 -607
- package/dist/analyst/index.js.map +1 -1
- package/dist/{benchmark-YDrpumqB.js → benchmark-CYtcIF2V.js} +299 -159
- package/dist/benchmark-CYtcIF2V.js.map +1 -0
- package/dist/{benchmark-CHX4orG7.d.ts → benchmark-DDVdWcwA.d.ts} +67 -15
- package/dist/benchmark-DDVdWcwA.d.ts.map +1 -0
- package/dist/benchmark-command-BKfjOBJ5.js +4537 -0
- package/dist/benchmark-command-BKfjOBJ5.js.map +1 -0
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-DCLkQOmc.js → benchmarks-zxhy1QV3.js} +4 -3
- package/dist/{benchmarks-DCLkQOmc.js.map → benchmarks-zxhy1QV3.js.map} +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +4 -3
- package/dist/{campaign-lgObcHFC.js → campaign-DrS6_hLd.js} +19 -11
- package/dist/campaign-DrS6_hLd.js.map +1 -0
- package/dist/canonical-D011XM8r.js +86 -0
- package/dist/canonical-D011XM8r.js.map +1 -0
- package/dist/cli.js +10 -3
- package/dist/cli.js.map +1 -1
- package/dist/{client-C8L6h6Wf.d.ts → client-BohnDFBq.d.ts} +4 -4
- package/dist/{client-C8L6h6Wf.d.ts.map → client-BohnDFBq.d.ts.map} +1 -1
- package/dist/{completion-verifier-DSyRNVzU.d.ts → completion-verifier-IPoP4fQO.d.ts} +178 -4
- package/dist/completion-verifier-IPoP4fQO.d.ts.map +1 -0
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +9 -8
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/{cost-ledger-D-5_-dhi.js → cost-ledger-CZ9diLxY.js} +90 -45
- package/dist/cost-ledger-CZ9diLxY.js.map +1 -0
- package/dist/{cost-ledger-D2o6JOrL.d.ts → cost-ledger-DKgyIWRj.d.ts} +6 -2
- package/dist/cost-ledger-DKgyIWRj.d.ts.map +1 -0
- package/dist/default-registry-B8vf7Rmf.d.ts +118 -0
- package/dist/default-registry-B8vf7Rmf.d.ts.map +1 -0
- package/dist/default-registry-BgJJItGr.js +2364 -0
- package/dist/default-registry-BgJJItGr.js.map +1 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js +344 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js.map +1 -0
- package/dist/{eval-campaign-CHqfLnff.js → eval-campaign-BmptJj50.js} +2 -2
- package/dist/{eval-campaign-CHqfLnff.js.map → eval-campaign-BmptJj50.js.map} +1 -1
- package/dist/exact-types-MaaFcllV.d.ts +234 -0
- package/dist/exact-types-MaaFcllV.d.ts.map +1 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts +32 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts.map +1 -0
- package/dist/{extract-usage-p-56bh8q.js → extract-usage-DZs601Va.js} +2 -2
- package/dist/{extract-usage-p-56bh8q.js.map → extract-usage-DZs601Va.js.map} +1 -1
- package/dist/{feedback-trajectory-N_F0PwHz.d.ts → feedback-trajectory-BJUWOkJM.d.ts} +3 -2
- package/dist/feedback-trajectory-BJUWOkJM.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/{hf-dataset-DBJXXoY1.js → hf-dataset-XggBupCr.js} +2 -2
- package/dist/{hf-dataset-DBJXXoY1.js.map → hf-dataset-XggBupCr.js.map} +1 -1
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-C-Pr4OWg.d.ts → index-BTm_P9aC.d.ts} +12 -11
- package/dist/index-BTm_P9aC.d.ts.map +1 -0
- package/dist/{index-U3RHOShi.d.ts → index-CWOPCJiw.d.ts} +2 -2
- package/dist/{index-U3RHOShi.d.ts.map → index-CWOPCJiw.d.ts.map} +1 -1
- package/dist/{index-DRNl6g_N.d.ts → index-CtR1xh4V.d.ts} +3 -3
- package/dist/{index-DRNl6g_N.d.ts.map → index-CtR1xh4V.d.ts.map} +1 -1
- package/dist/index-DEb46kc6.d.ts.map +1 -1
- package/dist/{index-BnP1QJUv.d.ts → index-_66rVpwN.d.ts} +5 -5
- package/dist/{index-BnP1QJUv.d.ts.map → index-_66rVpwN.d.ts.map} +1 -1
- package/dist/index.d.ts +35 -55
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +55 -514
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-B9ooYH_g.d.ts → insight-report-Bu5Wi9tG.d.ts} +4 -4
- package/dist/{insight-report-B9ooYH_g.d.ts.map → insight-report-Bu5Wi9tG.d.ts.map} +1 -1
- package/dist/{integrity-CKxosZ5Z.d.ts → integrity-COTh3DTH.d.ts} +2 -2
- package/dist/{integrity-CKxosZ5Z.d.ts.map → integrity-COTh3DTH.d.ts.map} +1 -1
- package/dist/kind-factory-CFxA0JQX.js +2133 -0
- package/dist/kind-factory-CFxA0JQX.js.map +1 -0
- package/dist/ledger-core/index.js +2 -1
- package/dist/{ledger-core-t6sItivm.js → ledger-core-Dxz0Rkwa.js} +210 -99
- package/dist/ledger-core-Dxz0Rkwa.js.map +1 -0
- package/dist/{llm-client-DKB25jV8.js → llm-client-bkztEfIx.js} +5 -5
- package/dist/llm-client-bkztEfIx.js.map +1 -0
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{proposal-findings-DCawte-y.js → proposal-findings-2GIUo1et.js} +2 -68
- package/dist/proposal-findings-2GIUo1et.js.map +1 -0
- package/dist/{release-report-CofgVNZt.d.ts → release-report-fZarvIm-.d.ts} +3 -3
- package/dist/{release-report-CofgVNZt.d.ts.map → release-report-fZarvIm-.d.ts.map} +1 -1
- package/dist/{replay-K8FaC0CB.d.ts → replay-DjG4IG60.d.ts} +34 -143
- package/dist/replay-DjG4IG60.d.ts.map +1 -0
- package/dist/{replay-Bju0T8Ls.js → replay-SA4OB7O7.js} +48 -136
- package/dist/replay-SA4OB7O7.js.map +1 -0
- package/dist/reporting.d.ts +4 -4
- package/dist/{researcher-Da0Wj-bt.d.ts → researcher-BxhtGfKa.d.ts} +5 -5
- package/dist/{researcher-Da0Wj-bt.d.ts.map → researcher-BxhtGfKa.d.ts.map} +1 -1
- package/dist/{reward-hacking-CQ3hTCO3.d.ts → reward-hacking-CqSLiV51.d.ts} +2 -2
- package/dist/{reward-hacking-CQ3hTCO3.d.ts.map → reward-hacking-CqSLiV51.d.ts.map} +1 -1
- package/dist/rl.d.ts +5 -5
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-DQFl0UXA.js → rollout-8nj3mYvx.js} +2 -2
- package/dist/{rollout-DQFl0UXA.js.map → rollout-8nj3mYvx.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C4sztLR3.d.ts → rubric-predictive-validity-DQBQj6uV.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C4sztLR3.d.ts.map → rubric-predictive-validity-DQBQj6uV.d.ts.map} +1 -1
- package/dist/{run-evidence-BDIircdA.d.ts → run-evidence-C4RcRQT5.d.ts} +3 -3
- package/dist/{run-evidence-BDIircdA.d.ts.map → run-evidence-C4RcRQT5.d.ts.map} +1 -1
- package/dist/{run-record-BPCa2rQ8.d.ts → run-record-CztDMXVF.d.ts} +2 -2
- package/dist/{run-record-BPCa2rQ8.d.ts.map → run-record-CztDMXVF.d.ts.map} +1 -1
- package/dist/{semantic-concept-judge-Bz64IckK.js → semantic-concept-judge-BuIJ9IfB.js} +49 -6
- package/dist/semantic-concept-judge-BuIJ9IfB.js.map +1 -0
- package/dist/{server-KjXZZUDX.js → server-DaCpLfi0.js} +3 -3
- package/dist/{server-KjXZZUDX.js.map → server-DaCpLfi0.js.map} +1 -1
- package/dist/single-run-lock-BTTtPZ9N.js +989 -0
- package/dist/single-run-lock-BTTtPZ9N.js.map +1 -0
- package/dist/{skill-usage-CFDLLlhF.d.ts → skill-usage-B-BFS8M2.d.ts} +65 -40
- package/dist/skill-usage-B-BFS8M2.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-f4o9sUT4.js → skillopt-optimization-method-BbGnCC53.js} +20 -979
- package/dist/skillopt-optimization-method-BbGnCC53.js.map +1 -0
- package/dist/{skillopt-optimization-method-BpbnlvAZ.d.ts → skillopt-optimization-method-_s0Tub7Y.d.ts} +11 -39
- package/dist/skillopt-optimization-method-_s0Tub7Y.d.ts.map +1 -0
- package/dist/{statistics-_7P642CN.d.ts → statistics-B5d0Zd-z.d.ts} +2 -2
- package/dist/{statistics-_7P642CN.d.ts.map → statistics-B5d0Zd-z.d.ts.map} +1 -1
- package/dist/store-otlp-DX4fGIcf.js +757 -0
- package/dist/store-otlp-DX4fGIcf.js.map +1 -0
- package/dist/{summary-report-DHipz9Kx.d.ts → summary-report-Cg7BifAM.d.ts} +3 -3
- package/dist/{summary-report-DHipz9Kx.d.ts.map → summary-report-Cg7BifAM.d.ts.map} +1 -1
- package/dist/tool-groups-CdYq22lX.d.ts +258 -0
- package/dist/tool-groups-CdYq22lX.d.ts.map +1 -0
- package/dist/traces.d.ts +7 -6
- package/dist/traces.js +5 -4
- package/dist/{types-CTGbIm57.d.ts → types-BBFNHxSK.d.ts} +5 -5
- package/dist/{types-CTGbIm57.d.ts.map → types-BBFNHxSK.d.ts.map} +1 -1
- package/dist/{types-CKswbJGO.d.ts → types-DoEYskCd.d.ts} +5 -5
- package/dist/{types-CKswbJGO.d.ts.map → types-DoEYskCd.d.ts.map} +1 -1
- package/dist/{types-CTvKfr5F.d.ts → types-uPrS6mD-.d.ts} +2 -2
- package/dist/{types-CTvKfr5F.d.ts.map → types-uPrS6mD-.d.ts.map} +1 -1
- package/dist/usage-receipt-CgxMEBZq.js +134 -0
- package/dist/usage-receipt-CgxMEBZq.js.map +1 -0
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/trace-analysis.md +191 -385
- package/package.json +5 -4
- package/dist/analyze-runs-PVtnfjvA.d.ts +0 -72
- package/dist/analyze-runs-PVtnfjvA.d.ts.map +0 -1
- package/dist/benchmark-CHX4orG7.d.ts.map +0 -1
- package/dist/benchmark-YDrpumqB.js.map +0 -1
- package/dist/campaign-lgObcHFC.js.map +0 -1
- package/dist/completion-verifier-DSyRNVzU.d.ts.map +0 -1
- package/dist/concurrency-MUjT7VjM.js +0 -109
- package/dist/concurrency-MUjT7VjM.js.map +0 -1
- package/dist/cost-ledger-D-5_-dhi.js.map +0 -1
- package/dist/cost-ledger-D2o6JOrL.d.ts.map +0 -1
- package/dist/default-registry-CLXbRt0f.js +0 -2594
- package/dist/default-registry-CLXbRt0f.js.map +0 -1
- package/dist/default-registry-Dc5D_Loc.d.ts +0 -202
- package/dist/default-registry-Dc5D_Loc.d.ts.map +0 -1
- package/dist/feedback-trajectory-N_F0PwHz.d.ts.map +0 -1
- package/dist/index-C-Pr4OWg.d.ts.map +0 -1
- package/dist/ledger-core-t6sItivm.js.map +0 -1
- package/dist/llm-client-DKB25jV8.js.map +0 -1
- package/dist/proposal-findings-DCawte-y.js.map +0 -1
- package/dist/registry-BdM7SuTr.d.ts +0 -124
- package/dist/registry-BdM7SuTr.d.ts.map +0 -1
- package/dist/replay-Bju0T8Ls.js.map +0 -1
- package/dist/replay-K8FaC0CB.d.ts.map +0 -1
- package/dist/semantic-concept-judge-Bz64IckK.js.map +0 -1
- package/dist/skill-usage-CFDLLlhF.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-BpbnlvAZ.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-f4o9sUT4.js.map +0 -1
- package/dist/tools-DZk2Jn64.js +0 -1876
- package/dist/tools-DZk2Jn64.js.map +0 -1
package/dist/{skillopt-optimization-method-f4o9sUT4.js → skillopt-optimization-method-BbGnCC53.js}
RENAMED
|
@@ -1,21 +1,20 @@
|
|
|
1
1
|
import { c as ValidationError, i as JudgeError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
|
|
2
|
-
import {
|
|
3
|
-
import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-
|
|
4
|
-
import { a as clamp01, o as combineAbortSignals, t as assertProposalFindings } from "./proposal-findings-
|
|
5
|
-
import { n as
|
|
2
|
+
import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-CZ9diLxY.js";
|
|
3
|
+
import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-bkztEfIx.js";
|
|
4
|
+
import { a as clamp01, o as combineAbortSignals, t as assertProposalFindings } from "./proposal-findings-2GIUo1et.js";
|
|
5
|
+
import { C as safePathComponent, S as removeCredentialEnvironment, _ as assertJsonValue, b as isExternalTextCandidate, d as runWithCleanup, f as startExternalOptimizerModelProxy, g as assertExternalOptimizerModelBudget, h as sendJson, l as runExternalOptimizerProcess, m as listenLocal, n as createRunCostLedger, p as closeServer, r as fsCampaignStorage, t as acquireSingleRunLock, u as closeExternalOptimizerResources, v as assertNoCredentialValues, x as isRecord, y as isCandidateText } from "./single-run-lock-BTTtPZ9N.js";
|
|
6
|
+
import { p as mapConcurrent } from "./ledger-core-Dxz0Rkwa.js";
|
|
6
7
|
import { E as pairedBootstrap, H as weightedComposite, M as pairedRiskDifferenceScore, N as pairedSignTest, O as pairedDeltaTieFraction, T as pairedBinaryScale, d as confidenceInterval, j as pairedRiskDifferenceExact } from "./statistics-ByxzSiOM.js";
|
|
7
|
-
import { a as appendLedgerLine, d as tryAcquireAtomicFileLock, o as tryWithLedgerFileLock, u as probeAtomicFileLock } from "./ledger-core-t6sItivm.js";
|
|
8
8
|
import { t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
|
|
9
9
|
import { a as campaignCellExecutionEvidence, l as projectCampaignCellQuality, t as detectRewardHacking } from "./reward-hacking-GyN0kMd8.js";
|
|
10
|
-
import { createRequire } from "node:module";
|
|
11
10
|
import { z } from "zod";
|
|
12
11
|
import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
13
12
|
import { basename, isAbsolute, join } from "node:path";
|
|
14
13
|
import { createHash, randomBytes } from "node:crypto";
|
|
15
|
-
import { lstat, mkdir, mkdtemp, open, rm, writeFile } from "node:fs/promises";
|
|
16
|
-
import { homedir, tmpdir } from "node:os";
|
|
17
|
-
import { execSync, spawn } from "node:child_process";
|
|
18
14
|
import { createServer } from "node:http";
|
|
15
|
+
import { execSync } from "node:child_process";
|
|
16
|
+
import { mkdir } from "node:fs/promises";
|
|
17
|
+
import { homedir, tmpdir } from "node:os";
|
|
19
18
|
import { isDeepStrictEqual } from "node:util";
|
|
20
19
|
//#region src/judges.ts
|
|
21
20
|
/**
|
|
@@ -1804,115 +1803,6 @@ function designedCellIds(scenarios, reps) {
|
|
|
1804
1803
|
return ids;
|
|
1805
1804
|
}
|
|
1806
1805
|
//#endregion
|
|
1807
|
-
//#region src/campaign/external-optimizer-contracts.ts
|
|
1808
|
-
const MAX_TIMER_DELAY_MS$5 = 2147483647;
|
|
1809
|
-
function isCandidateText(value, maxChars) {
|
|
1810
|
-
return typeof value === "string" && value.trim().length > 0 && value.length <= maxChars;
|
|
1811
|
-
}
|
|
1812
|
-
function isExternalTextCandidate(value) {
|
|
1813
|
-
if (typeof value === "string") return value.trim().length > 0;
|
|
1814
|
-
if (!isRecord(value)) return false;
|
|
1815
|
-
const entries = Object.entries(value);
|
|
1816
|
-
return entries.length > 0 && entries.every(([name, content]) => name.trim().length > 0 && name.trim() === name && typeof content === "string");
|
|
1817
|
-
}
|
|
1818
|
-
function assertJsonValue(value, label, seen = /* @__PURE__ */ new Set()) {
|
|
1819
|
-
if (value === null || typeof value === "string" || typeof value === "boolean") return;
|
|
1820
|
-
if (typeof value === "number" && Number.isFinite(value)) return;
|
|
1821
|
-
if (Array.isArray(value)) {
|
|
1822
|
-
if (seen.has(value)) throw new Error(`${label} must be JSON-serializable`);
|
|
1823
|
-
seen.add(value);
|
|
1824
|
-
for (const item of value) assertJsonValue(item, label, seen);
|
|
1825
|
-
seen.delete(value);
|
|
1826
|
-
return;
|
|
1827
|
-
}
|
|
1828
|
-
if (typeof value === "object") {
|
|
1829
|
-
if (seen.has(value)) throw new Error(`${label} must be JSON-serializable`);
|
|
1830
|
-
const prototype = Object.getPrototypeOf(value);
|
|
1831
|
-
if (prototype !== Object.prototype && prototype !== null) throw new Error(`${label} must be JSON-serializable`);
|
|
1832
|
-
seen.add(value);
|
|
1833
|
-
for (const item of Object.values(value)) assertJsonValue(item, label, seen);
|
|
1834
|
-
seen.delete(value);
|
|
1835
|
-
return;
|
|
1836
|
-
}
|
|
1837
|
-
throw new Error(`${label} must be JSON-serializable`);
|
|
1838
|
-
}
|
|
1839
|
-
function assertNoCredentialValues(value, path, credentialLocation = "runner.env") {
|
|
1840
|
-
if (Array.isArray(value)) {
|
|
1841
|
-
for (const [index, item] of value.entries()) assertNoCredentialValues(item, `${path}[${index}]`, credentialLocation);
|
|
1842
|
-
return;
|
|
1843
|
-
}
|
|
1844
|
-
if (!isRecord(value)) return;
|
|
1845
|
-
for (const [key, item] of Object.entries(value)) {
|
|
1846
|
-
if (isCredentialKey(key)) throw new Error(`${path}.${key} must be supplied through ${credentialLocation}`);
|
|
1847
|
-
assertNoCredentialValues(item, `${path}.${key}`, credentialLocation);
|
|
1848
|
-
}
|
|
1849
|
-
}
|
|
1850
|
-
function removeCredentialEnvironment(environment) {
|
|
1851
|
-
const sanitized = {};
|
|
1852
|
-
for (const [key, value] of Object.entries(environment)) if (value !== void 0 && !isCredentialKey(key)) sanitized[key] = value;
|
|
1853
|
-
return sanitized;
|
|
1854
|
-
}
|
|
1855
|
-
function assertExternalOptimizerModelBudget(value, label) {
|
|
1856
|
-
if (!value || typeof value !== "object") throw new Error(`${label} is required`);
|
|
1857
|
-
for (const [field, entry] of [
|
|
1858
|
-
["maxRequests", value.maxRequests],
|
|
1859
|
-
["maxRequestBytes", value.maxRequestBytes],
|
|
1860
|
-
["maxResponseBytes", value.maxResponseBytes],
|
|
1861
|
-
["maxOutputTokensPerRequest", value.maxOutputTokensPerRequest]
|
|
1862
|
-
]) if (!Number.isSafeInteger(entry) || entry <= 0) throw new Error(`${label}.${field} must be a positive safe integer`);
|
|
1863
|
-
if (!Number.isFinite(value.maxCostUsd) || value.maxCostUsd <= 0) throw new Error(`${label}.maxCostUsd must be positive and finite`);
|
|
1864
|
-
if (value.requestTimeoutMs !== void 0 && (!Number.isSafeInteger(value.requestTimeoutMs) || value.requestTimeoutMs <= 0 || value.requestTimeoutMs > MAX_TIMER_DELAY_MS$5)) throw new Error(`${label}.requestTimeoutMs must be between 1 and ${MAX_TIMER_DELAY_MS$5}`);
|
|
1865
|
-
costForTokenPricing(value.pricing, {
|
|
1866
|
-
inputTokens: 1,
|
|
1867
|
-
outputTokens: 1
|
|
1868
|
-
});
|
|
1869
|
-
}
|
|
1870
|
-
function safePathComponent(value) {
|
|
1871
|
-
return value.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
1872
|
-
}
|
|
1873
|
-
function isCredentialKey(key) {
|
|
1874
|
-
return key.replace(/([a-z0-9])([A-Z])/g, "$1_$2").replace(/-/g, "_").toLowerCase().split("_").some((segment) => [
|
|
1875
|
-
"auth",
|
|
1876
|
-
"authorization",
|
|
1877
|
-
"cookie",
|
|
1878
|
-
"credential",
|
|
1879
|
-
"credentials",
|
|
1880
|
-
"key",
|
|
1881
|
-
"password",
|
|
1882
|
-
"secret",
|
|
1883
|
-
"session",
|
|
1884
|
-
"token"
|
|
1885
|
-
].includes(segment));
|
|
1886
|
-
}
|
|
1887
|
-
function isRecord(value) {
|
|
1888
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1889
|
-
}
|
|
1890
|
-
//#endregion
|
|
1891
|
-
//#region src/campaign/external-optimizer-http.ts
|
|
1892
|
-
function listenLocal(server) {
|
|
1893
|
-
return new Promise((resolvePromise, reject) => {
|
|
1894
|
-
server.once("error", reject);
|
|
1895
|
-
server.listen(0, "127.0.0.1", () => {
|
|
1896
|
-
server.off("error", reject);
|
|
1897
|
-
const address = server.address();
|
|
1898
|
-
if (!address || typeof address === "string") {
|
|
1899
|
-
reject(/* @__PURE__ */ new Error("external optimizer callback did not bind a TCP port"));
|
|
1900
|
-
return;
|
|
1901
|
-
}
|
|
1902
|
-
resolvePromise(address.port);
|
|
1903
|
-
});
|
|
1904
|
-
});
|
|
1905
|
-
}
|
|
1906
|
-
function closeServer(server) {
|
|
1907
|
-
return new Promise((resolvePromise, reject) => {
|
|
1908
|
-
server.close((error) => error ? reject(error) : resolvePromise());
|
|
1909
|
-
});
|
|
1910
|
-
}
|
|
1911
|
-
function sendJson(response, status, body) {
|
|
1912
|
-
response.writeHead(status, { "content-type": "application/json; charset=utf-8" });
|
|
1913
|
-
response.end(JSON.stringify(body));
|
|
1914
|
-
}
|
|
1915
|
-
//#endregion
|
|
1916
1806
|
//#region src/campaign/external-optimizer-callback.ts
|
|
1917
1807
|
const MAX_CALLBACK_BODY_BYTES = 1e6;
|
|
1918
1808
|
async function startExternalOptimizerCallback(args) {
|
|
@@ -1925,7 +1815,7 @@ async function startExternalOptimizerCallback(args) {
|
|
|
1925
1815
|
const activeHandlers = /* @__PURE__ */ new Set();
|
|
1926
1816
|
const server = createServer((request, response) => {
|
|
1927
1817
|
if (!accepting) {
|
|
1928
|
-
sendJsonIfOpen
|
|
1818
|
+
sendJsonIfOpen(response, 503, { error: "external optimizer callback is closing" });
|
|
1929
1819
|
return;
|
|
1930
1820
|
}
|
|
1931
1821
|
const controller = new AbortController();
|
|
@@ -1973,7 +1863,7 @@ async function startExternalOptimizerCallback(args) {
|
|
|
1973
1863
|
const closingServer = closeServer(server);
|
|
1974
1864
|
server.closeIdleConnections?.();
|
|
1975
1865
|
for (const controller of activeControllers) controller.abort();
|
|
1976
|
-
const [serverResult] = await Promise.allSettled([closingServer, waitForActiveHandlers
|
|
1866
|
+
const [serverResult] = await Promise.allSettled([closingServer, waitForActiveHandlers(activeHandlers)]);
|
|
1977
1867
|
if (activeControllers.size !== 0 || activeHandlers.size !== 0) throw new Error("external optimizer callback closed with active request work");
|
|
1978
1868
|
if (serverResult?.status === "rejected") throw serverResult.reason;
|
|
1979
1869
|
}
|
|
@@ -1981,34 +1871,34 @@ async function startExternalOptimizerCallback(args) {
|
|
|
1981
1871
|
async function handleCallback(request, response, signal, args, nextEvaluation) {
|
|
1982
1872
|
try {
|
|
1983
1873
|
if (request.method !== "POST" || request.url !== "/evaluate") {
|
|
1984
|
-
sendJsonIfOpen
|
|
1874
|
+
sendJsonIfOpen(response, 404, { error: "not found" });
|
|
1985
1875
|
return;
|
|
1986
1876
|
}
|
|
1987
1877
|
if (request.headers.authorization !== `Bearer ${args.token}`) {
|
|
1988
|
-
sendJsonIfOpen
|
|
1878
|
+
sendJsonIfOpen(response, 401, { error: "unauthorized" });
|
|
1989
1879
|
return;
|
|
1990
1880
|
}
|
|
1991
1881
|
const body = await readJson(request);
|
|
1992
1882
|
if (!isRecord(body) || !isExternalTextCandidate(body.candidate) || typeof body.exampleId !== "string") {
|
|
1993
|
-
sendJsonIfOpen
|
|
1883
|
+
sendJsonIfOpen(response, 400, { error: "candidate and exampleId are required strings" });
|
|
1994
1884
|
return;
|
|
1995
1885
|
}
|
|
1996
1886
|
if (nextEvaluation() === void 0) {
|
|
1997
|
-
sendJsonIfOpen
|
|
1887
|
+
sendJsonIfOpen(response, 429, { error: "evaluation limit reached" });
|
|
1998
1888
|
return;
|
|
1999
1889
|
}
|
|
2000
|
-
sendJsonIfOpen
|
|
1890
|
+
sendJsonIfOpen(response, 200, await args.evaluate({
|
|
2001
1891
|
candidate: body.candidate,
|
|
2002
1892
|
exampleId: body.exampleId
|
|
2003
1893
|
}, signal));
|
|
2004
1894
|
} catch {
|
|
2005
|
-
sendJsonIfOpen
|
|
1895
|
+
sendJsonIfOpen(response, 500, { error: "evaluation failed" });
|
|
2006
1896
|
}
|
|
2007
1897
|
}
|
|
2008
|
-
async function waitForActiveHandlers
|
|
1898
|
+
async function waitForActiveHandlers(activeHandlers) {
|
|
2009
1899
|
while (activeHandlers.size > 0) await Promise.allSettled([...activeHandlers]);
|
|
2010
1900
|
}
|
|
2011
|
-
function sendJsonIfOpen
|
|
1901
|
+
function sendJsonIfOpen(response, status, body) {
|
|
2012
1902
|
if (response.destroyed || response.writableEnded) return;
|
|
2013
1903
|
sendJson(response, status, body);
|
|
2014
1904
|
}
|
|
@@ -2042,681 +1932,6 @@ function assertCallbackConfig(args) {
|
|
|
2042
1932
|
if (typeof args.evaluate !== "function") throw new Error("external optimizer callback: evaluate must be a function");
|
|
2043
1933
|
}
|
|
2044
1934
|
//#endregion
|
|
2045
|
-
//#region src/campaign/external-optimizer-model-proxy.ts
|
|
2046
|
-
const MODEL_PROXY_PATHS = /* @__PURE__ */ new Set(["/v1/chat/completions", "/v1/responses"]);
|
|
2047
|
-
/**
|
|
2048
|
-
* Put an OpenAI-compatible optimizer behind the shared cost ledger.
|
|
2049
|
-
*
|
|
2050
|
-
* The child process receives only a loopback URL and an ephemeral token.
|
|
2051
|
-
* Provider credentials stay in this process. Every request reserves its
|
|
2052
|
-
* conservative byte-count input bound plus the provider-enforced output cap
|
|
2053
|
-
* before it leaves the machine.
|
|
2054
|
-
*/
|
|
2055
|
-
async function startExternalOptimizerModelProxy(args) {
|
|
2056
|
-
assertModelProxyConfig(args);
|
|
2057
|
-
args.signal?.throwIfAborted();
|
|
2058
|
-
const token = randomLocalToken();
|
|
2059
|
-
const fetchImpl = args.fetchImpl ?? fetch;
|
|
2060
|
-
let requestCount = 0;
|
|
2061
|
-
let successfulCompletionCount = 0;
|
|
2062
|
-
let totalRequestCount = args.initialUsage?.requests ?? 0;
|
|
2063
|
-
let committedForBudget = args.initialUsage?.costUsd ?? 0;
|
|
2064
|
-
let reservedForBudget = 0;
|
|
2065
|
-
let accepting = true;
|
|
2066
|
-
let closePromise;
|
|
2067
|
-
const activeControllers = /* @__PURE__ */ new Set();
|
|
2068
|
-
const activeHandlers = /* @__PURE__ */ new Set();
|
|
2069
|
-
const server = createServer((request, response) => {
|
|
2070
|
-
if (!accepting) {
|
|
2071
|
-
sendJsonIfOpen(response, 503, { error: "external optimizer model proxy is closing" });
|
|
2072
|
-
return;
|
|
2073
|
-
}
|
|
2074
|
-
const controller = new AbortController();
|
|
2075
|
-
const abortRequest = () => {
|
|
2076
|
-
request.destroy();
|
|
2077
|
-
response.destroy();
|
|
2078
|
-
};
|
|
2079
|
-
activeControllers.add(controller);
|
|
2080
|
-
controller.signal.addEventListener("abort", abortRequest, { once: true });
|
|
2081
|
-
let handler;
|
|
2082
|
-
handler = handleModelProxyRequest({
|
|
2083
|
-
request,
|
|
2084
|
-
response,
|
|
2085
|
-
controller,
|
|
2086
|
-
token,
|
|
2087
|
-
args,
|
|
2088
|
-
fetchImpl,
|
|
2089
|
-
nextReservation: (maximumCostUsd) => {
|
|
2090
|
-
if (totalRequestCount >= args.budget.maxRequests) return {
|
|
2091
|
-
accepted: false,
|
|
2092
|
-
reason: "optimizer model request limit reached"
|
|
2093
|
-
};
|
|
2094
|
-
if (committedForBudget + reservedForBudget + maximumCostUsd > args.budget.maxCostUsd + Number.EPSILON) return {
|
|
2095
|
-
accepted: false,
|
|
2096
|
-
reason: "optimizer model cost limit reached"
|
|
2097
|
-
};
|
|
2098
|
-
requestCount += 1;
|
|
2099
|
-
totalRequestCount += 1;
|
|
2100
|
-
reservedForBudget += maximumCostUsd;
|
|
2101
|
-
return { accepted: true };
|
|
2102
|
-
},
|
|
2103
|
-
settleReservation: (maximumCostUsd, chargedCostUsd) => {
|
|
2104
|
-
reservedForBudget = Math.max(0, reservedForBudget - maximumCostUsd);
|
|
2105
|
-
committedForBudget += chargedCostUsd;
|
|
2106
|
-
},
|
|
2107
|
-
recordSuccessfulCompletion: () => {
|
|
2108
|
-
successfulCompletionCount += 1;
|
|
2109
|
-
}
|
|
2110
|
-
}).finally(() => {
|
|
2111
|
-
controller.signal.removeEventListener("abort", abortRequest);
|
|
2112
|
-
activeControllers.delete(controller);
|
|
2113
|
-
activeHandlers.delete(handler);
|
|
2114
|
-
});
|
|
2115
|
-
activeHandlers.add(handler);
|
|
2116
|
-
handler.catch(() => void 0);
|
|
2117
|
-
});
|
|
2118
|
-
const port = await listenLocal(server);
|
|
2119
|
-
const close = () => {
|
|
2120
|
-
closePromise ??= closeModelProxy();
|
|
2121
|
-
return closePromise;
|
|
2122
|
-
};
|
|
2123
|
-
const onAbort = () => {
|
|
2124
|
-
close().catch(() => void 0);
|
|
2125
|
-
};
|
|
2126
|
-
args.signal?.addEventListener("abort", onAbort, { once: true });
|
|
2127
|
-
if (args.signal?.aborted) onAbort();
|
|
2128
|
-
return {
|
|
2129
|
-
baseUrl: `http://127.0.0.1:${port}/v1`,
|
|
2130
|
-
apiKey: token,
|
|
2131
|
-
requestAttempts: () => requestCount,
|
|
2132
|
-
successfulCompletions: () => successfulCompletionCount,
|
|
2133
|
-
close
|
|
2134
|
-
};
|
|
2135
|
-
async function closeModelProxy() {
|
|
2136
|
-
args.signal?.removeEventListener("abort", onAbort);
|
|
2137
|
-
accepting = false;
|
|
2138
|
-
const closingServer = closeServer(server);
|
|
2139
|
-
server.closeIdleConnections?.();
|
|
2140
|
-
for (const controller of activeControllers) controller.abort();
|
|
2141
|
-
const [serverResult] = await Promise.allSettled([closingServer, waitForActiveHandlers(activeHandlers)]);
|
|
2142
|
-
if (activeControllers.size !== 0 || activeHandlers.size !== 0) throw new Error("external optimizer model proxy closed with active request work");
|
|
2143
|
-
if (serverResult?.status === "rejected") throw serverResult.reason;
|
|
2144
|
-
}
|
|
2145
|
-
}
|
|
2146
|
-
async function handleModelProxyRequest(args) {
|
|
2147
|
-
const { controller, request, response } = args;
|
|
2148
|
-
try {
|
|
2149
|
-
const path = request.url ? new URL(request.url, "http://127.0.0.1").pathname : "";
|
|
2150
|
-
if (request.method !== "POST" || !MODEL_PROXY_PATHS.has(path)) {
|
|
2151
|
-
sendJsonIfOpen(response, 404, { error: "not found" });
|
|
2152
|
-
return;
|
|
2153
|
-
}
|
|
2154
|
-
if (request.headers.authorization !== `Bearer ${args.token}`) {
|
|
2155
|
-
sendJsonIfOpen(response, 401, { error: "unauthorized" });
|
|
2156
|
-
return;
|
|
2157
|
-
}
|
|
2158
|
-
const body = await readBody(request, args.args.budget.maxRequestBytes);
|
|
2159
|
-
const parsed = parseModelProxyRequest(body, args.args.model, args.args.budget);
|
|
2160
|
-
const maximumUsage = conservativeMaximumUsage(body.byteLength, parsed.maxOutputTokens, args.args.budget.pricing);
|
|
2161
|
-
const maximumCostUsd = costForTokenPricing(args.args.budget.pricing, maximumUsage);
|
|
2162
|
-
const reservation = args.nextReservation(maximumCostUsd);
|
|
2163
|
-
if (!reservation.accepted) {
|
|
2164
|
-
sendJsonIfOpen(response, 429, { error: reservation.reason });
|
|
2165
|
-
return;
|
|
2166
|
-
}
|
|
2167
|
-
const timeout = setTimeout(() => controller.abort(), args.args.budget.requestTimeoutMs ?? 3e5);
|
|
2168
|
-
let chargedForBudget = maximumCostUsd;
|
|
2169
|
-
try {
|
|
2170
|
-
const paid = await args.args.costLedger.runPaidCall({
|
|
2171
|
-
channel: "optimizer",
|
|
2172
|
-
phase: args.args.phase,
|
|
2173
|
-
actor: args.args.actor,
|
|
2174
|
-
...args.args.tags ? { tags: args.args.tags } : {},
|
|
2175
|
-
model: args.args.model,
|
|
2176
|
-
maximumCharge: {
|
|
2177
|
-
customTokenPricing: args.args.budget.pricing,
|
|
2178
|
-
...maximumUsage
|
|
2179
|
-
},
|
|
2180
|
-
execute: async () => forwardModelProxyRequest({
|
|
2181
|
-
fetchImpl: args.fetchImpl,
|
|
2182
|
-
upstreamBaseUrl: args.args.upstreamBaseUrl,
|
|
2183
|
-
upstreamApiKey: args.args.upstreamApiKey,
|
|
2184
|
-
path,
|
|
2185
|
-
body,
|
|
2186
|
-
model: args.args.model,
|
|
2187
|
-
pricing: args.args.budget.pricing,
|
|
2188
|
-
maxResponseBytes: args.args.budget.maxResponseBytes,
|
|
2189
|
-
signal: controller.signal
|
|
2190
|
-
}),
|
|
2191
|
-
receipt: (result) => result.receipt,
|
|
2192
|
-
receiptFromError: () => ({
|
|
2193
|
-
model: args.args.model,
|
|
2194
|
-
inputTokens: 0,
|
|
2195
|
-
outputTokens: 0,
|
|
2196
|
-
costUnknown: true,
|
|
2197
|
-
usageUnknown: true
|
|
2198
|
-
})
|
|
2199
|
-
});
|
|
2200
|
-
if (!paid.succeeded) {
|
|
2201
|
-
chargedForBudget = paid.receipt ? paid.receipt.usageUnknown || paid.receipt.costUnknown ? maximumCostUsd : paid.receipt.costUsd : 0;
|
|
2202
|
-
sendJsonIfOpen(response, isAbortError(paid.error) ? 504 : paid.error instanceof ProviderResponseTooLargeError ? 502 : 429, { error: paid.error.message });
|
|
2203
|
-
return;
|
|
2204
|
-
}
|
|
2205
|
-
chargedForBudget = paid.value.usageComplete ? paid.receipt.costUsd : maximumCostUsd;
|
|
2206
|
-
if (!paid.value.usageComplete) {
|
|
2207
|
-
sendJsonIfOpen(response, 502, { error: "optimizer model response omitted complete token usage" });
|
|
2208
|
-
return;
|
|
2209
|
-
}
|
|
2210
|
-
if (paid.value.status >= 200 && paid.value.status < 300) args.recordSuccessfulCompletion();
|
|
2211
|
-
if (response.destroyed || response.writableEnded) return;
|
|
2212
|
-
response.writeHead(paid.value.status, {
|
|
2213
|
-
"content-type": paid.value.contentType,
|
|
2214
|
-
"content-length": String(paid.value.body.byteLength)
|
|
2215
|
-
});
|
|
2216
|
-
response.end(paid.value.body);
|
|
2217
|
-
} finally {
|
|
2218
|
-
clearTimeout(timeout);
|
|
2219
|
-
args.settleReservation(maximumCostUsd, chargedForBudget);
|
|
2220
|
-
}
|
|
2221
|
-
} catch (error) {
|
|
2222
|
-
sendJsonIfOpen(response, error instanceof RequestBodyTooLargeError ? 413 : error instanceof Error && isAbortError(error) ? 503 : 400, { error: toErrorMessage(error) });
|
|
2223
|
-
}
|
|
2224
|
-
}
|
|
2225
|
-
async function waitForActiveHandlers(activeHandlers) {
|
|
2226
|
-
while (activeHandlers.size > 0) await Promise.allSettled([...activeHandlers]);
|
|
2227
|
-
}
|
|
2228
|
-
function sendJsonIfOpen(response, status, body) {
|
|
2229
|
-
if (response.destroyed || response.writableEnded) return;
|
|
2230
|
-
sendJson(response, status, body);
|
|
2231
|
-
}
|
|
2232
|
-
async function forwardModelProxyRequest(args) {
|
|
2233
|
-
const upstream = modelProxyUpstreamUrl(args.upstreamBaseUrl, args.path);
|
|
2234
|
-
const response = await args.fetchImpl(upstream, {
|
|
2235
|
-
method: "POST",
|
|
2236
|
-
headers: {
|
|
2237
|
-
authorization: `Bearer ${args.upstreamApiKey}`,
|
|
2238
|
-
"content-type": "application/json"
|
|
2239
|
-
},
|
|
2240
|
-
body: args.body.buffer.slice(args.body.byteOffset, args.body.byteOffset + args.body.byteLength),
|
|
2241
|
-
signal: args.signal,
|
|
2242
|
-
redirect: "error"
|
|
2243
|
-
});
|
|
2244
|
-
const body = await readProviderResponseBody(response, args.maxResponseBytes);
|
|
2245
|
-
const usage = parseProviderUsage(body);
|
|
2246
|
-
return {
|
|
2247
|
-
status: response.status,
|
|
2248
|
-
contentType: response.headers.get("content-type") ?? "application/json",
|
|
2249
|
-
body,
|
|
2250
|
-
receipt: usage ? {
|
|
2251
|
-
model: args.model,
|
|
2252
|
-
...usage,
|
|
2253
|
-
...usage.actualCostUsd === void 0 ? { customTokenPricing: args.pricing } : {}
|
|
2254
|
-
} : {
|
|
2255
|
-
model: args.model,
|
|
2256
|
-
inputTokens: 0,
|
|
2257
|
-
outputTokens: 0,
|
|
2258
|
-
costUnknown: true,
|
|
2259
|
-
usageUnknown: true
|
|
2260
|
-
},
|
|
2261
|
-
usageComplete: usage !== void 0
|
|
2262
|
-
};
|
|
2263
|
-
}
|
|
2264
|
-
function parseModelProxyRequest(body, expectedModel, budget) {
|
|
2265
|
-
let value;
|
|
2266
|
-
try {
|
|
2267
|
-
value = JSON.parse(Buffer.from(body).toString("utf8"));
|
|
2268
|
-
} catch {
|
|
2269
|
-
throw new Error("optimizer model request must be valid JSON");
|
|
2270
|
-
}
|
|
2271
|
-
if (!isRecord(value)) throw new Error("optimizer model request must be a JSON object");
|
|
2272
|
-
if (value.model !== expectedModel) throw new Error(`optimizer model request must use configured model '${expectedModel}'`);
|
|
2273
|
-
if (value.stream !== void 0 && value.stream !== false) throw new Error("streaming optimizer model requests are not supported");
|
|
2274
|
-
for (const field of [
|
|
2275
|
-
"n",
|
|
2276
|
-
"best_of",
|
|
2277
|
-
"candidate_count",
|
|
2278
|
-
"num_candidates"
|
|
2279
|
-
]) if (value[field] !== void 0 && value[field] !== 1) throw new Error(`optimizer model request ${field} must be 1 when supplied`);
|
|
2280
|
-
const suppliedMaximums = [
|
|
2281
|
-
value.max_output_tokens,
|
|
2282
|
-
value.max_completion_tokens,
|
|
2283
|
-
value.max_tokens
|
|
2284
|
-
].filter((maximum) => maximum !== void 0);
|
|
2285
|
-
if (suppliedMaximums.length === 0 || suppliedMaximums.some((maximum) => !Number.isSafeInteger(maximum) || maximum <= 0)) throw new Error("optimizer model request requires a positive output-token limit");
|
|
2286
|
-
const maxOutputTokens = Math.max(...suppliedMaximums);
|
|
2287
|
-
if (maxOutputTokens > budget.maxOutputTokensPerRequest) throw new Error("optimizer model request exceeds maxOutputTokensPerRequest");
|
|
2288
|
-
return { maxOutputTokens };
|
|
2289
|
-
}
|
|
2290
|
-
function modelProxyUpstreamUrl(baseUrl, requestPath) {
|
|
2291
|
-
const upstream = new URL(baseUrl);
|
|
2292
|
-
const basePath = upstream.pathname.replace(/\/+$/, "");
|
|
2293
|
-
upstream.pathname = `${basePath}${basePath.endsWith("/v1") ? requestPath.replace(/^\/v1/, "") : requestPath}`;
|
|
2294
|
-
return upstream.toString();
|
|
2295
|
-
}
|
|
2296
|
-
function parseProviderUsage(body) {
|
|
2297
|
-
let value;
|
|
2298
|
-
try {
|
|
2299
|
-
value = JSON.parse(Buffer.from(body).toString("utf8"));
|
|
2300
|
-
} catch {
|
|
2301
|
-
return;
|
|
2302
|
-
}
|
|
2303
|
-
if (!isRecord(value) || !isRecord(value.usage)) return void 0;
|
|
2304
|
-
const usage = value.usage;
|
|
2305
|
-
const totalInputTokens = usage.input_tokens ?? usage.prompt_tokens;
|
|
2306
|
-
const outputTokens = usage.output_tokens ?? usage.completion_tokens;
|
|
2307
|
-
if (!Number.isSafeInteger(totalInputTokens) || totalInputTokens < 0 || !Number.isSafeInteger(outputTokens) || outputTokens < 0) return;
|
|
2308
|
-
const inputDetails = isRecord(usage.input_tokens_details) ? usage.input_tokens_details : isRecord(usage.prompt_tokens_details) ? usage.prompt_tokens_details : {};
|
|
2309
|
-
const outputDetails = isRecord(usage.output_tokens_details) ? usage.output_tokens_details : isRecord(usage.completion_tokens_details) ? usage.completion_tokens_details : {};
|
|
2310
|
-
const cachedTokens = optionalTokenCount(inputDetails, [
|
|
2311
|
-
"cached_tokens",
|
|
2312
|
-
"cache_read_tokens",
|
|
2313
|
-
"cache_read_input_tokens"
|
|
2314
|
-
]);
|
|
2315
|
-
const cacheWriteTokens = optionalTokenCount(inputDetails, [
|
|
2316
|
-
"cache_write_tokens",
|
|
2317
|
-
"cache_creation_tokens",
|
|
2318
|
-
"cache_creation_input_tokens"
|
|
2319
|
-
]);
|
|
2320
|
-
const reasoningTokens = optionalTokenCount(outputDetails, ["reasoning_tokens"]);
|
|
2321
|
-
const actualCostUsd = typeof usage.cost === "number" && Number.isFinite(usage.cost) && usage.cost >= 0 ? usage.cost : void 0;
|
|
2322
|
-
if (cachedTokens === INVALID_TOKEN_COUNT || cacheWriteTokens === INVALID_TOKEN_COUNT || reasoningTokens === INVALID_TOKEN_COUNT) return;
|
|
2323
|
-
const classifiedInputTokens = (cachedTokens ?? 0) + (cacheWriteTokens ?? 0);
|
|
2324
|
-
if (classifiedInputTokens > totalInputTokens || (reasoningTokens ?? 0) > outputTokens) return;
|
|
2325
|
-
return {
|
|
2326
|
-
inputTokens: totalInputTokens - classifiedInputTokens,
|
|
2327
|
-
outputTokens,
|
|
2328
|
-
...cachedTokens === void 0 ? {} : { cachedTokens },
|
|
2329
|
-
...cacheWriteTokens === void 0 ? {} : { cacheWriteTokens },
|
|
2330
|
-
...reasoningTokens === void 0 ? {} : { reasoningTokens },
|
|
2331
|
-
...actualCostUsd === void 0 ? {} : { actualCostUsd }
|
|
2332
|
-
};
|
|
2333
|
-
}
|
|
2334
|
-
const INVALID_TOKEN_COUNT = Symbol("invalid-token-count");
|
|
2335
|
-
function optionalTokenCount(details, fields) {
|
|
2336
|
-
let found;
|
|
2337
|
-
for (const field of fields) {
|
|
2338
|
-
const value = details[field];
|
|
2339
|
-
if (value === void 0 || value === null) continue;
|
|
2340
|
-
if (!Number.isSafeInteger(value) || value < 0) return INVALID_TOKEN_COUNT;
|
|
2341
|
-
if (found !== void 0 && found !== value) return INVALID_TOKEN_COUNT;
|
|
2342
|
-
found = value;
|
|
2343
|
-
}
|
|
2344
|
-
return found;
|
|
2345
|
-
}
|
|
2346
|
-
function conservativeMaximumUsage(inputTokenUpperBound, outputTokenUpperBound, pricing) {
|
|
2347
|
-
const inputRates = [
|
|
2348
|
-
pricing.inputUsdPerMillion,
|
|
2349
|
-
pricing.cachedInputUsdPerMillion ?? pricing.inputUsdPerMillion,
|
|
2350
|
-
pricing.cacheWriteUsdPerMillion ?? pricing.inputUsdPerMillion
|
|
2351
|
-
];
|
|
2352
|
-
const mostExpensiveInputClass = inputRates.indexOf(Math.max(...inputRates));
|
|
2353
|
-
return {
|
|
2354
|
-
inputTokens: mostExpensiveInputClass === 0 ? inputTokenUpperBound : 0,
|
|
2355
|
-
...mostExpensiveInputClass === 1 ? { cachedTokens: inputTokenUpperBound } : {},
|
|
2356
|
-
...mostExpensiveInputClass === 2 ? { cacheWriteTokens: inputTokenUpperBound } : {},
|
|
2357
|
-
outputTokens: outputTokenUpperBound
|
|
2358
|
-
};
|
|
2359
|
-
}
|
|
2360
|
-
async function readProviderResponseBody(response, maxBytes) {
|
|
2361
|
-
const declaredLength = response.headers.get("content-length");
|
|
2362
|
-
if (declaredLength !== null) {
|
|
2363
|
-
const parsedLength = Number(declaredLength);
|
|
2364
|
-
if (Number.isFinite(parsedLength) && parsedLength > maxBytes) throw new ProviderResponseTooLargeError();
|
|
2365
|
-
}
|
|
2366
|
-
if (!response.body) return /* @__PURE__ */ new Uint8Array();
|
|
2367
|
-
const reader = response.body.getReader();
|
|
2368
|
-
const chunks = [];
|
|
2369
|
-
let totalBytes = 0;
|
|
2370
|
-
try {
|
|
2371
|
-
while (true) {
|
|
2372
|
-
const next = await reader.read();
|
|
2373
|
-
if (next.done) break;
|
|
2374
|
-
totalBytes += next.value.byteLength;
|
|
2375
|
-
if (totalBytes > maxBytes) {
|
|
2376
|
-
await reader.cancel();
|
|
2377
|
-
throw new ProviderResponseTooLargeError();
|
|
2378
|
-
}
|
|
2379
|
-
chunks.push(next.value);
|
|
2380
|
-
}
|
|
2381
|
-
} finally {
|
|
2382
|
-
reader.releaseLock();
|
|
2383
|
-
}
|
|
2384
|
-
const body = new Uint8Array(totalBytes);
|
|
2385
|
-
let offset = 0;
|
|
2386
|
-
for (const chunk of chunks) {
|
|
2387
|
-
body.set(chunk, offset);
|
|
2388
|
-
offset += chunk.byteLength;
|
|
2389
|
-
}
|
|
2390
|
-
return body;
|
|
2391
|
-
}
|
|
2392
|
-
function assertModelProxyConfig(args) {
|
|
2393
|
-
for (const [label, value] of [
|
|
2394
|
-
["upstreamBaseUrl", args.upstreamBaseUrl],
|
|
2395
|
-
["upstreamApiKey", args.upstreamApiKey],
|
|
2396
|
-
["model", args.model],
|
|
2397
|
-
["phase", args.phase],
|
|
2398
|
-
["actor", args.actor]
|
|
2399
|
-
]) if (typeof value !== "string" || !value.trim() || value.trim() !== value) throw new Error(`external optimizer model proxy: ${label} must be trimmed and non-empty`);
|
|
2400
|
-
let parsed;
|
|
2401
|
-
try {
|
|
2402
|
-
parsed = new URL(args.upstreamBaseUrl);
|
|
2403
|
-
} catch {
|
|
2404
|
-
throw new Error("external optimizer model proxy: upstreamBaseUrl must be an HTTP(S) URL");
|
|
2405
|
-
}
|
|
2406
|
-
if (parsed.protocol !== "http:" && parsed.protocol !== "https:" || parsed.username || parsed.password || parsed.search || parsed.hash) throw new Error("external optimizer model proxy: upstreamBaseUrl must be an HTTP(S) URL without credentials, query, or fragment");
|
|
2407
|
-
assertExternalOptimizerModelBudget(args.budget, "external optimizer model proxy: budget");
|
|
2408
|
-
if (args.tags !== void 0) {
|
|
2409
|
-
for (const [key, value] of Object.entries(args.tags)) if (!key.trim() || key.trim() !== key || !value.trim() || value.trim() !== value) throw new Error("external optimizer model proxy: tags must be trimmed and non-empty");
|
|
2410
|
-
}
|
|
2411
|
-
if (args.initialUsage !== void 0) {
|
|
2412
|
-
if (!Number.isSafeInteger(args.initialUsage.requests) || args.initialUsage.requests < 0 || !Number.isFinite(args.initialUsage.costUsd) || args.initialUsage.costUsd < 0) throw new Error("external optimizer model proxy: initialUsage must contain non-negative requests and cost");
|
|
2413
|
-
if (args.initialUsage.requests > args.budget.maxRequests || args.initialUsage.costUsd > args.budget.maxCostUsd + Number.EPSILON) throw new Error("external optimizer model proxy: initialUsage exceeds the configured budget");
|
|
2414
|
-
}
|
|
2415
|
-
}
|
|
2416
|
-
var RequestBodyTooLargeError = class extends Error {};
|
|
2417
|
-
var ProviderResponseTooLargeError = class extends Error {
|
|
2418
|
-
constructor() {
|
|
2419
|
-
super("optimizer model response exceeds maxResponseBytes");
|
|
2420
|
-
}
|
|
2421
|
-
};
|
|
2422
|
-
function readBody(request, maximumBytes) {
|
|
2423
|
-
return new Promise((resolvePromise, reject) => {
|
|
2424
|
-
let size = 0;
|
|
2425
|
-
let tooLarge = false;
|
|
2426
|
-
const chunks = [];
|
|
2427
|
-
request.on("data", (chunk) => {
|
|
2428
|
-
if (tooLarge) return;
|
|
2429
|
-
size += chunk.byteLength;
|
|
2430
|
-
if (size > maximumBytes) {
|
|
2431
|
-
tooLarge = true;
|
|
2432
|
-
chunks.length = 0;
|
|
2433
|
-
return;
|
|
2434
|
-
}
|
|
2435
|
-
chunks.push(chunk);
|
|
2436
|
-
});
|
|
2437
|
-
request.on("error", reject);
|
|
2438
|
-
request.on("end", () => {
|
|
2439
|
-
if (tooLarge) {
|
|
2440
|
-
reject(new RequestBodyTooLargeError("optimizer model request body too large"));
|
|
2441
|
-
return;
|
|
2442
|
-
}
|
|
2443
|
-
resolvePromise(new Uint8Array(Buffer.concat(chunks)));
|
|
2444
|
-
});
|
|
2445
|
-
});
|
|
2446
|
-
}
|
|
2447
|
-
function randomLocalToken() {
|
|
2448
|
-
return randomBytes(32).toString("hex");
|
|
2449
|
-
}
|
|
2450
|
-
function isAbortError(error) {
|
|
2451
|
-
return error.name === "AbortError" || error.message.toLowerCase().includes("abort");
|
|
2452
|
-
}
|
|
2453
|
-
function toErrorMessage(error) {
|
|
2454
|
-
return error instanceof Error ? error.message : String(error);
|
|
2455
|
-
}
|
|
2456
|
-
//#endregion
|
|
2457
|
-
//#region src/campaign/external-optimizer-resources.ts
|
|
2458
|
-
async function runWithCleanup(args) {
|
|
2459
|
-
let value;
|
|
2460
|
-
let operationError;
|
|
2461
|
-
try {
|
|
2462
|
-
value = await args.run();
|
|
2463
|
-
} catch (cause) {
|
|
2464
|
-
operationError = toError(cause);
|
|
2465
|
-
}
|
|
2466
|
-
let cleanupError;
|
|
2467
|
-
try {
|
|
2468
|
-
await args.cleanup();
|
|
2469
|
-
} catch (cause) {
|
|
2470
|
-
cleanupError = toError(cause);
|
|
2471
|
-
}
|
|
2472
|
-
if (operationError && cleanupError) throw new AggregateError([operationError, cleanupError], `${operationError.message}; ${args.label} cleanup failed: ${cleanupError.message}`);
|
|
2473
|
-
if (operationError) throw operationError;
|
|
2474
|
-
if (cleanupError) throw new Error(`${args.label} cleanup failed: ${cleanupError.message}`, { cause: cleanupError });
|
|
2475
|
-
return value;
|
|
2476
|
-
}
|
|
2477
|
-
async function closeExternalOptimizerResources(args) {
|
|
2478
|
-
const errors = (await Promise.allSettled([...args.modelProxy ? [args.modelProxy.close()] : [], args.callback.close()])).flatMap((result) => result.status === "rejected" ? [result.reason] : []);
|
|
2479
|
-
if (errors.length > 0) throw new AggregateError(errors, `${args.label}: failed to close optimizer resources`);
|
|
2480
|
-
}
|
|
2481
|
-
function toError(cause) {
|
|
2482
|
-
return cause instanceof Error ? cause : new Error(String(cause));
|
|
2483
|
-
}
|
|
2484
|
-
//#endregion
|
|
2485
|
-
//#region src/campaign/external-optimizer-subprocess.ts
|
|
2486
|
-
const MAX_PROCESS_OUTPUT_CHARS = 64e3;
|
|
2487
|
-
const MAX_PROCESS_INPUT_BYTES = 64 * 1024 * 1024;
|
|
2488
|
-
const MAX_PROCESS_RESULT_BYTES = 4 * 1024 * 1024;
|
|
2489
|
-
const PROCESS_TERMINATION_GRACE_MS = 5e3;
|
|
2490
|
-
const PROCESS_TERMINATION_POLL_MS = 25;
|
|
2491
|
-
const PROCESS_KILL_WAIT_MS = 1e3;
|
|
2492
|
-
const MAX_TIMER_DELAY_MS$4 = 2147483647;
|
|
2493
|
-
const PROCESS_OUTPUT_HEAD_CHARS = MAX_PROCESS_OUTPUT_CHARS / 2;
|
|
2494
|
-
async function runExternalOptimizerProcess(args) {
|
|
2495
|
-
if (!Number.isSafeInteger(args.timeoutMs) || args.timeoutMs <= 0 || args.timeoutMs > MAX_TIMER_DELAY_MS$4) throw new Error(`${args.label} timeoutMs must be between 1 and ${MAX_TIMER_DELAY_MS$4}`);
|
|
2496
|
-
if (process.platform === "win32") throw new Error(`${args.label} requires POSIX process-group cleanup; run the optimizer through WSL or Linux`);
|
|
2497
|
-
throwIfAborted(args.signal, args.label);
|
|
2498
|
-
const dir = await mkdtemp(join(tmpdir(), args.tempPrefix));
|
|
2499
|
-
const inputPath = join(dir, "input.json");
|
|
2500
|
-
const outputPath = join(dir, "output.json");
|
|
2501
|
-
return runWithCleanup({
|
|
2502
|
-
label: `${args.label} temporary directory`,
|
|
2503
|
-
run: async () => {
|
|
2504
|
-
const serializedInput = JSON.stringify(args.input);
|
|
2505
|
-
if (serializedInput === void 0) throw new Error(`${args.label} input must be JSON-serializable`);
|
|
2506
|
-
const inputJson = `${serializedInput}\n`;
|
|
2507
|
-
const inputBytes = Buffer.byteLength(inputJson);
|
|
2508
|
-
if (inputBytes > MAX_PROCESS_INPUT_BYTES) throw new Error(`${args.label} input exceeds ${MAX_PROCESS_INPUT_BYTES} bytes (${inputBytes})`);
|
|
2509
|
-
await writeFile(inputPath, inputJson);
|
|
2510
|
-
const command = args.runner?.command ?? "python";
|
|
2511
|
-
const commandArgs = [
|
|
2512
|
-
...args.runner?.args ?? ["-m", args.module],
|
|
2513
|
-
"--input",
|
|
2514
|
-
inputPath,
|
|
2515
|
-
"--output",
|
|
2516
|
-
outputPath
|
|
2517
|
-
];
|
|
2518
|
-
await runProcess({
|
|
2519
|
-
label: args.label,
|
|
2520
|
-
command,
|
|
2521
|
-
args: commandArgs,
|
|
2522
|
-
cwd: dir,
|
|
2523
|
-
env: args.runner?.env,
|
|
2524
|
-
timeoutMs: args.timeoutMs,
|
|
2525
|
-
signal: args.signal
|
|
2526
|
-
});
|
|
2527
|
-
throwIfAborted(args.signal, args.label);
|
|
2528
|
-
const raw = JSON.parse(await readBoundedTextFile(outputPath, MAX_PROCESS_RESULT_BYTES, `${args.label} output`));
|
|
2529
|
-
throwIfAborted(args.signal, args.label);
|
|
2530
|
-
if (!isRecord(raw)) throw new Error(`${args.label} output must be a JSON object`);
|
|
2531
|
-
return raw;
|
|
2532
|
-
},
|
|
2533
|
-
cleanup: () => rm(dir, {
|
|
2534
|
-
recursive: true,
|
|
2535
|
-
force: true
|
|
2536
|
-
})
|
|
2537
|
-
});
|
|
2538
|
-
}
|
|
2539
|
-
async function readBoundedTextFile(path, maxBytes, label) {
|
|
2540
|
-
if (!(await lstat(path)).isFile()) throw new Error(`${label} must be a regular file`);
|
|
2541
|
-
const file = await open(path, "r");
|
|
2542
|
-
try {
|
|
2543
|
-
const buffer = Buffer.allocUnsafe(maxBytes + 1);
|
|
2544
|
-
let bytesRead = 0;
|
|
2545
|
-
while (bytesRead < buffer.byteLength) {
|
|
2546
|
-
const chunk = await file.read(buffer, bytesRead, buffer.byteLength - bytesRead, bytesRead);
|
|
2547
|
-
if (chunk.bytesRead === 0) break;
|
|
2548
|
-
bytesRead += chunk.bytesRead;
|
|
2549
|
-
}
|
|
2550
|
-
if (bytesRead > maxBytes) throw new Error(`${label} exceeds ${maxBytes} bytes`);
|
|
2551
|
-
return buffer.subarray(0, bytesRead).toString("utf8");
|
|
2552
|
-
} finally {
|
|
2553
|
-
await file.close();
|
|
2554
|
-
}
|
|
2555
|
-
}
|
|
2556
|
-
function runProcess(args) {
|
|
2557
|
-
throwIfAborted(args.signal, args.label);
|
|
2558
|
-
return new Promise((resolvePromise, reject) => {
|
|
2559
|
-
const child = spawn(args.command, args.args, {
|
|
2560
|
-
cwd: args.cwd,
|
|
2561
|
-
env: {
|
|
2562
|
-
...safeProcessEnvironment(),
|
|
2563
|
-
...args.env
|
|
2564
|
-
},
|
|
2565
|
-
stdio: [
|
|
2566
|
-
"ignore",
|
|
2567
|
-
"pipe",
|
|
2568
|
-
"pipe"
|
|
2569
|
-
],
|
|
2570
|
-
detached: process.platform !== "win32",
|
|
2571
|
-
windowsHide: true
|
|
2572
|
-
});
|
|
2573
|
-
const stdout = createProcessOutputCapture();
|
|
2574
|
-
const stderr = createProcessOutputCapture();
|
|
2575
|
-
let settled = false;
|
|
2576
|
-
let timeout;
|
|
2577
|
-
const onAbort = () => finish(abortReason(args.signal, args.label));
|
|
2578
|
-
const finish = (error) => {
|
|
2579
|
-
if (settled) return;
|
|
2580
|
-
settled = true;
|
|
2581
|
-
if (timeout) clearTimeout(timeout);
|
|
2582
|
-
args.signal?.removeEventListener("abort", onAbort);
|
|
2583
|
-
terminateProcessTree(child).then(() => {
|
|
2584
|
-
if (error) reject(error);
|
|
2585
|
-
else resolvePromise();
|
|
2586
|
-
}, (cleanupError) => {
|
|
2587
|
-
if (error) {
|
|
2588
|
-
const cleanup = cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError));
|
|
2589
|
-
reject(new AggregateError([error, cleanup], `${error.message}; ${args.label} process cleanup failed: ${cleanup.message}`));
|
|
2590
|
-
return;
|
|
2591
|
-
}
|
|
2592
|
-
reject(/* @__PURE__ */ new Error(`${args.label} process cleanup failed: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`));
|
|
2593
|
-
});
|
|
2594
|
-
};
|
|
2595
|
-
timeout = setTimeout(() => {
|
|
2596
|
-
finish(/* @__PURE__ */ new Error(`${args.label} exceeded ${args.timeoutMs}ms`));
|
|
2597
|
-
}, args.timeoutMs);
|
|
2598
|
-
args.signal?.addEventListener("abort", onAbort, { once: true });
|
|
2599
|
-
if (args.signal?.aborted) onAbort();
|
|
2600
|
-
child.stdout.on("data", (chunk) => {
|
|
2601
|
-
appendProcessOutput(stdout, chunk);
|
|
2602
|
-
});
|
|
2603
|
-
child.stderr.on("data", (chunk) => {
|
|
2604
|
-
appendProcessOutput(stderr, chunk);
|
|
2605
|
-
});
|
|
2606
|
-
child.on("error", (error) => {
|
|
2607
|
-
finish(/* @__PURE__ */ new Error(`${args.label} could not start: ${error.message}`));
|
|
2608
|
-
});
|
|
2609
|
-
child.on("close", (code) => {
|
|
2610
|
-
if (code === 0) {
|
|
2611
|
-
finish();
|
|
2612
|
-
return;
|
|
2613
|
-
}
|
|
2614
|
-
finish(/* @__PURE__ */ new Error(`${args.label} exited ${String(code)}. stderr=${summarizeProcessOutput(stderr)} stdout=${summarizeProcessOutput(stdout)}`));
|
|
2615
|
-
});
|
|
2616
|
-
});
|
|
2617
|
-
}
|
|
2618
|
-
function throwIfAborted(signal, label) {
|
|
2619
|
-
if (signal?.aborted) throw abortReason(signal, label);
|
|
2620
|
-
}
|
|
2621
|
-
function abortReason(signal, label) {
|
|
2622
|
-
return signal.reason instanceof Error ? signal.reason : new Error(`${label} aborted`, { cause: signal.reason });
|
|
2623
|
-
}
|
|
2624
|
-
function safeProcessEnvironment() {
|
|
2625
|
-
return Object.fromEntries([
|
|
2626
|
-
"PATH",
|
|
2627
|
-
"HOME",
|
|
2628
|
-
"USER",
|
|
2629
|
-
"LOGNAME",
|
|
2630
|
-
"SHELL",
|
|
2631
|
-
"TMPDIR",
|
|
2632
|
-
"TMP",
|
|
2633
|
-
"TEMP",
|
|
2634
|
-
"LANG",
|
|
2635
|
-
"LANGUAGE",
|
|
2636
|
-
"LC_ALL",
|
|
2637
|
-
"VIRTUAL_ENV",
|
|
2638
|
-
"PYTHONPATH",
|
|
2639
|
-
"SYSTEMROOT",
|
|
2640
|
-
"COMSPEC",
|
|
2641
|
-
"PATHEXT",
|
|
2642
|
-
"WINDIR"
|
|
2643
|
-
].flatMap((name) => {
|
|
2644
|
-
const value = process.env[name];
|
|
2645
|
-
return value === void 0 ? [] : [[name, value]];
|
|
2646
|
-
}));
|
|
2647
|
-
}
|
|
2648
|
-
async function terminateProcessTree(child) {
|
|
2649
|
-
const pid = child.pid;
|
|
2650
|
-
if (!pid) return;
|
|
2651
|
-
if (process.platform === "win32") throw new Error("native Windows process-tree cleanup is unsupported");
|
|
2652
|
-
if (!signalProcessGroup(pid, "SIGTERM")) return;
|
|
2653
|
-
if (await waitForProcessGroupExit(pid, PROCESS_TERMINATION_GRACE_MS)) return;
|
|
2654
|
-
if (!signalProcessGroup(pid, "SIGKILL")) return;
|
|
2655
|
-
if (await waitForProcessGroupExit(pid, PROCESS_KILL_WAIT_MS)) return;
|
|
2656
|
-
throw new Error(`process group ${pid} remained alive after SIGKILL`);
|
|
2657
|
-
}
|
|
2658
|
-
function signalProcessGroup(pid, signal) {
|
|
2659
|
-
try {
|
|
2660
|
-
process.kill(-pid, signal);
|
|
2661
|
-
return true;
|
|
2662
|
-
} catch (error) {
|
|
2663
|
-
if (isMissingProcessError(error)) return false;
|
|
2664
|
-
throw error;
|
|
2665
|
-
}
|
|
2666
|
-
}
|
|
2667
|
-
async function waitForProcessGroupExit(pid, timeoutMs) {
|
|
2668
|
-
const deadline = Date.now() + timeoutMs;
|
|
2669
|
-
while (processGroupExists(pid)) {
|
|
2670
|
-
const remainingMs = deadline - Date.now();
|
|
2671
|
-
if (remainingMs <= 0) return false;
|
|
2672
|
-
await new Promise((resolve) => setTimeout(resolve, Math.min(PROCESS_TERMINATION_POLL_MS, remainingMs)));
|
|
2673
|
-
}
|
|
2674
|
-
return true;
|
|
2675
|
-
}
|
|
2676
|
-
function processGroupExists(pid) {
|
|
2677
|
-
try {
|
|
2678
|
-
process.kill(-pid, 0);
|
|
2679
|
-
return true;
|
|
2680
|
-
} catch (error) {
|
|
2681
|
-
if (isMissingProcessError(error)) return false;
|
|
2682
|
-
throw error;
|
|
2683
|
-
}
|
|
2684
|
-
}
|
|
2685
|
-
function isMissingProcessError(error) {
|
|
2686
|
-
return typeof error === "object" && error !== null && "code" in error && error.code === "ESRCH";
|
|
2687
|
-
}
|
|
2688
|
-
function createProcessOutputCapture() {
|
|
2689
|
-
return {
|
|
2690
|
-
full: "",
|
|
2691
|
-
head: "",
|
|
2692
|
-
tail: "",
|
|
2693
|
-
totalChars: 0
|
|
2694
|
-
};
|
|
2695
|
-
}
|
|
2696
|
-
function appendProcessOutput(capture, chunk) {
|
|
2697
|
-
const text = chunk.toString();
|
|
2698
|
-
capture.totalChars += text.length;
|
|
2699
|
-
if (capture.full !== void 0) {
|
|
2700
|
-
const combined = `${capture.full}${text}`;
|
|
2701
|
-
if (combined.length <= MAX_PROCESS_OUTPUT_CHARS) {
|
|
2702
|
-
capture.full = combined;
|
|
2703
|
-
return;
|
|
2704
|
-
}
|
|
2705
|
-
capture.head = combined.slice(0, PROCESS_OUTPUT_HEAD_CHARS);
|
|
2706
|
-
capture.tail = combined.slice(-32e3);
|
|
2707
|
-
capture.full = void 0;
|
|
2708
|
-
return;
|
|
2709
|
-
}
|
|
2710
|
-
capture.tail = `${capture.tail}${text}`.slice(-32e3);
|
|
2711
|
-
}
|
|
2712
|
-
function summarizeProcessOutput(capture, max = 4e3) {
|
|
2713
|
-
const compact = (capture.full ?? `${capture.head}\n...[${capture.totalChars - capture.head.length - capture.tail.length} chars omitted]...\n${capture.tail}`).trim().replace(/\s+/g, " ");
|
|
2714
|
-
if (compact.length <= max) return compact;
|
|
2715
|
-
const headChars = Math.floor(max / 3);
|
|
2716
|
-
const tailChars = max - headChars;
|
|
2717
|
-
return `${compact.slice(0, headChars)}...[${compact.length - max} chars omitted]...${compact.slice(-tailChars)}`;
|
|
2718
|
-
}
|
|
2719
|
-
//#endregion
|
|
2720
1935
|
//#region src/campaign/external-optimizer-run-budget.ts
|
|
2721
1936
|
function externalOptimizerRunKey(input) {
|
|
2722
1937
|
if (!input.attemptId.trim() || input.attemptId.trim() !== input.attemptId) throw new Error("external optimizer attemptId must be trimmed and non-empty");
|
|
@@ -2806,127 +2021,6 @@ function resolveRunDir(runDir, repo) {
|
|
|
2806
2021
|
return join(tangleTracesRoot(), r, "runs", runDir);
|
|
2807
2022
|
}
|
|
2808
2023
|
//#endregion
|
|
2809
|
-
//#region src/campaign/search-ledger-errors.ts
|
|
2810
|
-
/** Base error for invalid search-ledger input or operations. */
|
|
2811
|
-
var SearchLedgerError = class extends ValidationError {};
|
|
2812
|
-
/** Error raised when durable search-ledger data fails an integrity check. */
|
|
2813
|
-
var SearchLedgerIntegrityError = class extends SearchLedgerError {};
|
|
2814
|
-
/** Error raised when an event identifier is reused with different content. */
|
|
2815
|
-
var SearchLedgerConflictError = class extends SearchLedgerError {};
|
|
2816
|
-
//#endregion
|
|
2817
|
-
//#region src/campaign/search-ledger-file.ts
|
|
2818
|
-
/** Campaign binding of the generic journal file layer to search-ledger errors. */
|
|
2819
|
-
const SEARCH_LEDGER_FILE_CONTEXT = {
|
|
2820
|
-
subject: "search ledger",
|
|
2821
|
-
integrityError: (message, options) => new SearchLedgerIntegrityError(message, options)
|
|
2822
|
-
};
|
|
2823
|
-
function appendSearchLedgerLine(path, line) {
|
|
2824
|
-
appendLedgerLine(path, line, SEARCH_LEDGER_FILE_CONTEXT);
|
|
2825
|
-
}
|
|
2826
|
-
function tryWithSearchLedgerFileLock(ledgerPath, run) {
|
|
2827
|
-
return tryWithLedgerFileLock(ledgerPath, SEARCH_LEDGER_FILE_CONTEXT, run);
|
|
2828
|
-
}
|
|
2829
|
-
//#endregion
|
|
2830
|
-
//#region src/campaign/storage.ts
|
|
2831
|
-
/** Node-filesystem storage — the default. Lazily requires `node:fs` so the
|
|
2832
|
-
* module imports cleanly in non-Node runtimes (where the caller passes
|
|
2833
|
-
* `inMemoryCampaignStorage` instead and never constructs this).
|
|
2834
|
-
*
|
|
2835
|
-
* `createRequire(import.meta.url)` is the ESM-native lazy require — a bare
|
|
2836
|
-
* `require` is a ReferenceError under `"type": "module"`, which is exactly
|
|
2837
|
-
* the shape this package publishes. */
|
|
2838
|
-
function fsCampaignStorage() {
|
|
2839
|
-
const { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } = createRequire(import.meta.url)("node:fs");
|
|
2840
|
-
return {
|
|
2841
|
-
ensureDir(dir) {
|
|
2842
|
-
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
2843
|
-
},
|
|
2844
|
-
exists(path) {
|
|
2845
|
-
return existsSync(path);
|
|
2846
|
-
},
|
|
2847
|
-
read(path) {
|
|
2848
|
-
try {
|
|
2849
|
-
return readFileSync(path, "utf8");
|
|
2850
|
-
} catch {
|
|
2851
|
-
return;
|
|
2852
|
-
}
|
|
2853
|
-
},
|
|
2854
|
-
write(path, content) {
|
|
2855
|
-
writeFileSync(path, content);
|
|
2856
|
-
},
|
|
2857
|
-
append(path, content, expectedBytes) {
|
|
2858
|
-
const result = tryWithSearchLedgerFileLock(path, () => {
|
|
2859
|
-
let actualBytes = 0;
|
|
2860
|
-
try {
|
|
2861
|
-
actualBytes = statSync(path).size;
|
|
2862
|
-
} catch (error) {
|
|
2863
|
-
if (error.code !== "ENOENT") throw error;
|
|
2864
|
-
}
|
|
2865
|
-
if (actualBytes !== expectedBytes) return void 0;
|
|
2866
|
-
appendSearchLedgerLine(path, content);
|
|
2867
|
-
return expectedBytes + Buffer.byteLength(content);
|
|
2868
|
-
});
|
|
2869
|
-
return result.acquired ? result.value : void 0;
|
|
2870
|
-
}
|
|
2871
|
-
};
|
|
2872
|
-
}
|
|
2873
|
-
/** In-memory storage for filesystem-less runtimes. Artifacts + trace spans
|
|
2874
|
-
* live in a `Map` for the duration of the run; the `CampaignResult` is
|
|
2875
|
-
* fully populated, but nothing is persisted to disk. */
|
|
2876
|
-
function inMemoryCampaignStorage() {
|
|
2877
|
-
const files = /* @__PURE__ */ new Map();
|
|
2878
|
-
const dirs = /* @__PURE__ */ new Set();
|
|
2879
|
-
return {
|
|
2880
|
-
ensureDir(dir) {
|
|
2881
|
-
dirs.add(dir);
|
|
2882
|
-
},
|
|
2883
|
-
exists(path) {
|
|
2884
|
-
return files.has(path) || dirs.has(path);
|
|
2885
|
-
},
|
|
2886
|
-
read(path) {
|
|
2887
|
-
const value = files.get(path);
|
|
2888
|
-
if (value === void 0) return void 0;
|
|
2889
|
-
return typeof value === "string" ? value : new TextDecoder().decode(value);
|
|
2890
|
-
},
|
|
2891
|
-
write(path, content) {
|
|
2892
|
-
files.set(path, content);
|
|
2893
|
-
},
|
|
2894
|
-
append(path, content, expectedBytes) {
|
|
2895
|
-
const current = files.get(path);
|
|
2896
|
-
const currentText = current === void 0 ? "" : typeof current === "string" ? current : new TextDecoder().decode(current);
|
|
2897
|
-
const currentBytes = new TextEncoder().encode(currentText).byteLength;
|
|
2898
|
-
if (currentBytes !== expectedBytes) return void 0;
|
|
2899
|
-
files.set(path, `${currentText}${content}`);
|
|
2900
|
-
return currentBytes + new TextEncoder().encode(content).byteLength;
|
|
2901
|
-
}
|
|
2902
|
-
};
|
|
2903
|
-
}
|
|
2904
|
-
/** Open the durable spend account stored beside a logical run. */
|
|
2905
|
-
function createRunCostLedger(input) {
|
|
2906
|
-
const path = join(input.runDir, "cost-ledger.jsonl");
|
|
2907
|
-
input.storage.ensureDir(input.runDir);
|
|
2908
|
-
return new CostLedger({
|
|
2909
|
-
costCeilingUsd: input.costCeilingUsd,
|
|
2910
|
-
persistence: {
|
|
2911
|
-
read: () => {
|
|
2912
|
-
const stored = input.storage.read(path);
|
|
2913
|
-
if (stored === void 0 && input.storage.exists(path)) throw new Error(`CostLedger: cannot read existing event log '${path}'`);
|
|
2914
|
-
const events = stored ?? "";
|
|
2915
|
-
return {
|
|
2916
|
-
revision: String(new TextEncoder().encode(events).byteLength),
|
|
2917
|
-
events
|
|
2918
|
-
};
|
|
2919
|
-
},
|
|
2920
|
-
append: (expectedRevision, event) => {
|
|
2921
|
-
const expectedBytes = Number(expectedRevision);
|
|
2922
|
-
if (!Number.isSafeInteger(expectedBytes) || expectedBytes < 0) throw new Error(`CostLedger: invalid storage revision '${expectedRevision}'`);
|
|
2923
|
-
const next = input.storage.append(path, event, expectedBytes);
|
|
2924
|
-
return next === void 0 ? void 0 : String(next);
|
|
2925
|
-
}
|
|
2926
|
-
}
|
|
2927
|
-
});
|
|
2928
|
-
}
|
|
2929
|
-
//#endregion
|
|
2930
2024
|
//#region src/campaign/run-campaign.ts
|
|
2931
2025
|
/**
|
|
2932
2026
|
* `runCampaign` — Pass A substrate primitive. ONE function that orchestrates
|
|
@@ -4654,59 +3748,6 @@ function assertComparisonCost(cost, label) {
|
|
|
4654
3748
|
if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`);
|
|
4655
3749
|
}
|
|
4656
3750
|
//#endregion
|
|
4657
|
-
//#region src/campaign/single-run-lock.ts
|
|
4658
|
-
/**
|
|
4659
|
-
* Single-run lock for evaluations that share one mutable environment.
|
|
4660
|
-
*
|
|
4661
|
-
* Two concurrent runs against a shared stateful gym silently corrupt each
|
|
4662
|
-
* other: each resets/mutates environment state mid-cell of the other, and
|
|
4663
|
-
* every score from both becomes garbage that LOOKS like worker variance
|
|
4664
|
-
* (agent-lab R357 burned hours on flip-flopping scores before tracing them
|
|
4665
|
-
* to exactly this). The fix is a pid lockfile: refuse to start while a live
|
|
4666
|
-
* holder exists, reclaim stale locks whose pid is gone, release only if the
|
|
4667
|
-
* lock is still ours.
|
|
4668
|
-
*
|
|
4669
|
-
* `alsoCheck` exists because independent runners can guard the same shared
|
|
4670
|
-
* resource with differently named lockfiles; a runner must respect all of
|
|
4671
|
-
* them even though it writes only its own.
|
|
4672
|
-
*/
|
|
4673
|
-
function assertAvailable(path) {
|
|
4674
|
-
const unavailable = probeAtomicFileLock({ lockPath: path });
|
|
4675
|
-
if (unavailable) throw unavailableError(path, unavailable);
|
|
4676
|
-
}
|
|
4677
|
-
function unavailableError(path, unavailable) {
|
|
4678
|
-
if (unavailable.reason === "recovery") return /* @__PURE__ */ new Error(`single-run lock recovery is already in progress (${path}); refusing a concurrent run`);
|
|
4679
|
-
return /* @__PURE__ */ new Error(`single-run lock held by live pid ${unavailable.holder.pid} (${path}); refusing a concurrent run on the shared resource`);
|
|
4680
|
-
}
|
|
4681
|
-
/**
|
|
4682
|
-
* Acquire the lock or throw naming the live holder. A stale lock (holder pid
|
|
4683
|
-
* no longer running) is reclaimed by one contender. An interrupted reclaim
|
|
4684
|
-
* leaves a marker that fails closed instead of admitting overlapping runs.
|
|
4685
|
-
*/
|
|
4686
|
-
function acquireSingleRunLock(opts) {
|
|
4687
|
-
const pid = opts.pid ?? process.pid;
|
|
4688
|
-
if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("single-run lock pid must be a positive integer");
|
|
4689
|
-
for (const path of opts.alsoCheck ?? []) assertAvailable(path);
|
|
4690
|
-
const acquisition = tryAcquireAtomicFileLock({
|
|
4691
|
-
lockPath: opts.lockPath,
|
|
4692
|
-
pid
|
|
4693
|
-
});
|
|
4694
|
-
if (!acquisition.acquired) throw unavailableError(opts.lockPath, acquisition);
|
|
4695
|
-
const release = () => {
|
|
4696
|
-
try {
|
|
4697
|
-
acquisition.lock.release();
|
|
4698
|
-
} catch {}
|
|
4699
|
-
};
|
|
4700
|
-
try {
|
|
4701
|
-
for (const path of opts.alsoCheck ?? []) assertAvailable(path);
|
|
4702
|
-
} catch (error) {
|
|
4703
|
-
release();
|
|
4704
|
-
throw error;
|
|
4705
|
-
}
|
|
4706
|
-
if (opts.releaseOnExit ?? true) process.on("exit", release);
|
|
4707
|
-
return { release };
|
|
4708
|
-
}
|
|
4709
|
-
//#endregion
|
|
4710
3751
|
//#region src/campaign/external-text-optimization.ts
|
|
4711
3752
|
const DEFAULT_MAX_CANDIDATE_CHARS = 2e5;
|
|
4712
3753
|
const DEFAULT_MAX_EVIDENCE_CHARS = 1e5;
|
|
@@ -8119,6 +7160,6 @@ function skillOptOptimizationMethod(config) {
|
|
|
8119
7160
|
};
|
|
8120
7161
|
}
|
|
8121
7162
|
//#endregion
|
|
8122
|
-
export {
|
|
7163
|
+
export { campaignScenarioIdentity as $, assertOptimizationResult as A, pairedDeltaTest as At, surfaceHash as B, detectScale as C, paretoFrontier as Ct, runCanaries as D, decidePairedPromotion as Dt, pairHoldout as E, recoverTruncatedJson as Et, assertComponentSurface as F, summarizeBackendIntegrity as Ft, buildReflectionPrompt as G, campaignMeanComposite as H, codeSurfaceIdentityMaterial as I, JudgeParseError as It, runCampaign as J, parseReflectionResponse as K, componentSurfaceIdentityMaterial as L, costFromLedgerSummary as M, assertRealAgentReceipts as Mt, optimizationTokenUsageFromSummary as N, assertRealBackend as Nt, composeGate as O, pairedDecisionShape as Ot, assertCodeSurfaceIdentity as P, summarizeAgentReceiptIntegrity as Pt, assertCampaignSplitIdentity as Q, renderSurfaceDiff as R, defaultProductionGate as S, dominates as St, heldoutSignificance as T, scalarScore as Tt, compareRankKeys as U, campaignBreakdown as V, DEFAULT_MUTATION_PRIMITIVES as W, tangleTracesRoot as X, resolveRunDir as Y, assertCampaignDesign as Z, buildEvidenceVector as _, canonicalJson as _t, emitLoopProvenance as a, createReferenceEquivalenceJudge as at, powerPreflight as b, inMemoryVerdictCache as bt, provenanceRecordPath as c, redTeamDataset as ct, runImprovementLoop as d, toolNamesForRun as dt, campaignSplitDigest as et, runOptimization as f, Dataset as ft, gepaOptimizationMethod as g, cachedJudge as gt, labelTrustRank as h, llmJudge as ht, canonicalDigest as i, REFERENCE_EQUIVALENCE_JUDGE_VERSION as it, compareOptimizationMethods as j, BackendIntegrityError as jt, externalTextOptimizationMethod as k, minimumPairsForPairedDeltaTest as kt, provenanceSpansPath as l, redTeamReport as lt, isProposedCandidate as m, hashScenarios as mt, buildLoopProvenanceRecord as n, openAutoPr as nt, loopProvenanceArgsFromResult as o, runReferenceEquivalenceJudge as ot, runEval as p, HoldoutLockedError as pt, planCampaignRun as q, campaignMeasurementDigest as r, REFERENCE_EQUIVALENCE_INPUT_LIMITS as rt, loopProvenanceSpans as s, DEFAULT_RED_TEAM_CORPUS as st, skillOptOptimizationMethod as t, campaignSplitDigestFromIdentities as tt, verifyLoopProvenanceRecord as u, scoreRedTeamOutput as ut, paretoPolicy as v, contentHash as vt, dimensionRegressions as w, paretoFrontierWithCrowding as wt, heldOutGate as x, crowdingDistance as xt, paretoSignificanceGate as y, fileVerdictCache as yt, surfaceContentHash as z };
|
|
8123
7164
|
|
|
8124
|
-
//# sourceMappingURL=skillopt-optimization-method-
|
|
7165
|
+
//# sourceMappingURL=skillopt-optimization-method-BbGnCC53.js.map
|