@tangle-network/agent-eval 0.144.9 → 0.144.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +6 -0
- package/dist/analyst/index.d.ts +1 -1
- package/dist/analyst/index.js +7 -7
- package/dist/{benchmark-B181aMF9.js → benchmark-CWeqGl7x.js} +2 -2
- package/dist/{benchmark-B181aMF9.js.map → benchmark-CWeqGl7x.js.map} +1 -1
- package/dist/{benchmark-command-Dtym4pA-.js → benchmark-command-BteMFN62.js} +8 -8
- package/dist/{benchmark-command-Dtym4pA-.js.map → benchmark-command-BteMFN62.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-Ce4Ae5fX.js → benchmarks-Dzs8CKb1.js} +4 -4
- package/dist/{benchmarks-Ce4Ae5fX.js.map → benchmarks-Dzs8CKb1.js.map} +1 -1
- package/dist/campaign/index.d.ts +4 -4
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-FG3thH2m.js → campaign-C2TTzQII.js} +4 -4
- package/dist/{campaign-FG3thH2m.js.map → campaign-C2TTzQII.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/contract/index.d.ts +1 -1
- package/dist/contract/index.js +6 -6
- package/dist/{default-registry-BaQXW1Ow.js → default-registry-BmktKy8r.js} +3 -3
- package/dist/{default-registry-BaQXW1Ow.js.map → default-registry-BmktKy8r.js.map} +1 -1
- package/dist/{dspy-rlm-engine-BiN49gK6.js → dspy-rlm-engine-DbTk4JdR.js} +3 -3
- package/dist/{dspy-rlm-engine-BiN49gK6.js.map → dspy-rlm-engine-DbTk4JdR.js.map} +1 -1
- package/dist/{extract-usage-BW27f3XW.js → extract-usage-CdZdoj1s.js} +2 -2
- package/dist/{extract-usage-BW27f3XW.js.map → extract-usage-CdZdoj1s.js.map} +1 -1
- package/dist/{index-CBEMKO8d.d.ts → index-DPPGNJ_R.d.ts} +2 -2
- package/dist/{index-CBEMKO8d.d.ts.map → index-DPPGNJ_R.d.ts.map} +1 -1
- package/dist/{index-CLKMJTJF2.d.ts → index-YE4KdKbO2.d.ts} +3 -3
- package/dist/{index-CLKMJTJF2.d.ts.map → index-YE4KdKbO2.d.ts.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +11 -11
- package/dist/{kind-factory-BHIgPmzS.js → kind-factory-B8-r8-y8.js} +3 -16
- package/dist/kind-factory-B8-r8-y8.js.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/{replay-B3gACG_H.js → replay-CohS93nE.js} +4 -4
- package/dist/{replay-B3gACG_H.js.map → replay-CohS93nE.js.map} +1 -1
- package/dist/{semantic-concept-judge-Bmrq6yqU.js → semantic-concept-judge-D1z-KepS.js} +2 -2
- package/dist/{semantic-concept-judge-Bmrq6yqU.js.map → semantic-concept-judge-D1z-KepS.js.map} +1 -1
- package/dist/{single-run-lock-BMQEv1wG.js → single-run-lock-DFWHEB09.js} +123 -123
- package/dist/{single-run-lock-BMQEv1wG.js.map → single-run-lock-DFWHEB09.js.map} +1 -1
- package/dist/{skillopt-optimization-method-Dtzp7QpP.js → skillopt-optimization-method-CQdVeM8k.js} +513 -134
- package/dist/skillopt-optimization-method-CQdVeM8k.js.map +1 -0
- package/dist/{skillopt-optimization-method-D8wysvU1.d.ts → skillopt-optimization-method-USDKhxSA.d.ts} +87 -3
- package/dist/skillopt-optimization-method-USDKhxSA.d.ts.map +1 -0
- package/dist/{store-otlp-CKtTpRhv.js → store-otlp-Dw8PPIlL.js} +2 -2
- package/dist/{store-otlp-CKtTpRhv.js.map → store-otlp-Dw8PPIlL.js.map} +1 -1
- package/dist/traces.js +4 -4
- package/dist/{usage-receipt-EVI8B8Xu.js → usage-receipt-t7vAzCRQ.js} +15 -2
- package/dist/usage-receipt-t7vAzCRQ.js.map +1 -0
- package/docs/campaign-proposers.md +4 -0
- package/package.json +1 -1
- package/dist/kind-factory-BHIgPmzS.js.map +0 -1
- package/dist/skillopt-optimization-method-D8wysvU1.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-Dtzp7QpP.js.map +0 -1
- package/dist/usage-receipt-EVI8B8Xu.js.map +0 -1
package/dist/{skillopt-optimization-method-Dtzp7QpP.js → skillopt-optimization-method-CQdVeM8k.js}
RENAMED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { c as ValidationError, i as JudgeError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
|
|
2
2
|
import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-DMFxsLKr.js";
|
|
3
3
|
import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-DzvMUsS_.js";
|
|
4
|
+
import { a as deepFreezeCanonicalJson } from "./usage-receipt-t7vAzCRQ.js";
|
|
4
5
|
import { a as clamp01, o as combineAbortSignals, t as assertProposalFindings } from "./proposal-findings-2GIUo1et.js";
|
|
5
|
-
import { C as isExternalTextCandidate, D as resolveExternalOptimizerProcessLimits, E as resolveExternalOptimizerCallbackLimits, O as safePathComponent, S as isCandidateText, T as removeCredentialEnvironment, b as assertJsonValue, d as closeExternalOptimizerResources, f as runWithCleanup,
|
|
6
|
+
import { C as isExternalTextCandidate, D as resolveExternalOptimizerProcessLimits, E as resolveExternalOptimizerCallbackLimits, O as safePathComponent, S as isCandidateText, T as removeCredentialEnvironment, b as assertJsonValue, d as closeExternalOptimizerResources, f as runWithCleanup, i as fsCampaignStorage, n as startExternalOptimizerCallback, p as startExternalOptimizerModelProxy, r as createRunCostLedger, t as acquireSingleRunLock, u as runExternalOptimizerProcess, w as isRecord, x as assertNoCredentialValues, y as assertExternalOptimizerModelBudget } from "./single-run-lock-DFWHEB09.js";
|
|
6
7
|
import { n as canonicalJson, r as contentHash } from "./verdict-cache-BCcOh0kF.js";
|
|
7
8
|
import { p as mapConcurrent } from "./ledger-core-DXZIqu17.js";
|
|
8
9
|
import { E as pairedBootstrap, H as weightedComposite, d as confidenceInterval } from "./statistics-ByxzSiOM.js";
|
|
@@ -1413,73 +1414,241 @@ function designedCellIds(scenarios, reps) {
|
|
|
1413
1414
|
return ids;
|
|
1414
1415
|
}
|
|
1415
1416
|
//#endregion
|
|
1416
|
-
//#region src/campaign/external-optimizer-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
const
|
|
1435
|
-
|
|
1436
|
-
|
|
1417
|
+
//#region src/campaign/external-optimizer-observations.ts
|
|
1418
|
+
/**
|
|
1419
|
+
* Read and verify the exact callback observation artifact addressed by method provenance.
|
|
1420
|
+
*
|
|
1421
|
+
* The reader checks the raw SHA-256, canonical JSONL bytes, sequence, candidate
|
|
1422
|
+
* identities, and summary counts before it returns any candidate.
|
|
1423
|
+
* This proves that the bytes match the supplied summary. The caller remains
|
|
1424
|
+
* responsible for obtaining that summary from trusted provenance.
|
|
1425
|
+
*/
|
|
1426
|
+
function readExternalOptimizerObservationArtifact(input) {
|
|
1427
|
+
const { summary } = input;
|
|
1428
|
+
assertObservationSummary(summary);
|
|
1429
|
+
const storage = input.storage ?? fsCampaignStorage();
|
|
1430
|
+
const contents = storage.read(summary.path);
|
|
1431
|
+
if (contents === void 0) {
|
|
1432
|
+
const state = storage.exists(summary.path) ? "unreadable" : "missing";
|
|
1433
|
+
throw new Error(`external optimizer observation artifact is ${state} at '${summary.path}'`);
|
|
1434
|
+
}
|
|
1435
|
+
const sha256 = `sha256:${createHash("sha256").update(contents).digest("hex")}`;
|
|
1436
|
+
if (sha256 !== summary.sha256) throw new Error(`external optimizer observation artifact digest mismatch at '${summary.path}': expected ${summary.sha256}, got ${sha256}`);
|
|
1437
|
+
if (contents.length > 0 && !contents.endsWith("\n")) throw new Error(`external optimizer observation artifact must end with a newline at '${summary.path}'`);
|
|
1438
|
+
const observations = contents.length === 0 ? [] : parseObservationLines(contents, summary.path);
|
|
1439
|
+
const candidates = [];
|
|
1440
|
+
const proposed = /* @__PURE__ */ new Map();
|
|
1441
|
+
const evaluationNumbers = /* @__PURE__ */ new Set();
|
|
1442
|
+
let evaluations = 0;
|
|
1443
|
+
let failedEvaluations = 0;
|
|
1444
|
+
let refusals = 0;
|
|
1445
|
+
for (const [index, observation] of observations.entries()) {
|
|
1446
|
+
if (observation.sequence !== index + 1) throw new Error(`external optimizer observation artifact expected sequence ${index + 1}, got ${observation.sequence}`);
|
|
1447
|
+
if (observation.kind === "proposal") {
|
|
1448
|
+
assertCandidateIdentity(observation.candidate, observation.candidateHash, observation.sequence);
|
|
1449
|
+
if (proposed.has(observation.candidateHash)) throw new Error(`external optimizer observation artifact repeats candidate ${observation.candidateHash}`);
|
|
1450
|
+
const candidate = {
|
|
1451
|
+
candidate: structuredClone(observation.candidate),
|
|
1452
|
+
candidateHash: observation.candidateHash,
|
|
1453
|
+
candidateDigest: `sha256:${observation.candidateHash}`,
|
|
1454
|
+
proposalSequence: observation.sequence,
|
|
1455
|
+
provenance: {
|
|
1456
|
+
path: summary.path,
|
|
1457
|
+
sha256: summary.sha256
|
|
1458
|
+
}
|
|
1459
|
+
};
|
|
1460
|
+
proposed.set(observation.candidateHash, candidate);
|
|
1461
|
+
candidates.push(candidate);
|
|
1462
|
+
continue;
|
|
1463
|
+
}
|
|
1464
|
+
if (observation.kind === "evaluation") {
|
|
1465
|
+
evaluations += 1;
|
|
1466
|
+
if (evaluationNumbers.has(observation.evaluationNumber)) throw new Error(`external optimizer observation artifact repeats evaluation number ${observation.evaluationNumber}`);
|
|
1467
|
+
evaluationNumbers.add(observation.evaluationNumber);
|
|
1468
|
+
assertObservedCandidate(observation, proposed);
|
|
1469
|
+
assertJsonValue(observation.response, "external optimizer evaluation response");
|
|
1470
|
+
continue;
|
|
1471
|
+
}
|
|
1472
|
+
refusals += 1;
|
|
1473
|
+
if (observation.reason !== "invalid-request" && observation.reason !== "evaluation-limit" && observation.reason !== "evaluation-failed") throw new Error(`external optimizer observation artifact has invalid refusal reason at sequence ${observation.sequence}`);
|
|
1474
|
+
if (observation.reason === "evaluation-failed") failedEvaluations += 1;
|
|
1475
|
+
const candidateFields = [observation.candidate, observation.candidateHash];
|
|
1476
|
+
if (observation.reason !== "invalid-request" && candidateFields.some((value) => value === void 0)) throw new Error(`external optimizer observation artifact omits a refused candidate at sequence ${observation.sequence}`);
|
|
1477
|
+
if (candidateFields.some((value) => value !== void 0)) {
|
|
1478
|
+
if (candidateFields.some((value) => value === void 0)) throw new Error(`external optimizer observation artifact has incomplete candidate identity at sequence ${observation.sequence}`);
|
|
1479
|
+
assertObservedCandidate(observation, proposed);
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
const acceptedEvaluations = evaluations + failedEvaluations;
|
|
1483
|
+
for (const evaluationNumber of evaluationNumbers) if (evaluationNumber > acceptedEvaluations) throw new Error(`external optimizer observation artifact has out-of-range evaluation number ${evaluationNumber}`);
|
|
1484
|
+
if (candidates.length !== summary.submittedCandidates || evaluations !== summary.evaluations || refusals !== summary.refusals) throw new Error(`external optimizer observation artifact counts disagree at '${summary.path}': expected ${summary.submittedCandidates}/${summary.evaluations}/${summary.refusals}, got ${candidates.length}/${evaluations}/${refusals}`);
|
|
1485
|
+
return deepFreezeCanonicalJson({
|
|
1486
|
+
summary: structuredClone(summary),
|
|
1487
|
+
observations,
|
|
1488
|
+
candidates
|
|
1437
1489
|
});
|
|
1490
|
+
}
|
|
1491
|
+
/** Append-only observation log for one external-optimizer attempt. */
|
|
1492
|
+
function openExternalOptimizerObservationLog(input) {
|
|
1493
|
+
if (input.storage.read(input.path) !== void 0 || input.storage.exists(input.path)) throw new Error(`external optimizer observation log already exists at '${input.path}'`);
|
|
1494
|
+
input.storage.write(input.path, "");
|
|
1495
|
+
let revision = 0;
|
|
1496
|
+
const counts = {
|
|
1497
|
+
submittedCandidates: 0,
|
|
1498
|
+
evaluations: 0,
|
|
1499
|
+
refusals: 0
|
|
1500
|
+
};
|
|
1438
1501
|
return {
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1502
|
+
observe(observation) {
|
|
1503
|
+
const expectedSequence = counts.submittedCandidates + counts.evaluations + counts.refusals + 1;
|
|
1504
|
+
if (observation.sequence !== expectedSequence) throw new Error(`external optimizer observation log expected sequence ${expectedSequence}, got ${observation.sequence}`);
|
|
1505
|
+
const line = `${canonicalJson(observation)}\n`;
|
|
1506
|
+
const next = input.storage.append(input.path, line, revision);
|
|
1507
|
+
if (next === void 0) throw new Error(`external optimizer observation log changed concurrently at '${input.path}'`);
|
|
1508
|
+
revision = next;
|
|
1509
|
+
if (observation.kind === "proposal") counts.submittedCandidates += 1;
|
|
1510
|
+
else if (observation.kind === "evaluation") counts.evaluations += 1;
|
|
1511
|
+
else counts.refusals += 1;
|
|
1512
|
+
},
|
|
1513
|
+
summary() {
|
|
1514
|
+
const contents = input.storage.read(input.path) ?? "";
|
|
1515
|
+
const bytes = new TextEncoder().encode(contents);
|
|
1516
|
+
if (bytes.byteLength !== revision) throw new Error(`external optimizer observation log revision changed at '${input.path}'`);
|
|
1517
|
+
return {
|
|
1518
|
+
scope: "callback-submitted-candidates",
|
|
1519
|
+
path: input.path,
|
|
1520
|
+
sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
|
|
1521
|
+
...counts
|
|
1522
|
+
};
|
|
1459
1523
|
}
|
|
1460
1524
|
};
|
|
1461
1525
|
}
|
|
1462
|
-
function
|
|
1463
|
-
if (
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1526
|
+
function assertObservationSummary(summary) {
|
|
1527
|
+
if (summary.scope !== "callback-submitted-candidates") throw new Error("external optimizer observation summary has an invalid scope");
|
|
1528
|
+
if (typeof summary.path !== "string" || summary.path.trim().length === 0) throw new Error("external optimizer observation summary has an invalid path");
|
|
1529
|
+
if (!/^sha256:[a-f0-9]{64}$/u.test(summary.sha256)) throw new Error("external optimizer observation summary has an invalid SHA-256");
|
|
1530
|
+
for (const [field, value] of [
|
|
1531
|
+
["submittedCandidates", summary.submittedCandidates],
|
|
1532
|
+
["evaluations", summary.evaluations],
|
|
1533
|
+
["refusals", summary.refusals]
|
|
1534
|
+
]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`external optimizer observation summary ${field} must be a non-negative integer`);
|
|
1535
|
+
}
|
|
1536
|
+
function parseObservationLines(contents, path) {
|
|
1537
|
+
return contents.slice(0, -1).split("\n").map((line, index) => {
|
|
1538
|
+
let observation;
|
|
1471
1539
|
try {
|
|
1472
|
-
|
|
1540
|
+
observation = JSON.parse(line);
|
|
1473
1541
|
} catch (cause) {
|
|
1474
|
-
throw new Error(`external optimizer
|
|
1542
|
+
throw new Error(`external optimizer observation artifact has invalid JSON at '${path}' line ${index + 1}`, { cause });
|
|
1475
1543
|
}
|
|
1476
|
-
|
|
1477
|
-
|
|
1544
|
+
assertObservationShape(observation, index + 1);
|
|
1545
|
+
if (canonicalJson(observation) !== line) throw new Error(`external optimizer observation artifact is not canonical at '${path}' line ${index + 1}`);
|
|
1546
|
+
return observation;
|
|
1547
|
+
});
|
|
1548
|
+
}
|
|
1549
|
+
function assertObservationShape(value, line) {
|
|
1550
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`external optimizer observation artifact line ${line} must be an object`);
|
|
1551
|
+
const observation = value;
|
|
1552
|
+
if (!Number.isSafeInteger(observation.sequence) || Number(observation.sequence) <= 0) throw new Error(`external optimizer observation artifact line ${line} has an invalid sequence`);
|
|
1553
|
+
if (observation.kind === "proposal") {
|
|
1554
|
+
assertExactKeys$1(observation, [
|
|
1555
|
+
"candidate",
|
|
1556
|
+
"candidateHash",
|
|
1557
|
+
"kind",
|
|
1558
|
+
"sequence"
|
|
1559
|
+
], line);
|
|
1560
|
+
assertCandidateFields(observation, line);
|
|
1561
|
+
return;
|
|
1478
1562
|
}
|
|
1479
|
-
if (
|
|
1563
|
+
if (observation.kind === "evaluation") {
|
|
1564
|
+
assertExactKeys$1(observation, [
|
|
1565
|
+
"candidate",
|
|
1566
|
+
"candidateHash",
|
|
1567
|
+
"evaluationNumber",
|
|
1568
|
+
"exampleId",
|
|
1569
|
+
"kind",
|
|
1570
|
+
"response",
|
|
1571
|
+
"sequence"
|
|
1572
|
+
], line);
|
|
1573
|
+
assertCandidateFields(observation, line);
|
|
1574
|
+
if (typeof observation.exampleId !== "string") throw new Error(`external optimizer observation artifact line ${line} has an invalid exampleId`);
|
|
1575
|
+
if (!Number.isSafeInteger(observation.evaluationNumber) || Number(observation.evaluationNumber) <= 0) throw new Error(`external optimizer observation artifact line ${line} has an invalid evaluationNumber`);
|
|
1576
|
+
assertJsonValue(observation.response, `external optimizer observation artifact line ${line}`);
|
|
1577
|
+
return;
|
|
1578
|
+
}
|
|
1579
|
+
if (observation.kind === "refusal") {
|
|
1580
|
+
assertAllowedKeys(observation, [
|
|
1581
|
+
"candidate",
|
|
1582
|
+
"candidateHash",
|
|
1583
|
+
"exampleId",
|
|
1584
|
+
"kind",
|
|
1585
|
+
"reason",
|
|
1586
|
+
"sequence"
|
|
1587
|
+
], line);
|
|
1588
|
+
if (typeof observation.reason !== "string") throw new Error(`external optimizer observation artifact line ${line} has an invalid reason`);
|
|
1589
|
+
if (observation.candidate !== void 0 || observation.candidateHash !== void 0) assertCandidateFields(observation, line);
|
|
1590
|
+
if (observation.exampleId !== void 0 && typeof observation.exampleId !== "string") throw new Error(`external optimizer observation artifact line ${line} has an invalid exampleId`);
|
|
1591
|
+
return;
|
|
1592
|
+
}
|
|
1593
|
+
throw new Error(`external optimizer observation artifact line ${line} has an invalid kind`);
|
|
1594
|
+
}
|
|
1595
|
+
function assertCandidateFields(observation, line) {
|
|
1596
|
+
if (!isExternalTextCandidate(observation.candidate)) throw new Error(`external optimizer observation artifact line ${line} has an invalid candidate`);
|
|
1597
|
+
if (typeof observation.candidateHash !== "string" || !/^[a-f0-9]{64}$/u.test(observation.candidateHash)) throw new Error(`external optimizer observation artifact line ${line} has an invalid candidateHash`);
|
|
1598
|
+
}
|
|
1599
|
+
function assertExactKeys$1(value, expected, line) {
|
|
1600
|
+
assertAllowedKeys(value, expected, line);
|
|
1601
|
+
for (const key of expected) if (!(key in value)) throw new Error(`external optimizer observation artifact line ${line} is missing ${key}`);
|
|
1602
|
+
}
|
|
1603
|
+
function assertAllowedKeys(value, allowed, line) {
|
|
1604
|
+
const extras = Object.keys(value).filter((key) => !allowed.includes(key));
|
|
1605
|
+
if (extras.length > 0) throw new Error(`external optimizer observation artifact line ${line} has unexpected field ${extras[0]}`);
|
|
1606
|
+
}
|
|
1607
|
+
function assertCandidateIdentity(candidate, candidateHash, sequence) {
|
|
1608
|
+
if (candidateHash !== contentHash({
|
|
1609
|
+
kind: "external-text-candidate",
|
|
1610
|
+
candidate
|
|
1611
|
+
})) throw new Error(`external optimizer observation artifact candidate hash mismatch at sequence ${sequence}`);
|
|
1612
|
+
}
|
|
1613
|
+
function assertObservedCandidate(observation, proposed) {
|
|
1614
|
+
assertCandidateIdentity(observation.candidate, observation.candidateHash, observation.sequence);
|
|
1615
|
+
const candidate = proposed.get(observation.candidateHash);
|
|
1616
|
+
if (!candidate) throw new Error(`external optimizer observation artifact references an unproposed candidate at sequence ${observation.sequence}`);
|
|
1617
|
+
if (!isDeepStrictEqual(observation.candidate, candidate.candidate)) throw new Error(`external optimizer observation artifact changes candidate bytes at sequence ${observation.sequence}`);
|
|
1618
|
+
}
|
|
1619
|
+
/** Append-only opaque Runtime execution records for one optimizer attempt. */
|
|
1620
|
+
function openExternalOptimizerExecutionLog(input) {
|
|
1621
|
+
if (input.storage.read(input.path) !== void 0 || input.storage.exists(input.path)) throw new Error(`external optimizer execution log already exists at '${input.path}'`);
|
|
1622
|
+
input.storage.write(input.path, "");
|
|
1623
|
+
let revision = 0;
|
|
1624
|
+
const counts = {
|
|
1625
|
+
calls: 0,
|
|
1626
|
+
succeeded: 0,
|
|
1627
|
+
failed: 0
|
|
1628
|
+
};
|
|
1480
1629
|
return {
|
|
1481
|
-
|
|
1482
|
-
|
|
1630
|
+
observe(observation) {
|
|
1631
|
+
if (observation.sequence !== counts.calls + 1) throw new Error(`external optimizer execution log expected sequence ${counts.calls + 1}, got ${observation.sequence}`);
|
|
1632
|
+
const line = `${canonicalJson(observation)}\n`;
|
|
1633
|
+
const next = input.storage.append(input.path, line, revision);
|
|
1634
|
+
if (next === void 0) throw new Error(`external optimizer execution log changed concurrently at '${input.path}'`);
|
|
1635
|
+
revision = next;
|
|
1636
|
+
counts.calls += 1;
|
|
1637
|
+
if (observation.succeeded) counts.succeeded += 1;
|
|
1638
|
+
else counts.failed += 1;
|
|
1639
|
+
},
|
|
1640
|
+
summary() {
|
|
1641
|
+
const contents = input.storage.read(input.path) ?? "";
|
|
1642
|
+
const bytes = new TextEncoder().encode(contents);
|
|
1643
|
+
if (bytes.byteLength !== revision) throw new Error(`external optimizer execution log revision changed at '${input.path}'`);
|
|
1644
|
+
if (counts.calls !== counts.succeeded + counts.failed) throw new Error(`external optimizer execution log counts disagree at '${input.path}'`);
|
|
1645
|
+
return {
|
|
1646
|
+
scope: "runtime-model-calls",
|
|
1647
|
+
path: input.path,
|
|
1648
|
+
sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
|
|
1649
|
+
...counts
|
|
1650
|
+
};
|
|
1651
|
+
}
|
|
1483
1652
|
};
|
|
1484
1653
|
}
|
|
1485
1654
|
//#endregion
|
|
@@ -2830,6 +2999,76 @@ function deepFreeze$1(value) {
|
|
|
2830
2999
|
return value;
|
|
2831
3000
|
}
|
|
2832
3001
|
//#endregion
|
|
3002
|
+
//#region src/campaign/external-optimizer-run-budget.ts
|
|
3003
|
+
function externalOptimizerRunKey(input) {
|
|
3004
|
+
if (!input.attemptId.trim() || input.attemptId.trim() !== input.attemptId) throw new Error("external optimizer attemptId must be trimmed and non-empty");
|
|
3005
|
+
if (typeof input.resumeEnabled !== "boolean") throw new Error("external optimizer resumeEnabled must be a boolean");
|
|
3006
|
+
const compatible = externalOptimizerCompatibleRunKey(input.material);
|
|
3007
|
+
return input.resumeEnabled ? compatible : `${compatible}-${input.attemptId}`;
|
|
3008
|
+
}
|
|
3009
|
+
function externalOptimizerCompatibleRunKey(material) {
|
|
3010
|
+
return contentHash(material);
|
|
3011
|
+
}
|
|
3012
|
+
function openExternalOptimizerRunBudget(input) {
|
|
3013
|
+
if (!input.runKey.trim() || input.runKey.trim() !== input.runKey) throw new Error("external optimizer runKey must be trimmed and non-empty");
|
|
3014
|
+
if (!input.attemptId.trim() || input.attemptId.trim() !== input.attemptId) throw new Error("external optimizer attemptId must be trimmed and non-empty");
|
|
3015
|
+
if (!Number.isSafeInteger(input.maxEvaluations) || input.maxEvaluations <= 0) throw new Error("external optimizer maxEvaluations must be a positive safe integer");
|
|
3016
|
+
const budgetDir = `${input.runDir}/budgets`;
|
|
3017
|
+
const statePath = `${budgetDir}/${input.runKey}.jsonl`;
|
|
3018
|
+
input.storage.ensureDir(budgetDir);
|
|
3019
|
+
const runTags = Object.freeze({ optimizerRun: input.runKey });
|
|
3020
|
+
const attemptTags = Object.freeze({
|
|
3021
|
+
optimizerRun: input.runKey,
|
|
3022
|
+
optimizerAttempt: input.attemptId
|
|
3023
|
+
});
|
|
3024
|
+
return {
|
|
3025
|
+
runKey: input.runKey,
|
|
3026
|
+
attemptId: input.attemptId,
|
|
3027
|
+
runTags,
|
|
3028
|
+
attemptTags,
|
|
3029
|
+
acceptedEvaluations: () => readState(input.storage.read(statePath), input.maxEvaluations, statePath).accepted,
|
|
3030
|
+
acceptEvaluation() {
|
|
3031
|
+
for (let attempt = 0; attempt < 16; attempt += 1) {
|
|
3032
|
+
const current = input.storage.read(statePath) ?? "";
|
|
3033
|
+
const state = readState(current, input.maxEvaluations, statePath);
|
|
3034
|
+
if (state.accepted >= input.maxEvaluations) return void 0;
|
|
3035
|
+
if (!input.storage.append) throw new Error("external optimizer evaluation budgets require appendable storage");
|
|
3036
|
+
const next = state.accepted + 1;
|
|
3037
|
+
const event = `${JSON.stringify({
|
|
3038
|
+
maxEvaluations: input.maxEvaluations,
|
|
3039
|
+
accepted: next
|
|
3040
|
+
})}\n`;
|
|
3041
|
+
const expectedBytes = new TextEncoder().encode(current).byteLength;
|
|
3042
|
+
if (input.storage.append(statePath, event, expectedBytes) !== void 0) return next;
|
|
3043
|
+
}
|
|
3044
|
+
throw new Error(`external optimizer evaluation counter for '${input.runKey}' was updated concurrently`);
|
|
3045
|
+
}
|
|
3046
|
+
};
|
|
3047
|
+
}
|
|
3048
|
+
function readState(text, maxEvaluations, statePath) {
|
|
3049
|
+
if (text === void 0 || text === "") return {
|
|
3050
|
+
maxEvaluations,
|
|
3051
|
+
accepted: 0
|
|
3052
|
+
};
|
|
3053
|
+
let accepted = 0;
|
|
3054
|
+
for (const line of text.split("\n")) {
|
|
3055
|
+
if (!line) continue;
|
|
3056
|
+
let value;
|
|
3057
|
+
try {
|
|
3058
|
+
value = JSON.parse(line);
|
|
3059
|
+
} catch (cause) {
|
|
3060
|
+
throw new Error(`external optimizer evaluation state is invalid at '${statePath}'`, { cause });
|
|
3061
|
+
}
|
|
3062
|
+
if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).sort().join(",") !== "accepted,maxEvaluations" || value.maxEvaluations !== maxEvaluations || value.accepted !== accepted + 1) throw new Error(`external optimizer evaluation state does not match at '${statePath}'`);
|
|
3063
|
+
accepted += 1;
|
|
3064
|
+
}
|
|
3065
|
+
if (accepted > maxEvaluations) throw new Error(`external optimizer evaluation state exceeds its limit at '${statePath}'`);
|
|
3066
|
+
return {
|
|
3067
|
+
maxEvaluations,
|
|
3068
|
+
accepted
|
|
3069
|
+
};
|
|
3070
|
+
}
|
|
3071
|
+
//#endregion
|
|
2833
3072
|
//#region src/campaign/external-text-optimization-contract.ts
|
|
2834
3073
|
const MAX_TIMER_DELAY_MS$3 = 2147483647;
|
|
2835
3074
|
function assertExternalTextOptimizationConfig(config) {
|
|
@@ -2924,6 +3163,191 @@ function assertExternalCostAccounting(value, calls, name) {
|
|
|
2924
3163
|
if (accounting.kind !== "external" || typeof accounting.reason !== "string" || !accounting.reason.trim() || accounting.reason.trim() !== accounting.reason) throw new Error(`${name}: optimizer returned invalid costAccounting`);
|
|
2925
3164
|
}
|
|
2926
3165
|
//#endregion
|
|
3166
|
+
//#region src/campaign/gepa-candidate-population.ts
|
|
3167
|
+
/**
|
|
3168
|
+
* Read GEPA's exact candidate graph from the artifact addressed by method provenance.
|
|
3169
|
+
*
|
|
3170
|
+
* The reader checks the supplied digest, declared byte count, run identity,
|
|
3171
|
+
* candidate surfaces, parent graph, selection scores, and configured bounds.
|
|
3172
|
+
* This proves that the bytes match the supplied summary. The caller remains
|
|
3173
|
+
* responsible for obtaining that summary from trusted method provenance.
|
|
3174
|
+
*/
|
|
3175
|
+
function readGepaCandidatePopulationArtifact(input) {
|
|
3176
|
+
assertGepaCandidatePopulationSummary(input.summary);
|
|
3177
|
+
const scenarioIds = scenarioIdSet(input.summary.scenarioIds);
|
|
3178
|
+
const storage = input.storage ?? fsCampaignStorage();
|
|
3179
|
+
const contents = storage.read(input.summary.path);
|
|
3180
|
+
if (contents === void 0) {
|
|
3181
|
+
const state = storage.exists(input.summary.path) ? "unreadable" : "missing";
|
|
3182
|
+
throw new Error(`GEPA candidate population artifact is ${state} at '${input.summary.path}'`);
|
|
3183
|
+
}
|
|
3184
|
+
const bytes = new TextEncoder().encode(contents).byteLength;
|
|
3185
|
+
if (bytes !== input.summary.bytes) throw new Error(`GEPA candidate population byte count mismatch at '${input.summary.path}': expected ${input.summary.bytes}, got ${bytes}`);
|
|
3186
|
+
if (BigInt(bytes) > maximumArtifactBytes(input.summary.maxCandidates, input.summary.maxCandidateChars, scenarioIds)) throw new Error(`GEPA candidate population exceeds its configured bounds at '${input.summary.path}'`);
|
|
3187
|
+
const sha256 = `sha256:${createHash("sha256").update(contents).digest("hex")}`;
|
|
3188
|
+
if (sha256 !== input.summary.sha256) throw new Error(`GEPA candidate population digest mismatch at '${input.summary.path}': expected ${input.summary.sha256}, got ${sha256}`);
|
|
3189
|
+
let raw;
|
|
3190
|
+
try {
|
|
3191
|
+
raw = JSON.parse(contents);
|
|
3192
|
+
} catch (cause) {
|
|
3193
|
+
throw new Error(`GEPA candidate population is not JSON at '${input.summary.path}'`, { cause });
|
|
3194
|
+
}
|
|
3195
|
+
assertExactKeys(raw, [
|
|
3196
|
+
"bestIndex",
|
|
3197
|
+
"candidates",
|
|
3198
|
+
"runId",
|
|
3199
|
+
"schemaVersion",
|
|
3200
|
+
"scope"
|
|
3201
|
+
], "artifact");
|
|
3202
|
+
if (raw.schemaVersion !== 1 || raw.scope !== "gepa-candidate-population") throw new Error("GEPA candidate population has an unsupported schema");
|
|
3203
|
+
if (raw.runId !== input.summary.runId) throw new Error("GEPA candidate population run ID differs from its summary");
|
|
3204
|
+
if (!Array.isArray(raw.candidates) || raw.candidates.length === 0) throw new Error("GEPA candidate population must contain candidates");
|
|
3205
|
+
if (raw.candidates.length !== input.summary.candidates || raw.candidates.length > input.summary.maxCandidates) throw new Error("GEPA candidate population count differs from its summary or configured bound");
|
|
3206
|
+
const bestIndex = raw.bestIndex;
|
|
3207
|
+
if (typeof bestIndex !== "number" || !Number.isSafeInteger(bestIndex) || bestIndex < 0 || bestIndex >= raw.candidates.length || bestIndex !== input.summary.bestIndex) throw new Error("GEPA candidate population has an invalid best index");
|
|
3208
|
+
const candidates = raw.candidates.map((candidate, index) => parseCandidate({
|
|
3209
|
+
candidate,
|
|
3210
|
+
index,
|
|
3211
|
+
maxCandidateChars: input.summary.maxCandidateChars,
|
|
3212
|
+
scenarioIds,
|
|
3213
|
+
expectsComponents: input.summary.surfaceKind === "components"
|
|
3214
|
+
}));
|
|
3215
|
+
return deepFreezeCanonicalJson({
|
|
3216
|
+
summary: structuredClone(input.summary),
|
|
3217
|
+
runId: raw.runId,
|
|
3218
|
+
bestIndex,
|
|
3219
|
+
candidates
|
|
3220
|
+
});
|
|
3221
|
+
}
|
|
3222
|
+
function assertGepaCandidatePopulationSummary(value) {
|
|
3223
|
+
assertExactKeys(value, [
|
|
3224
|
+
"bestIndex",
|
|
3225
|
+
"bytes",
|
|
3226
|
+
"candidates",
|
|
3227
|
+
"maxCandidateChars",
|
|
3228
|
+
"maxCandidates",
|
|
3229
|
+
"path",
|
|
3230
|
+
"runId",
|
|
3231
|
+
"scenarioIds",
|
|
3232
|
+
"scope",
|
|
3233
|
+
"sha256",
|
|
3234
|
+
"surfaceKind"
|
|
3235
|
+
], "summary");
|
|
3236
|
+
if (value.scope !== "gepa-candidate-population") throw new Error("GEPA candidate population summary has an invalid scope");
|
|
3237
|
+
if (typeof value.path !== "string" || value.path.trim().length === 0) throw new Error("GEPA candidate population summary has an invalid path");
|
|
3238
|
+
if (typeof value.sha256 !== "string" || !/^sha256:[a-f0-9]{64}$/u.test(value.sha256)) throw new Error("GEPA candidate population summary has an invalid SHA-256");
|
|
3239
|
+
if (typeof value.runId !== "string" || value.runId.trim().length === 0) throw new Error("GEPA candidate population summary has an invalid run ID");
|
|
3240
|
+
const bytes = value.bytes;
|
|
3241
|
+
const candidates = value.candidates;
|
|
3242
|
+
const maxCandidates = value.maxCandidates;
|
|
3243
|
+
const maxCandidateChars = value.maxCandidateChars;
|
|
3244
|
+
assertPositiveSafeInteger$2(bytes, "summary.bytes");
|
|
3245
|
+
assertPositiveSafeInteger$2(candidates, "summary.candidates");
|
|
3246
|
+
assertPositiveSafeInteger$2(maxCandidates, "summary.maxCandidates");
|
|
3247
|
+
assertPositiveSafeInteger$2(maxCandidateChars, "summary.maxCandidateChars");
|
|
3248
|
+
if (candidates > maxCandidates) throw new Error("GEPA candidate population summary exceeds its configured candidate bound");
|
|
3249
|
+
scenarioIdSet(value.scenarioIds);
|
|
3250
|
+
if (value.surfaceKind !== "text" && value.surfaceKind !== "components") throw new Error("GEPA candidate population summary has an invalid surface kind");
|
|
3251
|
+
const bestIndex = value.bestIndex;
|
|
3252
|
+
if (typeof bestIndex !== "number" || !Number.isSafeInteger(bestIndex) || bestIndex < 0 || bestIndex >= candidates) throw new Error("GEPA candidate population summary has an invalid best index");
|
|
3253
|
+
}
|
|
3254
|
+
function parseCandidate(input) {
|
|
3255
|
+
const { candidate, index } = input;
|
|
3256
|
+
assertExactKeys(candidate, [
|
|
3257
|
+
"aggregateScore",
|
|
3258
|
+
"candidate",
|
|
3259
|
+
"discoveryEvaluationCount",
|
|
3260
|
+
"index",
|
|
3261
|
+
"parentIndices",
|
|
3262
|
+
"selectionScores"
|
|
3263
|
+
], `candidate ${index}`);
|
|
3264
|
+
if (candidate.index !== index) throw new Error(`GEPA candidate population expected index ${index}`);
|
|
3265
|
+
if (!isExternalTextCandidate(candidate.candidate) || input.expectsComponents !== (typeof candidate.candidate !== "string") || candidateChars(candidate.candidate) > input.maxCandidateChars) throw new Error(`GEPA candidate ${index} has an invalid surface`);
|
|
3266
|
+
const parentIndices = parseParents(candidate.parentIndices, index);
|
|
3267
|
+
const selectionScores = parseSelectionScores(candidate.selectionScores, input.scenarioIds, index);
|
|
3268
|
+
const aggregateScore = parseAggregateScore(candidate.aggregateScore, selectionScores, index);
|
|
3269
|
+
const discoveryEvaluationCount = candidate.discoveryEvaluationCount;
|
|
3270
|
+
if (typeof discoveryEvaluationCount !== "number" || !Number.isSafeInteger(discoveryEvaluationCount) || discoveryEvaluationCount < 0) throw new Error(`GEPA candidate ${index} has an invalid discovery evaluation count`);
|
|
3271
|
+
const candidateHash = contentHash({
|
|
3272
|
+
kind: "external-text-candidate",
|
|
3273
|
+
candidate: candidate.candidate
|
|
3274
|
+
});
|
|
3275
|
+
return {
|
|
3276
|
+
index,
|
|
3277
|
+
candidate: structuredClone(candidate.candidate),
|
|
3278
|
+
candidateHash,
|
|
3279
|
+
candidateDigest: `sha256:${candidateHash}`,
|
|
3280
|
+
parentIndices,
|
|
3281
|
+
aggregateScore,
|
|
3282
|
+
selectionScores,
|
|
3283
|
+
discoveryEvaluationCount
|
|
3284
|
+
};
|
|
3285
|
+
}
|
|
3286
|
+
function parseParents(value, index) {
|
|
3287
|
+
if (!Array.isArray(value) || value.length === 0) throw new Error(`GEPA candidate ${index} has invalid parents`);
|
|
3288
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3289
|
+
for (const parent of value) {
|
|
3290
|
+
if (parent === null) {
|
|
3291
|
+
if (index !== 0) throw new Error(`GEPA candidate ${index} has a null parent`);
|
|
3292
|
+
} else if (!Number.isSafeInteger(parent) || parent < 0 || parent >= index) throw new Error(`GEPA candidate ${index} has an invalid parent index`);
|
|
3293
|
+
if (seen.has(parent)) throw new Error(`GEPA candidate ${index} repeats a parent index`);
|
|
3294
|
+
seen.add(parent);
|
|
3295
|
+
}
|
|
3296
|
+
if (index === 0 && (value.length !== 1 || value[0] !== null)) throw new Error("GEPA root candidate must have one null parent");
|
|
3297
|
+
return [...value];
|
|
3298
|
+
}
|
|
3299
|
+
function parseSelectionScores(value, scenarioIds, candidateIndex) {
|
|
3300
|
+
if (!Array.isArray(value)) throw new Error(`GEPA candidate ${candidateIndex} has invalid selection scores`);
|
|
3301
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3302
|
+
let prior = "";
|
|
3303
|
+
return value.map((row, index) => {
|
|
3304
|
+
assertExactKeys(row, ["scenarioId", "score"], `candidate ${candidateIndex} score ${index}`);
|
|
3305
|
+
if (typeof row.scenarioId !== "string" || !scenarioIds.has(row.scenarioId) || seen.has(row.scenarioId) || index > 0 && row.scenarioId <= prior) throw new Error(`GEPA candidate ${candidateIndex} has invalid selection score identity`);
|
|
3306
|
+
if (typeof row.score !== "number" || !Number.isFinite(row.score)) throw new Error(`GEPA candidate ${candidateIndex} has an invalid selection score`);
|
|
3307
|
+
seen.add(row.scenarioId);
|
|
3308
|
+
prior = row.scenarioId;
|
|
3309
|
+
return {
|
|
3310
|
+
scenarioId: row.scenarioId,
|
|
3311
|
+
score: row.score
|
|
3312
|
+
};
|
|
3313
|
+
});
|
|
3314
|
+
}
|
|
3315
|
+
function parseAggregateScore(value, scores, candidateIndex) {
|
|
3316
|
+
if (scores.length === 0) {
|
|
3317
|
+
if (value !== null) throw new Error(`GEPA candidate ${candidateIndex} has a score without selection evidence`);
|
|
3318
|
+
return null;
|
|
3319
|
+
}
|
|
3320
|
+
if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`GEPA candidate ${candidateIndex} has an invalid aggregate score`);
|
|
3321
|
+
const mean = scores.reduce((total, row) => total + row.score, 0) / scores.length;
|
|
3322
|
+
if (Math.abs(value - mean) > Math.max(1e-12, Math.abs(mean) * 1e-9)) throw new Error(`GEPA candidate ${candidateIndex} aggregate score differs from its selection scores`);
|
|
3323
|
+
return value;
|
|
3324
|
+
}
|
|
3325
|
+
function candidateChars(candidate) {
|
|
3326
|
+
return typeof candidate === "string" ? candidate.length : JSON.stringify(candidate).length;
|
|
3327
|
+
}
|
|
3328
|
+
function scenarioIdSet(values) {
|
|
3329
|
+
if (!Array.isArray(values)) throw new Error("scenarioIds must be an array");
|
|
3330
|
+
const result = /* @__PURE__ */ new Set();
|
|
3331
|
+
for (const value of values) {
|
|
3332
|
+
if (typeof value !== "string" || value.length === 0 || result.has(value)) throw new Error("scenarioIds must contain unique non-empty strings");
|
|
3333
|
+
result.add(value);
|
|
3334
|
+
}
|
|
3335
|
+
return result;
|
|
3336
|
+
}
|
|
3337
|
+
function maximumArtifactBytes(maxCandidates, maxCandidateChars, scenarioIds) {
|
|
3338
|
+
const scenarioChars = [...scenarioIds].reduce((total, id) => total + id.length, 0);
|
|
3339
|
+
const candidates = BigInt(maxCandidates);
|
|
3340
|
+
return 8192n + candidates * (BigInt(maxCandidateChars) * 6n + BigInt(scenarioChars) * 6n + BigInt(scenarioIds.size) * 128n + candidates * 32n + 8192n);
|
|
3341
|
+
}
|
|
3342
|
+
function assertExactKeys(value, expected, label) {
|
|
3343
|
+
if (!isRecord(value)) throw new Error(`GEPA candidate population ${label} must be an object`);
|
|
3344
|
+
const actual = Object.keys(value).sort();
|
|
3345
|
+
if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) throw new Error(`GEPA candidate population ${label} has unexpected fields`);
|
|
3346
|
+
}
|
|
3347
|
+
function assertPositiveSafeInteger$2(value, label) {
|
|
3348
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`GEPA candidate population ${label} must be a positive safe integer`);
|
|
3349
|
+
}
|
|
3350
|
+
//#endregion
|
|
2927
3351
|
//#region src/campaign/presets/compare-optimization-methods.ts
|
|
2928
3352
|
/**
|
|
2929
3353
|
* Compare optimization methods on shared train, selection, and test data.
|
|
@@ -3178,6 +3602,14 @@ function assertOptimizationProvenance(methodName, value) {
|
|
|
3178
3602
|
"refusals"
|
|
3179
3603
|
]) if (!Number.isSafeInteger(value.observations[field]) || value.observations[field] < 0) fail(`observations.${field}`);
|
|
3180
3604
|
}
|
|
3605
|
+
if (value.gepaCandidatePopulation !== void 0) {
|
|
3606
|
+
try {
|
|
3607
|
+
assertGepaCandidatePopulationSummary(value.gepaCandidatePopulation);
|
|
3608
|
+
} catch {
|
|
3609
|
+
fail("gepaCandidatePopulation");
|
|
3610
|
+
}
|
|
3611
|
+
if (value.gepaCandidatePopulation.runId !== value.runId) fail("gepaCandidatePopulation.runId");
|
|
3612
|
+
}
|
|
3181
3613
|
if (value.modelExecutions !== void 0) {
|
|
3182
3614
|
if (value.modelExecutions.scope !== "runtime-model-calls" || typeof value.modelExecutions.path !== "string" || !value.modelExecutions.path.trim() || typeof value.modelExecutions.sha256 !== "string" || !/^sha256:[0-9a-f]{64}$/.test(value.modelExecutions.sha256)) fail("modelExecutions");
|
|
3183
3615
|
for (const field of [
|
|
@@ -4493,78 +4925,6 @@ function assertExternalOptimizerCompletionCount(upstream, requestAttempts, succe
|
|
|
4493
4925
|
if (upstream.requestAttempts !== void 0 && upstream.requestAttempts !== requestAttempts) throw new Error(`${name}: ${optimizer} reported ${upstream.requestAttempts} model attempts but the proxy received ${requestAttempts}`);
|
|
4494
4926
|
}
|
|
4495
4927
|
//#endregion
|
|
4496
|
-
//#region src/campaign/external-optimizer-observations.ts
|
|
4497
|
-
/** Append-only observation log for one external-optimizer attempt. */
|
|
4498
|
-
function openExternalOptimizerObservationLog(input) {
|
|
4499
|
-
if (input.storage.read(input.path) !== void 0 || input.storage.exists(input.path)) throw new Error(`external optimizer observation log already exists at '${input.path}'`);
|
|
4500
|
-
input.storage.write(input.path, "");
|
|
4501
|
-
let revision = 0;
|
|
4502
|
-
const counts = {
|
|
4503
|
-
submittedCandidates: 0,
|
|
4504
|
-
evaluations: 0,
|
|
4505
|
-
refusals: 0
|
|
4506
|
-
};
|
|
4507
|
-
return {
|
|
4508
|
-
observe(observation) {
|
|
4509
|
-
const expectedSequence = counts.submittedCandidates + counts.evaluations + counts.refusals + 1;
|
|
4510
|
-
if (observation.sequence !== expectedSequence) throw new Error(`external optimizer observation log expected sequence ${expectedSequence}, got ${observation.sequence}`);
|
|
4511
|
-
const line = `${canonicalJson(observation)}\n`;
|
|
4512
|
-
const next = input.storage.append(input.path, line, revision);
|
|
4513
|
-
if (next === void 0) throw new Error(`external optimizer observation log changed concurrently at '${input.path}'`);
|
|
4514
|
-
revision = next;
|
|
4515
|
-
if (observation.kind === "proposal") counts.submittedCandidates += 1;
|
|
4516
|
-
else if (observation.kind === "evaluation") counts.evaluations += 1;
|
|
4517
|
-
else counts.refusals += 1;
|
|
4518
|
-
},
|
|
4519
|
-
summary() {
|
|
4520
|
-
const contents = input.storage.read(input.path) ?? "";
|
|
4521
|
-
const bytes = new TextEncoder().encode(contents);
|
|
4522
|
-
if (bytes.byteLength !== revision) throw new Error(`external optimizer observation log revision changed at '${input.path}'`);
|
|
4523
|
-
return {
|
|
4524
|
-
scope: "callback-submitted-candidates",
|
|
4525
|
-
path: input.path,
|
|
4526
|
-
sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
|
|
4527
|
-
...counts
|
|
4528
|
-
};
|
|
4529
|
-
}
|
|
4530
|
-
};
|
|
4531
|
-
}
|
|
4532
|
-
/** Append-only opaque Runtime execution records for one optimizer attempt. */
|
|
4533
|
-
function openExternalOptimizerExecutionLog(input) {
|
|
4534
|
-
if (input.storage.read(input.path) !== void 0 || input.storage.exists(input.path)) throw new Error(`external optimizer execution log already exists at '${input.path}'`);
|
|
4535
|
-
input.storage.write(input.path, "");
|
|
4536
|
-
let revision = 0;
|
|
4537
|
-
const counts = {
|
|
4538
|
-
calls: 0,
|
|
4539
|
-
succeeded: 0,
|
|
4540
|
-
failed: 0
|
|
4541
|
-
};
|
|
4542
|
-
return {
|
|
4543
|
-
observe(observation) {
|
|
4544
|
-
if (observation.sequence !== counts.calls + 1) throw new Error(`external optimizer execution log expected sequence ${counts.calls + 1}, got ${observation.sequence}`);
|
|
4545
|
-
const line = `${canonicalJson(observation)}\n`;
|
|
4546
|
-
const next = input.storage.append(input.path, line, revision);
|
|
4547
|
-
if (next === void 0) throw new Error(`external optimizer execution log changed concurrently at '${input.path}'`);
|
|
4548
|
-
revision = next;
|
|
4549
|
-
counts.calls += 1;
|
|
4550
|
-
if (observation.succeeded) counts.succeeded += 1;
|
|
4551
|
-
else counts.failed += 1;
|
|
4552
|
-
},
|
|
4553
|
-
summary() {
|
|
4554
|
-
const contents = input.storage.read(input.path) ?? "";
|
|
4555
|
-
const bytes = new TextEncoder().encode(contents);
|
|
4556
|
-
if (bytes.byteLength !== revision) throw new Error(`external optimizer execution log revision changed at '${input.path}'`);
|
|
4557
|
-
if (counts.calls !== counts.succeeded + counts.failed) throw new Error(`external optimizer execution log counts disagree at '${input.path}'`);
|
|
4558
|
-
return {
|
|
4559
|
-
scope: "runtime-model-calls",
|
|
4560
|
-
path: input.path,
|
|
4561
|
-
sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
|
|
4562
|
-
...counts
|
|
4563
|
-
};
|
|
4564
|
-
}
|
|
4565
|
-
};
|
|
4566
|
-
}
|
|
4567
|
-
//#endregion
|
|
4568
4928
|
//#region src/campaign/external-optimizer-run-config.ts
|
|
4569
4929
|
function externalOptimizerRunnerIdentity(runner, module) {
|
|
4570
4930
|
return {
|
|
@@ -4827,7 +5187,7 @@ function addEvaluationLimit(total, increment) {
|
|
|
4827
5187
|
}
|
|
4828
5188
|
//#endregion
|
|
4829
5189
|
//#region src/campaign/gepa-optimization-result.ts
|
|
4830
|
-
function assertGepaBridgeOutput(result, name, maxCandidateChars, recipeKind, maxEvaluations, expectsComponents) {
|
|
5190
|
+
function assertGepaBridgeOutput(result, name, maxCandidateChars, recipeKind, maxEvaluations, maxPopulationCandidates, scenarioIds, expectsComponents, requiresCandidatePopulation) {
|
|
4831
5191
|
if (result.recipeKind !== recipeKind) throw new Error(`${name}: GEPA bridge reported recipe '${String(result.recipeKind)}'`);
|
|
4832
5192
|
if (!isGepaCandidate(result.bestCandidate, maxCandidateChars) || expectsComponents !== (typeof result.bestCandidate !== "string")) throw new Error(`${name}: GEPA bridge returned an invalid candidate`);
|
|
4833
5193
|
if (!Number.isFinite(result.bestScore)) throw new Error(`${name}: GEPA bridge returned an invalid bestScore`);
|
|
@@ -4840,6 +5200,11 @@ function assertGepaBridgeOutput(result, name, maxCandidateChars, recipeKind, max
|
|
|
4840
5200
|
assertExternalOptimizerPackageSource(result.upstream, "gepa", name, "GEPA");
|
|
4841
5201
|
if (typeof result.runId !== "string" || result.runId.length === 0 || result.runId !== result.runId.trim()) throw new Error(`${name}: GEPA bridge returned an invalid runId`);
|
|
4842
5202
|
if (typeof result.resumed !== "boolean") throw new Error(`${name}: GEPA bridge returned an invalid resumed flag`);
|
|
5203
|
+
if (result.candidatePopulation !== void 0) {
|
|
5204
|
+
assertGepaCandidatePopulationSummary(result.candidatePopulation);
|
|
5205
|
+
if (result.candidatePopulation.runId !== result.runId) throw new Error(`${name}: GEPA candidate population has a different run ID`);
|
|
5206
|
+
if (result.candidatePopulation.maxCandidates !== maxPopulationCandidates || result.candidatePopulation.maxCandidateChars !== maxCandidateChars || result.candidatePopulation.surfaceKind !== (expectsComponents ? "components" : "text") || result.candidatePopulation.scenarioIds.length !== scenarioIds.length || result.candidatePopulation.scenarioIds.some((scenarioId, index) => scenarioId !== scenarioIds[index])) throw new Error(`${name}: GEPA candidate population differs from its configured bounds`);
|
|
5207
|
+
} else if (requiresCandidatePopulation) throw new Error(`${name}: GEPA bridge omitted the official candidate population`);
|
|
4843
5208
|
}
|
|
4844
5209
|
function isGepaCandidate(value, maxChars) {
|
|
4845
5210
|
if (!isExternalTextCandidate(value)) return false;
|
|
@@ -4893,6 +5258,8 @@ function gepaOptimizationMethod(config) {
|
|
|
4893
5258
|
const trainSet = input.trainScenarios.map((scenario) => describeExternalScenario(scenario, "GEPA", maxEvidenceChars, config.describeScenario));
|
|
4894
5259
|
const selectionSet = input.selectionScenarios.map((scenario) => describeExternalScenario(scenario, "GEPA", maxEvidenceChars, config.describeScenario));
|
|
4895
5260
|
const evaluationLimit = gepaRecipeEvaluationLimit(config.recipe, input.selectionScenarios.length);
|
|
5261
|
+
const maxPopulationCandidates = Math.min(Number.MAX_SAFE_INTEGER, evaluationLimit + 1);
|
|
5262
|
+
const populationScenarioIds = (selectionSet.length > 0 ? selectionSet : trainSet).map((scenario) => scenario.id);
|
|
4896
5263
|
const runMaterial = {
|
|
4897
5264
|
optimizer: "gepa",
|
|
4898
5265
|
runtime: runtimeIdentity,
|
|
@@ -4908,6 +5275,7 @@ function gepaOptimizationMethod(config) {
|
|
|
4908
5275
|
trainSet,
|
|
4909
5276
|
selectionSet,
|
|
4910
5277
|
maxCandidateChars,
|
|
5278
|
+
maxPopulationCandidates,
|
|
4911
5279
|
maxEvidenceChars,
|
|
4912
5280
|
evaluationCallbackLimits: resolveExternalOptimizerCallbackLimits(config.evaluationCallbackLimits),
|
|
4913
5281
|
optimizerModel: config.optimizer ? {
|
|
@@ -5021,6 +5389,7 @@ function gepaOptimizationMethod(config) {
|
|
|
5021
5389
|
trainSet,
|
|
5022
5390
|
selectionSet,
|
|
5023
5391
|
maxCandidateChars,
|
|
5392
|
+
maxPopulationCandidates,
|
|
5024
5393
|
maxEvidenceChars,
|
|
5025
5394
|
outputDir,
|
|
5026
5395
|
...modelProxy && config.optimizer ? { modelProxy: {
|
|
@@ -5043,7 +5412,7 @@ function gepaOptimizationMethod(config) {
|
|
|
5043
5412
|
cleanup: closeResources
|
|
5044
5413
|
});
|
|
5045
5414
|
signal?.throwIfAborted();
|
|
5046
|
-
assertGepaBridgeOutput(result, name, maxCandidateChars, config.recipe.kind, evaluationLimit, expectsComponents);
|
|
5415
|
+
assertGepaBridgeOutput(result, name, maxCandidateChars, config.recipe.kind, evaluationLimit, maxPopulationCandidates, populationScenarioIds, expectsComponents, config.recipe.kind === "engine" && config.recipe.run.engine === "gepa");
|
|
5047
5416
|
assertExternalOptimizerRunBinding({
|
|
5048
5417
|
label: name,
|
|
5049
5418
|
runtime: runtimeIdentity,
|
|
@@ -5055,6 +5424,15 @@ function gepaOptimizationMethod(config) {
|
|
|
5055
5424
|
resumed: result.resumed
|
|
5056
5425
|
});
|
|
5057
5426
|
if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: GEPA reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
|
|
5427
|
+
if (result.candidatePopulation) {
|
|
5428
|
+
const population = readGepaCandidatePopulationArtifact({ summary: result.candidatePopulation });
|
|
5429
|
+
const selected = population.candidates[population.bestIndex];
|
|
5430
|
+
const selectedHash = contentHash({
|
|
5431
|
+
kind: "external-text-candidate",
|
|
5432
|
+
candidate: result.bestCandidate
|
|
5433
|
+
});
|
|
5434
|
+
if (selected?.candidateHash !== selectedHash) throw new Error(`${name}: GEPA candidate population identifies a different winner`);
|
|
5435
|
+
}
|
|
5058
5436
|
const evaluationCost = costFromLedgerSummary(costLedger.summary({
|
|
5059
5437
|
phase: "gepa.external-evaluation",
|
|
5060
5438
|
tags: runBudget.runTags
|
|
@@ -5115,6 +5493,7 @@ function gepaOptimizationMethod(config) {
|
|
|
5115
5493
|
artifactDir: outputDir,
|
|
5116
5494
|
...tokenUsage ? { tokenUsage } : {},
|
|
5117
5495
|
observations: observationLog.summary(),
|
|
5496
|
+
...result.candidatePopulation ? { gepaCandidatePopulation: result.candidatePopulation } : {},
|
|
5118
5497
|
...executionLog ? { modelExecutions: executionLog.summary() } : {}
|
|
5119
5498
|
}
|
|
5120
5499
|
};
|
|
@@ -6528,6 +6907,6 @@ function skillOptOptimizationMethod(config) {
|
|
|
6528
6907
|
};
|
|
6529
6908
|
}
|
|
6530
6909
|
//#endregion
|
|
6531
|
-
export {
|
|
6910
|
+
export { REFERENCE_EQUIVALENCE_INPUT_LIMITS as $, assertComponentSurface as A, buildReflectionPrompt as B, combineComparisonCosts as C, JudgeParseError as Ct, readGepaCandidatePopulationArtifact as D, optimizationTokenUsageFromSummary as E, surfaceHash as F, tangleTracesRoot as G, planCampaignRun as H, campaignBreakdown as I, assertCampaignSplitIdentity as J, readExternalOptimizerObservationArtifact as K, campaignMeanComposite as L, componentSurfaceIdentityMaterial as M, renderSurfaceDiff as N, decodeExternalTextCandidate as O, surfaceContentHash as P, openAutoPr as Q, compareRankKeys as R, assertOptimizationResult as S, summarizeBackendIntegrity as St, costFromLedgerSummary as T, runCampaign as U, parseReflectionResponse as V, resolveRunDir as W, campaignSplitDigest as X, campaignScenarioIdentity as Y, campaignSplitDigestFromIdentities as Z, heldOutGate as _, recoverTruncatedJson as _t, emitLoopProvenance as a, redTeamReport as at, composeGate as b, assertRealBackend as bt, provenanceRecordPath as c, Dataset as ct, runImprovementLoop as d, llmJudge as dt, REFERENCE_EQUIVALENCE_JUDGE_VERSION as et, runOptimization as f, crowdingDistance as ft, gepaOptimizationMethod as g, scalarScore as gt, labelTrustRank as h, paretoFrontierWithCrowding as ht, canonicalDigest as i, redTeamDataset as it, codeSurfaceIdentityMaterial as j, assertCodeSurfaceIdentity as k, provenanceSpansPath as l, HoldoutLockedError as lt, isProposedCandidate as m, paretoFrontier as mt, buildLoopProvenanceRecord as n, runReferenceEquivalenceJudge as nt, loopProvenanceArgsFromResult as o, scoreRedTeamOutput as ot, runEval as p, dominates as pt, assertCampaignDesign as q, campaignMeasurementDigest as r, DEFAULT_RED_TEAM_CORPUS as rt, loopProvenanceSpans as s, toolNamesForRun as st, skillOptOptimizationMethod as t, createReferenceEquivalenceJudge as tt, verifyLoopProvenanceRecord as u, hashScenarios as ut, defaultProductionGate as v, BackendIntegrityError as vt, compareOptimizationMethods as w, externalTextOptimizationMethod as x, summarizeAgentReceiptIntegrity as xt, runCanaries as y, assertRealAgentReceipts as yt, DEFAULT_MUTATION_PRIMITIVES as z };
|
|
6532
6911
|
|
|
6533
|
-
//# sourceMappingURL=skillopt-optimization-method-
|
|
6912
|
+
//# sourceMappingURL=skillopt-optimization-method-CQdVeM8k.js.map
|