@tangle-network/agent-runtime 0.105.0 → 0.106.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/activation-CRjVfOiy.js +203 -0
- package/dist/activation-CRjVfOiy.js.map +1 -0
- package/dist/activation-DyWB0K6E.d.ts +98 -0
- package/dist/agent.d.ts +216 -234
- package/dist/agent.js +615 -547
- package/dist/agent.js.map +1 -1
- package/dist/analyst-loop-DGdtVENg.js +517 -0
- package/dist/analyst-loop-DGdtVENg.js.map +1 -0
- package/dist/analyst-loop.d.ts +10 -43
- package/dist/analyst-loop.js +2 -11
- package/dist/backends-CiOCyRHb.js +743 -0
- package/dist/backends-CiOCyRHb.js.map +1 -0
- package/dist/candidate-execution/index.d.ts +4 -192
- package/dist/candidate-execution/index.js +5 -66
- package/dist/candidate-execution-DqMbv1rU.js +488 -0
- package/dist/candidate-execution-DqMbv1rU.js.map +1 -0
- package/dist/coder-yhVWbdWc.js +40 -0
- package/dist/coder-yhVWbdWc.js.map +1 -0
- package/dist/conversation-BpLQZGPH.js +1265 -0
- package/dist/conversation-BpLQZGPH.js.map +1 -0
- package/dist/conversation.d.ts +2 -804
- package/dist/conversation.js +2 -55
- package/dist/environment-provider-CfLUmKmf.js +1168 -0
- package/dist/environment-provider-CfLUmKmf.js.map +1 -0
- package/dist/environment-provider-IUGU3epE.d.ts +964 -0
- package/dist/environment-provider.d.ts +2 -78
- package/dist/environment-provider.js +2 -18
- package/dist/errors-DEAvWQPy.js +111 -0
- package/dist/errors-DEAvWQPy.js.map +1 -0
- package/dist/improvement-adapter-Cr5TBSji.d.ts +166 -0
- package/dist/improvement-cycle-C095kS_g.js +2624 -0
- package/dist/improvement-cycle-C095kS_g.js.map +1 -0
- package/dist/index-BGdxAwfO.d.ts +838 -0
- package/dist/index-C-FYUuFG.d.ts +193 -0
- package/dist/index-CbNM40xG.d.ts +140 -0
- package/dist/index-DruSKs3Z.d.ts +7224 -0
- package/dist/index-Efjb3nrQ.d.ts +671 -0
- package/dist/index.d.ts +14 -1058
- package/dist/index.js +2106 -2082
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +768 -889
- package/dist/intelligence.js +1466 -1367
- package/dist/intelligence.js.map +1 -1
- package/dist/{kb-gate-CwHO0vz6.d.ts → kb-gate-C8z2juK8.d.ts} +33 -31
- package/dist/kb-gate-DpaSwXVx.js +80 -0
- package/dist/kb-gate-DpaSwXVx.js.map +1 -0
- package/dist/knowledge-Ku-mPP4f.js +433 -0
- package/dist/knowledge-Ku-mPP4f.js.map +1 -0
- package/dist/knowledge.d.ts +2 -150
- package/dist/knowledge.js +2 -40
- package/dist/local-harness-Dh8PJ0ot.d.ts +455 -0
- package/dist/loop-runner-bin-BXsFEUcf.d.ts +162 -0
- package/dist/loop-runner-bin-CRagyuNb.js +268 -0
- package/dist/loop-runner-bin-CRagyuNb.js.map +1 -0
- package/dist/loop-runner-bin.d.ts +2 -15
- package/dist/loop-runner-bin.js +2 -24
- package/dist/loops.d.ts +5 -4164
- package/dist/loops.js +8 -419
- package/dist/mcp/bin.d.ts +1 -1
- package/dist/mcp/bin.js +116 -146
- package/dist/mcp/bin.js.map +1 -1
- package/dist/mcp/index.d.ts +295 -436
- package/dist/mcp/index.js +1242 -1177
- package/dist/mcp/index.js.map +1 -1
- package/dist/mcp/memory-bin.d.ts +1 -1
- package/dist/mcp/memory-bin.js +37 -16
- package/dist/mcp/memory-bin.js.map +1 -1
- package/dist/memory-server-DL6cE2Ag.js +234 -0
- package/dist/memory-server-DL6cE2Ag.js.map +1 -0
- package/dist/model-resolution-Btd9iIKV.js +98 -0
- package/dist/model-resolution-Btd9iIKV.js.map +1 -0
- package/dist/openai-tools-B68JaOCx.d.ts +24 -0
- package/dist/openai-tools-CKLy1C7M.js +44 -0
- package/dist/openai-tools-CKLy1C7M.js.map +1 -0
- package/dist/otel-export-D34IQW1C.js +843 -0
- package/dist/otel-export-D34IQW1C.js.map +1 -0
- package/dist/platform.d.ts +188 -185
- package/dist/platform.js +185 -218
- package/dist/platform.js.map +1 -1
- package/dist/prepare-B53sPV7M.js +4292 -0
- package/dist/prepare-B53sPV7M.js.map +1 -0
- package/dist/primeintellect/index.d.ts +149 -145
- package/dist/primeintellect/index.js +626 -1003
- package/dist/primeintellect/index.js.map +1 -1
- package/dist/profiles.d.ts +296 -490
- package/dist/profiles.js +722 -749
- package/dist/profiles.js.map +1 -1
- package/dist/protected-model-port-BLPH5VVv.js +722 -0
- package/dist/protected-model-port-BLPH5VVv.js.map +1 -0
- package/dist/protected-model-port-BP6Z4eau.d.ts +283 -0
- package/dist/protected-redaction--F3v1oo8.js +200 -0
- package/dist/protected-redaction--F3v1oo8.js.map +1 -0
- package/dist/redact-D-u-rrcn.js +82 -0
- package/dist/redact-D-u-rrcn.js.map +1 -0
- package/dist/redact-kbIOjfjM.d.ts +452 -0
- package/dist/researcher-CoVqNhfI.js +433 -0
- package/dist/researcher-CoVqNhfI.js.map +1 -0
- package/dist/runtime-5zM5XxLr.js +5408 -0
- package/dist/runtime-5zM5XxLr.js.map +1 -0
- package/dist/sandbox-events-DeI5xX8P.js +316 -0
- package/dist/sandbox-events-DeI5xX8P.js.map +1 -0
- package/dist/spawn-journal-CwPvKUTa.js +456 -0
- package/dist/spawn-journal-CwPvKUTa.js.map +1 -0
- package/dist/structural-rollout-IXUEplky.js +1445 -0
- package/dist/structural-rollout-IXUEplky.js.map +1 -0
- package/dist/substrate-B0TYNrXn.js +27 -0
- package/dist/substrate-B0TYNrXn.js.map +1 -0
- package/dist/{substrate-Dyiqmd3o.d.ts → substrate-BcnuSHXm.d.ts} +57 -54
- package/dist/supervise-BiRutHS9.js +7321 -0
- package/dist/supervise-BiRutHS9.js.map +1 -0
- package/dist/supervisor-DTKhF-RV.js +3826 -0
- package/dist/supervisor-DTKhF-RV.js.map +1 -0
- package/dist/testing.d.ts +4 -3
- package/dist/testing.js +2478 -2637
- package/dist/testing.js.map +1 -1
- package/dist/tool-server-RcWgLIsL.js +141 -0
- package/dist/tool-server-RcWgLIsL.js.map +1 -0
- package/dist/trace-propagation-CJJC7SVB.js +57 -0
- package/dist/trace-propagation-CJJC7SVB.js.map +1 -0
- package/dist/types-BaIpb0XT.d.ts +492 -0
- package/dist/types-BevOjfTY.d.ts +765 -0
- package/dist/types-C9j4qg6l.d.ts +500 -0
- package/dist/types-Cja5OTGh.d.ts +190 -0
- package/dist/util-MVgdwuIS.js +145 -0
- package/dist/util-MVgdwuIS.js.map +1 -0
- package/dist/workspace-archive-BosQiotA.js +575 -0
- package/dist/workspace-archive-BosQiotA.js.map +1 -0
- package/package.json +44 -56
- package/skills/build-with-agent-runtime/SKILL.md +1 -0
- package/dist/activation-Cm-9MMX1.d.ts +0 -69
- package/dist/analyst-loop.js.map +0 -1
- package/dist/backends-BO4Jqoe2.d.ts +0 -134
- package/dist/candidate-execution/index.js.map +0 -1
- package/dist/chunk-2KGAN2HM.js +0 -1256
- package/dist/chunk-2KGAN2HM.js.map +0 -1
- package/dist/chunk-2VZYRB55.js +0 -834
- package/dist/chunk-2VZYRB55.js.map +0 -1
- package/dist/chunk-3MDZX7YU.js +0 -125
- package/dist/chunk-3MDZX7YU.js.map +0 -1
- package/dist/chunk-5IBQAPVB.js +0 -29
- package/dist/chunk-5IBQAPVB.js.map +0 -1
- package/dist/chunk-5QOB7H74.js +0 -387
- package/dist/chunk-5QOB7H74.js.map +0 -1
- package/dist/chunk-C5RTIBNZ.js +0 -48
- package/dist/chunk-C5RTIBNZ.js.map +0 -1
- package/dist/chunk-CTRA64LY.js +0 -680
- package/dist/chunk-CTRA64LY.js.map +0 -1
- package/dist/chunk-DPEUKJRO.js +0 -27
- package/dist/chunk-DPEUKJRO.js.map +0 -1
- package/dist/chunk-FD2MBMOH.js +0 -494
- package/dist/chunk-FD2MBMOH.js.map +0 -1
- package/dist/chunk-H5QPIZNX.js +0 -907
- package/dist/chunk-H5QPIZNX.js.map +0 -1
- package/dist/chunk-HLKC4UYB.js +0 -3514
- package/dist/chunk-HLKC4UYB.js.map +0 -1
- package/dist/chunk-HNP72PNU.js +0 -192
- package/dist/chunk-HNP72PNU.js.map +0 -1
- package/dist/chunk-ISPWRSEC.js +0 -408
- package/dist/chunk-ISPWRSEC.js.map +0 -1
- package/dist/chunk-KRBFHMV6.js +0 -1228
- package/dist/chunk-KRBFHMV6.js.map +0 -1
- package/dist/chunk-M22PL4GH.js +0 -171
- package/dist/chunk-M22PL4GH.js.map +0 -1
- package/dist/chunk-NURIQBJQ.js +0 -72
- package/dist/chunk-NURIQBJQ.js.map +0 -1
- package/dist/chunk-OPVWXJ2H.js +0 -75
- package/dist/chunk-OPVWXJ2H.js.map +0 -1
- package/dist/chunk-PZZKQVQV.js +0 -52
- package/dist/chunk-PZZKQVQV.js.map +0 -1
- package/dist/chunk-Q2JSAVQ3.js +0 -671
- package/dist/chunk-Q2JSAVQ3.js.map +0 -1
- package/dist/chunk-RDOAVVHY.js +0 -699
- package/dist/chunk-RDOAVVHY.js.map +0 -1
- package/dist/chunk-SGKPNBXE.js +0 -60
- package/dist/chunk-SGKPNBXE.js.map +0 -1
- package/dist/chunk-SMQXZGLZ.js +0 -351
- package/dist/chunk-SMQXZGLZ.js.map +0 -1
- package/dist/chunk-UHEZW5BU.js +0 -2918
- package/dist/chunk-UHEZW5BU.js.map +0 -1
- package/dist/chunk-UPWGXKXB.js +0 -254
- package/dist/chunk-UPWGXKXB.js.map +0 -1
- package/dist/chunk-VISA6CI3.js +0 -200
- package/dist/chunk-VISA6CI3.js.map +0 -1
- package/dist/chunk-WMTCUOQL.js +0 -474
- package/dist/chunk-WMTCUOQL.js.map +0 -1
- package/dist/chunk-WRTOVNN4.js +0 -4728
- package/dist/chunk-WRTOVNN4.js.map +0 -1
- package/dist/chunk-WSTRQZYQ.js +0 -5576
- package/dist/chunk-WSTRQZYQ.js.map +0 -1
- package/dist/chunk-XBG2W2VW.js +0 -6434
- package/dist/chunk-XBG2W2VW.js.map +0 -1
- package/dist/chunk-YEJR7IXO.js +0 -68
- package/dist/chunk-YEJR7IXO.js.map +0 -1
- package/dist/chunk-YJZA2BIK.js +0 -1338
- package/dist/chunk-YJZA2BIK.js.map +0 -1
- package/dist/completion-gate-BMy5LGoP.d.ts +0 -256
- package/dist/conversation.js.map +0 -1
- package/dist/coordination-BZZSVYpZ.d.ts +0 -2570
- package/dist/environment-provider.js.map +0 -1
- package/dist/improvement-adapter-HAZz-7vK.d.ts +0 -203
- package/dist/knowledge.js.map +0 -1
- package/dist/local-harness-BDNcl6jI.d.ts +0 -175
- package/dist/loop-runner-bin-68aoM9-7.d.ts +0 -199
- package/dist/loop-runner-bin.js.map +0 -1
- package/dist/loops.js.map +0 -1
- package/dist/openai-tools-fnj6SRVg.d.ts +0 -46
- package/dist/protected-model-port-B4t-OQjL.d.ts +0 -271
- package/dist/redact-BuFjxBUJ.d.ts +0 -481
- package/dist/sanitize-DEbPNtyI.d.ts +0 -328
- package/dist/structural-rollout-DEh6CEsa.d.ts +0 -446
- package/dist/supervise-4_48wwvy.d.ts +0 -379
- package/dist/types-BCoemcXU.d.ts +0 -802
- package/dist/types-BwoZWq-i.d.ts +0 -510
- package/dist/types-Cbp72qyu.d.ts +0 -491
- package/dist/types-CmYCMbFT.d.ts +0 -203
- package/dist/types-CvNwMzQt.d.ts +0 -957
- package/dist/worktree-fanout-DxhAWr5Z.d.ts +0 -999
|
@@ -0,0 +1,2624 @@
|
|
|
1
|
+
import { i as ConfigError } from "./errors-DEAvWQPy.js";
|
|
2
|
+
import { $ as parseExactAgentProfile, G as agentCandidateProfileAsAgentProfile, J as candidateMaterializerHarness, Q as parseAgentCandidateProfileActivation, Y as createAgentCandidateProfileActivation, Z as omitUndefinedObjectFields, ct as omitTopLevelDigest, d as verifiedResourceTextByDigest, f as verifyAgentCandidateBundle, it as canonicalCandidateDocument, n as executePreparedAgentCandidate, nt as canonicalCandidateBytes, q as assertCandidateProfileBinding, rt as canonicalCandidateDigest$1, st as immutableCandidateValue, t as prepareAgentCandidateExecution, ut as verifyCanonicalCandidateDocument } from "./prepare-B53sPV7M.js";
|
|
3
|
+
import { N as runLocalHarness, T as runSettledCommand, j as harnessInvocation } from "./supervisor-DTKhF-RV.js";
|
|
4
|
+
import { E as optimizerMethod, a as defaultStructuralRolloutPolicy } from "./structural-rollout-IXUEplky.js";
|
|
5
|
+
import { t as runAnalystLoop } from "./analyst-loop-DGdtVENg.js";
|
|
6
|
+
import { canonicalJson, makeFinding } from "@tangle-network/agent-eval";
|
|
7
|
+
import { campaignSplitDigest, compareOptimizationMethods, gitWorktreeAdapter, verifyCodeSurface } from "@tangle-network/agent-eval/campaign";
|
|
8
|
+
import { agentCandidateMaterializationReceiptSchema, agentCandidateRunReceiptSchema, agentImprovementActivationSchema, agentImprovementProposalSchema, agentImprovementReviewSchema, agentProfileDiffSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileModelHintsSchema, agentProfileSchema, candidateExecutionEvidenceSchema, changedProfileImprovementSurfaces, defineAgentProfileDiff, sha256DigestSchema } from "@tangle-network/agent-interface";
|
|
9
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
10
|
+
import { applyWorkspacePlan, materializeCandidateProfile, materializeProfile } from "@tangle-network/agent-profile-materialize";
|
|
11
|
+
import { existsSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
12
|
+
import { basename, join, resolve, sep } from "node:path";
|
|
13
|
+
import { spawnSync } from "node:child_process";
|
|
14
|
+
import { measuredComparisonFromCandidateExperiment, runCandidateExperiment, sealCandidateExperiment, selfImprove, verifyCandidateExperiment, verifyCandidateExperimentComparison } from "@tangle-network/agent-eval/contract";
|
|
15
|
+
import { assertNoJudgeVerdict } from "@tangle-network/agent-eval/analyst";
|
|
16
|
+
//#region src/improvement/agentic-generator.ts
|
|
17
|
+
/**
|
|
18
|
+
*
|
|
19
|
+
* `agenticGenerator` — the full-agentic `CandidateGenerator`. It runs a real
|
|
20
|
+
* coding harness (claude / codex / opencode) inside the candidate worktree the
|
|
21
|
+
* driver already created, letting the agent read the codebase + the research
|
|
22
|
+
* report and make the change in place. The driver then commits the worktree
|
|
23
|
+
* into a `CodeSurface`.
|
|
24
|
+
*
|
|
25
|
+
* Mechanism: identical to the proven Phase-2.8 in-process executor — spawn the
|
|
26
|
+
* harness as a subprocess with `cwd` = the worktree, on the same filesystem,
|
|
27
|
+
* so edits land in place (no sandbox-mount round-trip). `runLocalHarness` is
|
|
28
|
+
* the verified primitive. The OUTER sandbox is the improvement loop's own
|
|
29
|
+
* execution context; the generator does not nest a second sandbox per
|
|
30
|
+
* candidate (which would reintroduce a host↔sandbox worktree-transport
|
|
31
|
+
* problem that does not need solving here).
|
|
32
|
+
*
|
|
33
|
+
* `maxShots` is the DEPTH dial — a multi-shot verify-in-session loop, NOT the
|
|
34
|
+
* kernel `runAgentRounds`. Each shot runs one full harness session in the (persistent)
|
|
35
|
+
* worktree; between shots the loop refines based on what the last shot produced:
|
|
36
|
+
* - empty tree → "you changed nothing, make the edits" → retry
|
|
37
|
+
* - dirty + `verify` fails → feed the verifier's failure into the next shot
|
|
38
|
+
* (the worktree persists, so the harness RESUMES atop its own failing
|
|
39
|
+
* edits with the error in hand — no `--resume` session plumbing needed,
|
|
40
|
+
* and harness-agnostic across claude/codex/opencode)
|
|
41
|
+
* - dirty + `verify` ok (or no verifier configured) → return the candidate
|
|
42
|
+
* A candidate that never verifies within `maxShots` is discarded (`applied:
|
|
43
|
+
* false`), never shipped — if you configured a verifier, a non-passing tree is
|
|
44
|
+
* not a candidate. With no verifier the legacy behavior holds: first dirty shot
|
|
45
|
+
* is the candidate.
|
|
46
|
+
*
|
|
47
|
+
* @experimental
|
|
48
|
+
*/
|
|
49
|
+
const RAW_TRACE_ANALYST_ID = "raw-trace-distiller";
|
|
50
|
+
const RAW_TRACE_AREA = "raw-trace-context";
|
|
51
|
+
const RAW_TRACE_DIAGNOSIS_PATH = ".improve/raw-trace-diagnosis.md";
|
|
52
|
+
/** Dedicated ephemeral root for generic author-profile files. Every declared
|
|
53
|
+
* file must live below this root so cleanup cannot alter candidate-owned files. */
|
|
54
|
+
const AGENTIC_PROFILE_RESOURCE_ROOT = ".agent-runtime-profile-resources";
|
|
55
|
+
/** Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. */
|
|
56
|
+
function agenticGenerator(opts = {}) {
|
|
57
|
+
const harness = opts.harness ?? "claude";
|
|
58
|
+
if (opts.codexReproducible && harness !== "codex") throw new Error("agenticGenerator: codexReproducible requires harness 'codex'");
|
|
59
|
+
if (opts.codexReproducible && !opts.profile) throw new Error("agenticGenerator: codexReproducible requires an explicit author profile");
|
|
60
|
+
if (opts.codexReadDeniedPaths && !opts.codexReproducible) throw new Error("agenticGenerator: codexReadDeniedPaths requires codexReproducible");
|
|
61
|
+
if (opts.maximumCharge && !opts.codexReproducible) throw new Error("agenticGenerator: maximumCharge requires codexReproducible");
|
|
62
|
+
const profileResourcePlan = opts.codexReproducible ? authorProfileResourcePlan(opts.profile) : null;
|
|
63
|
+
const buildPrompt = opts.buildPrompt ?? defaultBuildPrompt;
|
|
64
|
+
const run = opts.runHarness ?? runLocalHarness;
|
|
65
|
+
const dirty = opts.isDirty ?? worktreeDirty;
|
|
66
|
+
const verify = opts.verify;
|
|
67
|
+
return {
|
|
68
|
+
kind: `agentic:${harness}`,
|
|
69
|
+
proposesWithoutFindings: true,
|
|
70
|
+
async generate({ worktreePath, report, findings, maxShots, signal, generation, candidateIndex, costLedger, costPhase }) {
|
|
71
|
+
signal.throwIfAborted();
|
|
72
|
+
let reproducibleCostLedger;
|
|
73
|
+
if (opts.codexReproducible) {
|
|
74
|
+
if (!costLedger) throw new Error("agenticGenerator: reproducible Codex requires the run-wide CostLedger supplied by agent-eval");
|
|
75
|
+
reproducibleCostLedger = costLedger;
|
|
76
|
+
}
|
|
77
|
+
const basePrompt = appendProfileResourcePaths(buildPrompt({
|
|
78
|
+
report,
|
|
79
|
+
findings
|
|
80
|
+
}), profileResourcePlan);
|
|
81
|
+
const needsRawTraceEvidence = requiresRawTraceEvidence(findings);
|
|
82
|
+
const shots = Math.max(1, maxShots);
|
|
83
|
+
let attemptNote = "";
|
|
84
|
+
for (let shot = 0; shot < shots; shot++) {
|
|
85
|
+
signal.throwIfAborted();
|
|
86
|
+
const taskPrompt = attemptNote ? `${basePrompt}\n\n${attemptNote}` : basePrompt;
|
|
87
|
+
const invocation = opts.profile ? harnessInvocation(harness, opts.profile, taskPrompt, {
|
|
88
|
+
dangerouslySkipPermissions: harness === "claude",
|
|
89
|
+
...opts.codexReproducible ? { codexReproducible: true } : {}
|
|
90
|
+
}) : void 0;
|
|
91
|
+
const exactPrompt = invocation?.prompt ?? taskPrompt;
|
|
92
|
+
const readDeniedPaths = typeof opts.codexReadDeniedPaths === "function" ? opts.codexReadDeniedPaths(worktreePath) : opts.codexReadDeniedPaths;
|
|
93
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
94
|
+
let harnessResult = null;
|
|
95
|
+
let profileWorkspaceReceipt = null;
|
|
96
|
+
let costReceipt = null;
|
|
97
|
+
let costCallId = null;
|
|
98
|
+
let shotError = null;
|
|
99
|
+
try {
|
|
100
|
+
const execute = async (executionSignal) => {
|
|
101
|
+
harnessResult = await withAuthorProfileResources(profileResourcePlan, worktreePath, async (receipt) => {
|
|
102
|
+
profileWorkspaceReceipt = receipt;
|
|
103
|
+
const result = await run({
|
|
104
|
+
harness,
|
|
105
|
+
cwd: worktreePath,
|
|
106
|
+
taskPrompt,
|
|
107
|
+
...invocation ? { invocation: {
|
|
108
|
+
command: invocation.command,
|
|
109
|
+
args: invocation.args
|
|
110
|
+
} } : {},
|
|
111
|
+
dangerouslySkipPermissions: harness === "claude",
|
|
112
|
+
...opts.codexReproducible ? { codexReproducible: true } : {},
|
|
113
|
+
...readDeniedPaths ? { codexReadDeniedPaths: readDeniedPaths } : {},
|
|
114
|
+
timeoutMs: opts.timeoutMs,
|
|
115
|
+
signal: executionSignal
|
|
116
|
+
});
|
|
117
|
+
harnessResult = result;
|
|
118
|
+
return result;
|
|
119
|
+
});
|
|
120
|
+
const failure = shotFailure(harnessResult, exactPrompt, opts.codexReproducible === true);
|
|
121
|
+
if (failure) throw failure;
|
|
122
|
+
return harnessResult;
|
|
123
|
+
};
|
|
124
|
+
if (reproducibleCostLedger) {
|
|
125
|
+
const model = opts.profile?.model?.default;
|
|
126
|
+
if (!model) throw new Error("agenticGenerator: reproducible Codex requires profile.model.default");
|
|
127
|
+
const paid = await reproducibleCostLedger.runPaidCall({
|
|
128
|
+
channel: "driver",
|
|
129
|
+
phase: costPhase ?? "search.proposal",
|
|
130
|
+
actor: `agentic-generator:${harness}`,
|
|
131
|
+
model,
|
|
132
|
+
tags: {
|
|
133
|
+
generation: String(generation ?? -1),
|
|
134
|
+
candidateIndex: String(candidateIndex ?? -1),
|
|
135
|
+
shot: String(shot + 1)
|
|
136
|
+
},
|
|
137
|
+
signal,
|
|
138
|
+
...opts.maximumCharge ? { maximumCharge: opts.maximumCharge } : {},
|
|
139
|
+
execute,
|
|
140
|
+
receipt: (result) => costReceiptFromHarness(result, model),
|
|
141
|
+
receiptFromError: () => harnessResult?.usage ? costReceiptFromHarness(harnessResult, model) : void 0
|
|
142
|
+
});
|
|
143
|
+
costCallId = paid.callId ?? null;
|
|
144
|
+
costReceipt = paid.receipt ?? null;
|
|
145
|
+
if (!paid.succeeded) throw paid.error;
|
|
146
|
+
harnessResult = paid.value;
|
|
147
|
+
} else harnessResult = await execute(signal);
|
|
148
|
+
} catch (cause) {
|
|
149
|
+
shotError = cause instanceof Error ? cause : new Error(String(cause));
|
|
150
|
+
}
|
|
151
|
+
const execution = shotExecutionSnapshot(harnessResult);
|
|
152
|
+
const receipt = shotReceipt({
|
|
153
|
+
generation,
|
|
154
|
+
candidateIndex,
|
|
155
|
+
shot,
|
|
156
|
+
maxShots: shots,
|
|
157
|
+
harness,
|
|
158
|
+
profile: opts.profile,
|
|
159
|
+
prompt: exactPrompt,
|
|
160
|
+
startedAt,
|
|
161
|
+
completedAt: /* @__PURE__ */ new Date(),
|
|
162
|
+
result: execution,
|
|
163
|
+
profileWorkspaceReceipt,
|
|
164
|
+
costCallId,
|
|
165
|
+
costReceipt,
|
|
166
|
+
error: shotError
|
|
167
|
+
});
|
|
168
|
+
await emitShotReceipt(opts.onShotCompleted, receipt, execution, shotError);
|
|
169
|
+
signal.throwIfAborted();
|
|
170
|
+
if (!execution) throw new Error("agenticGenerator: author shot completed without a harness result");
|
|
171
|
+
let worktreeChanged;
|
|
172
|
+
try {
|
|
173
|
+
worktreeChanged = dirty(worktreePath);
|
|
174
|
+
} catch (cause) {
|
|
175
|
+
signal.throwIfAborted();
|
|
176
|
+
return rethrowShotSetupError(opts.onShotDisposition, receipt, worktreePath, "worktree-inspection", cause);
|
|
177
|
+
}
|
|
178
|
+
if (!worktreeChanged) {
|
|
179
|
+
signal.throwIfAborted();
|
|
180
|
+
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
181
|
+
kind: "clean",
|
|
182
|
+
worktreePath
|
|
183
|
+
});
|
|
184
|
+
signal.throwIfAborted();
|
|
185
|
+
attemptNote = EMPTY_TREE_NOTE;
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
if (needsRawTraceEvidence) {
|
|
189
|
+
let problem;
|
|
190
|
+
try {
|
|
191
|
+
problem = rawTraceEvidenceProblem(worktreePath, findings);
|
|
192
|
+
} catch (cause) {
|
|
193
|
+
signal.throwIfAborted();
|
|
194
|
+
return rethrowShotSetupError(opts.onShotDisposition, receipt, worktreePath, "raw-trace-evidence", cause);
|
|
195
|
+
}
|
|
196
|
+
if (problem) {
|
|
197
|
+
signal.throwIfAborted();
|
|
198
|
+
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
199
|
+
kind: "rejected",
|
|
200
|
+
worktreePath,
|
|
201
|
+
stage: "raw-trace-evidence",
|
|
202
|
+
feedback: problem
|
|
203
|
+
});
|
|
204
|
+
signal.throwIfAborted();
|
|
205
|
+
attemptNote = problem;
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
if (!verify) {
|
|
210
|
+
signal.throwIfAborted();
|
|
211
|
+
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
212
|
+
kind: "accepted",
|
|
213
|
+
worktreePath,
|
|
214
|
+
verified: false
|
|
215
|
+
});
|
|
216
|
+
signal.throwIfAborted();
|
|
217
|
+
return acceptedCandidate(findings);
|
|
218
|
+
}
|
|
219
|
+
let result;
|
|
220
|
+
try {
|
|
221
|
+
signal.throwIfAborted();
|
|
222
|
+
result = await verify(worktreePath, signal);
|
|
223
|
+
signal.throwIfAborted();
|
|
224
|
+
} catch (cause) {
|
|
225
|
+
signal.throwIfAborted();
|
|
226
|
+
return rethrowShotSetupError(opts.onShotDisposition, receipt, worktreePath, "verification", cause);
|
|
227
|
+
}
|
|
228
|
+
if (result.ok) {
|
|
229
|
+
signal.throwIfAborted();
|
|
230
|
+
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
231
|
+
kind: "accepted",
|
|
232
|
+
worktreePath,
|
|
233
|
+
verified: true
|
|
234
|
+
});
|
|
235
|
+
signal.throwIfAborted();
|
|
236
|
+
return acceptedCandidate(findings);
|
|
237
|
+
}
|
|
238
|
+
signal.throwIfAborted();
|
|
239
|
+
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
240
|
+
kind: "rejected",
|
|
241
|
+
worktreePath,
|
|
242
|
+
stage: "verification",
|
|
243
|
+
feedback: result.feedback ?? null
|
|
244
|
+
});
|
|
245
|
+
signal.throwIfAborted();
|
|
246
|
+
attemptNote = failureNote(result.feedback);
|
|
247
|
+
}
|
|
248
|
+
return {
|
|
249
|
+
applied: false,
|
|
250
|
+
summary: ""
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
function authorProfileResourcePlan(profile) {
|
|
256
|
+
const resources = profile.resources;
|
|
257
|
+
if (!resources) return null;
|
|
258
|
+
const unsupportedKinds = [
|
|
259
|
+
resources.tools?.length ? "tools" : null,
|
|
260
|
+
resources.skills?.length ? "skills" : null,
|
|
261
|
+
resources.agents?.length ? "agents" : null
|
|
262
|
+
].filter((kind) => kind !== null);
|
|
263
|
+
if (unsupportedKinds.length > 0) throw new Error(`agenticGenerator: reproducible Codex author resources support files only; unsupported: ${unsupportedKinds.join(", ")}`);
|
|
264
|
+
if (!resources.files || resources.files.length === 0) return null;
|
|
265
|
+
const plan = materializeProfile({
|
|
266
|
+
name: profile.name,
|
|
267
|
+
resources: { files: resources.files }
|
|
268
|
+
}, "codex");
|
|
269
|
+
if (plan.unsupported.length > 0) throw new Error(`agenticGenerator: author profile files could not be materialized: ${plan.unsupported.map((item) => item.reason).join("; ")}`);
|
|
270
|
+
if (Object.keys(plan.env).length > 0 || plan.flags.length > 0) throw new Error("agenticGenerator: generic author profile files unexpectedly changed spawn values");
|
|
271
|
+
const virtualRoot = resolve("/", AGENTIC_PROFILE_RESOURCE_ROOT);
|
|
272
|
+
const seen = /* @__PURE__ */ new Set();
|
|
273
|
+
for (const file of plan.files) {
|
|
274
|
+
const target = resolve("/", file.relPath);
|
|
275
|
+
if (!target.startsWith(`${virtualRoot}${sep}`)) throw new Error(`agenticGenerator: author profile file must be below ${AGENTIC_PROFILE_RESOURCE_ROOT}: ${file.relPath}`);
|
|
276
|
+
if (seen.has(target)) throw new Error(`agenticGenerator: duplicate author profile file path: ${file.relPath}`);
|
|
277
|
+
seen.add(target);
|
|
278
|
+
}
|
|
279
|
+
return plan;
|
|
280
|
+
}
|
|
281
|
+
function appendProfileResourcePaths(prompt, plan) {
|
|
282
|
+
if (!plan) return prompt;
|
|
283
|
+
return [
|
|
284
|
+
prompt,
|
|
285
|
+
"",
|
|
286
|
+
"Profile resource files available for this shot:",
|
|
287
|
+
...plan.files.map((file) => `- ${file.relPath}`)
|
|
288
|
+
].join("\n");
|
|
289
|
+
}
|
|
290
|
+
async function withAuthorProfileResources(plan, worktreePath, run) {
|
|
291
|
+
if (!plan) return run(null);
|
|
292
|
+
const rootPath = resolve(worktreePath, AGENTIC_PROFILE_RESOURCE_ROOT);
|
|
293
|
+
if (existsSync(rootPath)) throw new Error(`agenticGenerator: ephemeral author profile root already exists: ${AGENTIC_PROFILE_RESOURCE_ROOT}`);
|
|
294
|
+
let value;
|
|
295
|
+
let primaryError;
|
|
296
|
+
try {
|
|
297
|
+
value = await run(applyWorkspacePlan(plan, worktreePath));
|
|
298
|
+
} catch (cause) {
|
|
299
|
+
primaryError = cause;
|
|
300
|
+
}
|
|
301
|
+
let cleanupError;
|
|
302
|
+
try {
|
|
303
|
+
rmSync(rootPath, {
|
|
304
|
+
recursive: true,
|
|
305
|
+
force: true
|
|
306
|
+
});
|
|
307
|
+
if (existsSync(rootPath)) throw new Error(`agenticGenerator: ephemeral author profile root survived cleanup: ${AGENTIC_PROFILE_RESOURCE_ROOT}`);
|
|
308
|
+
} catch (cause) {
|
|
309
|
+
cleanupError = cause;
|
|
310
|
+
}
|
|
311
|
+
if (primaryError !== void 0 && cleanupError !== void 0) throw new AggregateError([primaryError, cleanupError], "agenticGenerator: author shot and profile resource cleanup both failed");
|
|
312
|
+
if (primaryError !== void 0) throw primaryError;
|
|
313
|
+
if (cleanupError !== void 0) throw cleanupError;
|
|
314
|
+
return value;
|
|
315
|
+
}
|
|
316
|
+
async function emitShotReceipt(callback, receipt, execution, primaryError) {
|
|
317
|
+
try {
|
|
318
|
+
await callback?.(receipt, execution);
|
|
319
|
+
} catch (callbackError) {
|
|
320
|
+
if (primaryError !== null) throw new AggregateError([primaryError, callbackError], "agenticGenerator: author shot failed and its receipt could not be persisted");
|
|
321
|
+
throw callbackError;
|
|
322
|
+
}
|
|
323
|
+
if (primaryError !== null) throw primaryError;
|
|
324
|
+
}
|
|
325
|
+
async function emitShotDisposition(callback, receipt, disposition) {
|
|
326
|
+
await callback?.(receipt, disposition);
|
|
327
|
+
}
|
|
328
|
+
async function rethrowShotSetupError(callback, receipt, worktreePath, stage, cause) {
|
|
329
|
+
const error = cause instanceof Error ? cause : new Error(String(cause));
|
|
330
|
+
try {
|
|
331
|
+
await emitShotDisposition(callback, receipt, {
|
|
332
|
+
kind: "setup-error",
|
|
333
|
+
worktreePath,
|
|
334
|
+
stage,
|
|
335
|
+
error: {
|
|
336
|
+
name: error.name,
|
|
337
|
+
message: error.message
|
|
338
|
+
}
|
|
339
|
+
});
|
|
340
|
+
} catch (callbackError) {
|
|
341
|
+
throw new AggregateError([cause, callbackError], "agenticGenerator: shot processing failed and its worktree disposition could not be persisted");
|
|
342
|
+
}
|
|
343
|
+
throw cause;
|
|
344
|
+
}
|
|
345
|
+
function shotExecutionSnapshot(result) {
|
|
346
|
+
if (result === null) return null;
|
|
347
|
+
const usage = result.usage ? Object.freeze({ ...result.usage }) : void 0;
|
|
348
|
+
const evidence = result.evidence ? Object.freeze({
|
|
349
|
+
...result.evidence,
|
|
350
|
+
readDeniedPaths: Object.freeze([...result.evidence.readDeniedPaths]),
|
|
351
|
+
policy: Object.freeze({ ...result.evidence.policy })
|
|
352
|
+
}) : void 0;
|
|
353
|
+
return Object.freeze({
|
|
354
|
+
exitCode: result.exitCode,
|
|
355
|
+
stdout: result.stdout,
|
|
356
|
+
stderr: result.stderr,
|
|
357
|
+
killedBySignal: result.killedBySignal,
|
|
358
|
+
durationMs: result.durationMs,
|
|
359
|
+
timedOut: result.timedOut,
|
|
360
|
+
...result.aborted !== void 0 ? { aborted: result.aborted } : {},
|
|
361
|
+
...usage ? { usage } : {},
|
|
362
|
+
...evidence ? { evidence } : {}
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
function shotReceipt(input) {
|
|
366
|
+
const error = input.error ? {
|
|
367
|
+
name: input.error instanceof Error ? input.error.name : "Error",
|
|
368
|
+
message: input.error instanceof Error ? input.error.message : String(input.error)
|
|
369
|
+
} : null;
|
|
370
|
+
const result = input.result;
|
|
371
|
+
const costBasis = costBasisFor(input.costReceipt);
|
|
372
|
+
return {
|
|
373
|
+
generation: input.generation ?? null,
|
|
374
|
+
candidateIndex: input.candidateIndex ?? null,
|
|
375
|
+
shot: input.shot + 1,
|
|
376
|
+
maxShots: input.maxShots,
|
|
377
|
+
harness: input.harness,
|
|
378
|
+
model: input.profile?.model?.default ?? null,
|
|
379
|
+
reasoningEffort: input.profile?.model?.reasoningEffort ?? null,
|
|
380
|
+
promptSha256: sha256(input.prompt),
|
|
381
|
+
startedAt: input.startedAt.toISOString(),
|
|
382
|
+
completedAt: input.completedAt.toISOString(),
|
|
383
|
+
durationMs: result?.durationMs ?? input.completedAt.getTime() - input.startedAt.getTime(),
|
|
384
|
+
exitCode: result?.exitCode ?? null,
|
|
385
|
+
timedOut: result?.timedOut ?? false,
|
|
386
|
+
aborted: result?.aborted ?? false,
|
|
387
|
+
killedBySignal: result?.killedBySignal ?? null,
|
|
388
|
+
stdoutBytes: result ? Buffer.byteLength(result.stdout) : null,
|
|
389
|
+
stdoutSha256: result ? sha256(result.stdout) : null,
|
|
390
|
+
stderrBytes: result ? Buffer.byteLength(result.stderr) : null,
|
|
391
|
+
stderrSha256: result ? sha256(result.stderr) : null,
|
|
392
|
+
usage: result?.usage ? { ...result.usage } : null,
|
|
393
|
+
profileWorkspacePlanDigest: input.profileWorkspaceReceipt?.workspacePlanDigest ?? null,
|
|
394
|
+
profileWorkspaceFileCount: input.profileWorkspaceReceipt?.written.length ?? 0,
|
|
395
|
+
costCallId: input.costCallId,
|
|
396
|
+
costBasis,
|
|
397
|
+
costUsd: costBasis === "unknown" ? null : input.costReceipt?.costUsd ?? null,
|
|
398
|
+
costUsdKnown: costBasis === "provider-reported",
|
|
399
|
+
evidence: result?.evidence ? {
|
|
400
|
+
...result.evidence,
|
|
401
|
+
readDeniedPaths: [...result.evidence.readDeniedPaths],
|
|
402
|
+
policy: { ...result.evidence.policy }
|
|
403
|
+
} : null,
|
|
404
|
+
error
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
function costBasisFor(receipt) {
|
|
408
|
+
if (receipt === null || receipt.costUnknown) return "unknown";
|
|
409
|
+
return receipt.actualCostUsd === void 0 ? "estimated-pricing" : "provider-reported";
|
|
410
|
+
}
|
|
411
|
+
function shotFailure(result, exactPrompt, codexReproducible) {
|
|
412
|
+
if (result.timedOut) return /* @__PURE__ */ new Error("agenticGenerator: author shot timed out");
|
|
413
|
+
if (result.aborted) return /* @__PURE__ */ new Error("agenticGenerator: author shot was cancelled by the caller");
|
|
414
|
+
if (result.killedBySignal) return /* @__PURE__ */ new Error(`agenticGenerator: author shot was killed by ${result.killedBySignal}`);
|
|
415
|
+
if (result.exitCode !== 0) return /* @__PURE__ */ new Error(`agenticGenerator: author shot exited with code ${String(result.exitCode)}`);
|
|
416
|
+
if (!codexReproducible) return null;
|
|
417
|
+
if (!result.usage || !result.evidence) return /* @__PURE__ */ new Error("agenticGenerator: reproducible Codex shot completed without usage or execution evidence");
|
|
418
|
+
const expectedPromptSha256 = sha256(exactPrompt).slice(7);
|
|
419
|
+
if (result.evidence.requestedPromptSha256 !== expectedPromptSha256) return /* @__PURE__ */ new Error("agenticGenerator: reproducible Codex prompt evidence does not match the exact authored prompt");
|
|
420
|
+
return null;
|
|
421
|
+
}
|
|
422
|
+
function costReceiptFromHarness(result, model) {
|
|
423
|
+
if (!result.usage) throw new Error("agenticGenerator: author shot did not report terminal token usage");
|
|
424
|
+
return {
|
|
425
|
+
model,
|
|
426
|
+
inputTokens: result.usage.inputTokens - result.usage.cachedInputTokens,
|
|
427
|
+
outputTokens: result.usage.outputTokens,
|
|
428
|
+
...result.usage.cachedInputTokens > 0 ? { cachedTokens: result.usage.cachedInputTokens } : {}
|
|
429
|
+
};
|
|
430
|
+
}
|
|
431
|
+
function sha256(value) {
|
|
432
|
+
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
433
|
+
}
|
|
434
|
+
/** Turn the analyst's findings (+ optional report) into a concrete coder task —
|
|
435
|
+
* the senior scientific-method framing shared with the tool/MCP build prompts. */
|
|
436
|
+
function defaultBuildPrompt(args) {
|
|
437
|
+
const lines = [
|
|
438
|
+
"You are improving this codebase based on an evaluation analysis: real runs failed, an",
|
|
439
|
+
"analyst distilled the findings below, and your change will be measured on held-out tasks",
|
|
440
|
+
"against the unchanged baseline — only a real lift promotes it.",
|
|
441
|
+
"",
|
|
442
|
+
optimizerMethod,
|
|
443
|
+
"",
|
|
444
|
+
"THE SURFACE — what a deliverable change looks like here:",
|
|
445
|
+
"- edit this codebase in place: the smallest coherent change set that fully tests your",
|
|
446
|
+
" hypothesis about the dominant failure mode (see the method above — no unrelated edits,",
|
|
447
|
+
" they confound the measurement),",
|
|
448
|
+
"- keep the diff reviewable: a reviewer should be able to trace every hunk back to a finding,",
|
|
449
|
+
"- do not commit — leave changes in the working tree.",
|
|
450
|
+
"",
|
|
451
|
+
"FINDINGS — ranked evidence from real failed runs:"
|
|
452
|
+
];
|
|
453
|
+
for (const f of args.findings) {
|
|
454
|
+
const where = f.subject ? ` [${f.subject}]` : "";
|
|
455
|
+
lines.push(`- (${f.severity})${where} ${f.claim}`);
|
|
456
|
+
if (f.recommended_action) lines.push(` → ${f.recommended_action}`);
|
|
457
|
+
}
|
|
458
|
+
if (requiresRawTraceEvidence(args.findings)) lines.push("", "Raw trace evidence requirement:", `- Inspect at least one raw trace path named above before editing.`, `- Write ${RAW_TRACE_DIAGNOSIS_PATH} in this worktree.`, "- Include the exact trace path(s) inspected, the failure mechanism, and the code change made.", "- A candidate without this file, or with only this file changed, is discarded.");
|
|
459
|
+
return lines.join("\n");
|
|
460
|
+
}
|
|
461
|
+
const EMPTY_TREE_NOTE = "NOTE: your previous attempt left the working tree unchanged. Make the concrete file edits now.";
|
|
462
|
+
/** Next-shot feedback when the worktree is dirty but failed verification. The
|
|
463
|
+
* edits persist on disk, so the harness resumes atop them — tell it to fix in
|
|
464
|
+
* place, not start over. Verifier detail is truncated to keep the prompt bounded. */
|
|
465
|
+
function failureNote(feedback) {
|
|
466
|
+
const detail = feedback?.trim();
|
|
467
|
+
return [
|
|
468
|
+
"NOTE: your edits are in the working tree but verification FAILED.",
|
|
469
|
+
"Fix the problem in place — build on your existing edits, do not revert them.",
|
|
470
|
+
detail ? `Verifier output:\n${truncate$1(detail, 4e3)}` : "No verifier detail was captured."
|
|
471
|
+
].join("\n");
|
|
472
|
+
}
|
|
473
|
+
function rawTraceEvidenceProblem(worktreePath, findings) {
|
|
474
|
+
if (worktreeChangedPaths(worktreePath).filter((path) => path !== RAW_TRACE_DIAGNOSIS_PATH).length === 0) return [`NOTE: raw-trace mode requires a real code/config edit in addition to ${RAW_TRACE_DIAGNOSIS_PATH}.`, "Your previous attempt only changed the diagnosis artifact. Inspect the cited traces and make the causal code change."].join("\n");
|
|
475
|
+
const diagnosisPath = join(worktreePath, RAW_TRACE_DIAGNOSIS_PATH);
|
|
476
|
+
if (!existsSync(diagnosisPath)) return [`NOTE: raw-trace mode requires ${RAW_TRACE_DIAGNOSIS_PATH}.`, "Before retrying, inspect at least one cited spans.jsonl/cached-result.json/artifact path, then write the diagnosis file with the exact path, failure mechanism, and code change."].join("\n");
|
|
477
|
+
const body = readFileSync(diagnosisPath, "utf8");
|
|
478
|
+
const evidencePaths = traceEvidencePaths(findings);
|
|
479
|
+
if (evidencePaths.length > 0 && !evidencePaths.some((path) => body.includes(path))) return [`${RAW_TRACE_DIAGNOSIS_PATH} exists, but it does not cite any exact raw trace path from the findings.`, `Cite at least one of these inspected paths exactly: ${evidencePaths.slice(0, 5).join(", ")}`].join("\n");
|
|
480
|
+
return null;
|
|
481
|
+
}
|
|
482
|
+
function requiresRawTraceEvidence(findings) {
|
|
483
|
+
return findings.some((finding) => {
|
|
484
|
+
const f = finding;
|
|
485
|
+
return f.analyst_id === RAW_TRACE_ANALYST_ID || f.area === RAW_TRACE_AREA;
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
function traceEvidencePaths(findings) {
|
|
489
|
+
const out = [];
|
|
490
|
+
for (const finding of findings) {
|
|
491
|
+
const refs = finding.evidence_refs;
|
|
492
|
+
if (!Array.isArray(refs)) continue;
|
|
493
|
+
for (const ref of refs) {
|
|
494
|
+
if (!ref || typeof ref !== "object") continue;
|
|
495
|
+
const uri = ref.uri;
|
|
496
|
+
if (typeof uri === "string" && uri.length > 0) out.push(uri);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
return [...new Set(out)];
|
|
500
|
+
}
|
|
501
|
+
/** A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other
|
|
502
|
+
* exit ⇒ failed with stdout+stderr as feedback. The common case — verify by
|
|
503
|
+
* `tsc --noEmit`, `pnpm build`, or a test command. A timeout is treated as a
|
|
504
|
+
* FAILED candidate (a change that hangs the build is a bad change); a missing
|
|
505
|
+
* binary or spawn fault throws (a setup bug, not a failed candidate — no
|
|
506
|
+
* silent fallback). */
|
|
507
|
+
function commandVerifier(command, args = [], timeoutMs = 3e5) {
|
|
508
|
+
return async (worktreePath, signal) => {
|
|
509
|
+
let result;
|
|
510
|
+
try {
|
|
511
|
+
result = await runSettledCommand({
|
|
512
|
+
command,
|
|
513
|
+
args,
|
|
514
|
+
cwd: worktreePath,
|
|
515
|
+
timeoutMs,
|
|
516
|
+
...signal ? { signal } : {}
|
|
517
|
+
});
|
|
518
|
+
} catch (err) {
|
|
519
|
+
signal?.throwIfAborted();
|
|
520
|
+
if (err.code === "ENOENT") throw new Error(`commandVerifier: '${command}' not found in PATH (setup bug, not a failed candidate)`);
|
|
521
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
522
|
+
throw new Error(`commandVerifier: '${command}' failed to spawn: ${reason}`);
|
|
523
|
+
}
|
|
524
|
+
if (result.timedOut || result.killedBySignal) return {
|
|
525
|
+
ok: false,
|
|
526
|
+
feedback: `verifier '${command}' ${result.killedBySignal ? `killed by ${result.killedBySignal}` : "timed out"} after ${timeoutMs}ms`
|
|
527
|
+
};
|
|
528
|
+
if (result.exitCode === 0) return { ok: true };
|
|
529
|
+
const out = `${result.stdout}${result.stderr}`.trim();
|
|
530
|
+
return {
|
|
531
|
+
ok: false,
|
|
532
|
+
feedback: out.length > 0 ? out : `exit ${result.exitCode}`
|
|
533
|
+
};
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
/** A one-line summary for the commit message, derived from the findings. */
|
|
537
|
+
function summarizeFindings(findings) {
|
|
538
|
+
if (findings.length === 0) return "agentic improvement";
|
|
539
|
+
if (findings.length === 1) return `agentic: ${truncate$1(findings[0].claim, 64)}`;
|
|
540
|
+
return `agentic: ${findings.length} findings addressed`;
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* The accepted-shot result: summary (commit message) + the attribution pair
|
|
544
|
+
* the driver wraps into a `ProposedCandidate`, so the candidate stays
|
|
545
|
+
* attributable through `GenerationRecord` and the emitted provenance instead
|
|
546
|
+
* of landing as an anonymous surface.
|
|
547
|
+
*/
|
|
548
|
+
function acceptedCandidate(findings) {
|
|
549
|
+
const summary = summarizeFindings(findings);
|
|
550
|
+
return {
|
|
551
|
+
applied: true,
|
|
552
|
+
summary,
|
|
553
|
+
label: slugify(summary.split("\n", 1)[0] ?? summary),
|
|
554
|
+
rationale: boundedRationale(summary, findings)
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
/** Short slug from the summary's first line — the candidate's human label. */
|
|
558
|
+
function slugify(line) {
|
|
559
|
+
return truncate$1(line.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "agentic-improvement", 48);
|
|
560
|
+
}
|
|
561
|
+
/** Bounded "because Z" for the candidate: the findings it addressed, or the
|
|
562
|
+
* summary itself when the change was proposed from raw repo/trace context. */
|
|
563
|
+
function boundedRationale(summary, findings) {
|
|
564
|
+
if (findings.length === 0) return summary;
|
|
565
|
+
return truncate$1(findings.map((finding) => `(${finding.severity}) ${finding.claim}`).join("; "), 400);
|
|
566
|
+
}
|
|
567
|
+
function truncate$1(s, n) {
|
|
568
|
+
return s.length <= n ? s : `${s.slice(0, n - 1)}…`;
|
|
569
|
+
}
|
|
570
|
+
/** Non-empty `git status --porcelain` ⇒ the harness changed the worktree.
|
|
571
|
+
* Fails loud: the worktree is a fresh checkout, so a git error here means
|
|
572
|
+
* something is genuinely broken (git missing, corrupt index, killed mid-run).
|
|
573
|
+
* Folding that into `false` would silently discard a candidate and mask the
|
|
574
|
+
* real failure — forbidden by the no-silent-fallbacks doctrine. */
|
|
575
|
+
function worktreeDirty(worktreePath) {
|
|
576
|
+
return worktreeChangedPaths(worktreePath).length > 0;
|
|
577
|
+
}
|
|
578
|
+
function worktreeChangedPaths(worktreePath) {
|
|
579
|
+
const result = spawnSync("git", [
|
|
580
|
+
"status",
|
|
581
|
+
"--porcelain",
|
|
582
|
+
"--untracked-files=all"
|
|
583
|
+
], {
|
|
584
|
+
cwd: worktreePath,
|
|
585
|
+
encoding: "utf-8"
|
|
586
|
+
});
|
|
587
|
+
if (result.error) throw new Error(`agenticGenerator: git status failed to spawn in ${worktreePath}: ${result.error.message}`);
|
|
588
|
+
if (result.status !== 0) throw new Error(`agenticGenerator: git status exited ${result.status} in ${worktreePath}: ${result.stderr.trim()}`);
|
|
589
|
+
return result.stdout.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => line.slice(3).trim());
|
|
590
|
+
}
|
|
591
|
+
//#endregion
|
|
592
|
+
//#region src/improvement/findings.ts
|
|
593
|
+
/**
|
|
594
|
+
* Typed-findings accessor — the one place `unknown[]` findings become
|
|
595
|
+
* `AnalystFinding[]`.
|
|
596
|
+
*
|
|
597
|
+
* agent-eval's `ProposeContext.findings` is `TFindings[] = unknown[]` on the
|
|
598
|
+
* wire: the loop threads whatever the previous `analyzeGeneration` producer (or
|
|
599
|
+
* the caller's static seed) returned. Consumers that need the typed envelope
|
|
600
|
+
* (`claim`/`severity`/`recommended_action`) were down-casting with a bare
|
|
601
|
+
* `as AnalystFinding[]` — a lie at runtime whenever the seed was a raw string
|
|
602
|
+
* or an ad-hoc digest, which then rendered `undefined` into build prompts.
|
|
603
|
+
*
|
|
604
|
+
* `toAnalystFindings` replaces that cast: real findings pass through
|
|
605
|
+
* unchanged (structural guard, fail-closed), and non-conforming values are
|
|
606
|
+
* LIFTED into a real `AnalystFinding` envelope via `makeFinding` — the most
|
|
607
|
+
* actionable text becomes the claim, the original value rides in `metadata.raw`
|
|
608
|
+
* — so everything downstream of the accessor handles exactly one shape.
|
|
609
|
+
*/
|
|
610
|
+
const SEVERITIES = /* @__PURE__ */ new Set([
|
|
611
|
+
"critical",
|
|
612
|
+
"high",
|
|
613
|
+
"medium",
|
|
614
|
+
"low",
|
|
615
|
+
"info"
|
|
616
|
+
]);
|
|
617
|
+
/** Analyst id stamped on findings lifted from untyped seed values. */
|
|
618
|
+
const LIFTED_FINDING_ANALYST_ID = "lifted-seed";
|
|
619
|
+
/** Structural guard for the schema-versioned `AnalystFinding` envelope.
|
|
620
|
+
* Strict on the identity fields `makeFinding` always populates — a partial
|
|
621
|
+
* look-alike is lifted (re-enveloped), not trusted. */
|
|
622
|
+
function isAnalystFinding(value) {
|
|
623
|
+
if (!value || typeof value !== "object") return false;
|
|
624
|
+
const o = value;
|
|
625
|
+
return o.schema_version === "1.0.0" && typeof o.finding_id === "string" && typeof o.analyst_id === "string" && typeof o.severity === "string" && SEVERITIES.has(o.severity) && typeof o.area === "string" && typeof o.claim === "string" && typeof o.confidence === "number" && Array.isArray(o.evidence_refs);
|
|
626
|
+
}
|
|
627
|
+
/** The most actionable text of an untyped finding-ish value — mirrors the
|
|
628
|
+
* extraction order agent-eval's curator proposers use (`recommended_action` >
|
|
629
|
+
* `claim` > `lesson` > `notes` > `text` > `message`), so the two ends of the
|
|
630
|
+
* wire read the same field first. */
|
|
631
|
+
function liftedClaim(value) {
|
|
632
|
+
if (typeof value === "string") return value.trim() || null;
|
|
633
|
+
if (value && typeof value === "object") {
|
|
634
|
+
const o = value;
|
|
635
|
+
for (const key of [
|
|
636
|
+
"recommended_action",
|
|
637
|
+
"claim",
|
|
638
|
+
"lesson",
|
|
639
|
+
"notes",
|
|
640
|
+
"text",
|
|
641
|
+
"message"
|
|
642
|
+
]) {
|
|
643
|
+
const v = o[key];
|
|
644
|
+
if (typeof v === "string" && v.trim()) return v.trim();
|
|
645
|
+
}
|
|
646
|
+
try {
|
|
647
|
+
const json = JSON.stringify(value);
|
|
648
|
+
if (json && json !== "{}" && json !== "[]") return json.length > 400 ? `${json.slice(0, 399)}…` : json;
|
|
649
|
+
} catch {}
|
|
650
|
+
}
|
|
651
|
+
return null;
|
|
652
|
+
}
|
|
653
|
+
/**
|
|
654
|
+
* Normalize a mixed `unknown[]` findings array to `AnalystFinding[]`:
|
|
655
|
+
* conforming findings pass through by reference; strings and finding-ish
|
|
656
|
+
* objects are lifted into envelopes (claim = most actionable text, original
|
|
657
|
+
* value under `metadata.raw`); values with no extractable text are dropped.
|
|
658
|
+
* Never throws — a malformed seed must not kill a proposal round.
|
|
659
|
+
*/
|
|
660
|
+
function toAnalystFindings(findings, opts = {}) {
|
|
661
|
+
const analystId = opts.analystId ?? "lifted-seed";
|
|
662
|
+
const area = opts.area ?? "seed";
|
|
663
|
+
const out = [];
|
|
664
|
+
for (const f of findings) {
|
|
665
|
+
if (isAnalystFinding(f)) {
|
|
666
|
+
out.push(f);
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
const claim = liftedClaim(f);
|
|
670
|
+
if (!claim) continue;
|
|
671
|
+
out.push(makeFinding({
|
|
672
|
+
analyst_id: analystId,
|
|
673
|
+
severity: "info",
|
|
674
|
+
area,
|
|
675
|
+
confidence: .5,
|
|
676
|
+
claim,
|
|
677
|
+
evidence_refs: [],
|
|
678
|
+
...f && typeof f === "object" ? { metadata: { raw: f } } : {}
|
|
679
|
+
}));
|
|
680
|
+
}
|
|
681
|
+
return out;
|
|
682
|
+
}
|
|
683
|
+
//#endregion
|
|
684
|
+
//#region src/improvement/cleanup.ts
|
|
685
|
+
async function rethrowAfterCleanup(cause, cleanup, context) {
|
|
686
|
+
const cleanupErrors = [];
|
|
687
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
688
|
+
try {
|
|
689
|
+
await cleanup();
|
|
690
|
+
} catch (cleanupCause) {
|
|
691
|
+
cleanupErrors.push(cleanupCause);
|
|
692
|
+
continue;
|
|
693
|
+
}
|
|
694
|
+
if (cleanupErrors.length === 0) throw cause;
|
|
695
|
+
throw new AggregateError([cause, ...cleanupErrors], `${context}; cleanup retry succeeded`);
|
|
696
|
+
}
|
|
697
|
+
throw new AggregateError([cause, ...cleanupErrors], `${context}; cleanup failed`);
|
|
698
|
+
}
|
|
699
|
+
//#endregion
|
|
700
|
+
//#region src/improvement/improve-result.ts
|
|
701
|
+
function copyImproveCost(cost) {
|
|
702
|
+
return {
|
|
703
|
+
totalCostUsd: cost.totalCostUsd,
|
|
704
|
+
accountingComplete: cost.accountingComplete,
|
|
705
|
+
incompleteReasons: [...cost.incompleteReasons]
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
//#endregion
|
|
709
|
+
//#region src/improvement/improvement-driver.ts
|
|
710
|
+
/**
|
|
711
|
+
* Code-only candidate driver for Runtime-owned git worktrees.
|
|
712
|
+
*
|
|
713
|
+
* A `CandidateGenerator` edits an isolated checkout. This driver finalizes each
|
|
714
|
+
* accepted edit as a `CodeSurface` and disposes rejected worktrees.
|
|
715
|
+
*
|
|
716
|
+
* @experimental
|
|
717
|
+
*/
|
|
718
|
+
/** Build the code-only proposer used internally by `improve({ surface: 'code' })`. */
|
|
719
|
+
function improvementDriver(opts) {
|
|
720
|
+
const baseRef = opts.baseRef ?? "main";
|
|
721
|
+
const owned = /* @__PURE__ */ new Map();
|
|
722
|
+
return {
|
|
723
|
+
kind: `improvement:${opts.generator.kind}`,
|
|
724
|
+
async propose(ctx) {
|
|
725
|
+
const findings = resolveFindings(ctx);
|
|
726
|
+
if (findings.length === 0 && ctx.report === void 0 && !opts.generator.proposesWithoutFindings) return [];
|
|
727
|
+
const surfaces = [];
|
|
728
|
+
const incumbent = verifiedCodeIncumbent(ctx.currentSurface);
|
|
729
|
+
const proposalBaseRef = incumbent?.baseCommit ?? baseRef;
|
|
730
|
+
for (let i = 0; i < ctx.populationSize; i++) {
|
|
731
|
+
if (ctx.signal.aborted) break;
|
|
732
|
+
const wt = await opts.worktree.create({
|
|
733
|
+
baseRef: proposalBaseRef,
|
|
734
|
+
label: `${opts.generator.kind}-gen${ctx.generation}-cand${i}`
|
|
735
|
+
});
|
|
736
|
+
owned.set(wt.path, wt);
|
|
737
|
+
try {
|
|
738
|
+
if (incumbent) advanceToIncumbent(wt, incumbent);
|
|
739
|
+
const { applied, summary, label, rationale } = await opts.generator.generate({
|
|
740
|
+
worktreePath: wt.path,
|
|
741
|
+
report: ctx.report,
|
|
742
|
+
findings,
|
|
743
|
+
dataset: ctx.dataset,
|
|
744
|
+
maxShots: ctx.maxImprovementShots ?? 1,
|
|
745
|
+
signal: ctx.signal,
|
|
746
|
+
generation: ctx.generation,
|
|
747
|
+
candidateIndex: i,
|
|
748
|
+
...ctx.costLedger ? { costLedger: ctx.costLedger } : {},
|
|
749
|
+
...ctx.costPhase ? { costPhase: ctx.costPhase } : {}
|
|
750
|
+
});
|
|
751
|
+
if (!applied) {
|
|
752
|
+
await opts.worktree.discard(wt);
|
|
753
|
+
owned.delete(wt.path);
|
|
754
|
+
continue;
|
|
755
|
+
}
|
|
756
|
+
const surface = await opts.worktree.finalize(wt, summary);
|
|
757
|
+
surfaces.push(label || rationale ? {
|
|
758
|
+
surface,
|
|
759
|
+
label: label ?? "",
|
|
760
|
+
rationale: rationale ?? ""
|
|
761
|
+
} : surface);
|
|
762
|
+
owned.delete(wt.path);
|
|
763
|
+
owned.set(surface.worktreeRef, wt);
|
|
764
|
+
} catch (err) {
|
|
765
|
+
return rethrowAfterCleanup(err, async () => {
|
|
766
|
+
await opts.worktree.discard(wt);
|
|
767
|
+
owned.delete(wt.path);
|
|
768
|
+
}, `improvementDriver: ${err instanceof Error ? err.message : String(err)}`);
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
return surfaces;
|
|
772
|
+
},
|
|
773
|
+
async cleanup(retainWorktreeRefs = []) {
|
|
774
|
+
const retained = new Set(retainWorktreeRefs);
|
|
775
|
+
const errors = [];
|
|
776
|
+
for (const [worktreeRef, worktree] of owned) {
|
|
777
|
+
if (retained.has(worktreeRef)) continue;
|
|
778
|
+
try {
|
|
779
|
+
await opts.worktree.discard(worktree);
|
|
780
|
+
owned.delete(worktreeRef);
|
|
781
|
+
} catch (cause) {
|
|
782
|
+
errors.push(cause);
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
if (errors.length > 0) throw new AggregateError(errors, "improvementDriver: failed to discard candidate worktrees");
|
|
786
|
+
}
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
/** A code incumbent must still match the immutable identity that was measured. */
|
|
790
|
+
function verifiedCodeIncumbent(surface) {
|
|
791
|
+
if (typeof surface !== "object" || surface.kind !== "code") return void 0;
|
|
792
|
+
verifyCodeSurface(surface);
|
|
793
|
+
return surface;
|
|
794
|
+
}
|
|
795
|
+
/** Start at the root commit recorded by the incumbent, then fast-forward the
|
|
796
|
+
* fresh branch to the incumbent commit. The worktree stays clean for the
|
|
797
|
+
* generator while `finalize()` still emits one cumulative root-to-candidate
|
|
798
|
+
* patch that can be applied or rolled back independently of prior branches. */
|
|
799
|
+
function advanceToIncumbent(worktree, incumbent) {
|
|
800
|
+
if (worktree.baseCommit !== incumbent.baseCommit || worktree.baseTree !== incumbent.baseTree) throw new Error("improvementDriver: candidate worktree does not match incumbent base identity");
|
|
801
|
+
if (worktree.baseCommit === incumbent.candidateCommit) return;
|
|
802
|
+
const merge = spawnSync("git", [
|
|
803
|
+
"merge",
|
|
804
|
+
"--ff-only",
|
|
805
|
+
incumbent.candidateCommit
|
|
806
|
+
], {
|
|
807
|
+
cwd: worktree.path,
|
|
808
|
+
encoding: "utf8"
|
|
809
|
+
});
|
|
810
|
+
if (merge.error) throw new Error(`improvementDriver: failed to start candidate from incumbent: ${merge.error.message}`);
|
|
811
|
+
if (merge.status !== 0) throw new Error(`improvementDriver: could not fast-forward candidate to incumbent ${incumbent.candidateCommit}: ${merge.stderr.trim()}`);
|
|
812
|
+
const head = spawnSync("git", [
|
|
813
|
+
"rev-parse",
|
|
814
|
+
"--verify",
|
|
815
|
+
"HEAD"
|
|
816
|
+
], {
|
|
817
|
+
cwd: worktree.path,
|
|
818
|
+
encoding: "utf8"
|
|
819
|
+
});
|
|
820
|
+
if (head.error || head.status !== 0 || head.stdout.trim() !== incumbent.candidateCommit) throw new Error("improvementDriver: candidate worktree did not reach the incumbent commit");
|
|
821
|
+
}
|
|
822
|
+
/** Phase-2 report carries `findings` when present; else fall back to the
|
|
823
|
+
* loop's `ctx.findings`. The report is opaque to the substrate, so probe it
|
|
824
|
+
* structurally. Both paths run through `toAnalystFindings` — the wire is
|
|
825
|
+
* `unknown[]` at runtime regardless of the generic (static seeds, legacy
|
|
826
|
+
* digests), and a bare cast here fed `undefined` claims into build prompts. */
|
|
827
|
+
function resolveFindings(ctx) {
|
|
828
|
+
const report = ctx.report;
|
|
829
|
+
if (report && typeof report === "object" && "findings" in report) {
|
|
830
|
+
const f = report.findings;
|
|
831
|
+
if (Array.isArray(f) && f.length > 0) {
|
|
832
|
+
const lifted = toAnalystFindings(f, {
|
|
833
|
+
analystId: "report-findings",
|
|
834
|
+
area: "report"
|
|
835
|
+
});
|
|
836
|
+
if (lifted.length > 0) return lifted;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
return toAnalystFindings(ctx.findings ?? [], {
|
|
840
|
+
analystId: "loop-context",
|
|
841
|
+
area: "seed"
|
|
842
|
+
});
|
|
843
|
+
}
|
|
844
|
+
//#endregion
|
|
845
|
+
//#region src/improvement/rollout-policy.ts
|
|
846
|
+
/** The profile extensions namespace the policy persists under. */
|
|
847
|
+
const ROLLOUT_POLICY_EXTENSION = "structural-rollout";
|
|
848
|
+
const isBoundedInt = (v, min) => typeof v === "number" && Number.isInteger(v) && v >= min;
|
|
849
|
+
/** Parse a serialized policy surface. Returns `undefined` for non-strings,
|
|
850
|
+
* malformed JSON, or values outside the policy invariants. Unknown fields are
|
|
851
|
+
* dropped; supported optional fields are preserved. */
|
|
852
|
+
function parseRolloutPolicy(surface) {
|
|
853
|
+
if (typeof surface !== "string" || surface.trim().length === 0) return void 0;
|
|
854
|
+
let raw;
|
|
855
|
+
try {
|
|
856
|
+
raw = JSON.parse(surface);
|
|
857
|
+
} catch {
|
|
858
|
+
return;
|
|
859
|
+
}
|
|
860
|
+
return normalizeRolloutPolicy(raw);
|
|
861
|
+
}
|
|
862
|
+
/** Normalize an untyped policy bag (a parsed surface or a profile extension) into
|
|
863
|
+
* a full `StructuralRolloutPolicy`, defaults merged. Returns `undefined` when any
|
|
864
|
+
* present dial violates the policy invariants (mirrors `resolvePolicy`: integer
|
|
865
|
+
* k ≥ 1, repairRounds ≥ 0, testgen ≥ 0) — a corrupt config must read as "not
|
|
866
|
+
* configured", never as a fabricated recipe. */
|
|
867
|
+
function normalizeRolloutPolicy(raw) {
|
|
868
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return void 0;
|
|
869
|
+
const bag = raw;
|
|
870
|
+
const k = bag.k ?? defaultStructuralRolloutPolicy.k;
|
|
871
|
+
const repairRounds = bag.repairRounds ?? defaultStructuralRolloutPolicy.repairRounds;
|
|
872
|
+
const testgen = bag.testgen ?? defaultStructuralRolloutPolicy.testgen;
|
|
873
|
+
if (!isBoundedInt(k, 1) || !isBoundedInt(repairRounds, 0) || !isBoundedInt(testgen, 0)) return;
|
|
874
|
+
return {
|
|
875
|
+
k,
|
|
876
|
+
repairRounds,
|
|
877
|
+
testgen,
|
|
878
|
+
...typeof bag.diverse === "boolean" ? { diverse: bag.diverse } : {},
|
|
879
|
+
...typeof bag.temperature === "number" ? { temperature: bag.temperature } : {}
|
|
880
|
+
};
|
|
881
|
+
}
|
|
882
|
+
/** Stable serialization with fixed field order. */
|
|
883
|
+
function serializeRolloutPolicy(policy) {
|
|
884
|
+
return JSON.stringify({
|
|
885
|
+
k: policy.k,
|
|
886
|
+
repairRounds: policy.repairRounds,
|
|
887
|
+
testgen: policy.testgen,
|
|
888
|
+
...policy.diverse !== void 0 ? { diverse: policy.diverse } : {},
|
|
889
|
+
...policy.temperature !== void 0 ? { temperature: policy.temperature } : {}
|
|
890
|
+
});
|
|
891
|
+
}
|
|
892
|
+
/** Read the persisted policy off the profile. `undefined` when the profile does
|
|
893
|
+
* not opt into structural rollout. */
|
|
894
|
+
function structuralRolloutPolicyFromProfile(profile) {
|
|
895
|
+
const bag = profile.extensions?.[ROLLOUT_POLICY_EXTENSION];
|
|
896
|
+
if (bag === void 0) return void 0;
|
|
897
|
+
return normalizeRolloutPolicy(bag);
|
|
898
|
+
}
|
|
899
|
+
/** Persist a detached policy under the profile extension without mutating the input. */
|
|
900
|
+
function applyRolloutPolicyToProfile(profile, policy) {
|
|
901
|
+
const candidate = structuredClone(profile);
|
|
902
|
+
const bag = {
|
|
903
|
+
k: policy.k,
|
|
904
|
+
repairRounds: policy.repairRounds,
|
|
905
|
+
testgen: policy.testgen,
|
|
906
|
+
...policy.diverse !== void 0 ? { diverse: policy.diverse } : {},
|
|
907
|
+
...policy.temperature !== void 0 ? { temperature: policy.temperature } : {}
|
|
908
|
+
};
|
|
909
|
+
return {
|
|
910
|
+
...candidate,
|
|
911
|
+
extensions: {
|
|
912
|
+
...candidate.extensions,
|
|
913
|
+
[ROLLOUT_POLICY_EXTENSION]: bag
|
|
914
|
+
}
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
//#endregion
|
|
918
|
+
//#region src/improvement/profile-surface.ts
|
|
919
|
+
/** Extract the baseline optimized by a method and retain its structured value
|
|
920
|
+
* so external optimizers can inspect it for private fields before serialization. */
|
|
921
|
+
function prepareProfileSurface(profile, surface, skills, profileComponents) {
|
|
922
|
+
switch (surface) {
|
|
923
|
+
case "prompt": return {
|
|
924
|
+
surface: profile.prompt?.systemPrompt ?? "",
|
|
925
|
+
value: profile.prompt?.systemPrompt ?? ""
|
|
926
|
+
};
|
|
927
|
+
case "skills": {
|
|
928
|
+
const value = inlineSkill(profile, skills).content;
|
|
929
|
+
return {
|
|
930
|
+
surface: value,
|
|
931
|
+
value
|
|
932
|
+
};
|
|
933
|
+
}
|
|
934
|
+
case "tools": {
|
|
935
|
+
const value = profile.tools ?? {};
|
|
936
|
+
return {
|
|
937
|
+
surface: canonicalJson(value),
|
|
938
|
+
value
|
|
939
|
+
};
|
|
940
|
+
}
|
|
941
|
+
case "mcp": {
|
|
942
|
+
const value = profile.mcp ?? {};
|
|
943
|
+
return {
|
|
944
|
+
surface: canonicalJson(value),
|
|
945
|
+
value
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
case "hooks": {
|
|
949
|
+
const value = profile.hooks ?? {};
|
|
950
|
+
return {
|
|
951
|
+
surface: canonicalJson(value),
|
|
952
|
+
value
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
case "subagents": {
|
|
956
|
+
const value = profile.subagents ?? {};
|
|
957
|
+
return {
|
|
958
|
+
surface: canonicalJson(value),
|
|
959
|
+
value
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
case "agent-profile":
|
|
963
|
+
if (profileComponents) {
|
|
964
|
+
const value = profileComponents.read(profile);
|
|
965
|
+
return {
|
|
966
|
+
surface: componentSurface(value, "profileComponents.read"),
|
|
967
|
+
value
|
|
968
|
+
};
|
|
969
|
+
}
|
|
970
|
+
return {
|
|
971
|
+
surface: canonicalJson(profile),
|
|
972
|
+
value: profile
|
|
973
|
+
};
|
|
974
|
+
case "memory": {
|
|
975
|
+
const value = profileInstructions(profile);
|
|
976
|
+
return {
|
|
977
|
+
surface: value,
|
|
978
|
+
value
|
|
979
|
+
};
|
|
980
|
+
}
|
|
981
|
+
case "rollout-policy": {
|
|
982
|
+
const policy = structuralRolloutPolicyFromProfile(profile);
|
|
983
|
+
return {
|
|
984
|
+
surface: policy ? serializeRolloutPolicy(policy) : "",
|
|
985
|
+
value: policy ?? null
|
|
986
|
+
};
|
|
987
|
+
}
|
|
988
|
+
case "code": throw new ConfigError("improve(): code requires the isolated baseline created from opts.code.repoRoot");
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
function isCodeSurface(surface) {
|
|
992
|
+
return typeof surface === "object" && surface !== null && surface.kind === "code";
|
|
993
|
+
}
|
|
994
|
+
function isComponentSurface(surface) {
|
|
995
|
+
return typeof surface === "object" && surface !== null && surface.kind === "components";
|
|
996
|
+
}
|
|
997
|
+
function componentSurface(components, source) {
|
|
998
|
+
const entries = validateComponents(components, source);
|
|
999
|
+
return immutableCandidateValue({
|
|
1000
|
+
kind: "components",
|
|
1001
|
+
components: Object.fromEntries(entries)
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
function validateComponents(components, source) {
|
|
1005
|
+
if (typeof components !== "object" || components === null || Array.isArray(components)) throw new ConfigError(`improve(): ${source} must return a component record`);
|
|
1006
|
+
const entries = Object.entries(components);
|
|
1007
|
+
if (entries.length === 0) throw new ConfigError(`improve(): ${source} must return at least one component`);
|
|
1008
|
+
for (const [name, value] of entries) if (!name || name.trim() !== name || typeof value !== "string") throw new ConfigError(`improve(): ${source} must return trimmed component names with string values`);
|
|
1009
|
+
return entries;
|
|
1010
|
+
}
|
|
1011
|
+
/** Parse a JSON winner surface with a typed, contextual error. */
|
|
1012
|
+
function parseWinnerJson(winner, surface) {
|
|
1013
|
+
try {
|
|
1014
|
+
return JSON.parse(winner);
|
|
1015
|
+
} catch (cause) {
|
|
1016
|
+
throw new ConfigError(`improve(): the '${surface}' candidate is not valid JSON, so it cannot form a profile candidate: ${cause.message}`);
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
function assertCandidateSurfaceKind(surface, baseline, winner) {
|
|
1020
|
+
if (surface === "code") {
|
|
1021
|
+
if (isCodeSurface(winner)) return;
|
|
1022
|
+
throw new ConfigError(`improve(): the '${surface}' candidate returned an incompatible surface value`);
|
|
1023
|
+
}
|
|
1024
|
+
if (typeof baseline === "string") {
|
|
1025
|
+
if (typeof winner === "string") return;
|
|
1026
|
+
throw new ConfigError(`improve(): the '${surface}' candidate changed from a text surface to an incompatible surface value`);
|
|
1027
|
+
}
|
|
1028
|
+
if (!isComponentSurface(baseline) || !isComponentSurface(winner)) throw new ConfigError(`improve(): the '${surface}' candidate returned an incompatible surface value`);
|
|
1029
|
+
validateComponents(winner.components, `the '${surface}' candidate`);
|
|
1030
|
+
const baselineNames = Object.keys(baseline.components).sort();
|
|
1031
|
+
const winnerNames = Object.keys(winner.components).sort();
|
|
1032
|
+
if (baselineNames.length !== winnerNames.length || baselineNames.some((name, index) => name !== winnerNames[index])) throw new ConfigError(`improve(): the '${surface}' candidate must preserve the exact component names`);
|
|
1033
|
+
}
|
|
1034
|
+
/** Materialize a detached profile candidate without changing the baseline. */
|
|
1035
|
+
function materializeImprovementProfileCandidate(profile, surface, winner, skills, profileComponents) {
|
|
1036
|
+
let candidate;
|
|
1037
|
+
if (isComponentSurface(winner)) {
|
|
1038
|
+
if (surface !== "agent-profile" || !profileComponents) throw new ConfigError(`improve(): the '${surface}' candidate has no profile component mapping`);
|
|
1039
|
+
const winnerComponents = immutableCandidateValue({ ...winner.components });
|
|
1040
|
+
const validated = validateProfileCandidate(profileComponents.apply(profile, winnerComponents), surface);
|
|
1041
|
+
const materializedComponents = Object.fromEntries(validateComponents(profileComponents.read(validated), "profileComponents.read after apply"));
|
|
1042
|
+
const names = Object.keys(winnerComponents);
|
|
1043
|
+
if (names.length !== Object.keys(materializedComponents).length || names.some((name) => materializedComponents[name] !== winnerComponents[name])) throw new ConfigError("improve(): profileComponents.apply must round-trip every winning component exactly");
|
|
1044
|
+
return validated;
|
|
1045
|
+
}
|
|
1046
|
+
if (typeof winner !== "string") throw new ConfigError(`improve(): the '${surface}' candidate cannot form an AgentProfile`);
|
|
1047
|
+
switch (surface) {
|
|
1048
|
+
case "prompt":
|
|
1049
|
+
candidate = {
|
|
1050
|
+
...profile,
|
|
1051
|
+
prompt: {
|
|
1052
|
+
...profile.prompt,
|
|
1053
|
+
systemPrompt: winner
|
|
1054
|
+
}
|
|
1055
|
+
};
|
|
1056
|
+
break;
|
|
1057
|
+
case "skills": {
|
|
1058
|
+
const selectedSkill = inlineSkill(profile, skills);
|
|
1059
|
+
candidate = {
|
|
1060
|
+
...profile,
|
|
1061
|
+
resources: {
|
|
1062
|
+
...profile.resources,
|
|
1063
|
+
skills: profile.resources?.skills?.map((resource) => resource === selectedSkill ? {
|
|
1064
|
+
...resource,
|
|
1065
|
+
content: winner
|
|
1066
|
+
} : resource)
|
|
1067
|
+
}
|
|
1068
|
+
};
|
|
1069
|
+
break;
|
|
1070
|
+
}
|
|
1071
|
+
case "tools":
|
|
1072
|
+
candidate = {
|
|
1073
|
+
...profile,
|
|
1074
|
+
tools: parseWinnerJson(winner, surface)
|
|
1075
|
+
};
|
|
1076
|
+
break;
|
|
1077
|
+
case "mcp":
|
|
1078
|
+
candidate = {
|
|
1079
|
+
...profile,
|
|
1080
|
+
mcp: parseWinnerJson(winner, surface)
|
|
1081
|
+
};
|
|
1082
|
+
break;
|
|
1083
|
+
case "hooks":
|
|
1084
|
+
candidate = {
|
|
1085
|
+
...profile,
|
|
1086
|
+
hooks: parseWinnerJson(winner, surface)
|
|
1087
|
+
};
|
|
1088
|
+
break;
|
|
1089
|
+
case "subagents":
|
|
1090
|
+
candidate = {
|
|
1091
|
+
...profile,
|
|
1092
|
+
subagents: parseWinnerJson(winner, surface)
|
|
1093
|
+
};
|
|
1094
|
+
break;
|
|
1095
|
+
case "agent-profile":
|
|
1096
|
+
candidate = parseWinnerJson(winner, surface);
|
|
1097
|
+
break;
|
|
1098
|
+
case "memory":
|
|
1099
|
+
candidate = {
|
|
1100
|
+
...profile,
|
|
1101
|
+
resources: {
|
|
1102
|
+
...profile.resources,
|
|
1103
|
+
instructions: replaceProfileInstructions(profile, winner)
|
|
1104
|
+
}
|
|
1105
|
+
};
|
|
1106
|
+
break;
|
|
1107
|
+
case "rollout-policy": {
|
|
1108
|
+
const policy = normalizeRolloutPolicy(parseWinnerJson(winner, surface));
|
|
1109
|
+
if (!policy) throw new ConfigError(`improve(): the shipped 'rollout-policy' winner is not a valid StructuralRolloutPolicy (integer k >= 1, repairRounds >= 0, testgen >= 0), so it cannot be applied: ${winner}`);
|
|
1110
|
+
candidate = applyRolloutPolicyToProfile(profile, policy);
|
|
1111
|
+
break;
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
return validateProfileCandidate(candidate, surface);
|
|
1115
|
+
}
|
|
1116
|
+
function validateProfileCandidate(candidate, surface) {
|
|
1117
|
+
const parsed = agentProfileSchema.safeParse(candidate);
|
|
1118
|
+
if (!parsed.success) throw new ConfigError(`improve(): the '${surface}' candidate does not produce a valid AgentProfile: ${parsed.error.message}`);
|
|
1119
|
+
return immutableCandidateValue(parsed.data);
|
|
1120
|
+
}
|
|
1121
|
+
function createProfileCandidateMaterializer(profile, surface, baselineSurface, skills, profileComponents) {
|
|
1122
|
+
const baselineDigest = canonicalCandidateDigest$1(baselineSurface);
|
|
1123
|
+
if (profileComponents) {
|
|
1124
|
+
if (canonicalCandidateDigest$1(materializeImprovementProfileCandidate(profile, surface, baselineSurface, skills, profileComponents)) !== canonicalCandidateDigest$1(profile)) throw new ConfigError("improve(): profileComponents.apply(profile, profileComponents.read(profile)) must reproduce the complete baseline profile exactly");
|
|
1125
|
+
}
|
|
1126
|
+
const candidates = /* @__PURE__ */ new Map([[baselineDigest, profile]]);
|
|
1127
|
+
return (candidateSurface) => {
|
|
1128
|
+
assertCandidateSurfaceKind(surface, baselineSurface, candidateSurface);
|
|
1129
|
+
const digest = canonicalCandidateDigest$1(candidateSurface);
|
|
1130
|
+
const existing = candidates.get(digest);
|
|
1131
|
+
if (existing) return existing;
|
|
1132
|
+
const candidate = materializeImprovementProfileCandidate(profile, surface, immutableCandidateValue(candidateSurface), skills, profileComponents);
|
|
1133
|
+
candidates.set(digest, candidate);
|
|
1134
|
+
return candidate;
|
|
1135
|
+
};
|
|
1136
|
+
}
|
|
1137
|
+
function inlineSkill(profile, options) {
|
|
1138
|
+
const resourceName = options?.resourceName.trim();
|
|
1139
|
+
if (!resourceName) throw new ConfigError("improve(): surface 'skills' requires opts.skills.resourceName for one inline profile skill");
|
|
1140
|
+
assertFailClosedResources(profile, "skills");
|
|
1141
|
+
const matches = (profile.resources?.skills ?? []).filter((resource) => resource.name === resourceName);
|
|
1142
|
+
if (matches.length !== 1 || matches[0]?.kind !== "inline") throw new ConfigError(`improve(): skill '${resourceName}' must identify exactly one inline profile resource`);
|
|
1143
|
+
return matches[0];
|
|
1144
|
+
}
|
|
1145
|
+
function profileInstructions(profile) {
|
|
1146
|
+
assertFailClosedResources(profile, "memory");
|
|
1147
|
+
const instructions = profile.resources?.instructions;
|
|
1148
|
+
if (instructions === void 0) return "";
|
|
1149
|
+
if (typeof instructions === "string") return instructions;
|
|
1150
|
+
if (instructions.kind === "inline") return instructions.content;
|
|
1151
|
+
throw new ConfigError("improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact");
|
|
1152
|
+
}
|
|
1153
|
+
function replaceProfileInstructions(profile, content) {
|
|
1154
|
+
const instructions = profile.resources?.instructions;
|
|
1155
|
+
if (typeof instructions !== "object") return content;
|
|
1156
|
+
if (instructions.kind !== "inline") throw new ConfigError("improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact");
|
|
1157
|
+
return {
|
|
1158
|
+
...instructions,
|
|
1159
|
+
content
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
function assertFailClosedResources(profile, surface) {
|
|
1163
|
+
if (profile.resources?.failOnError !== true) throw new ConfigError(`improve(): surface '${surface}' requires profile.resources.failOnError: true`);
|
|
1164
|
+
}
|
|
1165
|
+
//#endregion
|
|
1166
|
+
//#region src/improvement/raw-trace-distiller.ts
|
|
1167
|
+
/**
|
|
1168
|
+
*
|
|
1169
|
+
* `rawTraceDistiller` — the meta-harness `analyzeGeneration` producer.
|
|
1170
|
+
*
|
|
1171
|
+
* The default `generationFailureDistiller` (in `improve.ts`) COMPRESSES each
|
|
1172
|
+
* generation's failing cells into ~1500-char structured findings before the next
|
|
1173
|
+
* proposal round. That is the ACE-style recipe: a small summary is the proposer's
|
|
1174
|
+
* whole view of what went wrong. This producer does the opposite — the
|
|
1175
|
+
* meta-harness recipe (yoonholee.com/meta-harness): it does NOT summarize. It
|
|
1176
|
+
* points the coding-agent proposer at the generation's RAW run traces already on
|
|
1177
|
+
* disk under `runDir` — the durable per-cell `spans.jsonl` event logs,
|
|
1178
|
+
* `cached-result.json` scores, and any artifacts the substrate persisted — and
|
|
1179
|
+
* instructs the agent to `grep`/`cat`/`ls` them to diagnose the failures itself
|
|
1180
|
+
* (up to the harness's full context, ~millions of tokens, vs a ~1500-char digest).
|
|
1181
|
+
*
|
|
1182
|
+
* It emits `AnalystFinding[]` so it drops into the SAME `opts.analyzeGeneration`
|
|
1183
|
+
* slot the default distiller uses, and renders through the same
|
|
1184
|
+
* `agenticGenerator` prompt path (`claim` + `recommended_action`). The findings
|
|
1185
|
+
* carry ABSOLUTE paths — the coding harness runs with `cwd` = a candidate
|
|
1186
|
+
* worktree, so a relative `runDir` would be uncattable from there.
|
|
1187
|
+
*
|
|
1188
|
+
* Runtime layout it reads (written by agent-eval's optimization loop):
|
|
1189
|
+
*
|
|
1190
|
+
* <runDir>/gen-<N>/ ← the generation dir (input.runDir)
|
|
1191
|
+
* candidate-<i>/ ← one candidate campaign (campaign.runDir)
|
|
1192
|
+
* <sanitized cellId>/ ← one scenario×rep cell
|
|
1193
|
+
* spans.jsonl ← the raw trace (event/span log)
|
|
1194
|
+
* cached-result.json ← the cell's score + artifact ref
|
|
1195
|
+
* <artifacts…> ← whatever the dispatch wrote
|
|
1196
|
+
*
|
|
1197
|
+
* @experimental
|
|
1198
|
+
*/
|
|
1199
|
+
const ANALYST_ID = "raw-trace-distiller";
|
|
1200
|
+
/** A cell counts as "failing" below this mean composite (matches the default
|
|
1201
|
+
* distiller's near-perfect threshold) or when it recorded an `error`. */
|
|
1202
|
+
const PASS_THRESHOLD = .999;
|
|
1203
|
+
/**
|
|
1204
|
+
* Build an `analyzeGeneration` producer that feeds the proposer RAW-TRACE
|
|
1205
|
+
* FILESYSTEM CONTEXT — paths into the prior generation's real run traces plus a
|
|
1206
|
+
* grep/cat-to-diagnose instruction — instead of a pre-summarized digest.
|
|
1207
|
+
*
|
|
1208
|
+
* Drop-in for `analyzeGeneration` on `improve({ surface: 'code' })`:
|
|
1209
|
+
*
|
|
1210
|
+
* await improve({
|
|
1211
|
+
* surface: 'code',
|
|
1212
|
+
* findings: seedFindings,
|
|
1213
|
+
* code: { repoRoot },
|
|
1214
|
+
* runDir: '/abs/run', // MUST be a real path — the traces live here
|
|
1215
|
+
* analyzeGeneration: rawTraceDistiller(),
|
|
1216
|
+
* scenarios, judge, agent,
|
|
1217
|
+
* })
|
|
1218
|
+
*/
|
|
1219
|
+
function rawTraceDistiller(options = {}) {
|
|
1220
|
+
const maxCandidates = options.maxCandidates ?? 12;
|
|
1221
|
+
const maxCellsPerCandidate = options.maxCellsPerCandidate ?? 8;
|
|
1222
|
+
const maxFilesPerCell = options.maxFilesPerCell ?? 24;
|
|
1223
|
+
return async (input) => {
|
|
1224
|
+
const genRoot = absoluteRunDir(options.runDir ?? input.runDir);
|
|
1225
|
+
const durable = isDurable(genRoot);
|
|
1226
|
+
const ranked = [...input.candidates].map((c) => ({
|
|
1227
|
+
surfaceHash: c.surfaceHash,
|
|
1228
|
+
composite: c.composite,
|
|
1229
|
+
campaignDir: absoluteRunDir(c.campaign.runDir),
|
|
1230
|
+
cells: failingCells(c.campaign, maxCellsPerCandidate, maxFilesPerCell)
|
|
1231
|
+
})).sort((a, b) => compareCandidateComposites(a.composite, b.composite)).slice(0, maxCandidates);
|
|
1232
|
+
const totalFailingCells = ranked.reduce((n, c) => n + c.cells.length, 0);
|
|
1233
|
+
if (totalFailingCells === 0) {
|
|
1234
|
+
if (options.fallbackFindings && options.fallbackFindings.length > 0) return options.fallbackFindings;
|
|
1235
|
+
return [makeFinding({
|
|
1236
|
+
analyst_id: ANALYST_ID,
|
|
1237
|
+
severity: "info",
|
|
1238
|
+
area: "raw-trace-context",
|
|
1239
|
+
confidence: 1,
|
|
1240
|
+
claim: `Generation ${input.generation} had no failing cells. The full raw run traces are on disk under ${genRoot}.`,
|
|
1241
|
+
recommended_action: `To keep improving, grep/cat the raw traces under ${genRoot} (per-cell spans.jsonl + cached-result.json) to find the weakest passing runs, then make a targeted harness-code edit.`,
|
|
1242
|
+
evidence_refs: [{
|
|
1243
|
+
kind: "artifact",
|
|
1244
|
+
uri: genRoot
|
|
1245
|
+
}],
|
|
1246
|
+
metadata: {
|
|
1247
|
+
generation: input.generation,
|
|
1248
|
+
runDir: genRoot,
|
|
1249
|
+
failingCells: 0
|
|
1250
|
+
}
|
|
1251
|
+
})];
|
|
1252
|
+
}
|
|
1253
|
+
const findings = [];
|
|
1254
|
+
findings.push(makeFinding({
|
|
1255
|
+
analyst_id: ANALYST_ID,
|
|
1256
|
+
severity: "high",
|
|
1257
|
+
area: "raw-trace-context",
|
|
1258
|
+
confidence: 1,
|
|
1259
|
+
claim: `Generation ${input.generation} produced ${totalFailingCells} failing/low-scoring cell(s) across ${ranked.length} candidate(s). Their FULL RAW run traces are on disk under ${genRoot} — the actual event logs (spans.jsonl), scores (cached-result.json), and artifacts, not a summary.${durable ? "" : " (WARNING: this run root does not exist on disk — it looks like an in-memory run; pass a real runDir to improve() to get raw-trace context.)"}`,
|
|
1260
|
+
recommended_action: `Do NOT rely on a pre-summarized finding. Before editing, DIAGNOSE from the raw traces: run \`grep\`/\`cat\`/\`ls\` over the trace files and directories named in the following findings to see exactly what each failing run did and why it scored low, then make the smallest harness-code edit that fixes the dominant failure. Start with \`grep -rIn "error" ${genRoot}\` then \`cat\` the spans.jsonl of the worst cell.`,
|
|
1261
|
+
evidence_refs: [{
|
|
1262
|
+
kind: "artifact",
|
|
1263
|
+
uri: genRoot
|
|
1264
|
+
}],
|
|
1265
|
+
metadata: {
|
|
1266
|
+
generation: input.generation,
|
|
1267
|
+
runDir: genRoot,
|
|
1268
|
+
failingCells: totalFailingCells,
|
|
1269
|
+
candidates: ranked.length
|
|
1270
|
+
}
|
|
1271
|
+
}));
|
|
1272
|
+
for (const cand of ranked) {
|
|
1273
|
+
if (cand.cells.length === 0) continue;
|
|
1274
|
+
const scenarioList = cand.cells.map((c) => c.scenarioId).join(", ");
|
|
1275
|
+
const fileLines = cand.cells.map((c) => {
|
|
1276
|
+
const header = ` cell ${c.scenarioId} (composite ${c.composite.toFixed(3)}${c.error ? `, error: ${truncate(c.error, 160)}` : ""}) — dir ${c.cellDir}`;
|
|
1277
|
+
const files = c.files.map((f) => ` - ${f}`).join("\n");
|
|
1278
|
+
const more = c.truncatedFiles ? `\n - …(ls ${c.cellDir} for the rest)` : "";
|
|
1279
|
+
return c.files.length > 0 ? `${header}\n${files}${more}` : header;
|
|
1280
|
+
}).join("\n");
|
|
1281
|
+
findings.push(makeFinding({
|
|
1282
|
+
analyst_id: ANALYST_ID,
|
|
1283
|
+
severity: cand.composite !== null && cand.composite < .5 ? "critical" : "high",
|
|
1284
|
+
area: "raw-trace-context",
|
|
1285
|
+
confidence: 1,
|
|
1286
|
+
subject: cand.surfaceHash,
|
|
1287
|
+
claim: `Candidate ${cand.surfaceHash} ${candidateCompositeDescription(cand.composite)} with ${cand.cells.length} failing cell(s) [${scenarioList}]. Its raw traces are under ${cand.campaignDir}.`,
|
|
1288
|
+
recommended_action: `grep/cat these raw trace files to diagnose WHY this candidate failed before editing:\n${fileLines}\nOr scan the whole candidate at once: \`grep -rIn . ${cand.campaignDir}\` and \`ls -R ${cand.campaignDir}\`.`,
|
|
1289
|
+
evidence_refs: [{
|
|
1290
|
+
kind: "artifact",
|
|
1291
|
+
uri: cand.campaignDir
|
|
1292
|
+
}, ...cand.cells.flatMap((c) => c.files.map((f) => ({
|
|
1293
|
+
kind: "artifact",
|
|
1294
|
+
uri: f
|
|
1295
|
+
})))],
|
|
1296
|
+
metadata: {
|
|
1297
|
+
surfaceHash: cand.surfaceHash,
|
|
1298
|
+
composite: cand.composite,
|
|
1299
|
+
campaignDir: cand.campaignDir,
|
|
1300
|
+
cells: cand.cells.map((c) => ({
|
|
1301
|
+
scenarioId: c.scenarioId,
|
|
1302
|
+
composite: c.composite,
|
|
1303
|
+
cellDir: c.cellDir,
|
|
1304
|
+
files: c.files,
|
|
1305
|
+
...c.error ? { error: c.error } : {}
|
|
1306
|
+
}))
|
|
1307
|
+
}
|
|
1308
|
+
}));
|
|
1309
|
+
}
|
|
1310
|
+
return findings;
|
|
1311
|
+
};
|
|
1312
|
+
}
|
|
1313
|
+
/** Keep unscored candidates visible without inventing a numeric aggregate. */
|
|
1314
|
+
function compareCandidateComposites(left, right) {
|
|
1315
|
+
if (left === null) return right === null ? 0 : 1;
|
|
1316
|
+
if (right === null) return -1;
|
|
1317
|
+
return left - right;
|
|
1318
|
+
}
|
|
1319
|
+
function candidateCompositeDescription(composite) {
|
|
1320
|
+
return composite === null ? "has no aggregate score" : `scored composite ${composite.toFixed(3)}`;
|
|
1321
|
+
}
|
|
1322
|
+
/** The failing cells of a candidate campaign, each with its on-disk trace files.
|
|
1323
|
+
* Mirrors the default distiller's per-cell composite (mean of judge composites,
|
|
1324
|
+
* 0 when a cell produced no judge score) and its failing predicate. */
|
|
1325
|
+
function failingCells(campaign, maxCells, maxFiles) {
|
|
1326
|
+
const campaignDir = absoluteRunDir(campaign.runDir);
|
|
1327
|
+
const durable = isDurable(campaignDir);
|
|
1328
|
+
const out = [];
|
|
1329
|
+
for (const cell of campaign.cells) {
|
|
1330
|
+
const scores = Object.values(cell.judgeScores ?? {});
|
|
1331
|
+
const composite = scores.length === 0 ? 0 : scores.reduce((sum, s) => sum + (s.composite ?? 0), 0) / scores.length;
|
|
1332
|
+
if (!cell.error && composite >= PASS_THRESHOLD) continue;
|
|
1333
|
+
const cellDir = join(campaignDir, sanitizeCellId(cell.cellId));
|
|
1334
|
+
const artifactPaths = artifactPathsForCell(campaign.artifactsByPath, cell.cellId);
|
|
1335
|
+
const discovered = durable ? listTraceFiles(cellDir) : [];
|
|
1336
|
+
const canonical = [join(cellDir, "spans.jsonl"), join(cellDir, "cached-result.json")];
|
|
1337
|
+
const files = dedupeSorted([
|
|
1338
|
+
...discovered,
|
|
1339
|
+
...artifactPaths,
|
|
1340
|
+
...canonical
|
|
1341
|
+
]);
|
|
1342
|
+
out.push({
|
|
1343
|
+
scenarioId: cell.scenarioId,
|
|
1344
|
+
composite: Number(composite.toFixed(3)),
|
|
1345
|
+
...cell.error ? { error: cell.error } : {},
|
|
1346
|
+
cellDir,
|
|
1347
|
+
files: files.slice(0, maxFiles),
|
|
1348
|
+
truncatedFiles: files.length > maxFiles
|
|
1349
|
+
});
|
|
1350
|
+
if (out.length >= maxCells) break;
|
|
1351
|
+
}
|
|
1352
|
+
return out;
|
|
1353
|
+
}
|
|
1354
|
+
/** Absolute paths of artifacts the campaign recorded for a cell. `artifactsByPath`
|
|
1355
|
+
* is keyed `${cellId}/${relPath}` → absolute path. */
|
|
1356
|
+
function artifactPathsForCell(artifactsByPath, cellId) {
|
|
1357
|
+
if (!artifactsByPath) return [];
|
|
1358
|
+
const prefix = `${cellId}/`;
|
|
1359
|
+
return Object.entries(artifactsByPath).filter(([key]) => key.startsWith(prefix)).map(([, absPath]) => resolve(absPath));
|
|
1360
|
+
}
|
|
1361
|
+
/** Real files directly under `dir` and one level of sub-directories (artifacts
|
|
1362
|
+
* are sometimes nested). Absolute paths, sorted. `[]` when the dir is absent,
|
|
1363
|
+
* stale, unreadable, or contains symlinked dirs — trace context is advisory and
|
|
1364
|
+
* the canonical anchors below still tell the proposer where to inspect. */
|
|
1365
|
+
function listTraceFiles(dir) {
|
|
1366
|
+
const out = [];
|
|
1367
|
+
for (const entry of safeReadDir(dir)) {
|
|
1368
|
+
const full = join(dir, entry.name);
|
|
1369
|
+
if (entry.isFile()) out.push(full);
|
|
1370
|
+
else if (!entry.isSymbolicLink() && entry.isDirectory()) {
|
|
1371
|
+
for (const sub of safeReadDir(full)) if (sub.isFile()) out.push(join(full, sub.name));
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
return out;
|
|
1375
|
+
}
|
|
1376
|
+
function safeReadDir(dir) {
|
|
1377
|
+
try {
|
|
1378
|
+
return readdirSync(dir, { withFileTypes: true });
|
|
1379
|
+
} catch {
|
|
1380
|
+
return [];
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
/** Substrate cell-dir sanitization — must match agent-eval's
|
|
1384
|
+
* `cellId.replace(/[^a-zA-Z0-9_-]/g, '_')` so the computed dir matches disk. */
|
|
1385
|
+
function sanitizeCellId(cellId) {
|
|
1386
|
+
return cellId.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
1387
|
+
}
|
|
1388
|
+
/** A run root is durable (has real files) when it is not an in-memory sentinel
|
|
1389
|
+
* and exists on disk. `mem://` runs keep everything in-process — no traces. */
|
|
1390
|
+
function isDurable(runDir) {
|
|
1391
|
+
return !runDir.startsWith("mem://") && existsSync(runDir);
|
|
1392
|
+
}
|
|
1393
|
+
/** Resolve a run dir to absolute (the coding harness runs from a worktree cwd, so
|
|
1394
|
+
* relative paths are uncattable there). `mem://` sentinels pass through untouched. */
|
|
1395
|
+
function absoluteRunDir(runDir) {
|
|
1396
|
+
return runDir.startsWith("mem://") ? runDir : resolve(runDir);
|
|
1397
|
+
}
|
|
1398
|
+
function dedupeSorted(paths) {
|
|
1399
|
+
return [...new Set(paths)].sort((a, b) => {
|
|
1400
|
+
const da = a.slice(0, a.length - basename(a).length);
|
|
1401
|
+
const db = b.slice(0, b.length - basename(b).length);
|
|
1402
|
+
return da === db ? basename(a).localeCompare(basename(b)) : da.localeCompare(db);
|
|
1403
|
+
});
|
|
1404
|
+
}
|
|
1405
|
+
function truncate(s, n) {
|
|
1406
|
+
return s.length <= n ? s : `${s.slice(0, n - 1)}…`;
|
|
1407
|
+
}
|
|
1408
|
+
//#endregion
|
|
1409
|
+
//#region src/improvement/code-execution.ts
|
|
1410
|
+
/** Slice bound for distilled judge notes: wide enough that a real traceback or
|
|
1411
|
+
* failing assertion survives intact. */
|
|
1412
|
+
const distilledNotesMaxChars = 1500;
|
|
1413
|
+
/** Slice bound for a cell's error string. Full text remains on the raw cell. */
|
|
1414
|
+
const distilledErrorMaxChars = 500;
|
|
1415
|
+
/** Distill failing cells into typed findings for the next proposal round. */
|
|
1416
|
+
function generationFailureDistiller(staticFindings) {
|
|
1417
|
+
const CAP = 12;
|
|
1418
|
+
return async (input) => {
|
|
1419
|
+
const failures = [];
|
|
1420
|
+
for (const candidate of input.candidates) for (const rawCell of candidate.campaign.cells) {
|
|
1421
|
+
const cell = rawCell;
|
|
1422
|
+
const scenario = String(cell.scenarioId ?? "unknown");
|
|
1423
|
+
const error = typeof cell.error === "string" ? cell.error : void 0;
|
|
1424
|
+
const judgeScores = cell.judgeScores && typeof cell.judgeScores === "object" ? Object.values(cell.judgeScores) : [];
|
|
1425
|
+
const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((sum, judge) => sum + (judge.composite ?? 0), 0) / judgeScores.length;
|
|
1426
|
+
if (!error && composite >= .999) continue;
|
|
1427
|
+
const notes = judgeScores.map((judge) => judge.notes).filter((note) => typeof note === "string" && note.length > 0).join("; ").slice(0, distilledNotesMaxChars);
|
|
1428
|
+
const claim = notes || (error ? `Scenario ${scenario} failed: ${error.slice(0, distilledErrorMaxChars)}` : "");
|
|
1429
|
+
failures.push({
|
|
1430
|
+
scenario,
|
|
1431
|
+
composite: Number(composite.toFixed(3)),
|
|
1432
|
+
notes,
|
|
1433
|
+
...claim ? { claim } : {},
|
|
1434
|
+
...error ? { error: error.slice(0, distilledErrorMaxChars) } : {}
|
|
1435
|
+
});
|
|
1436
|
+
}
|
|
1437
|
+
if (failures.length === 0) return staticFindings;
|
|
1438
|
+
failures.sort((left, right) => left.composite - right.composite);
|
|
1439
|
+
return failures.slice(0, CAP).map((failure) => makeFinding({
|
|
1440
|
+
analyst_id: "generation-failure-distiller",
|
|
1441
|
+
severity: failure.error !== void 0 || failure.composite < .5 ? "high" : "medium",
|
|
1442
|
+
area: "generation-failure",
|
|
1443
|
+
confidence: 1,
|
|
1444
|
+
subject: failure.scenario,
|
|
1445
|
+
claim: `Scenario ${failure.scenario} scored composite ${failure.composite}${failure.notes ? `: ${failure.notes}` : ""}${failure.error ? ` (error: ${failure.error})` : ""}`,
|
|
1446
|
+
evidence_refs: [],
|
|
1447
|
+
metadata: {
|
|
1448
|
+
scenario: failure.scenario,
|
|
1449
|
+
composite: failure.composite,
|
|
1450
|
+
...failure.notes ? { notes: failure.notes } : {},
|
|
1451
|
+
...failure.error !== void 0 ? { error: failure.error } : {}
|
|
1452
|
+
}
|
|
1453
|
+
}));
|
|
1454
|
+
};
|
|
1455
|
+
}
|
|
1456
|
+
/** Default code-run analysis: raw trace paths for durable runs, otherwise a
|
|
1457
|
+
* bounded digest of failed cells. */
|
|
1458
|
+
function defaultDistillerFor(opts, findings) {
|
|
1459
|
+
const durableRun = opts.runDir !== void 0 && !opts.runDir.startsWith("mem://");
|
|
1460
|
+
if (opts.rawTraceContext ?? durableRun) return rawTraceDistiller({ fallbackFindings: findings });
|
|
1461
|
+
return generationFailureDistiller(findings);
|
|
1462
|
+
}
|
|
1463
|
+
async function discardPreparedBaseline(worktree, baselineWorktree, cause) {
|
|
1464
|
+
return rethrowAfterCleanup(cause, () => worktree.discard(baselineWorktree), "improve(): code preparation failed");
|
|
1465
|
+
}
|
|
1466
|
+
/** Create a clean incumbent checkout and the candidate producer for a code run. */
|
|
1467
|
+
async function prepareCodeRun(code) {
|
|
1468
|
+
const baseRef = code.baseRef ?? "main";
|
|
1469
|
+
const worktree = code.worktree ?? gitWorktreeAdapter({
|
|
1470
|
+
repoRoot: code.repoRoot,
|
|
1471
|
+
...code.worktreeDir ? { worktreeDir: code.worktreeDir } : {}
|
|
1472
|
+
});
|
|
1473
|
+
const baselineWorktree = await worktree.create({
|
|
1474
|
+
baseRef,
|
|
1475
|
+
label: "incumbent-baseline"
|
|
1476
|
+
});
|
|
1477
|
+
try {
|
|
1478
|
+
const baseline = await worktree.finalize(baselineWorktree, "Incumbent code checkout");
|
|
1479
|
+
let baselineDiscarded = false;
|
|
1480
|
+
const managed = improvementDriver({
|
|
1481
|
+
worktree,
|
|
1482
|
+
generator: code.generator ?? agenticGenerator({
|
|
1483
|
+
...code.harness ? { harness: code.harness } : {},
|
|
1484
|
+
...code.verify ? { verify: code.verify } : {},
|
|
1485
|
+
...code.timeoutMs ? { timeoutMs: code.timeoutMs } : {}
|
|
1486
|
+
}),
|
|
1487
|
+
baseRef
|
|
1488
|
+
});
|
|
1489
|
+
return {
|
|
1490
|
+
baseline,
|
|
1491
|
+
proposer: managed,
|
|
1492
|
+
async cleanup(retainedWinner) {
|
|
1493
|
+
const errors = [];
|
|
1494
|
+
const retainedWorktreeRef = isCodeSurface(retainedWinner) ? retainedWinner.worktreeRef : void 0;
|
|
1495
|
+
try {
|
|
1496
|
+
await managed?.cleanup(retainedWorktreeRef ? [retainedWorktreeRef] : []);
|
|
1497
|
+
} catch (cause) {
|
|
1498
|
+
errors.push(cause);
|
|
1499
|
+
}
|
|
1500
|
+
if (!baselineDiscarded && retainedWorktreeRef !== baseline.worktreeRef) try {
|
|
1501
|
+
await worktree.discard(baselineWorktree);
|
|
1502
|
+
baselineDiscarded = true;
|
|
1503
|
+
} catch (cause) {
|
|
1504
|
+
errors.push(cause);
|
|
1505
|
+
}
|
|
1506
|
+
if (errors.length > 0) throw new AggregateError(errors, "improve(): failed to clean code improvement worktrees");
|
|
1507
|
+
}
|
|
1508
|
+
};
|
|
1509
|
+
} catch (cause) {
|
|
1510
|
+
return discardPreparedBaseline(worktree, baselineWorktree, cause);
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
function idempotentDispose(dispose) {
|
|
1514
|
+
let disposed = false;
|
|
1515
|
+
let inFlight;
|
|
1516
|
+
return async () => {
|
|
1517
|
+
if (disposed) return;
|
|
1518
|
+
if (inFlight) return inFlight;
|
|
1519
|
+
inFlight = (async () => {
|
|
1520
|
+
await dispose();
|
|
1521
|
+
disposed = true;
|
|
1522
|
+
})();
|
|
1523
|
+
try {
|
|
1524
|
+
await inFlight;
|
|
1525
|
+
} finally {
|
|
1526
|
+
inFlight = void 0;
|
|
1527
|
+
}
|
|
1528
|
+
};
|
|
1529
|
+
}
|
|
1530
|
+
async function runCodeImprovement(opts) {
|
|
1531
|
+
const { gate = "holdout", findings: inputFindings = [], rawTraceContext: _rawTraceContext, code, promotionGate, analyzeGeneration, surface: _surface, ...sharedOptions } = opts;
|
|
1532
|
+
const findings = [...inputFindings];
|
|
1533
|
+
const preparedCode = await prepareCodeRun(code);
|
|
1534
|
+
const budget = gate === "none" ? {
|
|
1535
|
+
...sharedOptions.budget,
|
|
1536
|
+
generations: 0
|
|
1537
|
+
} : { ...sharedOptions.budget };
|
|
1538
|
+
let raw;
|
|
1539
|
+
try {
|
|
1540
|
+
raw = await selfImprove({
|
|
1541
|
+
...sharedOptions,
|
|
1542
|
+
baselineSurface: preparedCode.baseline,
|
|
1543
|
+
proposer: preparedCode.proposer,
|
|
1544
|
+
budget,
|
|
1545
|
+
findings,
|
|
1546
|
+
...promotionGate !== void 0 ? { gate: promotionGate } : {},
|
|
1547
|
+
...analyzeGeneration === null ? {} : { analyzeGeneration: analyzeGeneration ?? defaultDistillerFor(opts, findings) }
|
|
1548
|
+
});
|
|
1549
|
+
} catch (cause) {
|
|
1550
|
+
return rethrowAfterCleanup(cause, () => preparedCode.cleanup(), "improve(): code improvement failed");
|
|
1551
|
+
}
|
|
1552
|
+
const winnerSurface = raw.winner.surface;
|
|
1553
|
+
assertCandidateSurfaceKind("code", preparedCode.baseline, winnerSurface);
|
|
1554
|
+
try {
|
|
1555
|
+
await preparedCode.cleanup(winnerSurface);
|
|
1556
|
+
} catch (cleanupCause) {
|
|
1557
|
+
try {
|
|
1558
|
+
await preparedCode.cleanup();
|
|
1559
|
+
} catch (finalCleanupCause) {
|
|
1560
|
+
throw new AggregateError([cleanupCause, finalCleanupCause], "improve(): code result cleanup failed, including the final all-worktree retry");
|
|
1561
|
+
}
|
|
1562
|
+
throw new AggregateError([cleanupCause], "improve(): code result cleanup failed; the final all-worktree retry succeeded");
|
|
1563
|
+
}
|
|
1564
|
+
const dispose = idempotentDispose(async () => preparedCode.cleanup());
|
|
1565
|
+
return {
|
|
1566
|
+
mode: "code",
|
|
1567
|
+
candidate: immutableCandidateValue({
|
|
1568
|
+
surface: "code",
|
|
1569
|
+
value: winnerSurface
|
|
1570
|
+
}),
|
|
1571
|
+
decision: raw.gateDecision,
|
|
1572
|
+
...raw.lift !== void 0 ? { lift: raw.lift } : {},
|
|
1573
|
+
cost: copyImproveCost(raw.cost),
|
|
1574
|
+
durationMs: raw.durationMs,
|
|
1575
|
+
lineage: Object.freeze({
|
|
1576
|
+
invocationId: raw.provenance.runId,
|
|
1577
|
+
runId: raw.provenance.runId,
|
|
1578
|
+
developmentSplitDigest: raw.provenance.evidence.search.splitDigest
|
|
1579
|
+
}),
|
|
1580
|
+
generationsExplored: raw.generationsExplored,
|
|
1581
|
+
raw,
|
|
1582
|
+
dispose
|
|
1583
|
+
};
|
|
1584
|
+
}
|
|
1585
|
+
//#endregion
|
|
1586
|
+
//#region src/improvement/method-controls.ts
|
|
1587
|
+
const methodRuntimeControls = Symbol("agent-runtime.improvement.method-runtime-controls");
|
|
1588
|
+
function withMethodRuntimeControls(method, controls) {
|
|
1589
|
+
return Object.freeze({
|
|
1590
|
+
...method,
|
|
1591
|
+
[methodRuntimeControls]: Object.freeze({ ...controls })
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1594
|
+
function methodRuntimeControlsOf(method) {
|
|
1595
|
+
return method[methodRuntimeControls];
|
|
1596
|
+
}
|
|
1597
|
+
//#endregion
|
|
1598
|
+
//#region src/improvement/method-cost.ts
|
|
1599
|
+
const EVALUATION_TAG = "runtimeEvaluationRef";
|
|
1600
|
+
const INVOCATION_TAG = "runtimeInvocationId";
|
|
1601
|
+
function methodInputWithScopedCost(input, scope, costAttribution = "invocation") {
|
|
1602
|
+
return Object.freeze({
|
|
1603
|
+
...input,
|
|
1604
|
+
costLedger: scopedCostLedger(input.costLedger, scope, costAttribution)
|
|
1605
|
+
});
|
|
1606
|
+
}
|
|
1607
|
+
function methodInvocationCostLedger(ledger, scope) {
|
|
1608
|
+
return scopedCostLedger(ledger, scope, "invocation");
|
|
1609
|
+
}
|
|
1610
|
+
function assertMethodCostRecorded(methodName, result, compatibleLedger, invocationLedger, costCeiling, costAttribution = "invocation") {
|
|
1611
|
+
const { cost } = result;
|
|
1612
|
+
const observed = costAttribution === "optimizer-run" && result.provenance?.runId ? compatibleLedger.summary({ tags: { optimizerRun: result.provenance.runId } }) : invocationLedger.summary();
|
|
1613
|
+
if (costCeiling !== void 0 && !cost.accountingComplete) throw new ConfigError(`improve(): method '${methodName}' returned incomplete cost accounting under costCeiling; refusing final scoring`);
|
|
1614
|
+
if (costCeiling !== void 0 && exceeds(cost.totalCostUsd, costCeiling)) throw new ConfigError(`improve(): method '${methodName}' reported cost $${cost.totalCostUsd} above costCeiling $${costCeiling}; refusing final scoring`);
|
|
1615
|
+
if (cost.accountingComplete && !observed.accountingComplete) throw new ConfigError(`improve(): method '${methodName}' reported complete cost accounting but its shared cost receipts are incomplete`);
|
|
1616
|
+
if (cost.accountingComplete && !approximatelyEqual$1(cost.totalCostUsd, observed.totalCostUsd)) throw new ConfigError(`improve(): method '${methodName}' reported $${cost.totalCostUsd} but recorded $${observed.totalCostUsd} through input.costLedger`);
|
|
1617
|
+
}
|
|
1618
|
+
function scopedCostLedger(parent, scope, costAttribution) {
|
|
1619
|
+
const evaluationTags = { [EVALUATION_TAG]: scope.evaluationRef };
|
|
1620
|
+
const invocationTags = {
|
|
1621
|
+
...evaluationTags,
|
|
1622
|
+
[INVOCATION_TAG]: scope.invocationId
|
|
1623
|
+
};
|
|
1624
|
+
const readTags = (filter) => costAttribution === "optimizer-run" && hasOptimizerRunFilter(filter) ? evaluationTags : invocationTags;
|
|
1625
|
+
return {
|
|
1626
|
+
costCeilingUsd: parent.costCeilingUsd,
|
|
1627
|
+
runPaidCall: (input) => parent.runPaidCall({
|
|
1628
|
+
...input,
|
|
1629
|
+
tags: {
|
|
1630
|
+
...input.tags ?? {},
|
|
1631
|
+
...invocationTags
|
|
1632
|
+
}
|
|
1633
|
+
}),
|
|
1634
|
+
reconcile: (...args) => parent.reconcile(...args),
|
|
1635
|
+
list: (filter) => parent.list(withTags(filter, readTags(filter))),
|
|
1636
|
+
listPending: (filter) => parent.listPending?.(withTags(filter, readTags(filter))) ?? [],
|
|
1637
|
+
summary: (filter) => parent.summary(withTags(filter, readTags(filter))),
|
|
1638
|
+
waitForIdle: (options = {}) => parent.waitForIdle?.({
|
|
1639
|
+
...options,
|
|
1640
|
+
filter: withTags(options.filter, readTags(options.filter))
|
|
1641
|
+
}) ?? Promise.resolve(parent.summary(withTags(options.filter, readTags(options.filter))).pendingCalls === 0),
|
|
1642
|
+
markCompleted: (count) => parent.markCompleted(count),
|
|
1643
|
+
costPerCompletedTask: () => parent.costPerCompletedTask()
|
|
1644
|
+
};
|
|
1645
|
+
}
|
|
1646
|
+
function hasOptimizerRunFilter(filter) {
|
|
1647
|
+
return typeof filter?.tags?.optimizerRun === "string" && filter.tags.optimizerRun.length > 0;
|
|
1648
|
+
}
|
|
1649
|
+
function withTags(filter, tags) {
|
|
1650
|
+
return {
|
|
1651
|
+
...filter ?? {},
|
|
1652
|
+
tags: {
|
|
1653
|
+
...filter?.tags ?? {},
|
|
1654
|
+
...tags
|
|
1655
|
+
}
|
|
1656
|
+
};
|
|
1657
|
+
}
|
|
1658
|
+
function approximatelyEqual$1(left, right) {
|
|
1659
|
+
const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 8;
|
|
1660
|
+
return Math.abs(left - right) <= tolerance;
|
|
1661
|
+
}
|
|
1662
|
+
function exceeds(value, limit) {
|
|
1663
|
+
const tolerance = Number.EPSILON * Math.max(1, Math.abs(value), Math.abs(limit)) * 8;
|
|
1664
|
+
return value - limit > tolerance;
|
|
1665
|
+
}
|
|
1666
|
+
//#endregion
|
|
1667
|
+
//#region src/improvement/method-identity.ts
|
|
1668
|
+
function buildMethodEvaluationIdentity(input) {
|
|
1669
|
+
const optimizationReps = input.optimizationRunOptions?.reps ?? 1;
|
|
1670
|
+
const developmentSplitDigest = canonicalCandidateDigest$1({
|
|
1671
|
+
train: campaignSplitDigest(input.trainScenarios, optimizationReps),
|
|
1672
|
+
selection: campaignSplitDigest(input.selectionScenarios, optimizationReps)
|
|
1673
|
+
});
|
|
1674
|
+
const judgeDescriptors = input.judges.map(judgeDescriptor);
|
|
1675
|
+
return {
|
|
1676
|
+
evaluationRef: canonicalCandidateDigest$1({
|
|
1677
|
+
executionRef: input.executionRef,
|
|
1678
|
+
baselineProfileDigest: input.baselineProfileDigest,
|
|
1679
|
+
coordinate: profileCoordinate(input.surface, input.baselineSurface, input.skills),
|
|
1680
|
+
candidateValidation: input.validateCandidate ? Function.prototype.toString.call(input.validateCandidate) : null,
|
|
1681
|
+
developmentSplitDigest,
|
|
1682
|
+
findings: input.findings,
|
|
1683
|
+
judges: judgeDescriptors,
|
|
1684
|
+
run: {
|
|
1685
|
+
seed: input.seed ?? 42,
|
|
1686
|
+
finalReps: input.reps ?? 1,
|
|
1687
|
+
optimizationReps,
|
|
1688
|
+
costCeiling: input.costCeiling ?? null,
|
|
1689
|
+
resumable: input.optimizationRunOptions?.resumable ?? true,
|
|
1690
|
+
maxConcurrency: input.optimizationRunOptions?.maxConcurrency ?? 2,
|
|
1691
|
+
abortOnCellError: input.optimizationRunOptions?.abortOnCellError ?? false,
|
|
1692
|
+
dispatchTimeoutMs: input.optimizationRunOptions?.dispatchTimeoutMs ?? null,
|
|
1693
|
+
dispatchShutdownTimeoutMs: input.optimizationRunOptions?.dispatchShutdownTimeoutMs ?? 5e3,
|
|
1694
|
+
tracing: input.optimizationRunOptions?.tracing ?? "on",
|
|
1695
|
+
expectUsage: input.optimizationRunOptions?.expectUsage ?? "warn",
|
|
1696
|
+
captureSource: input.optimizationRunOptions?.captureSource ?? null,
|
|
1697
|
+
captureSourceVersionHash: input.optimizationRunOptions?.captureSourceVersionHash ?? null
|
|
1698
|
+
}
|
|
1699
|
+
}),
|
|
1700
|
+
developmentSplitDigest,
|
|
1701
|
+
judgeDescriptors
|
|
1702
|
+
};
|
|
1703
|
+
}
|
|
1704
|
+
function profileCoordinate(surface, baselineSurface, skills) {
|
|
1705
|
+
return {
|
|
1706
|
+
surface,
|
|
1707
|
+
resourceName: surface === "skills" ? skills?.resourceName.trim() ?? null : null,
|
|
1708
|
+
componentNames: typeof baselineSurface === "object" && baselineSurface !== null && baselineSurface.kind === "components" ? Object.keys(baselineSurface.components).sort() : []
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
function judgeDescriptor(judge) {
|
|
1712
|
+
return {
|
|
1713
|
+
name: judge.name,
|
|
1714
|
+
dimensions: judge.dimensions.map((dimension) => ({ ...dimension })),
|
|
1715
|
+
declaredVersion: judge.judgeVersion ?? null,
|
|
1716
|
+
scoreImplementation: Function.prototype.toString.call(judge.score),
|
|
1717
|
+
appliesToImplementation: judge.appliesTo ? Function.prototype.toString.call(judge.appliesTo) : null
|
|
1718
|
+
};
|
|
1719
|
+
}
|
|
1720
|
+
//#endregion
|
|
1721
|
+
//#region src/improvement/method-execution.ts
|
|
1722
|
+
function resolveOptimizationMethod(source, context) {
|
|
1723
|
+
const method = typeof source === "function" ? source(context) : source;
|
|
1724
|
+
if (!method || typeof method !== "object" || typeof method.name !== "string" || method.name.trim() !== method.name || method.name.length === 0 || typeof method.optimize !== "function") throw new ConfigError("improve(): method must be a complete OptimizationMethod with a trimmed name and optimize(input)");
|
|
1725
|
+
return method;
|
|
1726
|
+
}
|
|
1727
|
+
function copyProvenance(provenance) {
|
|
1728
|
+
return immutableCandidateValue(provenance);
|
|
1729
|
+
}
|
|
1730
|
+
function validateExecutionRef(value) {
|
|
1731
|
+
const parsed = sha256DigestSchema.safeParse(value);
|
|
1732
|
+
if (!parsed.success) throw new ConfigError("improve(): executionRef must be a lowercase sha256:<64 hex> digest");
|
|
1733
|
+
return parsed.data;
|
|
1734
|
+
}
|
|
1735
|
+
async function runMethodImprovement(profile, opts) {
|
|
1736
|
+
const { surface = "prompt", executionRef: inputExecutionRef, method: methodSource, agent, validateCandidate, findings: inputFindings = [], skills, profileComponents, optimizationRunOptions, minimumLift = 0, ...comparisonOptions } = opts;
|
|
1737
|
+
if (!Number.isFinite(minimumLift) || minimumLift < 0) throw new ConfigError("improve(): minimumLift must be a finite number greater than or equal to 0");
|
|
1738
|
+
if (profileComponents && surface !== "agent-profile") throw new ConfigError("improve(): profileComponents is valid only with surface 'agent-profile'");
|
|
1739
|
+
const executionRef = validateExecutionRef(inputExecutionRef);
|
|
1740
|
+
const findings = [...inputFindings];
|
|
1741
|
+
const preparedSurface = prepareProfileSurface(profile, surface, skills, profileComponents);
|
|
1742
|
+
const baselineSurface = preparedSurface.surface;
|
|
1743
|
+
const baselineValue = immutableCandidateValue(preparedSurface.value);
|
|
1744
|
+
const baselineProfileDigest = canonicalCandidateDigest$1(profile);
|
|
1745
|
+
const identity = buildMethodEvaluationIdentity({
|
|
1746
|
+
executionRef,
|
|
1747
|
+
baselineProfileDigest,
|
|
1748
|
+
baselineSurface,
|
|
1749
|
+
surface,
|
|
1750
|
+
skills,
|
|
1751
|
+
validateCandidate,
|
|
1752
|
+
findings,
|
|
1753
|
+
trainScenarios: comparisonOptions.trainScenarios,
|
|
1754
|
+
selectionScenarios: comparisonOptions.selectionScenarios,
|
|
1755
|
+
judges: comparisonOptions.judges,
|
|
1756
|
+
seed: comparisonOptions.seed,
|
|
1757
|
+
reps: comparisonOptions.reps,
|
|
1758
|
+
costCeiling: comparisonOptions.costCeiling,
|
|
1759
|
+
optimizationRunOptions
|
|
1760
|
+
});
|
|
1761
|
+
const { evaluationRef, developmentSplitDigest } = identity;
|
|
1762
|
+
const dispatchRef = `improve:${evaluationRef}`;
|
|
1763
|
+
const identifiedJudges = comparisonOptions.judges.map((judge, index) => Object.freeze({
|
|
1764
|
+
...judge,
|
|
1765
|
+
judgeVersion: canonicalCandidateDigest$1({
|
|
1766
|
+
evaluationRef,
|
|
1767
|
+
descriptor: identity.judgeDescriptors[index]
|
|
1768
|
+
})
|
|
1769
|
+
}));
|
|
1770
|
+
const rawMaterializeProfile = createProfileCandidateMaterializer(profile, surface, baselineSurface, skills, profileComponents);
|
|
1771
|
+
const runtimeInvocationId = `runtime-optimization:${randomUUID()}`;
|
|
1772
|
+
const method = resolveOptimizationMethod(methodSource, {
|
|
1773
|
+
profile,
|
|
1774
|
+
evaluationRef,
|
|
1775
|
+
surface,
|
|
1776
|
+
baselineSurface,
|
|
1777
|
+
baselineValue,
|
|
1778
|
+
findings
|
|
1779
|
+
});
|
|
1780
|
+
const methodControls = methodRuntimeControlsOf(method);
|
|
1781
|
+
const baselineSurfaceDigest = canonicalCandidateDigest$1(baselineSurface);
|
|
1782
|
+
const validatedCandidates = /* @__PURE__ */ new Set();
|
|
1783
|
+
const materializeProfile = (candidateSurface) => {
|
|
1784
|
+
const candidate = rawMaterializeProfile(candidateSurface);
|
|
1785
|
+
const candidateDigest = canonicalCandidateDigest$1(candidateSurface);
|
|
1786
|
+
if (!validatedCandidates.has(candidateDigest)) {
|
|
1787
|
+
const prepared = prepareProfileSurface(candidate, surface, skills, profileComponents);
|
|
1788
|
+
const validationInput = Object.freeze({
|
|
1789
|
+
profile: candidate,
|
|
1790
|
+
surface,
|
|
1791
|
+
candidateSurface: immutableCandidateValue(candidateSurface),
|
|
1792
|
+
value: immutableCandidateValue(prepared.value),
|
|
1793
|
+
isBaseline: candidateDigest === baselineSurfaceDigest
|
|
1794
|
+
});
|
|
1795
|
+
methodControls?.validateCandidate(validationInput);
|
|
1796
|
+
validateCandidate?.(validationInput);
|
|
1797
|
+
validatedCandidates.add(candidateDigest);
|
|
1798
|
+
}
|
|
1799
|
+
return candidate;
|
|
1800
|
+
};
|
|
1801
|
+
materializeProfile(baselineSurface);
|
|
1802
|
+
const measuredMethod = {
|
|
1803
|
+
...method,
|
|
1804
|
+
async optimize(input) {
|
|
1805
|
+
const costScope = {
|
|
1806
|
+
evaluationRef,
|
|
1807
|
+
invocationId: runtimeInvocationId
|
|
1808
|
+
};
|
|
1809
|
+
const scopedInput = methodInputWithScopedCost(input, costScope, methodControls?.costAttribution);
|
|
1810
|
+
const invocationLedger = methodInvocationCostLedger(input.costLedger, costScope);
|
|
1811
|
+
const result = await method.optimize(scopedInput);
|
|
1812
|
+
assertMethodCostRecorded(method.name, result, scopedInput.costLedger, invocationLedger, comparisonOptions.costCeiling, methodControls?.costAttribution);
|
|
1813
|
+
materializeProfile(result.winnerSurface);
|
|
1814
|
+
return result;
|
|
1815
|
+
}
|
|
1816
|
+
};
|
|
1817
|
+
const startedAt = Date.now();
|
|
1818
|
+
const raw = await compareOptimizationMethods({
|
|
1819
|
+
...comparisonOptions,
|
|
1820
|
+
judges: identifiedJudges,
|
|
1821
|
+
dispatchRef,
|
|
1822
|
+
optimizationRunOptions: {
|
|
1823
|
+
...optimizationRunOptions ?? {},
|
|
1824
|
+
dispatchRef
|
|
1825
|
+
},
|
|
1826
|
+
methods: [measuredMethod],
|
|
1827
|
+
baselineSurface,
|
|
1828
|
+
dispatchWithSurface: (candidateSurface, scenario, ctx) => agent(materializeProfile(candidateSurface), scenario, ctx)
|
|
1829
|
+
});
|
|
1830
|
+
if (comparisonOptions.costCeiling !== void 0 && raw.totalCost.totalCostUsd > comparisonOptions.costCeiling) throw new ConfigError(`improve(): reported total cost $${raw.totalCost.totalCostUsd} exceeds costCeiling $${comparisonOptions.costCeiling}`);
|
|
1831
|
+
const score = raw.best;
|
|
1832
|
+
const winnerSurface = immutableCandidateValue(score.winnerSurface);
|
|
1833
|
+
assertCandidateSurfaceKind(surface, baselineSurface, winnerSurface);
|
|
1834
|
+
const candidateProfile = materializeProfile(winnerSurface);
|
|
1835
|
+
const candidate = Object.freeze({
|
|
1836
|
+
surface,
|
|
1837
|
+
value: winnerSurface,
|
|
1838
|
+
profile: candidateProfile
|
|
1839
|
+
});
|
|
1840
|
+
const cost = copyImproveCost(raw.totalCost);
|
|
1841
|
+
return {
|
|
1842
|
+
mode: "method",
|
|
1843
|
+
method: method.name,
|
|
1844
|
+
...score.provenance ? { provenance: copyProvenance(score.provenance) } : {},
|
|
1845
|
+
candidate,
|
|
1846
|
+
decision: cost.accountingComplete && score.liftCi.low > minimumLift ? "ship" : "hold",
|
|
1847
|
+
lift: score.lift,
|
|
1848
|
+
liftInterval: { ...score.liftCi },
|
|
1849
|
+
cost,
|
|
1850
|
+
durationMs: Date.now() - startedAt,
|
|
1851
|
+
lineage: Object.freeze({
|
|
1852
|
+
invocationId: runtimeInvocationId,
|
|
1853
|
+
runId: score.provenance?.runId ?? runtimeInvocationId,
|
|
1854
|
+
developmentSplitDigest,
|
|
1855
|
+
executionRef,
|
|
1856
|
+
baselineProfileDigest
|
|
1857
|
+
}),
|
|
1858
|
+
raw,
|
|
1859
|
+
async dispose() {}
|
|
1860
|
+
};
|
|
1861
|
+
}
|
|
1862
|
+
//#endregion
|
|
1863
|
+
//#region src/improvement/improve.ts
|
|
1864
|
+
async function improve(profileOrCode, opts) {
|
|
1865
|
+
if (opts === void 0) {
|
|
1866
|
+
const code = profileOrCode;
|
|
1867
|
+
if (code?.surface !== "code") throw new ConfigError("improve(): the one-argument form requires { surface: 'code', ... }");
|
|
1868
|
+
return runCodeImprovement(code);
|
|
1869
|
+
}
|
|
1870
|
+
if (opts.surface === "code") throw new ConfigError("improve(): code takes one argument: improve({ surface: 'code', ... })");
|
|
1871
|
+
const parsedProfile = agentProfileSchema.safeParse(profileOrCode);
|
|
1872
|
+
if (!parsedProfile.success) throw new ConfigError(`improve(): input is not a valid AgentProfile: ${parsedProfile.error.message}`);
|
|
1873
|
+
return runMethodImprovement(immutableCandidateValue(parsedProfile.data), opts);
|
|
1874
|
+
}
|
|
1875
|
+
//#endregion
|
|
1876
|
+
//#region src/intelligence/improvement-surfaces.ts
|
|
1877
|
+
const changedSurfaceOrder = [
|
|
1878
|
+
"prompt",
|
|
1879
|
+
"skills",
|
|
1880
|
+
"tools",
|
|
1881
|
+
"mcp",
|
|
1882
|
+
"hooks",
|
|
1883
|
+
"subagents",
|
|
1884
|
+
"agent-profile",
|
|
1885
|
+
"memory",
|
|
1886
|
+
"code",
|
|
1887
|
+
"knowledge"
|
|
1888
|
+
];
|
|
1889
|
+
/** Agent improvement surfaces delivered as exact `AgentProfileDiff` replacements. */
|
|
1890
|
+
const AGENT_IMPROVEMENT_PROFILE_SURFACES = [
|
|
1891
|
+
"prompt",
|
|
1892
|
+
"skills",
|
|
1893
|
+
"tools",
|
|
1894
|
+
"mcp",
|
|
1895
|
+
"hooks",
|
|
1896
|
+
"subagents"
|
|
1897
|
+
];
|
|
1898
|
+
function deriveChangedSurfaces(baselineBundle, candidateBundle) {
|
|
1899
|
+
if (baselineBundle.knowledge || candidateBundle.knowledge) assertKnowledgeCandidatePair(baselineBundle, candidateBundle);
|
|
1900
|
+
assertCodeCandidatePair(baselineBundle, candidateBundle);
|
|
1901
|
+
const baseline = improvementSurfaceValues(baselineBundle);
|
|
1902
|
+
const candidate = improvementSurfaceValues(candidateBundle);
|
|
1903
|
+
const changed = changedSurfaceOrder.filter((surface) => canonicalCandidateDigest$1(baseline[surface]) !== canonicalCandidateDigest$1(candidate[surface]));
|
|
1904
|
+
if (changed.length === 0) throw new Error("candidate experiment does not change an agent surface");
|
|
1905
|
+
return changed;
|
|
1906
|
+
}
|
|
1907
|
+
function assertAgentImprovementActivationTargets(surfaces, experiment, intent, targets) {
|
|
1908
|
+
if (isProfileImprovementExperiment(experiment)) assertProfileImprovementTargetsShareIdentity(targets);
|
|
1909
|
+
const expected = new Set(surfaces);
|
|
1910
|
+
const actual = new Set(targets.map((target) => target.surface));
|
|
1911
|
+
const sourceArm = intent === "activate-candidate" ? "baseline" : "candidate";
|
|
1912
|
+
if (!(isProfileImprovementExperiment(experiment) ? sameAgentImprovementSurfaceSet(surfaces, activationExperimentChangedSurfaces(experiment)) : sameOrderedSurfaces(surfaces, activationExperimentChangedSurfaces(experiment))) || targets.some((target) => !target.identity.trim()) || targets.some((target) => target.expectedBaseDigest !== activationTargetDigest(experiment, sourceArm, target.surface)) || targets.length !== surfaces.length || expected.size !== actual.size || [...expected].some((surface) => !actual.has(surface))) throw new Error("agent improvement activation targets must cover exactly the changed surfaces");
|
|
1913
|
+
}
|
|
1914
|
+
/** One opaque profile-state transition can change only one complete profile record. */
|
|
1915
|
+
function assertProfileImprovementTargetsShareIdentity(targets) {
|
|
1916
|
+
if (new Set(targets.map((target) => target.identity)).size !== 1) throw new Error("profile improvement activation targets must name one profile identity");
|
|
1917
|
+
}
|
|
1918
|
+
/** Bind caller-owned target identities to the exact source state Runtime measured. */
|
|
1919
|
+
function buildAgentImprovementActivationTargets(surfaces, experiment, intent, identities) {
|
|
1920
|
+
const sourceArm = intent === "activate-candidate" ? "baseline" : "candidate";
|
|
1921
|
+
const targets = identities.map((target) => ({
|
|
1922
|
+
...target,
|
|
1923
|
+
expectedBaseDigest: activationTargetDigest(experiment, sourceArm, target.surface)
|
|
1924
|
+
}));
|
|
1925
|
+
assertAgentImprovementActivationTargets(surfaces, experiment, intent, targets);
|
|
1926
|
+
return targets;
|
|
1927
|
+
}
|
|
1928
|
+
/** Exact host-owned profile state for every changed profile surface. */
|
|
1929
|
+
function agentProfileImprovementStateDigest(experiment, arm) {
|
|
1930
|
+
return experiment[arm].stateDigest;
|
|
1931
|
+
}
|
|
1932
|
+
/** Map Interface's current profile-improvement contract to Runtime-deliverable surfaces. */
|
|
1933
|
+
function profileImprovementChangedSurfaces(change) {
|
|
1934
|
+
const surfaces = changedProfileImprovementSurfaces(change);
|
|
1935
|
+
if (surfaces.length === 0 || !surfaces.every((surface) => surface === "prompt" || surface === "skills")) throw new Error("profile improvement experiment does not change a supported surface");
|
|
1936
|
+
return surfaces;
|
|
1937
|
+
}
|
|
1938
|
+
function agentImprovementTargetDigest(experiment, arm, surface) {
|
|
1939
|
+
if (surface === "knowledge") {
|
|
1940
|
+
const knowledge = assertKnowledgeCandidatePair(experiment.baseline, experiment.candidate);
|
|
1941
|
+
return arm === "baseline" ? knowledge.candidate.baseHash : knowledge.candidate.candidateHash;
|
|
1942
|
+
}
|
|
1943
|
+
if (surface === "code") assertCodeCandidatePair(experiment.baseline, experiment.candidate);
|
|
1944
|
+
return canonicalCandidateDigest$1(improvementSurfaceValues(experiment[arm])[surface]);
|
|
1945
|
+
}
|
|
1946
|
+
function agentImprovementTargetInput(bundle, surface) {
|
|
1947
|
+
return improvementSurfaceValues(bundle)[surface];
|
|
1948
|
+
}
|
|
1949
|
+
function activationTargetDigest(experiment, arm, surface) {
|
|
1950
|
+
if (isProfileImprovementExperiment(experiment)) {
|
|
1951
|
+
if (!activationExperimentChangedSurfaces(experiment).includes(surface)) throw new Error("profile improvement activation targets an unchanged surface");
|
|
1952
|
+
return agentProfileImprovementStateDigest(experiment, arm);
|
|
1953
|
+
}
|
|
1954
|
+
return agentImprovementTargetDigest(experiment, arm, surface);
|
|
1955
|
+
}
|
|
1956
|
+
function activationExperimentChangedSurfaces(experiment) {
|
|
1957
|
+
if (!isProfileImprovementExperiment(experiment)) return deriveChangedSurfaces(experiment.baseline, experiment.candidate);
|
|
1958
|
+
return profileImprovementChangedSurfaces(experiment.change);
|
|
1959
|
+
}
|
|
1960
|
+
function isProfileImprovementExperiment(experiment) {
|
|
1961
|
+
return experiment.kind === "agent-profile-improvement-experiment";
|
|
1962
|
+
}
|
|
1963
|
+
function sameOrderedSurfaces(left, right) {
|
|
1964
|
+
return left.length === right.length && left.every((surface, index) => surface === right[index]);
|
|
1965
|
+
}
|
|
1966
|
+
/** Profile-change surface order is presentation only; its measured contract is set-based. */
|
|
1967
|
+
function sameAgentImprovementSurfaceSet(left, right) {
|
|
1968
|
+
return left.length === right.length && left.every((surface) => right.includes(surface));
|
|
1969
|
+
}
|
|
1970
|
+
/** Return whether a measured surface can be delivered through an agent profile. */
|
|
1971
|
+
function isAgentImprovementProfileSurface(surface) {
|
|
1972
|
+
return AGENT_IMPROVEMENT_PROFILE_SURFACES.includes(surface);
|
|
1973
|
+
}
|
|
1974
|
+
/**
|
|
1975
|
+
* Return the canonical current-state input for one profile-deliverable improvement target.
|
|
1976
|
+
* Missing slots become `null`; tools and subagents include both their direct and resource slots.
|
|
1977
|
+
* Unrelated profile fields are excluded. The result matches `agentImprovementTargetInput` for the
|
|
1978
|
+
* same profile inside a candidate bundle.
|
|
1979
|
+
*/
|
|
1980
|
+
function agentImprovementProfileSurfaceInput(profile, surface) {
|
|
1981
|
+
return immutableCandidateValue(profileSurfaceInput(parseExactAgentProfile(omitUndefinedObjectFields(profile, "agent improvement profile"), "agent improvement profile"), surface));
|
|
1982
|
+
}
|
|
1983
|
+
function profileSurfaceInput(profile, surface) {
|
|
1984
|
+
switch (surface) {
|
|
1985
|
+
case "prompt": return { prompt: profile.prompt ?? null };
|
|
1986
|
+
case "skills": return profile.resources?.skills ?? null;
|
|
1987
|
+
case "tools": return {
|
|
1988
|
+
tools: profile.tools ?? null,
|
|
1989
|
+
resources: profile.resources?.tools ?? null
|
|
1990
|
+
};
|
|
1991
|
+
case "mcp": return profile.mcp ?? null;
|
|
1992
|
+
case "hooks": return profile.hooks ?? null;
|
|
1993
|
+
case "subagents": return {
|
|
1994
|
+
subagents: profile.subagents ?? null,
|
|
1995
|
+
resources: profile.resources?.agents ?? null
|
|
1996
|
+
};
|
|
1997
|
+
}
|
|
1998
|
+
}
|
|
1999
|
+
/** Return the `Sha256Digest` of one profile surface using Runtime's canonical candidate digest. */
|
|
2000
|
+
function agentImprovementProfileSurfaceDigest(profile, surface) {
|
|
2001
|
+
return canonicalCandidateDigest$1(agentImprovementProfileSurfaceInput(profile, surface));
|
|
2002
|
+
}
|
|
2003
|
+
/**
|
|
2004
|
+
* Replace one measured profile surface exactly, including array-valued resources.
|
|
2005
|
+
* Apply the returned diffs in order: a diff applies its set before its removal,
|
|
2006
|
+
* so exact replacement requires a reset record followed by a set record.
|
|
2007
|
+
*/
|
|
2008
|
+
function agentImprovementTargetProfileDiffs(target, options) {
|
|
2009
|
+
const { remove, set } = improvementSurfaceReplacement(target);
|
|
2010
|
+
const common = {
|
|
2011
|
+
kind: "agent-profile-diff",
|
|
2012
|
+
...options.source ? { source: options.source } : {},
|
|
2013
|
+
metadata: {
|
|
2014
|
+
...options.metadata ?? {},
|
|
2015
|
+
surface: target.surface
|
|
2016
|
+
}
|
|
2017
|
+
};
|
|
2018
|
+
const reset = agentProfileDiffSchema.parse(defineAgentProfileDiff({
|
|
2019
|
+
...common,
|
|
2020
|
+
id: `${options.id}:${target.surface}:reset`,
|
|
2021
|
+
title: `Replace active ${target.surface}`,
|
|
2022
|
+
remove
|
|
2023
|
+
}));
|
|
2024
|
+
if (!set) return [reset];
|
|
2025
|
+
return [reset, agentProfileDiffSchema.parse(defineAgentProfileDiff({
|
|
2026
|
+
...common,
|
|
2027
|
+
id: `${options.id}:${target.surface}:set`,
|
|
2028
|
+
title: `Activate measured ${target.surface}`,
|
|
2029
|
+
set
|
|
2030
|
+
}))];
|
|
2031
|
+
}
|
|
2032
|
+
function improvementSurfaceReplacement(target) {
|
|
2033
|
+
const value = target.desiredInput;
|
|
2034
|
+
switch (target.surface) {
|
|
2035
|
+
case "prompt": {
|
|
2036
|
+
const prompt = exactObject(value, ["prompt"], "prompt activation input").prompt;
|
|
2037
|
+
assertDefined(prompt, "prompt activation input.prompt");
|
|
2038
|
+
return {
|
|
2039
|
+
remove: { prompt: true },
|
|
2040
|
+
...prompt === null ? {} : { set: parseProfileSet({ prompt }) }
|
|
2041
|
+
};
|
|
2042
|
+
}
|
|
2043
|
+
case "skills":
|
|
2044
|
+
assertDefined(value, "skills activation input");
|
|
2045
|
+
return {
|
|
2046
|
+
remove: { resources: { skills: true } },
|
|
2047
|
+
...value === null ? {} : { set: parseProfileSet({ resources: { skills: value } }) }
|
|
2048
|
+
};
|
|
2049
|
+
case "tools": {
|
|
2050
|
+
const parsed = exactObject(value, ["tools", "resources"], "tools activation input");
|
|
2051
|
+
assertDefined(parsed.tools, "tools activation input.tools");
|
|
2052
|
+
assertDefined(parsed.resources, "tools activation input.resources");
|
|
2053
|
+
const set = {
|
|
2054
|
+
...parsed.tools === null ? {} : { tools: parsed.tools },
|
|
2055
|
+
...parsed.resources === null ? {} : { resources: { tools: parsed.resources } }
|
|
2056
|
+
};
|
|
2057
|
+
return {
|
|
2058
|
+
remove: {
|
|
2059
|
+
tools: true,
|
|
2060
|
+
resources: { tools: true }
|
|
2061
|
+
},
|
|
2062
|
+
...Object.keys(set).length === 0 ? {} : { set: parseProfileSet(set) }
|
|
2063
|
+
};
|
|
2064
|
+
}
|
|
2065
|
+
case "mcp":
|
|
2066
|
+
assertDefined(value, "mcp activation input");
|
|
2067
|
+
return {
|
|
2068
|
+
remove: { mcp: true },
|
|
2069
|
+
...value === null ? {} : { set: parseProfileSet({ mcp: value }) }
|
|
2070
|
+
};
|
|
2071
|
+
case "hooks":
|
|
2072
|
+
assertDefined(value, "hooks activation input");
|
|
2073
|
+
return {
|
|
2074
|
+
remove: { hooks: true },
|
|
2075
|
+
...value === null ? {} : { set: parseProfileSet({ hooks: value }) }
|
|
2076
|
+
};
|
|
2077
|
+
case "subagents": {
|
|
2078
|
+
const parsed = exactObject(value, ["subagents", "resources"], "subagents activation input");
|
|
2079
|
+
assertDefined(parsed.subagents, "subagents activation input.subagents");
|
|
2080
|
+
assertDefined(parsed.resources, "subagents activation input.resources");
|
|
2081
|
+
const set = {
|
|
2082
|
+
...parsed.subagents === null ? {} : { subagents: parsed.subagents },
|
|
2083
|
+
...parsed.resources === null ? {} : { resources: { agents: parsed.resources } }
|
|
2084
|
+
};
|
|
2085
|
+
return {
|
|
2086
|
+
remove: {
|
|
2087
|
+
subagents: true,
|
|
2088
|
+
resources: { agents: true }
|
|
2089
|
+
},
|
|
2090
|
+
...Object.keys(set).length === 0 ? {} : { set: parseProfileSet(set) }
|
|
2091
|
+
};
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
2095
|
+
function assertDefined(value, label) {
|
|
2096
|
+
if (value === void 0) throw new Error(`${label} must not be undefined`);
|
|
2097
|
+
}
|
|
2098
|
+
function exactObject(value, keys, label) {
|
|
2099
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`);
|
|
2100
|
+
const record = value;
|
|
2101
|
+
const actual = Object.keys(record).sort();
|
|
2102
|
+
const expected = [...keys].sort();
|
|
2103
|
+
if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) throw new Error(`${label} must contain exactly: ${expected.join(", ")}`);
|
|
2104
|
+
return record;
|
|
2105
|
+
}
|
|
2106
|
+
function parseProfileSet(value) {
|
|
2107
|
+
return agentProfileSchema.parse(value);
|
|
2108
|
+
}
|
|
2109
|
+
function assertKnowledgeCandidatePair(baselineBundle, candidateBundle) {
|
|
2110
|
+
const baseline = baselineBundle.knowledge;
|
|
2111
|
+
const candidate = candidateBundle.knowledge;
|
|
2112
|
+
if (!baseline && !candidate) throw new Error("knowledge candidate pair is not present");
|
|
2113
|
+
if (!baseline || !candidate || canonicalCandidateDigest$1({
|
|
2114
|
+
candidate: baseline.candidate,
|
|
2115
|
+
evaluation: baseline.evaluation
|
|
2116
|
+
}) !== canonicalCandidateDigest$1({
|
|
2117
|
+
candidate: candidate.candidate,
|
|
2118
|
+
evaluation: candidate.evaluation
|
|
2119
|
+
})) throw new Error("knowledge experiment arms must share one measured candidate and evaluation identity");
|
|
2120
|
+
return candidate;
|
|
2121
|
+
}
|
|
2122
|
+
function assertCodeCandidatePair(baselineBundle, candidateBundle) {
|
|
2123
|
+
const baseline = baselineBundle.code;
|
|
2124
|
+
const candidate = candidateBundle.code;
|
|
2125
|
+
if (canonicalCandidateDigest$1(baseline) === canonicalCandidateDigest$1(candidate) || candidate.kind !== "git-patch") return;
|
|
2126
|
+
const baselineTree = baseline.kind === "no-op" ? baseline.baseTree : baseline.kind === "git-patch" ? baseline.candidateTree : void 0;
|
|
2127
|
+
const baselineRepository = baseline.kind === "disabled" ? void 0 : baseline.repository;
|
|
2128
|
+
if (baselineTree !== candidate.baseTree || !baselineRepository || canonicalCandidateDigest$1(baselineRepository) !== canonicalCandidateDigest$1(candidate.repository)) throw new Error("code candidate must be based on the exact repository tree measured by the baseline arm");
|
|
2129
|
+
}
|
|
2130
|
+
function improvementSurfaceValues(bundle) {
|
|
2131
|
+
const profile = agentCandidateProfileAsAgentProfile(bundle.profile);
|
|
2132
|
+
return {
|
|
2133
|
+
prompt: agentImprovementProfileSurfaceInput(profile, "prompt"),
|
|
2134
|
+
skills: agentImprovementProfileSurfaceInput(profile, "skills"),
|
|
2135
|
+
tools: agentImprovementProfileSurfaceInput(profile, "tools"),
|
|
2136
|
+
mcp: agentImprovementProfileSurfaceInput(profile, "mcp"),
|
|
2137
|
+
hooks: agentImprovementProfileSurfaceInput(profile, "hooks"),
|
|
2138
|
+
subagents: agentImprovementProfileSurfaceInput(profile, "subagents"),
|
|
2139
|
+
"agent-profile": {
|
|
2140
|
+
profile: opaqueProfileSlice(profile),
|
|
2141
|
+
execution: bundle.execution
|
|
2142
|
+
},
|
|
2143
|
+
memory: {
|
|
2144
|
+
instructions: profile.resources?.instructions ?? null,
|
|
2145
|
+
executionPolicy: bundle.memory
|
|
2146
|
+
},
|
|
2147
|
+
code: bundle.code,
|
|
2148
|
+
knowledge: bundle.knowledge ?? null
|
|
2149
|
+
};
|
|
2150
|
+
}
|
|
2151
|
+
function opaqueProfileSlice(profile) {
|
|
2152
|
+
const { prompt: _prompt, tools: _tools, mcp: _mcp, hooks: _hooks, subagents: _subagents, resources, ...opaqueProfile } = profile;
|
|
2153
|
+
const { instructions: _instructions, skills: _skills, tools: _resourceTools, agents: _agents, ...opaqueResources } = resources ?? {};
|
|
2154
|
+
return {
|
|
2155
|
+
...opaqueProfile,
|
|
2156
|
+
...Object.keys(opaqueResources).length > 0 ? { resources: opaqueResources } : {}
|
|
2157
|
+
};
|
|
2158
|
+
}
|
|
2159
|
+
//#endregion
|
|
2160
|
+
//#region src/intelligence/optimization-receipt.ts
|
|
2161
|
+
const optimizationReceiptMetadataKey = "optimizationReceipt";
|
|
2162
|
+
/** Build a detached receipt only for methods backed by an identified external optimizer. */
|
|
2163
|
+
function createOptimizationActivationReceipt(improvement) {
|
|
2164
|
+
const provenance = improvement.provenance;
|
|
2165
|
+
if (!provenance) return void 0;
|
|
2166
|
+
const optimizerModel = provenance.optimizerModel;
|
|
2167
|
+
const candidateModel = improvement.candidate.profile.model;
|
|
2168
|
+
return canonicalCandidateDocument({
|
|
2169
|
+
kind: "optimization-activation-receipt",
|
|
2170
|
+
method: improvement.method,
|
|
2171
|
+
source: provenance.source,
|
|
2172
|
+
...provenance.bridge ? { bridge: provenance.bridge } : {},
|
|
2173
|
+
...provenance.modules ? { modules: provenance.modules } : {},
|
|
2174
|
+
...provenance.python ? { python: provenance.python } : {},
|
|
2175
|
+
...candidateModel || optimizerModel ? { models: {
|
|
2176
|
+
...candidateModel ? { candidate: candidateModel } : {},
|
|
2177
|
+
...optimizerModel ? { optimizer: optimizerModel } : {}
|
|
2178
|
+
} } : {},
|
|
2179
|
+
usage: {
|
|
2180
|
+
optimizerEvaluations: provenance.evaluationCount,
|
|
2181
|
+
...provenance.tokenUsage ? { optimizerTokens: provenance.tokenUsage } : {}
|
|
2182
|
+
},
|
|
2183
|
+
cost: {
|
|
2184
|
+
optimization: receiptCost(improvement.raw.optimizationCost),
|
|
2185
|
+
finalTest: receiptCost(improvement.raw.testCost),
|
|
2186
|
+
total: receiptCost(improvement.raw.totalCost)
|
|
2187
|
+
},
|
|
2188
|
+
invocation: {
|
|
2189
|
+
runtimeInvocationId: improvement.lineage.invocationId,
|
|
2190
|
+
optimizerRunId: provenance.runId,
|
|
2191
|
+
...provenance.compatibleRunId ? { compatibleOptimizerRunId: provenance.compatibleRunId } : {},
|
|
2192
|
+
resumed: provenance.resumed,
|
|
2193
|
+
artifactDir: provenance.artifactDir
|
|
2194
|
+
},
|
|
2195
|
+
developmentDataDigest: improvement.lineage.developmentSplitDigest
|
|
2196
|
+
}).value;
|
|
2197
|
+
}
|
|
2198
|
+
/** Add Runtime-owned optimizer evidence without aliasing caller metadata. */
|
|
2199
|
+
function attachOptimizationActivationReceipt(metadata, receipt) {
|
|
2200
|
+
assertNoCallerOptimizationReceipt(metadata);
|
|
2201
|
+
return immutableCandidateValue({
|
|
2202
|
+
...metadata ?? {},
|
|
2203
|
+
[optimizationReceiptMetadataKey]: receipt
|
|
2204
|
+
});
|
|
2205
|
+
}
|
|
2206
|
+
function assertNoCallerOptimizationReceipt(metadata) {
|
|
2207
|
+
if (metadata && Object.hasOwn(metadata, "optimizationReceipt")) throw new Error(`candidate metadata reserves '${optimizationReceiptMetadataKey}' for Runtime`);
|
|
2208
|
+
}
|
|
2209
|
+
/** Read and verify the optimizer evidence carried by a measured proposal. */
|
|
2210
|
+
function optimizationActivationReceiptFromMetadata(metadata) {
|
|
2211
|
+
const value = metadata?.[optimizationReceiptMetadataKey];
|
|
2212
|
+
if (value === void 0) return void 0;
|
|
2213
|
+
return immutableCandidateValue(parseOptimizationActivationReceipt(value));
|
|
2214
|
+
}
|
|
2215
|
+
function parseOptimizationActivationReceipt(value) {
|
|
2216
|
+
if (!isRecord(value) || value.kind !== "optimization-activation-receipt") throw new Error("optimization receipt must be an optimization-activation-receipt");
|
|
2217
|
+
if (!isNonEmptyString(value.method) || !isPackageSource(value.source) || value.bridge !== void 0 && !isPackageSource(value.bridge) || !isModules(value.modules) || !isPythonRuntime(value.python) || !isModels(value.models) || !isUsage(value.usage) || !isCost(value.cost) || !isInvocation(value.invocation) || !isSha256Digest(value.developmentDataDigest) || !isSha256Digest(value.digest)) throw new Error("optimization receipt contains invalid evidence");
|
|
2218
|
+
const receipt = value;
|
|
2219
|
+
if (canonicalCandidateDigest$1(omitTopLevelDigest(receipt)) !== receipt.digest) throw new Error("optimization receipt digest does not match its evidence");
|
|
2220
|
+
return receipt;
|
|
2221
|
+
}
|
|
2222
|
+
function isPackageSource(value) {
|
|
2223
|
+
if (!isRecord(value) || value.kind !== "package" || value.evidence !== "observed" && value.evidence !== "declared" || !isNonEmptyString(value.package) || !isNonEmptyString(value.version)) return false;
|
|
2224
|
+
return isOptionalNonEmptyString(value.sourceUrl) && isOptionalNonEmptyString(value.revision) && (value.sourceSha256 === void 0 || typeof value.sourceSha256 === "string" && /^[0-9a-f]{64}$/.test(value.sourceSha256));
|
|
2225
|
+
}
|
|
2226
|
+
function isModules(value) {
|
|
2227
|
+
return value === void 0 || Array.isArray(value) && value.every((module) => isRecord(module) && isNonEmptyString(module.module) && typeof module.sourceSha256 === "string" && /^[0-9a-f]{64}$/.test(module.sourceSha256));
|
|
2228
|
+
}
|
|
2229
|
+
function isPythonRuntime(value) {
|
|
2230
|
+
return value === void 0 || isRecord(value) && isNonEmptyString(value.implementation) && isNonEmptyString(value.version);
|
|
2231
|
+
}
|
|
2232
|
+
function isModels(value) {
|
|
2233
|
+
return value === void 0 || isRecord(value) && (value.candidate === void 0 || agentProfileModelHintsSchema.safeParse(value.candidate).success) && isOptionalNonEmptyString(value.optimizer) && (value.candidate !== void 0 || value.optimizer !== void 0);
|
|
2234
|
+
}
|
|
2235
|
+
function isUsage(value) {
|
|
2236
|
+
if (!isRecord(value) || !isNonNegativeInteger(value.optimizerEvaluations)) return false;
|
|
2237
|
+
if (value.optimizerTokens === void 0) return true;
|
|
2238
|
+
const tokens = value.optimizerTokens;
|
|
2239
|
+
if (!isRecord(tokens)) return false;
|
|
2240
|
+
const inputTokens = tokens.inputTokens;
|
|
2241
|
+
const outputTokens = tokens.outputTokens;
|
|
2242
|
+
const totalTokens = tokens.totalTokens;
|
|
2243
|
+
const calls = tokens.calls;
|
|
2244
|
+
if (!isNonNegativeInteger(inputTokens) || !isNonNegativeInteger(outputTokens) || !isNonNegativeInteger(totalTokens) || !isNonNegativeInteger(calls)) return false;
|
|
2245
|
+
if (!isOptionalNonNegativeInteger(tokens.cachedInputTokens) || !isOptionalNonNegativeInteger(tokens.cacheWriteInputTokens) || !isOptionalNonNegativeInteger(tokens.reasoningTokens)) return false;
|
|
2246
|
+
const cachedInputTokens = typeof tokens.cachedInputTokens === "number" ? tokens.cachedInputTokens : 0;
|
|
2247
|
+
const cacheWriteInputTokens = typeof tokens.cacheWriteInputTokens === "number" ? tokens.cacheWriteInputTokens : 0;
|
|
2248
|
+
const reasoningTokens = typeof tokens.reasoningTokens === "number" ? tokens.reasoningTokens : 0;
|
|
2249
|
+
return totalTokens === inputTokens + outputTokens && cachedInputTokens + cacheWriteInputTokens <= inputTokens && reasoningTokens <= outputTokens && (totalTokens === 0 || calls > 0);
|
|
2250
|
+
}
|
|
2251
|
+
function isCost(value) {
|
|
2252
|
+
if (!isRecord(value) || !isCostPart(value.optimization) || !isCostPart(value.finalTest) || !isCostPart(value.total)) return false;
|
|
2253
|
+
const optimization = value.optimization;
|
|
2254
|
+
const finalTest = value.finalTest;
|
|
2255
|
+
const total = value.total;
|
|
2256
|
+
return approximatelyEqual(total.totalUsd, optimization.totalUsd + finalTest.totalUsd) && total.accountingComplete === (optimization.accountingComplete && finalTest.accountingComplete);
|
|
2257
|
+
}
|
|
2258
|
+
function isCostPart(value) {
|
|
2259
|
+
return isRecord(value) && typeof value.totalUsd === "number" && Number.isFinite(value.totalUsd) && value.totalUsd >= 0 && typeof value.accountingComplete === "boolean" && Array.isArray(value.incompleteReasons) && value.incompleteReasons.every((reason) => typeof reason === "string");
|
|
2260
|
+
}
|
|
2261
|
+
function receiptCost(value) {
|
|
2262
|
+
return {
|
|
2263
|
+
totalUsd: value.totalCostUsd,
|
|
2264
|
+
accountingComplete: value.accountingComplete,
|
|
2265
|
+
incompleteReasons: [...value.incompleteReasons]
|
|
2266
|
+
};
|
|
2267
|
+
}
|
|
2268
|
+
function approximatelyEqual(left, right) {
|
|
2269
|
+
const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 8;
|
|
2270
|
+
return Math.abs(left - right) <= tolerance;
|
|
2271
|
+
}
|
|
2272
|
+
function isInvocation(value) {
|
|
2273
|
+
return isRecord(value) && isNonEmptyString(value.runtimeInvocationId) && isNonEmptyString(value.optimizerRunId) && isOptionalNonEmptyString(value.compatibleOptimizerRunId) && typeof value.resumed === "boolean" && isNonEmptyString(value.artifactDir);
|
|
2274
|
+
}
|
|
2275
|
+
function isRecord(value) {
|
|
2276
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
2277
|
+
}
|
|
2278
|
+
function isNonEmptyString(value) {
|
|
2279
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
2280
|
+
}
|
|
2281
|
+
function isOptionalNonEmptyString(value) {
|
|
2282
|
+
return value === void 0 || isNonEmptyString(value);
|
|
2283
|
+
}
|
|
2284
|
+
function isNonNegativeInteger(value) {
|
|
2285
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
2286
|
+
}
|
|
2287
|
+
function isOptionalNonNegativeInteger(value) {
|
|
2288
|
+
return value === void 0 || isNonNegativeInteger(value);
|
|
2289
|
+
}
|
|
2290
|
+
function isSha256Digest(value) {
|
|
2291
|
+
return typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value);
|
|
2292
|
+
}
|
|
2293
|
+
//#endregion
|
|
2294
|
+
//#region src/intelligence/improvement-cycle.ts
|
|
2295
|
+
/** A failed baseline or candidate cell with its complete Runtime failure result. */
|
|
2296
|
+
var AgentCandidateExperimentCellExecutionError = class extends Error {
|
|
2297
|
+
finalization;
|
|
2298
|
+
constructor(finalization) {
|
|
2299
|
+
super(`candidate experiment cell failed: ${finalization.reason}`);
|
|
2300
|
+
this.name = "AgentCandidateExperimentCellExecutionError";
|
|
2301
|
+
this.finalization = finalization;
|
|
2302
|
+
}
|
|
2303
|
+
};
|
|
2304
|
+
function sealAgentImprovementExperiment(material, improvement) {
|
|
2305
|
+
assertRuntimeOwnedExperimentFieldsAbsent(material);
|
|
2306
|
+
const candidateLineage = {
|
|
2307
|
+
source: "optimizer",
|
|
2308
|
+
parentDigests: [material.baseline.digest],
|
|
2309
|
+
runIds: [improvement.lineage.runId],
|
|
2310
|
+
developmentSplitDigest: improvement.lineage.developmentSplitDigest
|
|
2311
|
+
};
|
|
2312
|
+
return sealCandidateExperiment({
|
|
2313
|
+
...material,
|
|
2314
|
+
candidateLineage
|
|
2315
|
+
});
|
|
2316
|
+
}
|
|
2317
|
+
function assertRuntimeOwnedExperimentFieldsAbsent(material) {
|
|
2318
|
+
if (material === null || typeof material !== "object" || Array.isArray(material)) throw new Error("agent improvement experiment material must be an object");
|
|
2319
|
+
const supplied = ["candidateLineage", "digest"].filter((field) => Object.hasOwn(material, field));
|
|
2320
|
+
if (supplied.length > 0) throw new Error(`agent improvement experiment material must not supply Runtime-owned fields: ${supplied.join(", ")}`);
|
|
2321
|
+
}
|
|
2322
|
+
/** Execute both arms of one immutable experiment and derive its paired result. */
|
|
2323
|
+
async function runAgentCandidateExperiment(options) {
|
|
2324
|
+
const experiment = verifyCandidateExperiment(options.experiment);
|
|
2325
|
+
const measurements = await runCandidateExperiment({
|
|
2326
|
+
experiment,
|
|
2327
|
+
...options.maxConcurrency === void 0 ? {} : { maxConcurrency: options.maxConcurrency },
|
|
2328
|
+
...options.signal ? { signal: options.signal } : {},
|
|
2329
|
+
execute: async (input) => {
|
|
2330
|
+
const placement = await options.placeCell(input);
|
|
2331
|
+
return await executeAgentCandidateExperimentCell({
|
|
2332
|
+
...input,
|
|
2333
|
+
...placement
|
|
2334
|
+
});
|
|
2335
|
+
}
|
|
2336
|
+
});
|
|
2337
|
+
return {
|
|
2338
|
+
experiment,
|
|
2339
|
+
measurements,
|
|
2340
|
+
evaluation: createAgentImprovementMeasuredComparison({
|
|
2341
|
+
experiment,
|
|
2342
|
+
measurements,
|
|
2343
|
+
runId: options.runId,
|
|
2344
|
+
...options.candidate ? { candidate: options.candidate } : {},
|
|
2345
|
+
...options.generationsExplored === void 0 ? {} : { generationsExplored: options.generationsExplored },
|
|
2346
|
+
...options.searchDurationMs === void 0 ? {} : { searchDurationMs: options.searchDurationMs },
|
|
2347
|
+
...options.searchCostUsd === void 0 ? {} : { searchCostUsd: options.searchCostUsd },
|
|
2348
|
+
...options.metadata ? { metadata: options.metadata } : {}
|
|
2349
|
+
})
|
|
2350
|
+
};
|
|
2351
|
+
}
|
|
2352
|
+
/** Execute one exact arm, task, repetition, seed, and attempt through Runtime. */
|
|
2353
|
+
async function executeAgentCandidateExperimentCell(options) {
|
|
2354
|
+
const experiment = verifyCandidateExperiment(options.experiment);
|
|
2355
|
+
const bundle = experiment[options.arm];
|
|
2356
|
+
assertExactExperimentInput(options, experiment, bundle);
|
|
2357
|
+
const attempt = options.attempt ?? 1;
|
|
2358
|
+
if (attempt > options.task.attempt.maxAttempts) throw new Error("candidate experiment attempt exceeds the signed task policy");
|
|
2359
|
+
const runCell = canonicalCandidateDocument({
|
|
2360
|
+
kind: "agent-candidate-run-cell",
|
|
2361
|
+
experimentDigest: experiment.digest,
|
|
2362
|
+
arm: options.arm,
|
|
2363
|
+
bundleDigest: bundle.digest,
|
|
2364
|
+
suiteDigest: options.benchmarkCell.suiteDigest,
|
|
2365
|
+
taskDigest: options.task.digest,
|
|
2366
|
+
taskIndex: options.benchmarkCell.taskIndex,
|
|
2367
|
+
repetition: options.benchmarkCell.repetition,
|
|
2368
|
+
seed: options.seed,
|
|
2369
|
+
attempt
|
|
2370
|
+
}).value;
|
|
2371
|
+
const verified = await verifyAgentCandidateBundle(bundle, options.ports);
|
|
2372
|
+
const prepared = await prepareAgentCandidateExecution(verified, {
|
|
2373
|
+
executionId: options.executionId,
|
|
2374
|
+
runCell,
|
|
2375
|
+
benchmarkSuite: experiment.benchmark.suite,
|
|
2376
|
+
task: options.task,
|
|
2377
|
+
executionRoots: options.executionRoots,
|
|
2378
|
+
stagingRoots: options.stagingRoots
|
|
2379
|
+
}, options.ports, options.preparation);
|
|
2380
|
+
const finalization = await executePreparedAgentCandidate(prepared, options.execution);
|
|
2381
|
+
if (!finalization.succeeded) throw new AgentCandidateExperimentCellExecutionError(finalization);
|
|
2382
|
+
const evidence = canonicalCandidateDocument({
|
|
2383
|
+
kind: "agent-candidate-execution-evidence",
|
|
2384
|
+
materializationReceipt: prepared.materializationReceipt.value,
|
|
2385
|
+
receipt: finalization.receipt.value
|
|
2386
|
+
}).value;
|
|
2387
|
+
return verifyCandidateExecutionEvidence(evidence, {
|
|
2388
|
+
experiment,
|
|
2389
|
+
arm: options.arm,
|
|
2390
|
+
benchmarkCell: options.benchmarkCell,
|
|
2391
|
+
seed: options.seed,
|
|
2392
|
+
attempt,
|
|
2393
|
+
resolvedResources: verifiedResourceTextByDigest(verified)
|
|
2394
|
+
});
|
|
2395
|
+
}
|
|
2396
|
+
/** Delegate all statistics and promotion checks to agent-eval's receipt-based comparison. */
|
|
2397
|
+
function createAgentImprovementMeasuredComparison(options) {
|
|
2398
|
+
return verifyCandidateExperimentComparison(measuredComparisonFromCandidateExperiment(options));
|
|
2399
|
+
}
|
|
2400
|
+
/** Analyze, search, then remeasure the resulting exact candidate before proposing it. */
|
|
2401
|
+
async function proposeAgentImprovement(options) {
|
|
2402
|
+
assertNoCallerOptimizationReceipt(options.metadata);
|
|
2403
|
+
const analysis = await runAnalystLoop({
|
|
2404
|
+
...options.analysis,
|
|
2405
|
+
runId: options.runId
|
|
2406
|
+
});
|
|
2407
|
+
const findings = assertNoJudgeVerdict(analysis.analystResult.findings, "proposeAgentImprovement findings");
|
|
2408
|
+
const improvementInput = {
|
|
2409
|
+
...options.improvement,
|
|
2410
|
+
findings: [...options.improvement.findings ?? [], ...findings]
|
|
2411
|
+
};
|
|
2412
|
+
const improvement = improvementInput.surface === "code" ? await improve(improvementInput) : await improve(options.profile, improvementInput);
|
|
2413
|
+
try {
|
|
2414
|
+
if (improvement.decision !== "ship") throw new Error("agent improvement search did not produce a promotable candidate");
|
|
2415
|
+
const optimizationReceipt = improvement.mode === "method" ? createOptimizationActivationReceipt(improvement) : void 0;
|
|
2416
|
+
const experiment = sealAgentImprovementExperiment(await options.buildExperiment({
|
|
2417
|
+
analysis,
|
|
2418
|
+
improvement
|
|
2419
|
+
}), improvement);
|
|
2420
|
+
assertCandidateProfileBinding(options.profile, experiment.baseline.profile);
|
|
2421
|
+
assertImprovementCandidateBinding(improvement, experiment);
|
|
2422
|
+
const measured = await runAgentCandidateExperiment({
|
|
2423
|
+
experiment,
|
|
2424
|
+
runId: options.runId,
|
|
2425
|
+
placeCell: options.placeCell,
|
|
2426
|
+
...options.maxConcurrency === void 0 ? {} : { maxConcurrency: options.maxConcurrency },
|
|
2427
|
+
...options.signal ? { signal: options.signal } : {},
|
|
2428
|
+
...options.candidate ? { candidate: options.candidate } : {},
|
|
2429
|
+
...optimizationReceipt ? { metadata: attachOptimizationActivationReceipt(options.metadata, optimizationReceipt) } : options.metadata ? { metadata: options.metadata } : {},
|
|
2430
|
+
...improvement.generationsExplored === void 0 ? {} : { generationsExplored: improvement.generationsExplored },
|
|
2431
|
+
searchDurationMs: improvement.durationMs,
|
|
2432
|
+
searchCostUsd: improvement.cost.totalCostUsd
|
|
2433
|
+
});
|
|
2434
|
+
const proposal = createAgentImprovementProposal({
|
|
2435
|
+
runId: options.runId,
|
|
2436
|
+
findings,
|
|
2437
|
+
evaluation: measured.evaluation,
|
|
2438
|
+
...options.now ? { now: options.now } : {}
|
|
2439
|
+
});
|
|
2440
|
+
return {
|
|
2441
|
+
analysis,
|
|
2442
|
+
improvement,
|
|
2443
|
+
experiment,
|
|
2444
|
+
measurements: measured.measurements,
|
|
2445
|
+
proposal
|
|
2446
|
+
};
|
|
2447
|
+
} catch (cause) {
|
|
2448
|
+
return rethrowAfterCleanup(cause, () => improvement.dispose(), "proposeAgentImprovement failed");
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
function assertImprovementCandidateBinding(improvement, experiment) {
|
|
2452
|
+
const candidate = improvement.candidate;
|
|
2453
|
+
if (candidate.surface !== "code") {
|
|
2454
|
+
try {
|
|
2455
|
+
assertCandidateProfileBinding(candidate.profile, experiment.candidate.profile);
|
|
2456
|
+
} catch (cause) {
|
|
2457
|
+
throw new Error("candidate experiment does not contain the improvement winner", { cause });
|
|
2458
|
+
}
|
|
2459
|
+
return;
|
|
2460
|
+
}
|
|
2461
|
+
const surface = candidate.value;
|
|
2462
|
+
const code = experiment.candidate.code;
|
|
2463
|
+
if (typeof surface !== "object" || surface === null || surface.kind !== "code" || code.kind !== "git-patch" || code.baseCommit !== surface.baseCommit || code.baseTree !== surface.baseTree || code.candidateTree !== surface.candidateTree || code.patch.artifact.sha256 !== surface.patch.sha256 || code.patch.artifact.byteLength !== surface.patch.byteLength) throw new Error("candidate experiment does not contain the improvement winner");
|
|
2464
|
+
if (canonicalCandidateDigest$1(experiment.baseline.profile) !== canonicalCandidateDigest$1(experiment.candidate.profile)) throw new Error("code improvement candidate changed the agent profile");
|
|
2465
|
+
}
|
|
2466
|
+
/** Create the reviewable record only from a complete, recomputable experiment result. */
|
|
2467
|
+
function createAgentImprovementProposal(options) {
|
|
2468
|
+
const findings = assertNoJudgeVerdict([...options.findings], "createAgentImprovementProposal findings");
|
|
2469
|
+
const { evaluation, changedSurfaces } = validateShippableAgentImprovementEvaluation(options.evaluation, options.runId, "agent improvement proposal");
|
|
2470
|
+
return agentImprovementProposalSchema.parse(canonicalCandidateDocument({
|
|
2471
|
+
kind: "agent-improvement-proposal",
|
|
2472
|
+
runId: options.runId,
|
|
2473
|
+
changedSurfaces,
|
|
2474
|
+
proposedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
|
|
2475
|
+
findings: [...findings],
|
|
2476
|
+
evaluation
|
|
2477
|
+
}).value);
|
|
2478
|
+
}
|
|
2479
|
+
/** Persist a human or tenant-policy decision bound to one exact proposal. */
|
|
2480
|
+
function reviewAgentImprovementProposal(inputProposal, input) {
|
|
2481
|
+
const proposal = verifyAgentImprovementProposal(inputProposal);
|
|
2482
|
+
if (!input.reviewedBy.trim()) throw new Error("candidate review requires reviewedBy");
|
|
2483
|
+
if (!input.reason.trim()) throw new Error("candidate review requires a reason");
|
|
2484
|
+
if (input.decision === "approve" && proposal.evaluation.decision.outcome !== "ship") throw new Error("candidate cannot be approved without a passing experiment");
|
|
2485
|
+
const reviewedAt = (input.now ?? (() => /* @__PURE__ */ new Date()))().toISOString();
|
|
2486
|
+
if (Date.parse(reviewedAt) < Date.parse(proposal.proposedAt)) throw new Error("candidate review cannot predate its proposal");
|
|
2487
|
+
return agentImprovementReviewSchema.parse(canonicalCandidateDocument({
|
|
2488
|
+
kind: "agent-improvement-review",
|
|
2489
|
+
proposalDigest: proposal.digest,
|
|
2490
|
+
decision: input.decision,
|
|
2491
|
+
reviewedBy: input.reviewedBy,
|
|
2492
|
+
reviewedAt,
|
|
2493
|
+
reason: input.reason,
|
|
2494
|
+
...input.feedback === void 0 ? {} : { feedback: input.feedback }
|
|
2495
|
+
}).value);
|
|
2496
|
+
}
|
|
2497
|
+
/** Authorize product-owned writes only after the exact candidate was measured and approved. */
|
|
2498
|
+
function createAgentImprovementActivation(inputProposal, inputReview, options) {
|
|
2499
|
+
const proposal = verifyAgentImprovementProposal(inputProposal);
|
|
2500
|
+
const review = verifyAgentImprovementReview(inputReview);
|
|
2501
|
+
if (review.decision !== "approve" || review.proposalDigest !== proposal.digest) throw new Error("candidate activation requires an approval for the exact proposal");
|
|
2502
|
+
if (!options.fundingOwner.trim() || !options.authorizedBy.trim()) throw new Error("candidate activation authority must be non-empty");
|
|
2503
|
+
const experiment = proposal.evaluation.experiment;
|
|
2504
|
+
const authorizedAt = (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString();
|
|
2505
|
+
if (Date.parse(authorizedAt) < Date.parse(review.reviewedAt)) throw new Error("candidate activation cannot predate its approval");
|
|
2506
|
+
const targets = buildAgentImprovementActivationTargets(proposal.changedSurfaces, experiment, options.intent, options.targets);
|
|
2507
|
+
return agentImprovementActivationSchema.parse(canonicalCandidateDocument({
|
|
2508
|
+
kind: "agent-improvement-activation",
|
|
2509
|
+
proposalDigest: proposal.digest,
|
|
2510
|
+
reviewDigest: review.digest,
|
|
2511
|
+
experimentDigest: experiment.digest,
|
|
2512
|
+
candidateDigest: measuredCandidateDigest(proposal),
|
|
2513
|
+
intent: options.intent,
|
|
2514
|
+
targets,
|
|
2515
|
+
fundingOwner: options.fundingOwner,
|
|
2516
|
+
authorizedBy: options.authorizedBy,
|
|
2517
|
+
authorizedAt,
|
|
2518
|
+
expiresAt: options.expiresAt
|
|
2519
|
+
}).value);
|
|
2520
|
+
}
|
|
2521
|
+
/** Validate a proposal and recompute every binding to its measured experiment. */
|
|
2522
|
+
function verifyAgentImprovementProposal(input) {
|
|
2523
|
+
const proposal = verifyCanonicalCandidateDocument(agentImprovementProposalSchema.parse(input), "agent improvement proposal");
|
|
2524
|
+
const { changedSurfaces } = validateShippableAgentImprovementEvaluation(proposal.evaluation, proposal.runId, "agent improvement proposal");
|
|
2525
|
+
if (!(proposal.evaluation.kind === "agent-profile-improvement-measured-comparison" ? sameAgentImprovementSurfaceSet(proposal.changedSurfaces, changedSurfaces) : sameOrderedValues(proposal.changedSurfaces, changedSurfaces))) throw new Error("proposal changed surfaces do not match its exact experiment");
|
|
2526
|
+
assertNoJudgeDerivedProposalFindings(proposal.findings);
|
|
2527
|
+
return proposal;
|
|
2528
|
+
}
|
|
2529
|
+
/** Return a sealed bundle experiment; ordinary profile changes need a product-owned executor. */
|
|
2530
|
+
function requireSealedCandidateExperiment(proposal) {
|
|
2531
|
+
if (proposal.evaluation.kind !== "agent-improvement-measured-comparison") throw new Error("agent profile improvement activation requires a product profile-diff executor, not a sealed candidate bundle");
|
|
2532
|
+
return proposal.evaluation.experiment;
|
|
2533
|
+
}
|
|
2534
|
+
function measuredCandidateDigest(proposal) {
|
|
2535
|
+
return proposal.evaluation.kind === "agent-profile-improvement-measured-comparison" ? proposal.evaluation.experiment.candidate.stateDigest : proposal.evaluation.experiment.candidate.digest;
|
|
2536
|
+
}
|
|
2537
|
+
function validateShippableAgentImprovementEvaluation(input, runId, subject) {
|
|
2538
|
+
const evaluation = verifyAgentImprovementEvaluation(input);
|
|
2539
|
+
if (evaluation.decision.outcome !== "ship") throw new Error(`${subject} requires a passing experiment`);
|
|
2540
|
+
if (runId !== evaluation.provenance.runId) throw new Error("proposal runId does not match its measured experiment");
|
|
2541
|
+
return {
|
|
2542
|
+
evaluation,
|
|
2543
|
+
changedSurfaces: evaluation.kind === "agent-profile-improvement-measured-comparison" ? profileImprovementChangedSurfaces(evaluation.experiment.change) : deriveChangedSurfaces(evaluation.experiment.baseline, evaluation.experiment.candidate)
|
|
2544
|
+
};
|
|
2545
|
+
}
|
|
2546
|
+
/** Use each owning package's complete measurement validator before a proposal is persisted. */
|
|
2547
|
+
function verifyAgentImprovementEvaluation(input) {
|
|
2548
|
+
if (typeof input === "object" && input !== null && "kind" in input && input.kind === "agent-profile-improvement-measured-comparison") {
|
|
2549
|
+
const evaluation = agentProfileImprovementMeasuredComparisonSchema.parse(input);
|
|
2550
|
+
optimizationActivationReceiptFromMetadata(evaluation.metadata);
|
|
2551
|
+
return evaluation;
|
|
2552
|
+
}
|
|
2553
|
+
const evaluation = verifyCandidateExperimentComparison(input);
|
|
2554
|
+
optimizationActivationReceiptFromMetadata(evaluation.metadata);
|
|
2555
|
+
return evaluation;
|
|
2556
|
+
}
|
|
2557
|
+
/** Validate the canonical identity and wire shape of an improvement review. */
|
|
2558
|
+
function verifyAgentImprovementReview(input) {
|
|
2559
|
+
return verifyCanonicalCandidateDocument(agentImprovementReviewSchema.parse(input), "agent improvement review");
|
|
2560
|
+
}
|
|
2561
|
+
/** Validate activation authority against the exact proposal, review, experiment, and base state. */
|
|
2562
|
+
function verifyAgentImprovementActivation(input) {
|
|
2563
|
+
const proposal = verifyAgentImprovementProposal(input.proposal);
|
|
2564
|
+
const review = verifyAgentImprovementReview(input.review);
|
|
2565
|
+
const activation = verifyCanonicalCandidateDocument(agentImprovementActivationSchema.parse(input.activation), "agent improvement activation");
|
|
2566
|
+
const experiment = proposal.evaluation.experiment;
|
|
2567
|
+
if (review.decision !== "approve" || review.proposalDigest !== proposal.digest || activation.proposalDigest !== proposal.digest || activation.reviewDigest !== review.digest || activation.experimentDigest !== experiment.digest || activation.candidateDigest !== measuredCandidateDigest(proposal) || Date.parse(review.reviewedAt) < Date.parse(proposal.proposedAt) || Date.parse(activation.authorizedAt) < Date.parse(review.reviewedAt)) throw new Error("candidate activation does not bind the measured and approved candidate");
|
|
2568
|
+
assertAgentImprovementActivationTargets(proposal.changedSurfaces, experiment, activation.intent, activation.targets);
|
|
2569
|
+
return activation;
|
|
2570
|
+
}
|
|
2571
|
+
/** Recheck one Runtime receipt against its exact signed experiment cell. */
|
|
2572
|
+
function verifyCandidateExecutionEvidence(input, options) {
|
|
2573
|
+
const experiment = verifyCandidateExperiment(options.experiment);
|
|
2574
|
+
const bundle = experiment[options.arm];
|
|
2575
|
+
const task = experiment.benchmark.tasks[options.benchmarkCell.taskIndex];
|
|
2576
|
+
const index = options.benchmarkCell.taskIndex * experiment.benchmark.suite.reps + options.benchmarkCell.repetition;
|
|
2577
|
+
if (!task || options.benchmarkCell.suiteDigest !== experiment.benchmark.suite.digest || options.seed !== experiment.benchmark.suite.seeds[index]) throw new Error("candidate execution evidence points outside its signed experiment");
|
|
2578
|
+
const evidence = verifyCanonicalCandidateDocument(candidateExecutionEvidenceSchema.parse(input), "candidate execution evidence");
|
|
2579
|
+
const materialization = verifyCanonicalCandidateDocument(agentCandidateMaterializationReceiptSchema.parse(evidence.materializationReceipt), "candidate materialization receipt");
|
|
2580
|
+
const receipt = verifyCanonicalCandidateDocument(agentCandidateRunReceiptSchema.parse(evidence.receipt), "candidate run receipt");
|
|
2581
|
+
const plan = materialization.executionPlan;
|
|
2582
|
+
const cell = plan.material.runCell;
|
|
2583
|
+
const attempt = options.attempt ?? 1;
|
|
2584
|
+
if (cell.experimentDigest !== experiment.digest || cell.arm !== options.arm || cell.bundleDigest !== bundle.digest || cell.suiteDigest !== experiment.benchmark.suite.digest || cell.taskDigest !== task.digest || cell.taskIndex !== options.benchmarkCell.taskIndex || cell.repetition !== options.benchmarkCell.repetition || cell.seed !== options.seed || cell.attempt !== attempt || canonicalCandidateDigest$1(omitTopLevelDigest(cell)) !== cell.digest) throw new Error("candidate execution receipt substituted its signed experiment cell");
|
|
2585
|
+
assertCapturedInput(materialization.benchmark.suite, experiment.benchmark.suite, "benchmark suite");
|
|
2586
|
+
assertCapturedInput(materialization.benchmark.task, task, "benchmark task");
|
|
2587
|
+
assertEvidenceMaterialDigest(plan, "candidate execution plan");
|
|
2588
|
+
assertEvidenceMaterialDigest(materialization.profileActivation.profilePlan, "candidate profile plan");
|
|
2589
|
+
const expectedProfilePlan = materializeCandidateProfile(bundle.profile, candidateMaterializerHarness(materialization.harness), { resolvedResources: options.resolvedResources });
|
|
2590
|
+
const activation = parseAgentCandidateProfileActivation(materialization.profileActivation, materialization.profileActivation.profilePlan.digest);
|
|
2591
|
+
const regeneratedActivation = createAgentCandidateProfileActivation(expectedProfilePlan, materialization.profileActivation.profilePlan);
|
|
2592
|
+
if (activation.digest !== regeneratedActivation.digest) throw new Error("candidate profile activation does not match the experiment bundle");
|
|
2593
|
+
if (materialization.bundleDigest !== bundle.digest || receipt.bundleDigest !== bundle.digest || receipt.runCellDigest !== cell.digest || receipt.materializationReceiptDigest !== materialization.digest || receipt.executionPlanDigest !== plan.digest) throw new Error("candidate execution evidence does not bind one exact Runtime run");
|
|
2594
|
+
assertEvidenceMaterialDigest(receipt.modelSettlement, "candidate model settlement");
|
|
2595
|
+
assertEvidenceMaterialDigest(receipt.taskOutcome, "candidate task outcome");
|
|
2596
|
+
assertEvidenceMaterialDigest(receipt.benchmarkResult, "candidate benchmark result");
|
|
2597
|
+
return immutableCandidateValue(evidence);
|
|
2598
|
+
}
|
|
2599
|
+
function assertExactExperimentInput(input, experiment, bundle) {
|
|
2600
|
+
const task = experiment.benchmark.tasks[input.benchmarkCell.taskIndex];
|
|
2601
|
+
const index = input.benchmarkCell.taskIndex * experiment.benchmark.suite.reps + input.benchmarkCell.repetition;
|
|
2602
|
+
if (input.experiment.digest !== experiment.digest || input.bundle.digest !== bundle.digest || !task || input.task.digest !== task.digest || input.benchmarkCell.suiteDigest !== experiment.benchmark.suite.digest || input.seed !== experiment.benchmark.suite.seeds[index]) throw new Error("Runtime received a substituted candidate experiment cell");
|
|
2603
|
+
}
|
|
2604
|
+
function assertCapturedInput(captured, expected, label) {
|
|
2605
|
+
const bytes = canonicalCandidateBytes(omitTopLevelDigest(expected));
|
|
2606
|
+
if (captured.digest !== expected.digest || captured.material.sha256 !== expected.digest || captured.material.byteLength !== bytes.byteLength) throw new Error(`candidate materialization substituted its ${label}`);
|
|
2607
|
+
}
|
|
2608
|
+
function assertEvidenceMaterialDigest(evidence, label) {
|
|
2609
|
+
const bytes = canonicalCandidateBytes(evidence.material);
|
|
2610
|
+
if (canonicalCandidateDigest$1(evidence.material) !== evidence.digest || evidence.artifact.sha256 !== evidence.digest || evidence.artifact.byteLength !== bytes.byteLength) throw new Error(`${label} digest does not match its canonical material`);
|
|
2611
|
+
}
|
|
2612
|
+
function sameOrderedValues(left, right) {
|
|
2613
|
+
return left.length === right.length && left.every((value, index) => value === right[index]);
|
|
2614
|
+
}
|
|
2615
|
+
function assertNoJudgeDerivedProposalFindings(findings) {
|
|
2616
|
+
const leaked = findings.filter((finding) => finding.derived_from_judge === true);
|
|
2617
|
+
if (leaked.length === 0) return;
|
|
2618
|
+
const identifiers = leaked.map((finding) => typeof finding.finding_id === "string" ? finding.finding_id : "<unknown>");
|
|
2619
|
+
throw new Error(`agent improvement proposal findings: judge-derived findings cannot steer an improvement: [${identifiers.join(", ")}]`);
|
|
2620
|
+
}
|
|
2621
|
+
//#endregion
|
|
2622
|
+
export { applyRolloutPolicyToProfile as A, commandVerifier as B, assertProfileImprovementTargetsShareIdentity as C, withMethodRuntimeControls as D, improve as E, LIFTED_FINDING_ANALYST_ID as F, worktreeChangedPaths as G, rawTraceEvidenceProblem as H, isAnalystFinding as I, toAnalystFindings as L, parseRolloutPolicy as M, serializeRolloutPolicy as N, rawTraceDistiller as O, structuralRolloutPolicyFromProfile as P, AGENTIC_PROFILE_RESOURCE_ROOT as R, agentProfileImprovementStateDigest as S, isAgentImprovementProfileSurface as T, requiresRawTraceEvidence as U, defaultBuildPrompt as V, summarizeFindings as W, agentImprovementProfileSurfaceDigest as _, executeAgentCandidateExperimentCell as a, agentImprovementTargetInput as b, reviewAgentImprovementProposal as c, verifyAgentImprovementProposal as d, verifyAgentImprovementReview as f, AGENT_IMPROVEMENT_PROFILE_SURFACES as g, optimizationActivationReceiptFromMetadata as h, createAgentImprovementProposal as i, normalizeRolloutPolicy as j, ROLLOUT_POLICY_EXTENSION as k, runAgentCandidateExperiment as l, createOptimizationActivationReceipt as m, createAgentImprovementActivation as n, proposeAgentImprovement as o, verifyCandidateExecutionEvidence as p, createAgentImprovementMeasuredComparison as r, requireSealedCandidateExperiment as s, AgentCandidateExperimentCellExecutionError as t, verifyAgentImprovementActivation as u, agentImprovementProfileSurfaceInput as v, buildAgentImprovementActivationTargets as w, agentImprovementTargetProfileDiffs as x, agentImprovementTargetDigest as y, agenticGenerator as z };
|
|
2623
|
+
|
|
2624
|
+
//# sourceMappingURL=improvement-cycle-C095kS_g.js.map
|