@tangle-network/agent-runtime 0.105.0 → 0.106.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/activation-CRjVfOiy.js +203 -0
- package/dist/activation-CRjVfOiy.js.map +1 -0
- package/dist/activation-DyWB0K6E.d.ts +98 -0
- package/dist/agent.d.ts +216 -234
- package/dist/agent.js +615 -547
- package/dist/agent.js.map +1 -1
- package/dist/analyst-loop-DGdtVENg.js +517 -0
- package/dist/analyst-loop-DGdtVENg.js.map +1 -0
- package/dist/analyst-loop.d.ts +10 -43
- package/dist/analyst-loop.js +2 -11
- package/dist/backends-Dy8NujB4.js +713 -0
- package/dist/backends-Dy8NujB4.js.map +1 -0
- package/dist/candidate-execution/index.d.ts +4 -192
- package/dist/candidate-execution/index.js +5 -66
- package/dist/candidate-execution-DqMbv1rU.js +488 -0
- package/dist/candidate-execution-DqMbv1rU.js.map +1 -0
- package/dist/coder-yhVWbdWc.js +40 -0
- package/dist/coder-yhVWbdWc.js.map +1 -0
- package/dist/conversation-XucG_K3l.js +1265 -0
- package/dist/conversation-XucG_K3l.js.map +1 -0
- package/dist/conversation.d.ts +2 -804
- package/dist/conversation.js +2 -55
- package/dist/environment-provider-CfLUmKmf.js +1168 -0
- package/dist/environment-provider-CfLUmKmf.js.map +1 -0
- package/dist/environment-provider-IUGU3epE.d.ts +964 -0
- package/dist/environment-provider.d.ts +2 -78
- package/dist/environment-provider.js +2 -18
- package/dist/errors-DEAvWQPy.js +111 -0
- package/dist/errors-DEAvWQPy.js.map +1 -0
- package/dist/improvement-adapter-Cr5TBSji.d.ts +166 -0
- package/dist/improvement-cycle-C095kS_g.js +2624 -0
- package/dist/improvement-cycle-C095kS_g.js.map +1 -0
- package/dist/index-BGdxAwfO.d.ts +838 -0
- package/dist/index-C-FYUuFG.d.ts +193 -0
- package/dist/index-CbNM40xG.d.ts +140 -0
- package/dist/index-DruSKs3Z.d.ts +7224 -0
- package/dist/index-Efjb3nrQ.d.ts +671 -0
- package/dist/index.d.ts +14 -1058
- package/dist/index.js +2106 -2082
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +768 -889
- package/dist/intelligence.js +1466 -1367
- package/dist/intelligence.js.map +1 -1
- package/dist/{kb-gate-CwHO0vz6.d.ts → kb-gate-C8z2juK8.d.ts} +33 -31
- package/dist/kb-gate-DpaSwXVx.js +80 -0
- package/dist/kb-gate-DpaSwXVx.js.map +1 -0
- package/dist/knowledge-Ku-mPP4f.js +433 -0
- package/dist/knowledge-Ku-mPP4f.js.map +1 -0
- package/dist/knowledge.d.ts +2 -150
- package/dist/knowledge.js +2 -40
- package/dist/local-harness-Dh8PJ0ot.d.ts +455 -0
- package/dist/loop-runner-bin-BXsFEUcf.d.ts +162 -0
- package/dist/loop-runner-bin-BwjQ1NfH.js +268 -0
- package/dist/loop-runner-bin-BwjQ1NfH.js.map +1 -0
- package/dist/loop-runner-bin.d.ts +2 -15
- package/dist/loop-runner-bin.js +2 -24
- package/dist/loops.d.ts +5 -4164
- package/dist/loops.js +8 -419
- package/dist/mcp/bin.d.ts +1 -1
- package/dist/mcp/bin.js +116 -146
- package/dist/mcp/bin.js.map +1 -1
- package/dist/mcp/index.d.ts +295 -436
- package/dist/mcp/index.js +1242 -1177
- package/dist/mcp/index.js.map +1 -1
- package/dist/mcp/memory-bin.d.ts +1 -1
- package/dist/mcp/memory-bin.js +37 -16
- package/dist/mcp/memory-bin.js.map +1 -1
- package/dist/memory-server-DL6cE2Ag.js +234 -0
- package/dist/memory-server-DL6cE2Ag.js.map +1 -0
- package/dist/model-resolution-Btd9iIKV.js +98 -0
- package/dist/model-resolution-Btd9iIKV.js.map +1 -0
- package/dist/openai-tools-B68JaOCx.d.ts +24 -0
- package/dist/openai-tools-CKLy1C7M.js +44 -0
- package/dist/openai-tools-CKLy1C7M.js.map +1 -0
- package/dist/otel-export-D34IQW1C.js +843 -0
- package/dist/otel-export-D34IQW1C.js.map +1 -0
- package/dist/platform.d.ts +188 -185
- package/dist/platform.js +185 -218
- package/dist/platform.js.map +1 -1
- package/dist/prepare-B53sPV7M.js +4292 -0
- package/dist/prepare-B53sPV7M.js.map +1 -0
- package/dist/primeintellect/index.d.ts +149 -145
- package/dist/primeintellect/index.js +626 -1003
- package/dist/primeintellect/index.js.map +1 -1
- package/dist/profiles.d.ts +296 -490
- package/dist/profiles.js +722 -749
- package/dist/profiles.js.map +1 -1
- package/dist/protected-model-port-BLPH5VVv.js +722 -0
- package/dist/protected-model-port-BLPH5VVv.js.map +1 -0
- package/dist/protected-model-port-BP6Z4eau.d.ts +283 -0
- package/dist/protected-redaction--F3v1oo8.js +200 -0
- package/dist/protected-redaction--F3v1oo8.js.map +1 -0
- package/dist/redact-D-u-rrcn.js +82 -0
- package/dist/redact-D-u-rrcn.js.map +1 -0
- package/dist/redact-kbIOjfjM.d.ts +452 -0
- package/dist/researcher-CoVqNhfI.js +433 -0
- package/dist/researcher-CoVqNhfI.js.map +1 -0
- package/dist/runtime-BG0BziaP.js +5408 -0
- package/dist/runtime-BG0BziaP.js.map +1 -0
- package/dist/sandbox-events-DeI5xX8P.js +316 -0
- package/dist/sandbox-events-DeI5xX8P.js.map +1 -0
- package/dist/spawn-journal-CwPvKUTa.js +456 -0
- package/dist/spawn-journal-CwPvKUTa.js.map +1 -0
- package/dist/structural-rollout-IXUEplky.js +1445 -0
- package/dist/structural-rollout-IXUEplky.js.map +1 -0
- package/dist/substrate-B0TYNrXn.js +27 -0
- package/dist/substrate-B0TYNrXn.js.map +1 -0
- package/dist/{substrate-Dyiqmd3o.d.ts → substrate-BcnuSHXm.d.ts} +57 -54
- package/dist/supervise-BiRutHS9.js +7321 -0
- package/dist/supervise-BiRutHS9.js.map +1 -0
- package/dist/supervisor-DTKhF-RV.js +3826 -0
- package/dist/supervisor-DTKhF-RV.js.map +1 -0
- package/dist/testing.d.ts +4 -3
- package/dist/testing.js +2478 -2637
- package/dist/testing.js.map +1 -1
- package/dist/tool-server-RcWgLIsL.js +141 -0
- package/dist/tool-server-RcWgLIsL.js.map +1 -0
- package/dist/trace-propagation-CJJC7SVB.js +57 -0
- package/dist/trace-propagation-CJJC7SVB.js.map +1 -0
- package/dist/types-BaIpb0XT.d.ts +492 -0
- package/dist/types-BevOjfTY.d.ts +765 -0
- package/dist/types-C9j4qg6l.d.ts +500 -0
- package/dist/types-Cja5OTGh.d.ts +190 -0
- package/dist/util-MVgdwuIS.js +145 -0
- package/dist/util-MVgdwuIS.js.map +1 -0
- package/dist/workspace-archive-BosQiotA.js +575 -0
- package/dist/workspace-archive-BosQiotA.js.map +1 -0
- package/package.json +43 -56
- package/skills/build-with-agent-runtime/SKILL.md +1 -0
- package/dist/activation-Cm-9MMX1.d.ts +0 -69
- package/dist/analyst-loop.js.map +0 -1
- package/dist/backends-BO4Jqoe2.d.ts +0 -134
- package/dist/candidate-execution/index.js.map +0 -1
- package/dist/chunk-2KGAN2HM.js +0 -1256
- package/dist/chunk-2KGAN2HM.js.map +0 -1
- package/dist/chunk-2VZYRB55.js +0 -834
- package/dist/chunk-2VZYRB55.js.map +0 -1
- package/dist/chunk-3MDZX7YU.js +0 -125
- package/dist/chunk-3MDZX7YU.js.map +0 -1
- package/dist/chunk-5IBQAPVB.js +0 -29
- package/dist/chunk-5IBQAPVB.js.map +0 -1
- package/dist/chunk-5QOB7H74.js +0 -387
- package/dist/chunk-5QOB7H74.js.map +0 -1
- package/dist/chunk-C5RTIBNZ.js +0 -48
- package/dist/chunk-C5RTIBNZ.js.map +0 -1
- package/dist/chunk-CTRA64LY.js +0 -680
- package/dist/chunk-CTRA64LY.js.map +0 -1
- package/dist/chunk-DPEUKJRO.js +0 -27
- package/dist/chunk-DPEUKJRO.js.map +0 -1
- package/dist/chunk-FD2MBMOH.js +0 -494
- package/dist/chunk-FD2MBMOH.js.map +0 -1
- package/dist/chunk-H5QPIZNX.js +0 -907
- package/dist/chunk-H5QPIZNX.js.map +0 -1
- package/dist/chunk-HLKC4UYB.js +0 -3514
- package/dist/chunk-HLKC4UYB.js.map +0 -1
- package/dist/chunk-HNP72PNU.js +0 -192
- package/dist/chunk-HNP72PNU.js.map +0 -1
- package/dist/chunk-ISPWRSEC.js +0 -408
- package/dist/chunk-ISPWRSEC.js.map +0 -1
- package/dist/chunk-KRBFHMV6.js +0 -1228
- package/dist/chunk-KRBFHMV6.js.map +0 -1
- package/dist/chunk-M22PL4GH.js +0 -171
- package/dist/chunk-M22PL4GH.js.map +0 -1
- package/dist/chunk-NURIQBJQ.js +0 -72
- package/dist/chunk-NURIQBJQ.js.map +0 -1
- package/dist/chunk-OPVWXJ2H.js +0 -75
- package/dist/chunk-OPVWXJ2H.js.map +0 -1
- package/dist/chunk-PZZKQVQV.js +0 -52
- package/dist/chunk-PZZKQVQV.js.map +0 -1
- package/dist/chunk-Q2JSAVQ3.js +0 -671
- package/dist/chunk-Q2JSAVQ3.js.map +0 -1
- package/dist/chunk-RDOAVVHY.js +0 -699
- package/dist/chunk-RDOAVVHY.js.map +0 -1
- package/dist/chunk-SGKPNBXE.js +0 -60
- package/dist/chunk-SGKPNBXE.js.map +0 -1
- package/dist/chunk-SMQXZGLZ.js +0 -351
- package/dist/chunk-SMQXZGLZ.js.map +0 -1
- package/dist/chunk-UHEZW5BU.js +0 -2918
- package/dist/chunk-UHEZW5BU.js.map +0 -1
- package/dist/chunk-UPWGXKXB.js +0 -254
- package/dist/chunk-UPWGXKXB.js.map +0 -1
- package/dist/chunk-VISA6CI3.js +0 -200
- package/dist/chunk-VISA6CI3.js.map +0 -1
- package/dist/chunk-WMTCUOQL.js +0 -474
- package/dist/chunk-WMTCUOQL.js.map +0 -1
- package/dist/chunk-WRTOVNN4.js +0 -4728
- package/dist/chunk-WRTOVNN4.js.map +0 -1
- package/dist/chunk-WSTRQZYQ.js +0 -5576
- package/dist/chunk-WSTRQZYQ.js.map +0 -1
- package/dist/chunk-XBG2W2VW.js +0 -6434
- package/dist/chunk-XBG2W2VW.js.map +0 -1
- package/dist/chunk-YEJR7IXO.js +0 -68
- package/dist/chunk-YEJR7IXO.js.map +0 -1
- package/dist/chunk-YJZA2BIK.js +0 -1338
- package/dist/chunk-YJZA2BIK.js.map +0 -1
- package/dist/completion-gate-BMy5LGoP.d.ts +0 -256
- package/dist/conversation.js.map +0 -1
- package/dist/coordination-BZZSVYpZ.d.ts +0 -2570
- package/dist/environment-provider.js.map +0 -1
- package/dist/improvement-adapter-HAZz-7vK.d.ts +0 -203
- package/dist/knowledge.js.map +0 -1
- package/dist/local-harness-BDNcl6jI.d.ts +0 -175
- package/dist/loop-runner-bin-68aoM9-7.d.ts +0 -199
- package/dist/loop-runner-bin.js.map +0 -1
- package/dist/loops.js.map +0 -1
- package/dist/openai-tools-fnj6SRVg.d.ts +0 -46
- package/dist/protected-model-port-B4t-OQjL.d.ts +0 -271
- package/dist/redact-BuFjxBUJ.d.ts +0 -481
- package/dist/sanitize-DEbPNtyI.d.ts +0 -328
- package/dist/structural-rollout-DEh6CEsa.d.ts +0 -446
- package/dist/supervise-4_48wwvy.d.ts +0 -379
- package/dist/types-BCoemcXU.d.ts +0 -802
- package/dist/types-BwoZWq-i.d.ts +0 -510
- package/dist/types-Cbp72qyu.d.ts +0 -491
- package/dist/types-CmYCMbFT.d.ts +0 -203
- package/dist/types-CvNwMzQt.d.ts +0 -957
- package/dist/worktree-fanout-DxhAWr5Z.d.ts +0 -999
package/dist/chunk-UHEZW5BU.js
DELETED
|
@@ -1,2918 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
agentCandidateProfileAsAgentProfile,
|
|
3
|
-
assertCandidateProfileBinding,
|
|
4
|
-
candidateMaterializerHarness,
|
|
5
|
-
canonicalCandidateBytes,
|
|
6
|
-
canonicalCandidateDigest,
|
|
7
|
-
canonicalCandidateDocument,
|
|
8
|
-
createAgentCandidateProfileActivation,
|
|
9
|
-
executePreparedAgentCandidate,
|
|
10
|
-
immutableCandidateValue,
|
|
11
|
-
omitTopLevelDigest,
|
|
12
|
-
omitUndefinedObjectFields,
|
|
13
|
-
parseAgentCandidateProfileActivation,
|
|
14
|
-
parseExactAgentProfile,
|
|
15
|
-
prepareAgentCandidateExecution,
|
|
16
|
-
verifiedResourceTextByDigest,
|
|
17
|
-
verifyAgentCandidateBundle,
|
|
18
|
-
verifyCanonicalCandidateDocument
|
|
19
|
-
} from "./chunk-WSTRQZYQ.js";
|
|
20
|
-
import {
|
|
21
|
-
defaultStructuralRolloutPolicy,
|
|
22
|
-
optimizerMethod
|
|
23
|
-
} from "./chunk-YJZA2BIK.js";
|
|
24
|
-
import {
|
|
25
|
-
harnessInvocation,
|
|
26
|
-
runLocalHarness,
|
|
27
|
-
runSettledCommand
|
|
28
|
-
} from "./chunk-HLKC4UYB.js";
|
|
29
|
-
import {
|
|
30
|
-
runAnalystLoop
|
|
31
|
-
} from "./chunk-FD2MBMOH.js";
|
|
32
|
-
import {
|
|
33
|
-
ConfigError
|
|
34
|
-
} from "./chunk-YEJR7IXO.js";
|
|
35
|
-
|
|
36
|
-
// src/improvement/agentic-generator.ts
|
|
37
|
-
import { spawnSync } from "child_process";
|
|
38
|
-
import { createHash } from "crypto";
|
|
39
|
-
import { existsSync, readFileSync, rmSync } from "fs";
|
|
40
|
-
import { join, resolve, sep } from "path";
|
|
41
|
-
import {
|
|
42
|
-
applyWorkspacePlan,
|
|
43
|
-
materializeProfile
|
|
44
|
-
} from "@tangle-network/agent-profile-materialize";
|
|
45
|
-
var RAW_TRACE_ANALYST_ID = "raw-trace-distiller";
|
|
46
|
-
var RAW_TRACE_AREA = "raw-trace-context";
|
|
47
|
-
var RAW_TRACE_DIAGNOSIS_PATH = ".improve/raw-trace-diagnosis.md";
|
|
48
|
-
var AGENTIC_PROFILE_RESOURCE_ROOT = ".agent-runtime-profile-resources";
|
|
49
|
-
function agenticGenerator(opts = {}) {
|
|
50
|
-
const harness = opts.harness ?? "claude";
|
|
51
|
-
if (opts.codexReproducible && harness !== "codex") {
|
|
52
|
-
throw new Error("agenticGenerator: codexReproducible requires harness 'codex'");
|
|
53
|
-
}
|
|
54
|
-
if (opts.codexReproducible && !opts.profile) {
|
|
55
|
-
throw new Error("agenticGenerator: codexReproducible requires an explicit author profile");
|
|
56
|
-
}
|
|
57
|
-
if (opts.codexReadDeniedPaths && !opts.codexReproducible) {
|
|
58
|
-
throw new Error("agenticGenerator: codexReadDeniedPaths requires codexReproducible");
|
|
59
|
-
}
|
|
60
|
-
if (opts.maximumCharge && !opts.codexReproducible) {
|
|
61
|
-
throw new Error("agenticGenerator: maximumCharge requires codexReproducible");
|
|
62
|
-
}
|
|
63
|
-
const profileResourcePlan = opts.codexReproducible ? authorProfileResourcePlan(opts.profile) : null;
|
|
64
|
-
const buildPrompt = opts.buildPrompt ?? defaultBuildPrompt;
|
|
65
|
-
const run = opts.runHarness ?? runLocalHarness;
|
|
66
|
-
const dirty = opts.isDirty ?? worktreeDirty;
|
|
67
|
-
const verify = opts.verify;
|
|
68
|
-
return {
|
|
69
|
-
kind: `agentic:${harness}`,
|
|
70
|
-
// The seed repo + (in rawTraceContext mode) the raw-trace filesystem context
|
|
71
|
-
// are the change signal — an agentic coder proposes from them even when the
|
|
72
|
-
// distiller yielded zero findings. Without this, the code candidate driver's
|
|
73
|
-
// empty-findings guard short-circuits and generates ZERO candidates on the
|
|
74
|
-
// first (and, for a single-generation run, only) proposal round.
|
|
75
|
-
proposesWithoutFindings: true,
|
|
76
|
-
async generate({
|
|
77
|
-
worktreePath,
|
|
78
|
-
report,
|
|
79
|
-
findings,
|
|
80
|
-
maxShots,
|
|
81
|
-
signal,
|
|
82
|
-
generation,
|
|
83
|
-
candidateIndex,
|
|
84
|
-
costLedger,
|
|
85
|
-
costPhase
|
|
86
|
-
}) {
|
|
87
|
-
signal.throwIfAborted();
|
|
88
|
-
let reproducibleCostLedger;
|
|
89
|
-
if (opts.codexReproducible) {
|
|
90
|
-
if (!costLedger) {
|
|
91
|
-
throw new Error(
|
|
92
|
-
"agenticGenerator: reproducible Codex requires the run-wide CostLedger supplied by agent-eval"
|
|
93
|
-
);
|
|
94
|
-
}
|
|
95
|
-
reproducibleCostLedger = costLedger;
|
|
96
|
-
}
|
|
97
|
-
const basePrompt = appendProfileResourcePaths(
|
|
98
|
-
buildPrompt({ report, findings }),
|
|
99
|
-
profileResourcePlan
|
|
100
|
-
);
|
|
101
|
-
const needsRawTraceEvidence = requiresRawTraceEvidence(findings);
|
|
102
|
-
const shots = Math.max(1, maxShots);
|
|
103
|
-
let attemptNote = "";
|
|
104
|
-
for (let shot = 0; shot < shots; shot++) {
|
|
105
|
-
signal.throwIfAborted();
|
|
106
|
-
const taskPrompt = attemptNote ? `${basePrompt}
|
|
107
|
-
|
|
108
|
-
${attemptNote}` : basePrompt;
|
|
109
|
-
const invocation = opts.profile ? harnessInvocation(harness, opts.profile, taskPrompt, {
|
|
110
|
-
dangerouslySkipPermissions: harness === "claude",
|
|
111
|
-
...opts.codexReproducible ? { codexReproducible: true } : {}
|
|
112
|
-
}) : void 0;
|
|
113
|
-
const exactPrompt = invocation?.prompt ?? taskPrompt;
|
|
114
|
-
const readDeniedPaths = typeof opts.codexReadDeniedPaths === "function" ? opts.codexReadDeniedPaths(worktreePath) : opts.codexReadDeniedPaths;
|
|
115
|
-
const startedAt = /* @__PURE__ */ new Date();
|
|
116
|
-
let harnessResult = null;
|
|
117
|
-
let profileWorkspaceReceipt = null;
|
|
118
|
-
let costReceipt = null;
|
|
119
|
-
let costCallId = null;
|
|
120
|
-
let shotError = null;
|
|
121
|
-
try {
|
|
122
|
-
const execute = async (executionSignal) => {
|
|
123
|
-
harnessResult = await withAuthorProfileResources(
|
|
124
|
-
profileResourcePlan,
|
|
125
|
-
worktreePath,
|
|
126
|
-
async (receipt2) => {
|
|
127
|
-
profileWorkspaceReceipt = receipt2;
|
|
128
|
-
const result2 = await run({
|
|
129
|
-
harness,
|
|
130
|
-
cwd: worktreePath,
|
|
131
|
-
taskPrompt,
|
|
132
|
-
...invocation ? { invocation: { command: invocation.command, args: invocation.args } } : {},
|
|
133
|
-
// The candidate worktree is isolated and must be editable without an
|
|
134
|
-
// interactive permission prompt. Other runLocalHarness callers remain
|
|
135
|
-
// permission-safe by default.
|
|
136
|
-
dangerouslySkipPermissions: harness === "claude",
|
|
137
|
-
...opts.codexReproducible ? { codexReproducible: true } : {},
|
|
138
|
-
...readDeniedPaths ? { codexReadDeniedPaths: readDeniedPaths } : {},
|
|
139
|
-
timeoutMs: opts.timeoutMs,
|
|
140
|
-
signal: executionSignal
|
|
141
|
-
});
|
|
142
|
-
harnessResult = result2;
|
|
143
|
-
return result2;
|
|
144
|
-
}
|
|
145
|
-
);
|
|
146
|
-
const failure = shotFailure(harnessResult, exactPrompt, opts.codexReproducible === true);
|
|
147
|
-
if (failure) throw failure;
|
|
148
|
-
return harnessResult;
|
|
149
|
-
};
|
|
150
|
-
if (reproducibleCostLedger) {
|
|
151
|
-
const model = opts.profile?.model?.default;
|
|
152
|
-
if (!model) {
|
|
153
|
-
throw new Error("agenticGenerator: reproducible Codex requires profile.model.default");
|
|
154
|
-
}
|
|
155
|
-
const paid = await reproducibleCostLedger.runPaidCall({
|
|
156
|
-
channel: "driver",
|
|
157
|
-
phase: costPhase ?? "search.proposal",
|
|
158
|
-
actor: `agentic-generator:${harness}`,
|
|
159
|
-
model,
|
|
160
|
-
tags: {
|
|
161
|
-
generation: String(generation ?? -1),
|
|
162
|
-
candidateIndex: String(candidateIndex ?? -1),
|
|
163
|
-
shot: String(shot + 1)
|
|
164
|
-
},
|
|
165
|
-
signal,
|
|
166
|
-
...opts.maximumCharge ? { maximumCharge: opts.maximumCharge } : {},
|
|
167
|
-
execute,
|
|
168
|
-
receipt: (result2) => costReceiptFromHarness(result2, model),
|
|
169
|
-
receiptFromError: () => harnessResult?.usage ? costReceiptFromHarness(harnessResult, model) : void 0
|
|
170
|
-
});
|
|
171
|
-
costCallId = paid.callId ?? null;
|
|
172
|
-
costReceipt = paid.receipt ?? null;
|
|
173
|
-
if (!paid.succeeded) throw paid.error;
|
|
174
|
-
harnessResult = paid.value;
|
|
175
|
-
} else {
|
|
176
|
-
harnessResult = await execute(signal);
|
|
177
|
-
}
|
|
178
|
-
} catch (cause) {
|
|
179
|
-
shotError = cause instanceof Error ? cause : new Error(String(cause));
|
|
180
|
-
}
|
|
181
|
-
const execution = shotExecutionSnapshot(harnessResult);
|
|
182
|
-
const receipt = shotReceipt({
|
|
183
|
-
generation,
|
|
184
|
-
candidateIndex,
|
|
185
|
-
shot,
|
|
186
|
-
maxShots: shots,
|
|
187
|
-
harness,
|
|
188
|
-
profile: opts.profile,
|
|
189
|
-
prompt: exactPrompt,
|
|
190
|
-
startedAt,
|
|
191
|
-
completedAt: /* @__PURE__ */ new Date(),
|
|
192
|
-
result: execution,
|
|
193
|
-
profileWorkspaceReceipt,
|
|
194
|
-
costCallId,
|
|
195
|
-
costReceipt,
|
|
196
|
-
error: shotError
|
|
197
|
-
});
|
|
198
|
-
await emitShotReceipt(opts.onShotCompleted, receipt, execution, shotError);
|
|
199
|
-
signal.throwIfAborted();
|
|
200
|
-
if (!execution) {
|
|
201
|
-
throw new Error("agenticGenerator: author shot completed without a harness result");
|
|
202
|
-
}
|
|
203
|
-
let worktreeChanged;
|
|
204
|
-
try {
|
|
205
|
-
worktreeChanged = dirty(worktreePath);
|
|
206
|
-
} catch (cause) {
|
|
207
|
-
signal.throwIfAborted();
|
|
208
|
-
return rethrowShotSetupError(
|
|
209
|
-
opts.onShotDisposition,
|
|
210
|
-
receipt,
|
|
211
|
-
worktreePath,
|
|
212
|
-
"worktree-inspection",
|
|
213
|
-
cause
|
|
214
|
-
);
|
|
215
|
-
}
|
|
216
|
-
if (!worktreeChanged) {
|
|
217
|
-
signal.throwIfAborted();
|
|
218
|
-
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
219
|
-
kind: "clean",
|
|
220
|
-
worktreePath
|
|
221
|
-
});
|
|
222
|
-
signal.throwIfAborted();
|
|
223
|
-
attemptNote = EMPTY_TREE_NOTE;
|
|
224
|
-
continue;
|
|
225
|
-
}
|
|
226
|
-
if (needsRawTraceEvidence) {
|
|
227
|
-
let problem;
|
|
228
|
-
try {
|
|
229
|
-
problem = rawTraceEvidenceProblem(worktreePath, findings);
|
|
230
|
-
} catch (cause) {
|
|
231
|
-
signal.throwIfAborted();
|
|
232
|
-
return rethrowShotSetupError(
|
|
233
|
-
opts.onShotDisposition,
|
|
234
|
-
receipt,
|
|
235
|
-
worktreePath,
|
|
236
|
-
"raw-trace-evidence",
|
|
237
|
-
cause
|
|
238
|
-
);
|
|
239
|
-
}
|
|
240
|
-
if (problem) {
|
|
241
|
-
signal.throwIfAborted();
|
|
242
|
-
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
243
|
-
kind: "rejected",
|
|
244
|
-
worktreePath,
|
|
245
|
-
stage: "raw-trace-evidence",
|
|
246
|
-
feedback: problem
|
|
247
|
-
});
|
|
248
|
-
signal.throwIfAborted();
|
|
249
|
-
attemptNote = problem;
|
|
250
|
-
continue;
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
if (!verify) {
|
|
254
|
-
signal.throwIfAborted();
|
|
255
|
-
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
256
|
-
kind: "accepted",
|
|
257
|
-
worktreePath,
|
|
258
|
-
verified: false
|
|
259
|
-
});
|
|
260
|
-
signal.throwIfAborted();
|
|
261
|
-
return acceptedCandidate(findings);
|
|
262
|
-
}
|
|
263
|
-
let result;
|
|
264
|
-
try {
|
|
265
|
-
signal.throwIfAborted();
|
|
266
|
-
result = await verify(worktreePath, signal);
|
|
267
|
-
signal.throwIfAborted();
|
|
268
|
-
} catch (cause) {
|
|
269
|
-
signal.throwIfAborted();
|
|
270
|
-
return rethrowShotSetupError(
|
|
271
|
-
opts.onShotDisposition,
|
|
272
|
-
receipt,
|
|
273
|
-
worktreePath,
|
|
274
|
-
"verification",
|
|
275
|
-
cause
|
|
276
|
-
);
|
|
277
|
-
}
|
|
278
|
-
if (result.ok) {
|
|
279
|
-
signal.throwIfAborted();
|
|
280
|
-
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
281
|
-
kind: "accepted",
|
|
282
|
-
worktreePath,
|
|
283
|
-
verified: true
|
|
284
|
-
});
|
|
285
|
-
signal.throwIfAborted();
|
|
286
|
-
return acceptedCandidate(findings);
|
|
287
|
-
}
|
|
288
|
-
signal.throwIfAborted();
|
|
289
|
-
await emitShotDisposition(opts.onShotDisposition, receipt, {
|
|
290
|
-
kind: "rejected",
|
|
291
|
-
worktreePath,
|
|
292
|
-
stage: "verification",
|
|
293
|
-
feedback: result.feedback ?? null
|
|
294
|
-
});
|
|
295
|
-
signal.throwIfAborted();
|
|
296
|
-
attemptNote = failureNote(result.feedback);
|
|
297
|
-
}
|
|
298
|
-
return { applied: false, summary: "" };
|
|
299
|
-
}
|
|
300
|
-
};
|
|
301
|
-
}
|
|
302
|
-
function authorProfileResourcePlan(profile) {
|
|
303
|
-
const resources = profile.resources;
|
|
304
|
-
if (!resources) return null;
|
|
305
|
-
const unsupportedKinds = [
|
|
306
|
-
resources.tools?.length ? "tools" : null,
|
|
307
|
-
resources.skills?.length ? "skills" : null,
|
|
308
|
-
resources.agents?.length ? "agents" : null
|
|
309
|
-
].filter((kind) => kind !== null);
|
|
310
|
-
if (unsupportedKinds.length > 0) {
|
|
311
|
-
throw new Error(
|
|
312
|
-
`agenticGenerator: reproducible Codex author resources support files only; unsupported: ${unsupportedKinds.join(", ")}`
|
|
313
|
-
);
|
|
314
|
-
}
|
|
315
|
-
if (!resources.files || resources.files.length === 0) return null;
|
|
316
|
-
const plan = materializeProfile(
|
|
317
|
-
{
|
|
318
|
-
name: profile.name,
|
|
319
|
-
resources: { files: resources.files }
|
|
320
|
-
},
|
|
321
|
-
"codex"
|
|
322
|
-
);
|
|
323
|
-
if (plan.unsupported.length > 0) {
|
|
324
|
-
throw new Error(
|
|
325
|
-
`agenticGenerator: author profile files could not be materialized: ${plan.unsupported.map((item) => item.reason).join("; ")}`
|
|
326
|
-
);
|
|
327
|
-
}
|
|
328
|
-
if (Object.keys(plan.env).length > 0 || plan.flags.length > 0) {
|
|
329
|
-
throw new Error(
|
|
330
|
-
"agenticGenerator: generic author profile files unexpectedly changed spawn values"
|
|
331
|
-
);
|
|
332
|
-
}
|
|
333
|
-
const virtualRoot = resolve("/", AGENTIC_PROFILE_RESOURCE_ROOT);
|
|
334
|
-
const seen = /* @__PURE__ */ new Set();
|
|
335
|
-
for (const file of plan.files) {
|
|
336
|
-
const target = resolve("/", file.relPath);
|
|
337
|
-
if (!target.startsWith(`${virtualRoot}${sep}`)) {
|
|
338
|
-
throw new Error(
|
|
339
|
-
`agenticGenerator: author profile file must be below ${AGENTIC_PROFILE_RESOURCE_ROOT}: ${file.relPath}`
|
|
340
|
-
);
|
|
341
|
-
}
|
|
342
|
-
if (seen.has(target)) {
|
|
343
|
-
throw new Error(`agenticGenerator: duplicate author profile file path: ${file.relPath}`);
|
|
344
|
-
}
|
|
345
|
-
seen.add(target);
|
|
346
|
-
}
|
|
347
|
-
return plan;
|
|
348
|
-
}
|
|
349
|
-
function appendProfileResourcePaths(prompt, plan) {
|
|
350
|
-
if (!plan) return prompt;
|
|
351
|
-
return [
|
|
352
|
-
prompt,
|
|
353
|
-
"",
|
|
354
|
-
"Profile resource files available for this shot:",
|
|
355
|
-
...plan.files.map((file) => `- ${file.relPath}`)
|
|
356
|
-
].join("\n");
|
|
357
|
-
}
|
|
358
|
-
async function withAuthorProfileResources(plan, worktreePath, run) {
|
|
359
|
-
if (!plan) return run(null);
|
|
360
|
-
const rootPath = resolve(worktreePath, AGENTIC_PROFILE_RESOURCE_ROOT);
|
|
361
|
-
if (existsSync(rootPath)) {
|
|
362
|
-
throw new Error(
|
|
363
|
-
`agenticGenerator: ephemeral author profile root already exists: ${AGENTIC_PROFILE_RESOURCE_ROOT}`
|
|
364
|
-
);
|
|
365
|
-
}
|
|
366
|
-
let value;
|
|
367
|
-
let primaryError;
|
|
368
|
-
try {
|
|
369
|
-
const receipt = applyWorkspacePlan(plan, worktreePath);
|
|
370
|
-
value = await run(receipt);
|
|
371
|
-
} catch (cause) {
|
|
372
|
-
primaryError = cause;
|
|
373
|
-
}
|
|
374
|
-
let cleanupError;
|
|
375
|
-
try {
|
|
376
|
-
rmSync(rootPath, { recursive: true, force: true });
|
|
377
|
-
if (existsSync(rootPath)) {
|
|
378
|
-
throw new Error(
|
|
379
|
-
`agenticGenerator: ephemeral author profile root survived cleanup: ${AGENTIC_PROFILE_RESOURCE_ROOT}`
|
|
380
|
-
);
|
|
381
|
-
}
|
|
382
|
-
} catch (cause) {
|
|
383
|
-
cleanupError = cause;
|
|
384
|
-
}
|
|
385
|
-
if (primaryError !== void 0 && cleanupError !== void 0) {
|
|
386
|
-
throw new AggregateError(
|
|
387
|
-
[primaryError, cleanupError],
|
|
388
|
-
"agenticGenerator: author shot and profile resource cleanup both failed"
|
|
389
|
-
);
|
|
390
|
-
}
|
|
391
|
-
if (primaryError !== void 0) throw primaryError;
|
|
392
|
-
if (cleanupError !== void 0) throw cleanupError;
|
|
393
|
-
return value;
|
|
394
|
-
}
|
|
395
|
-
async function emitShotReceipt(callback, receipt, execution, primaryError) {
|
|
396
|
-
try {
|
|
397
|
-
await callback?.(receipt, execution);
|
|
398
|
-
} catch (callbackError) {
|
|
399
|
-
if (primaryError !== null) {
|
|
400
|
-
throw new AggregateError(
|
|
401
|
-
[primaryError, callbackError],
|
|
402
|
-
"agenticGenerator: author shot failed and its receipt could not be persisted"
|
|
403
|
-
);
|
|
404
|
-
}
|
|
405
|
-
throw callbackError;
|
|
406
|
-
}
|
|
407
|
-
if (primaryError !== null) throw primaryError;
|
|
408
|
-
}
|
|
409
|
-
async function emitShotDisposition(callback, receipt, disposition) {
|
|
410
|
-
await callback?.(receipt, disposition);
|
|
411
|
-
}
|
|
412
|
-
async function rethrowShotSetupError(callback, receipt, worktreePath, stage, cause) {
|
|
413
|
-
const error = cause instanceof Error ? cause : new Error(String(cause));
|
|
414
|
-
try {
|
|
415
|
-
await emitShotDisposition(callback, receipt, {
|
|
416
|
-
kind: "setup-error",
|
|
417
|
-
worktreePath,
|
|
418
|
-
stage,
|
|
419
|
-
error: { name: error.name, message: error.message }
|
|
420
|
-
});
|
|
421
|
-
} catch (callbackError) {
|
|
422
|
-
throw new AggregateError(
|
|
423
|
-
[cause, callbackError],
|
|
424
|
-
"agenticGenerator: shot processing failed and its worktree disposition could not be persisted"
|
|
425
|
-
);
|
|
426
|
-
}
|
|
427
|
-
throw cause;
|
|
428
|
-
}
|
|
429
|
-
function shotExecutionSnapshot(result) {
|
|
430
|
-
if (result === null) return null;
|
|
431
|
-
const usage = result.usage ? Object.freeze({ ...result.usage }) : void 0;
|
|
432
|
-
const evidence = result.evidence ? Object.freeze({
|
|
433
|
-
...result.evidence,
|
|
434
|
-
readDeniedPaths: Object.freeze([...result.evidence.readDeniedPaths]),
|
|
435
|
-
policy: Object.freeze({ ...result.evidence.policy })
|
|
436
|
-
}) : void 0;
|
|
437
|
-
return Object.freeze({
|
|
438
|
-
exitCode: result.exitCode,
|
|
439
|
-
stdout: result.stdout,
|
|
440
|
-
stderr: result.stderr,
|
|
441
|
-
killedBySignal: result.killedBySignal,
|
|
442
|
-
durationMs: result.durationMs,
|
|
443
|
-
timedOut: result.timedOut,
|
|
444
|
-
...result.aborted !== void 0 ? { aborted: result.aborted } : {},
|
|
445
|
-
...usage ? { usage } : {},
|
|
446
|
-
...evidence ? { evidence } : {}
|
|
447
|
-
});
|
|
448
|
-
}
|
|
449
|
-
function shotReceipt(input) {
|
|
450
|
-
const error = input.error ? {
|
|
451
|
-
name: input.error instanceof Error ? input.error.name : "Error",
|
|
452
|
-
message: input.error instanceof Error ? input.error.message : String(input.error)
|
|
453
|
-
} : null;
|
|
454
|
-
const result = input.result;
|
|
455
|
-
const costBasis = costBasisFor(input.costReceipt);
|
|
456
|
-
return {
|
|
457
|
-
generation: input.generation ?? null,
|
|
458
|
-
candidateIndex: input.candidateIndex ?? null,
|
|
459
|
-
shot: input.shot + 1,
|
|
460
|
-
maxShots: input.maxShots,
|
|
461
|
-
harness: input.harness,
|
|
462
|
-
model: input.profile?.model?.default ?? null,
|
|
463
|
-
reasoningEffort: input.profile?.model?.reasoningEffort ?? null,
|
|
464
|
-
promptSha256: sha256(input.prompt),
|
|
465
|
-
startedAt: input.startedAt.toISOString(),
|
|
466
|
-
completedAt: input.completedAt.toISOString(),
|
|
467
|
-
durationMs: result?.durationMs ?? input.completedAt.getTime() - input.startedAt.getTime(),
|
|
468
|
-
exitCode: result?.exitCode ?? null,
|
|
469
|
-
timedOut: result?.timedOut ?? false,
|
|
470
|
-
aborted: result?.aborted ?? false,
|
|
471
|
-
killedBySignal: result?.killedBySignal ?? null,
|
|
472
|
-
stdoutBytes: result ? Buffer.byteLength(result.stdout) : null,
|
|
473
|
-
stdoutSha256: result ? sha256(result.stdout) : null,
|
|
474
|
-
stderrBytes: result ? Buffer.byteLength(result.stderr) : null,
|
|
475
|
-
stderrSha256: result ? sha256(result.stderr) : null,
|
|
476
|
-
usage: result?.usage ? { ...result.usage } : null,
|
|
477
|
-
profileWorkspacePlanDigest: input.profileWorkspaceReceipt?.workspacePlanDigest ?? null,
|
|
478
|
-
profileWorkspaceFileCount: input.profileWorkspaceReceipt?.written.length ?? 0,
|
|
479
|
-
costCallId: input.costCallId,
|
|
480
|
-
costBasis,
|
|
481
|
-
costUsd: costBasis === "unknown" ? null : input.costReceipt?.costUsd ?? null,
|
|
482
|
-
costUsdKnown: costBasis === "provider-reported",
|
|
483
|
-
evidence: result?.evidence ? {
|
|
484
|
-
...result.evidence,
|
|
485
|
-
readDeniedPaths: [...result.evidence.readDeniedPaths],
|
|
486
|
-
policy: { ...result.evidence.policy }
|
|
487
|
-
} : null,
|
|
488
|
-
error
|
|
489
|
-
};
|
|
490
|
-
}
|
|
491
|
-
function costBasisFor(receipt) {
|
|
492
|
-
if (receipt === null || receipt.costUnknown) return "unknown";
|
|
493
|
-
return receipt.actualCostUsd === void 0 ? "estimated-pricing" : "provider-reported";
|
|
494
|
-
}
|
|
495
|
-
function shotFailure(result, exactPrompt, codexReproducible) {
|
|
496
|
-
if (result.timedOut) {
|
|
497
|
-
return new Error("agenticGenerator: author shot timed out");
|
|
498
|
-
}
|
|
499
|
-
if (result.aborted) {
|
|
500
|
-
return new Error("agenticGenerator: author shot was cancelled by the caller");
|
|
501
|
-
}
|
|
502
|
-
if (result.killedBySignal) {
|
|
503
|
-
return new Error(`agenticGenerator: author shot was killed by ${result.killedBySignal}`);
|
|
504
|
-
}
|
|
505
|
-
if (result.exitCode !== 0) {
|
|
506
|
-
return new Error(`agenticGenerator: author shot exited with code ${String(result.exitCode)}`);
|
|
507
|
-
}
|
|
508
|
-
if (!codexReproducible) return null;
|
|
509
|
-
if (!result.usage || !result.evidence) {
|
|
510
|
-
return new Error(
|
|
511
|
-
"agenticGenerator: reproducible Codex shot completed without usage or execution evidence"
|
|
512
|
-
);
|
|
513
|
-
}
|
|
514
|
-
const expectedPromptSha256 = sha256(exactPrompt).slice("sha256:".length);
|
|
515
|
-
if (result.evidence.requestedPromptSha256 !== expectedPromptSha256) {
|
|
516
|
-
return new Error(
|
|
517
|
-
"agenticGenerator: reproducible Codex prompt evidence does not match the exact authored prompt"
|
|
518
|
-
);
|
|
519
|
-
}
|
|
520
|
-
return null;
|
|
521
|
-
}
|
|
522
|
-
function costReceiptFromHarness(result, model) {
|
|
523
|
-
if (!result.usage) {
|
|
524
|
-
throw new Error("agenticGenerator: author shot did not report terminal token usage");
|
|
525
|
-
}
|
|
526
|
-
return {
|
|
527
|
-
model,
|
|
528
|
-
inputTokens: result.usage.inputTokens - result.usage.cachedInputTokens,
|
|
529
|
-
outputTokens: result.usage.outputTokens,
|
|
530
|
-
...result.usage.cachedInputTokens > 0 ? { cachedTokens: result.usage.cachedInputTokens } : {}
|
|
531
|
-
};
|
|
532
|
-
}
|
|
533
|
-
function sha256(value) {
|
|
534
|
-
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
535
|
-
}
|
|
536
|
-
function defaultBuildPrompt(args) {
|
|
537
|
-
const lines = [
|
|
538
|
-
"You are improving this codebase based on an evaluation analysis: real runs failed, an",
|
|
539
|
-
"analyst distilled the findings below, and your change will be measured on held-out tasks",
|
|
540
|
-
"against the unchanged baseline \u2014 only a real lift promotes it.",
|
|
541
|
-
"",
|
|
542
|
-
optimizerMethod,
|
|
543
|
-
"",
|
|
544
|
-
"THE SURFACE \u2014 what a deliverable change looks like here:",
|
|
545
|
-
"- edit this codebase in place: the smallest coherent change set that fully tests your",
|
|
546
|
-
" hypothesis about the dominant failure mode (see the method above \u2014 no unrelated edits,",
|
|
547
|
-
" they confound the measurement),",
|
|
548
|
-
"- keep the diff reviewable: a reviewer should be able to trace every hunk back to a finding,",
|
|
549
|
-
"- do not commit \u2014 leave changes in the working tree.",
|
|
550
|
-
"",
|
|
551
|
-
"FINDINGS \u2014 ranked evidence from real failed runs:"
|
|
552
|
-
];
|
|
553
|
-
for (const f of args.findings) {
|
|
554
|
-
const where = f.subject ? ` [${f.subject}]` : "";
|
|
555
|
-
lines.push(`- (${f.severity})${where} ${f.claim}`);
|
|
556
|
-
if (f.recommended_action) lines.push(` \u2192 ${f.recommended_action}`);
|
|
557
|
-
}
|
|
558
|
-
if (requiresRawTraceEvidence(args.findings)) {
|
|
559
|
-
lines.push(
|
|
560
|
-
"",
|
|
561
|
-
"Raw trace evidence requirement:",
|
|
562
|
-
`- Inspect at least one raw trace path named above before editing.`,
|
|
563
|
-
`- Write ${RAW_TRACE_DIAGNOSIS_PATH} in this worktree.`,
|
|
564
|
-
"- Include the exact trace path(s) inspected, the failure mechanism, and the code change made.",
|
|
565
|
-
"- A candidate without this file, or with only this file changed, is discarded."
|
|
566
|
-
);
|
|
567
|
-
}
|
|
568
|
-
return lines.join("\n");
|
|
569
|
-
}
|
|
570
|
-
var EMPTY_TREE_NOTE = "NOTE: your previous attempt left the working tree unchanged. Make the concrete file edits now.";
|
|
571
|
-
function failureNote(feedback) {
|
|
572
|
-
const detail = feedback?.trim();
|
|
573
|
-
return [
|
|
574
|
-
"NOTE: your edits are in the working tree but verification FAILED.",
|
|
575
|
-
"Fix the problem in place \u2014 build on your existing edits, do not revert them.",
|
|
576
|
-
detail ? `Verifier output:
|
|
577
|
-
${truncate(detail, 4e3)}` : "No verifier detail was captured."
|
|
578
|
-
].join("\n");
|
|
579
|
-
}
|
|
580
|
-
function rawTraceEvidenceProblem(worktreePath, findings) {
|
|
581
|
-
const changedPaths = worktreeChangedPaths(worktreePath);
|
|
582
|
-
const substantive = changedPaths.filter((path) => path !== RAW_TRACE_DIAGNOSIS_PATH);
|
|
583
|
-
if (substantive.length === 0) {
|
|
584
|
-
return [
|
|
585
|
-
`NOTE: raw-trace mode requires a real code/config edit in addition to ${RAW_TRACE_DIAGNOSIS_PATH}.`,
|
|
586
|
-
"Your previous attempt only changed the diagnosis artifact. Inspect the cited traces and make the causal code change."
|
|
587
|
-
].join("\n");
|
|
588
|
-
}
|
|
589
|
-
const diagnosisPath = join(worktreePath, RAW_TRACE_DIAGNOSIS_PATH);
|
|
590
|
-
if (!existsSync(diagnosisPath)) {
|
|
591
|
-
return [
|
|
592
|
-
`NOTE: raw-trace mode requires ${RAW_TRACE_DIAGNOSIS_PATH}.`,
|
|
593
|
-
"Before retrying, inspect at least one cited spans.jsonl/cached-result.json/artifact path, then write the diagnosis file with the exact path, failure mechanism, and code change."
|
|
594
|
-
].join("\n");
|
|
595
|
-
}
|
|
596
|
-
const body = readFileSync(diagnosisPath, "utf8");
|
|
597
|
-
const evidencePaths = traceEvidencePaths(findings);
|
|
598
|
-
if (evidencePaths.length > 0 && !evidencePaths.some((path) => body.includes(path))) {
|
|
599
|
-
return [
|
|
600
|
-
`${RAW_TRACE_DIAGNOSIS_PATH} exists, but it does not cite any exact raw trace path from the findings.`,
|
|
601
|
-
`Cite at least one of these inspected paths exactly: ${evidencePaths.slice(0, 5).join(", ")}`
|
|
602
|
-
].join("\n");
|
|
603
|
-
}
|
|
604
|
-
return null;
|
|
605
|
-
}
|
|
606
|
-
function requiresRawTraceEvidence(findings) {
|
|
607
|
-
return findings.some((finding) => {
|
|
608
|
-
const f = finding;
|
|
609
|
-
return f.analyst_id === RAW_TRACE_ANALYST_ID || f.area === RAW_TRACE_AREA;
|
|
610
|
-
});
|
|
611
|
-
}
|
|
612
|
-
function traceEvidencePaths(findings) {
|
|
613
|
-
const out = [];
|
|
614
|
-
for (const finding of findings) {
|
|
615
|
-
const refs = finding.evidence_refs;
|
|
616
|
-
if (!Array.isArray(refs)) continue;
|
|
617
|
-
for (const ref of refs) {
|
|
618
|
-
if (!ref || typeof ref !== "object") continue;
|
|
619
|
-
const uri = ref.uri;
|
|
620
|
-
if (typeof uri === "string" && uri.length > 0) out.push(uri);
|
|
621
|
-
}
|
|
622
|
-
}
|
|
623
|
-
return [...new Set(out)];
|
|
624
|
-
}
|
|
625
|
-
function commandVerifier(command, args = [], timeoutMs = 3e5) {
|
|
626
|
-
return async (worktreePath, signal) => {
|
|
627
|
-
let result;
|
|
628
|
-
try {
|
|
629
|
-
result = await runSettledCommand({
|
|
630
|
-
command,
|
|
631
|
-
args,
|
|
632
|
-
cwd: worktreePath,
|
|
633
|
-
timeoutMs,
|
|
634
|
-
...signal ? { signal } : {}
|
|
635
|
-
});
|
|
636
|
-
} catch (err) {
|
|
637
|
-
signal?.throwIfAborted();
|
|
638
|
-
const code = err.code;
|
|
639
|
-
if (code === "ENOENT") {
|
|
640
|
-
throw new Error(
|
|
641
|
-
`commandVerifier: '${command}' not found in PATH (setup bug, not a failed candidate)`
|
|
642
|
-
);
|
|
643
|
-
}
|
|
644
|
-
const reason = err instanceof Error ? err.message : String(err);
|
|
645
|
-
throw new Error(`commandVerifier: '${command}' failed to spawn: ${reason}`);
|
|
646
|
-
}
|
|
647
|
-
if (result.timedOut || result.killedBySignal) {
|
|
648
|
-
return {
|
|
649
|
-
ok: false,
|
|
650
|
-
feedback: `verifier '${command}' ${result.killedBySignal ? `killed by ${result.killedBySignal}` : "timed out"} after ${timeoutMs}ms`
|
|
651
|
-
};
|
|
652
|
-
}
|
|
653
|
-
if (result.exitCode === 0) return { ok: true };
|
|
654
|
-
const out = `${result.stdout}${result.stderr}`.trim();
|
|
655
|
-
return { ok: false, feedback: out.length > 0 ? out : `exit ${result.exitCode}` };
|
|
656
|
-
};
|
|
657
|
-
}
|
|
658
|
-
function summarizeFindings(findings) {
|
|
659
|
-
if (findings.length === 0) return "agentic improvement";
|
|
660
|
-
if (findings.length === 1) return `agentic: ${truncate(findings[0].claim, 64)}`;
|
|
661
|
-
return `agentic: ${findings.length} findings addressed`;
|
|
662
|
-
}
|
|
663
|
-
function acceptedCandidate(findings) {
|
|
664
|
-
const summary = summarizeFindings(findings);
|
|
665
|
-
return {
|
|
666
|
-
applied: true,
|
|
667
|
-
summary,
|
|
668
|
-
label: slugify(summary.split("\n", 1)[0] ?? summary),
|
|
669
|
-
rationale: boundedRationale(summary, findings)
|
|
670
|
-
};
|
|
671
|
-
}
|
|
672
|
-
function slugify(line) {
|
|
673
|
-
const slug = line.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
674
|
-
return truncate(slug || "agentic-improvement", 48);
|
|
675
|
-
}
|
|
676
|
-
function boundedRationale(summary, findings) {
|
|
677
|
-
if (findings.length === 0) return summary;
|
|
678
|
-
return truncate(
|
|
679
|
-
findings.map((finding) => `(${finding.severity}) ${finding.claim}`).join("; "),
|
|
680
|
-
400
|
|
681
|
-
);
|
|
682
|
-
}
|
|
683
|
-
function truncate(s, n) {
|
|
684
|
-
return s.length <= n ? s : `${s.slice(0, n - 1)}\u2026`;
|
|
685
|
-
}
|
|
686
|
-
function worktreeDirty(worktreePath) {
|
|
687
|
-
return worktreeChangedPaths(worktreePath).length > 0;
|
|
688
|
-
}
|
|
689
|
-
function worktreeChangedPaths(worktreePath) {
|
|
690
|
-
const result = spawnSync("git", ["status", "--porcelain", "--untracked-files=all"], {
|
|
691
|
-
cwd: worktreePath,
|
|
692
|
-
encoding: "utf-8"
|
|
693
|
-
});
|
|
694
|
-
if (result.error) {
|
|
695
|
-
throw new Error(
|
|
696
|
-
`agenticGenerator: git status failed to spawn in ${worktreePath}: ${result.error.message}`
|
|
697
|
-
);
|
|
698
|
-
}
|
|
699
|
-
if (result.status !== 0) {
|
|
700
|
-
throw new Error(
|
|
701
|
-
`agenticGenerator: git status exited ${result.status} in ${worktreePath}: ${result.stderr.trim()}`
|
|
702
|
-
);
|
|
703
|
-
}
|
|
704
|
-
return result.stdout.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => line.slice(3).trim());
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
// src/improvement/findings.ts
|
|
708
|
-
import { makeFinding } from "@tangle-network/agent-eval";
|
|
709
|
-
var SEVERITIES = /* @__PURE__ */ new Set(["critical", "high", "medium", "low", "info"]);
|
|
710
|
-
var LIFTED_FINDING_ANALYST_ID = "lifted-seed";
|
|
711
|
-
function isAnalystFinding(value) {
|
|
712
|
-
if (!value || typeof value !== "object") return false;
|
|
713
|
-
const o = value;
|
|
714
|
-
return o.schema_version === "1.0.0" && typeof o.finding_id === "string" && typeof o.analyst_id === "string" && typeof o.severity === "string" && SEVERITIES.has(o.severity) && typeof o.area === "string" && typeof o.claim === "string" && typeof o.confidence === "number" && Array.isArray(o.evidence_refs);
|
|
715
|
-
}
|
|
716
|
-
function liftedClaim(value) {
|
|
717
|
-
if (typeof value === "string") return value.trim() || null;
|
|
718
|
-
if (value && typeof value === "object") {
|
|
719
|
-
const o = value;
|
|
720
|
-
for (const key of ["recommended_action", "claim", "lesson", "notes", "text", "message"]) {
|
|
721
|
-
const v = o[key];
|
|
722
|
-
if (typeof v === "string" && v.trim()) return v.trim();
|
|
723
|
-
}
|
|
724
|
-
try {
|
|
725
|
-
const json = JSON.stringify(value);
|
|
726
|
-
if (json && json !== "{}" && json !== "[]") {
|
|
727
|
-
return json.length > 400 ? `${json.slice(0, 399)}\u2026` : json;
|
|
728
|
-
}
|
|
729
|
-
} catch {
|
|
730
|
-
}
|
|
731
|
-
}
|
|
732
|
-
return null;
|
|
733
|
-
}
|
|
734
|
-
function toAnalystFindings(findings, opts = {}) {
|
|
735
|
-
const analystId = opts.analystId ?? LIFTED_FINDING_ANALYST_ID;
|
|
736
|
-
const area = opts.area ?? "seed";
|
|
737
|
-
const out = [];
|
|
738
|
-
for (const f of findings) {
|
|
739
|
-
if (isAnalystFinding(f)) {
|
|
740
|
-
out.push(f);
|
|
741
|
-
continue;
|
|
742
|
-
}
|
|
743
|
-
const claim = liftedClaim(f);
|
|
744
|
-
if (!claim) continue;
|
|
745
|
-
out.push(
|
|
746
|
-
makeFinding({
|
|
747
|
-
analyst_id: analystId,
|
|
748
|
-
severity: "info",
|
|
749
|
-
area,
|
|
750
|
-
confidence: 0.5,
|
|
751
|
-
claim,
|
|
752
|
-
evidence_refs: [],
|
|
753
|
-
...f && typeof f === "object" ? { metadata: { raw: f } } : {}
|
|
754
|
-
})
|
|
755
|
-
);
|
|
756
|
-
}
|
|
757
|
-
return out;
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
// src/improvement/rollout-policy.ts
|
|
761
|
-
var ROLLOUT_POLICY_EXTENSION = "structural-rollout";
|
|
762
|
-
var isBoundedInt = (v, min) => typeof v === "number" && Number.isInteger(v) && v >= min;
|
|
763
|
-
function parseRolloutPolicy(surface) {
|
|
764
|
-
if (typeof surface !== "string" || surface.trim().length === 0) return void 0;
|
|
765
|
-
let raw;
|
|
766
|
-
try {
|
|
767
|
-
raw = JSON.parse(surface);
|
|
768
|
-
} catch {
|
|
769
|
-
return void 0;
|
|
770
|
-
}
|
|
771
|
-
return normalizeRolloutPolicy(raw);
|
|
772
|
-
}
|
|
773
|
-
function normalizeRolloutPolicy(raw) {
|
|
774
|
-
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return void 0;
|
|
775
|
-
const bag = raw;
|
|
776
|
-
const k = bag.k ?? defaultStructuralRolloutPolicy.k;
|
|
777
|
-
const repairRounds = bag.repairRounds ?? defaultStructuralRolloutPolicy.repairRounds;
|
|
778
|
-
const testgen = bag.testgen ?? defaultStructuralRolloutPolicy.testgen;
|
|
779
|
-
if (!isBoundedInt(k, 1) || !isBoundedInt(repairRounds, 0) || !isBoundedInt(testgen, 0)) {
|
|
780
|
-
return void 0;
|
|
781
|
-
}
|
|
782
|
-
return {
|
|
783
|
-
k,
|
|
784
|
-
repairRounds,
|
|
785
|
-
testgen,
|
|
786
|
-
...typeof bag.diverse === "boolean" ? { diverse: bag.diverse } : {},
|
|
787
|
-
...typeof bag.temperature === "number" ? { temperature: bag.temperature } : {}
|
|
788
|
-
};
|
|
789
|
-
}
|
|
790
|
-
function serializeRolloutPolicy(policy) {
|
|
791
|
-
return JSON.stringify({
|
|
792
|
-
k: policy.k,
|
|
793
|
-
repairRounds: policy.repairRounds,
|
|
794
|
-
testgen: policy.testgen,
|
|
795
|
-
...policy.diverse !== void 0 ? { diverse: policy.diverse } : {},
|
|
796
|
-
...policy.temperature !== void 0 ? { temperature: policy.temperature } : {}
|
|
797
|
-
});
|
|
798
|
-
}
|
|
799
|
-
function structuralRolloutPolicyFromProfile(profile) {
|
|
800
|
-
const bag = profile.extensions?.[ROLLOUT_POLICY_EXTENSION];
|
|
801
|
-
if (bag === void 0) return void 0;
|
|
802
|
-
return normalizeRolloutPolicy(bag);
|
|
803
|
-
}
|
|
804
|
-
function applyRolloutPolicyToProfile(profile, policy) {
|
|
805
|
-
const candidate = structuredClone(profile);
|
|
806
|
-
const bag = {
|
|
807
|
-
k: policy.k,
|
|
808
|
-
repairRounds: policy.repairRounds,
|
|
809
|
-
testgen: policy.testgen,
|
|
810
|
-
...policy.diverse !== void 0 ? { diverse: policy.diverse } : {},
|
|
811
|
-
...policy.temperature !== void 0 ? { temperature: policy.temperature } : {}
|
|
812
|
-
};
|
|
813
|
-
return {
|
|
814
|
-
...candidate,
|
|
815
|
-
extensions: { ...candidate.extensions, [ROLLOUT_POLICY_EXTENSION]: bag }
|
|
816
|
-
};
|
|
817
|
-
}
|
|
818
|
-
|
|
819
|
-
// src/improvement/raw-trace-distiller.ts
|
|
820
|
-
import { existsSync as existsSync2, readdirSync } from "fs";
|
|
821
|
-
import { basename, join as join2, resolve as resolve2 } from "path";
|
|
822
|
-
import { makeFinding as makeFinding2 } from "@tangle-network/agent-eval";
|
|
823
|
-
var ANALYST_ID = "raw-trace-distiller";
|
|
824
|
-
var PASS_THRESHOLD = 0.999;
|
|
825
|
-
function rawTraceDistiller(options = {}) {
|
|
826
|
-
const maxCandidates = options.maxCandidates ?? 12;
|
|
827
|
-
const maxCellsPerCandidate = options.maxCellsPerCandidate ?? 8;
|
|
828
|
-
const maxFilesPerCell = options.maxFilesPerCell ?? 24;
|
|
829
|
-
return async (input) => {
|
|
830
|
-
const genRoot = absoluteRunDir(options.runDir ?? input.runDir);
|
|
831
|
-
const durable = isDurable(genRoot);
|
|
832
|
-
const ranked = [...input.candidates].map((c) => ({
|
|
833
|
-
surfaceHash: c.surfaceHash,
|
|
834
|
-
composite: c.composite,
|
|
835
|
-
campaignDir: absoluteRunDir(c.campaign.runDir),
|
|
836
|
-
cells: failingCells(c.campaign, maxCellsPerCandidate, maxFilesPerCell)
|
|
837
|
-
})).sort((a, b) => a.composite - b.composite).slice(0, maxCandidates);
|
|
838
|
-
const totalFailingCells = ranked.reduce((n, c) => n + c.cells.length, 0);
|
|
839
|
-
if (totalFailingCells === 0) {
|
|
840
|
-
if (options.fallbackFindings && options.fallbackFindings.length > 0) {
|
|
841
|
-
return options.fallbackFindings;
|
|
842
|
-
}
|
|
843
|
-
return [
|
|
844
|
-
makeFinding2({
|
|
845
|
-
analyst_id: ANALYST_ID,
|
|
846
|
-
severity: "info",
|
|
847
|
-
area: "raw-trace-context",
|
|
848
|
-
confidence: 1,
|
|
849
|
-
claim: `Generation ${input.generation} had no failing cells. The full raw run traces are on disk under ${genRoot}.`,
|
|
850
|
-
recommended_action: `To keep improving, grep/cat the raw traces under ${genRoot} (per-cell spans.jsonl + cached-result.json) to find the weakest passing runs, then make a targeted harness-code edit.`,
|
|
851
|
-
evidence_refs: [{ kind: "artifact", uri: genRoot }],
|
|
852
|
-
metadata: { generation: input.generation, runDir: genRoot, failingCells: 0 }
|
|
853
|
-
})
|
|
854
|
-
];
|
|
855
|
-
}
|
|
856
|
-
const findings = [];
|
|
857
|
-
findings.push(
|
|
858
|
-
makeFinding2({
|
|
859
|
-
analyst_id: ANALYST_ID,
|
|
860
|
-
severity: "high",
|
|
861
|
-
area: "raw-trace-context",
|
|
862
|
-
confidence: 1,
|
|
863
|
-
claim: `Generation ${input.generation} produced ${totalFailingCells} failing/low-scoring cell(s) across ${ranked.length} candidate(s). Their FULL RAW run traces are on disk under ${genRoot} \u2014 the actual event logs (spans.jsonl), scores (cached-result.json), and artifacts, not a summary.${durable ? "" : " (WARNING: this run root does not exist on disk \u2014 it looks like an in-memory run; pass a real runDir to improve() to get raw-trace context.)"}`,
|
|
864
|
-
recommended_action: `Do NOT rely on a pre-summarized finding. Before editing, DIAGNOSE from the raw traces: run \`grep\`/\`cat\`/\`ls\` over the trace files and directories named in the following findings to see exactly what each failing run did and why it scored low, then make the smallest harness-code edit that fixes the dominant failure. Start with \`grep -rIn "error" ${genRoot}\` then \`cat\` the spans.jsonl of the worst cell.`,
|
|
865
|
-
evidence_refs: [{ kind: "artifact", uri: genRoot }],
|
|
866
|
-
metadata: {
|
|
867
|
-
generation: input.generation,
|
|
868
|
-
runDir: genRoot,
|
|
869
|
-
failingCells: totalFailingCells,
|
|
870
|
-
candidates: ranked.length
|
|
871
|
-
}
|
|
872
|
-
})
|
|
873
|
-
);
|
|
874
|
-
for (const cand of ranked) {
|
|
875
|
-
if (cand.cells.length === 0) continue;
|
|
876
|
-
const scenarioList = cand.cells.map((c) => c.scenarioId).join(", ");
|
|
877
|
-
const fileLines = cand.cells.map((c) => {
|
|
878
|
-
const header = ` cell ${c.scenarioId} (composite ${c.composite.toFixed(3)}${c.error ? `, error: ${truncate2(c.error, 160)}` : ""}) \u2014 dir ${c.cellDir}`;
|
|
879
|
-
const files = c.files.map((f) => ` - ${f}`).join("\n");
|
|
880
|
-
const more = c.truncatedFiles ? `
|
|
881
|
-
- \u2026(ls ${c.cellDir} for the rest)` : "";
|
|
882
|
-
return c.files.length > 0 ? `${header}
|
|
883
|
-
${files}${more}` : header;
|
|
884
|
-
}).join("\n");
|
|
885
|
-
findings.push(
|
|
886
|
-
makeFinding2({
|
|
887
|
-
analyst_id: ANALYST_ID,
|
|
888
|
-
severity: cand.composite < 0.5 ? "critical" : "high",
|
|
889
|
-
area: "raw-trace-context",
|
|
890
|
-
confidence: 1,
|
|
891
|
-
subject: cand.surfaceHash,
|
|
892
|
-
claim: `Candidate ${cand.surfaceHash} scored composite ${cand.composite.toFixed(3)} with ${cand.cells.length} failing cell(s) [${scenarioList}]. Its raw traces are under ${cand.campaignDir}.`,
|
|
893
|
-
recommended_action: `grep/cat these raw trace files to diagnose WHY this candidate failed before editing:
|
|
894
|
-
${fileLines}
|
|
895
|
-
Or scan the whole candidate at once: \`grep -rIn . ${cand.campaignDir}\` and \`ls -R ${cand.campaignDir}\`.`,
|
|
896
|
-
evidence_refs: [
|
|
897
|
-
{ kind: "artifact", uri: cand.campaignDir },
|
|
898
|
-
...cand.cells.flatMap(
|
|
899
|
-
(c) => c.files.map((f) => ({ kind: "artifact", uri: f }))
|
|
900
|
-
)
|
|
901
|
-
],
|
|
902
|
-
metadata: {
|
|
903
|
-
surfaceHash: cand.surfaceHash,
|
|
904
|
-
composite: cand.composite,
|
|
905
|
-
campaignDir: cand.campaignDir,
|
|
906
|
-
cells: cand.cells.map((c) => ({
|
|
907
|
-
scenarioId: c.scenarioId,
|
|
908
|
-
composite: c.composite,
|
|
909
|
-
cellDir: c.cellDir,
|
|
910
|
-
files: c.files,
|
|
911
|
-
...c.error ? { error: c.error } : {}
|
|
912
|
-
}))
|
|
913
|
-
}
|
|
914
|
-
})
|
|
915
|
-
);
|
|
916
|
-
}
|
|
917
|
-
return findings;
|
|
918
|
-
};
|
|
919
|
-
}
|
|
920
|
-
function failingCells(campaign, maxCells, maxFiles) {
|
|
921
|
-
const campaignDir = absoluteRunDir(campaign.runDir);
|
|
922
|
-
const durable = isDurable(campaignDir);
|
|
923
|
-
const out = [];
|
|
924
|
-
for (const cell of campaign.cells) {
|
|
925
|
-
const scores = Object.values(cell.judgeScores ?? {});
|
|
926
|
-
const composite = scores.length === 0 ? 0 : scores.reduce((sum, s) => sum + (s.composite ?? 0), 0) / scores.length;
|
|
927
|
-
if (!cell.error && composite >= PASS_THRESHOLD) continue;
|
|
928
|
-
const cellDir = join2(campaignDir, sanitizeCellId(cell.cellId));
|
|
929
|
-
const artifactPaths = artifactPathsForCell(campaign.artifactsByPath, cell.cellId);
|
|
930
|
-
const discovered = durable ? listTraceFiles(cellDir) : [];
|
|
931
|
-
const canonical = [join2(cellDir, "spans.jsonl"), join2(cellDir, "cached-result.json")];
|
|
932
|
-
const files = dedupeSorted([...discovered, ...artifactPaths, ...canonical]);
|
|
933
|
-
out.push({
|
|
934
|
-
scenarioId: cell.scenarioId,
|
|
935
|
-
composite: Number(composite.toFixed(3)),
|
|
936
|
-
...cell.error ? { error: cell.error } : {},
|
|
937
|
-
cellDir,
|
|
938
|
-
files: files.slice(0, maxFiles),
|
|
939
|
-
truncatedFiles: files.length > maxFiles
|
|
940
|
-
});
|
|
941
|
-
if (out.length >= maxCells) break;
|
|
942
|
-
}
|
|
943
|
-
return out;
|
|
944
|
-
}
|
|
945
|
-
function artifactPathsForCell(artifactsByPath, cellId) {
|
|
946
|
-
if (!artifactsByPath) return [];
|
|
947
|
-
const prefix = `${cellId}/`;
|
|
948
|
-
return Object.entries(artifactsByPath).filter(([key]) => key.startsWith(prefix)).map(([, absPath]) => resolve2(absPath));
|
|
949
|
-
}
|
|
950
|
-
function listTraceFiles(dir) {
|
|
951
|
-
const out = [];
|
|
952
|
-
for (const entry of safeReadDir(dir)) {
|
|
953
|
-
const full = join2(dir, entry.name);
|
|
954
|
-
if (entry.isFile()) {
|
|
955
|
-
out.push(full);
|
|
956
|
-
} else if (!entry.isSymbolicLink() && entry.isDirectory()) {
|
|
957
|
-
for (const sub of safeReadDir(full)) {
|
|
958
|
-
if (sub.isFile()) out.push(join2(full, sub.name));
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
}
|
|
962
|
-
return out;
|
|
963
|
-
}
|
|
964
|
-
function safeReadDir(dir) {
|
|
965
|
-
try {
|
|
966
|
-
return readdirSync(dir, { withFileTypes: true });
|
|
967
|
-
} catch {
|
|
968
|
-
return [];
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
function sanitizeCellId(cellId) {
|
|
972
|
-
return cellId.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
973
|
-
}
|
|
974
|
-
function isDurable(runDir) {
|
|
975
|
-
return !runDir.startsWith("mem://") && existsSync2(runDir);
|
|
976
|
-
}
|
|
977
|
-
function absoluteRunDir(runDir) {
|
|
978
|
-
return runDir.startsWith("mem://") ? runDir : resolve2(runDir);
|
|
979
|
-
}
|
|
980
|
-
function dedupeSorted(paths) {
|
|
981
|
-
return [...new Set(paths)].sort((a, b) => {
|
|
982
|
-
const da = a.slice(0, a.length - basename(a).length);
|
|
983
|
-
const db = b.slice(0, b.length - basename(b).length);
|
|
984
|
-
return da === db ? basename(a).localeCompare(basename(b)) : da.localeCompare(db);
|
|
985
|
-
});
|
|
986
|
-
}
|
|
987
|
-
function truncate2(s, n) {
|
|
988
|
-
return s.length <= n ? s : `${s.slice(0, n - 1)}\u2026`;
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
// src/improvement/improve.ts
|
|
992
|
-
import { agentProfileSchema as agentProfileSchema2 } from "@tangle-network/agent-interface";
|
|
993
|
-
|
|
994
|
-
// src/improvement/code-execution.ts
|
|
995
|
-
import { makeFinding as makeFinding3 } from "@tangle-network/agent-eval";
|
|
996
|
-
import {
|
|
997
|
-
gitWorktreeAdapter
|
|
998
|
-
} from "@tangle-network/agent-eval/campaign";
|
|
999
|
-
import {
|
|
1000
|
-
selfImprove
|
|
1001
|
-
} from "@tangle-network/agent-eval/contract";
|
|
1002
|
-
|
|
1003
|
-
// src/improvement/cleanup.ts
|
|
1004
|
-
async function rethrowAfterCleanup(cause, cleanup, context) {
|
|
1005
|
-
const cleanupErrors = [];
|
|
1006
|
-
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
1007
|
-
try {
|
|
1008
|
-
await cleanup();
|
|
1009
|
-
} catch (cleanupCause) {
|
|
1010
|
-
cleanupErrors.push(cleanupCause);
|
|
1011
|
-
continue;
|
|
1012
|
-
}
|
|
1013
|
-
if (cleanupErrors.length === 0) throw cause;
|
|
1014
|
-
throw new AggregateError([cause, ...cleanupErrors], `${context}; cleanup retry succeeded`);
|
|
1015
|
-
}
|
|
1016
|
-
throw new AggregateError([cause, ...cleanupErrors], `${context}; cleanup failed`);
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
// src/improvement/improve-result.ts
|
|
1020
|
-
function copyImproveCost(cost) {
|
|
1021
|
-
return {
|
|
1022
|
-
totalCostUsd: cost.totalCostUsd,
|
|
1023
|
-
accountingComplete: cost.accountingComplete,
|
|
1024
|
-
incompleteReasons: [...cost.incompleteReasons]
|
|
1025
|
-
};
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1028
|
-
// src/improvement/improvement-driver.ts
|
|
1029
|
-
import { spawnSync as spawnSync2 } from "child_process";
|
|
1030
|
-
import {
|
|
1031
|
-
verifyCodeSurface
|
|
1032
|
-
} from "@tangle-network/agent-eval/campaign";
|
|
1033
|
-
function improvementDriver(opts) {
|
|
1034
|
-
const baseRef = opts.baseRef ?? "main";
|
|
1035
|
-
const owned = /* @__PURE__ */ new Map();
|
|
1036
|
-
return {
|
|
1037
|
-
kind: `improvement:${opts.generator.kind}`,
|
|
1038
|
-
async propose(ctx) {
|
|
1039
|
-
const findings = resolveFindings(ctx);
|
|
1040
|
-
if (findings.length === 0 && ctx.report === void 0 && !opts.generator.proposesWithoutFindings) {
|
|
1041
|
-
return [];
|
|
1042
|
-
}
|
|
1043
|
-
const surfaces = [];
|
|
1044
|
-
const incumbent = verifiedCodeIncumbent(ctx.currentSurface);
|
|
1045
|
-
const proposalBaseRef = incumbent?.baseCommit ?? baseRef;
|
|
1046
|
-
for (let i = 0; i < ctx.populationSize; i++) {
|
|
1047
|
-
if (ctx.signal.aborted) break;
|
|
1048
|
-
const wt = await opts.worktree.create({
|
|
1049
|
-
baseRef: proposalBaseRef,
|
|
1050
|
-
label: `${opts.generator.kind}-gen${ctx.generation}-cand${i}`
|
|
1051
|
-
});
|
|
1052
|
-
owned.set(wt.path, wt);
|
|
1053
|
-
try {
|
|
1054
|
-
if (incumbent) advanceToIncumbent(wt, incumbent);
|
|
1055
|
-
const { applied, summary, label, rationale } = await opts.generator.generate({
|
|
1056
|
-
worktreePath: wt.path,
|
|
1057
|
-
report: ctx.report,
|
|
1058
|
-
findings,
|
|
1059
|
-
dataset: ctx.dataset,
|
|
1060
|
-
maxShots: ctx.maxImprovementShots ?? 1,
|
|
1061
|
-
signal: ctx.signal,
|
|
1062
|
-
generation: ctx.generation,
|
|
1063
|
-
candidateIndex: i,
|
|
1064
|
-
...ctx.costLedger ? { costLedger: ctx.costLedger } : {},
|
|
1065
|
-
...ctx.costPhase ? { costPhase: ctx.costPhase } : {}
|
|
1066
|
-
});
|
|
1067
|
-
if (!applied) {
|
|
1068
|
-
await opts.worktree.discard(wt);
|
|
1069
|
-
owned.delete(wt.path);
|
|
1070
|
-
continue;
|
|
1071
|
-
}
|
|
1072
|
-
const surface = await opts.worktree.finalize(wt, summary);
|
|
1073
|
-
surfaces.push(
|
|
1074
|
-
label || rationale ? { surface, label: label ?? "", rationale: rationale ?? "" } : surface
|
|
1075
|
-
);
|
|
1076
|
-
owned.delete(wt.path);
|
|
1077
|
-
owned.set(surface.worktreeRef, wt);
|
|
1078
|
-
} catch (err) {
|
|
1079
|
-
const failure = err instanceof Error ? err.message : String(err);
|
|
1080
|
-
return rethrowAfterCleanup(
|
|
1081
|
-
err,
|
|
1082
|
-
async () => {
|
|
1083
|
-
await opts.worktree.discard(wt);
|
|
1084
|
-
owned.delete(wt.path);
|
|
1085
|
-
},
|
|
1086
|
-
`improvementDriver: ${failure}`
|
|
1087
|
-
);
|
|
1088
|
-
}
|
|
1089
|
-
}
|
|
1090
|
-
return surfaces;
|
|
1091
|
-
},
|
|
1092
|
-
async cleanup(retainWorktreeRefs = []) {
|
|
1093
|
-
const retained = new Set(retainWorktreeRefs);
|
|
1094
|
-
const errors = [];
|
|
1095
|
-
for (const [worktreeRef, worktree] of owned) {
|
|
1096
|
-
if (retained.has(worktreeRef)) continue;
|
|
1097
|
-
try {
|
|
1098
|
-
await opts.worktree.discard(worktree);
|
|
1099
|
-
owned.delete(worktreeRef);
|
|
1100
|
-
} catch (cause) {
|
|
1101
|
-
errors.push(cause);
|
|
1102
|
-
}
|
|
1103
|
-
}
|
|
1104
|
-
if (errors.length > 0) {
|
|
1105
|
-
throw new AggregateError(errors, "improvementDriver: failed to discard candidate worktrees");
|
|
1106
|
-
}
|
|
1107
|
-
}
|
|
1108
|
-
};
|
|
1109
|
-
}
|
|
1110
|
-
function verifiedCodeIncumbent(surface) {
|
|
1111
|
-
if (typeof surface !== "object" || surface.kind !== "code") return void 0;
|
|
1112
|
-
verifyCodeSurface(surface);
|
|
1113
|
-
return surface;
|
|
1114
|
-
}
|
|
1115
|
-
function advanceToIncumbent(worktree, incumbent) {
|
|
1116
|
-
if (worktree.baseCommit !== incumbent.baseCommit || worktree.baseTree !== incumbent.baseTree) {
|
|
1117
|
-
throw new Error("improvementDriver: candidate worktree does not match incumbent base identity");
|
|
1118
|
-
}
|
|
1119
|
-
if (worktree.baseCommit === incumbent.candidateCommit) return;
|
|
1120
|
-
const merge = spawnSync2("git", ["merge", "--ff-only", incumbent.candidateCommit], {
|
|
1121
|
-
cwd: worktree.path,
|
|
1122
|
-
encoding: "utf8"
|
|
1123
|
-
});
|
|
1124
|
-
if (merge.error) {
|
|
1125
|
-
throw new Error(
|
|
1126
|
-
`improvementDriver: failed to start candidate from incumbent: ${merge.error.message}`
|
|
1127
|
-
);
|
|
1128
|
-
}
|
|
1129
|
-
if (merge.status !== 0) {
|
|
1130
|
-
throw new Error(
|
|
1131
|
-
`improvementDriver: could not fast-forward candidate to incumbent ${incumbent.candidateCommit}: ${merge.stderr.trim()}`
|
|
1132
|
-
);
|
|
1133
|
-
}
|
|
1134
|
-
const head = spawnSync2("git", ["rev-parse", "--verify", "HEAD"], {
|
|
1135
|
-
cwd: worktree.path,
|
|
1136
|
-
encoding: "utf8"
|
|
1137
|
-
});
|
|
1138
|
-
if (head.error || head.status !== 0 || head.stdout.trim() !== incumbent.candidateCommit) {
|
|
1139
|
-
throw new Error("improvementDriver: candidate worktree did not reach the incumbent commit");
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1142
|
-
function resolveFindings(ctx) {
|
|
1143
|
-
const report = ctx.report;
|
|
1144
|
-
if (report && typeof report === "object" && "findings" in report) {
|
|
1145
|
-
const f = report.findings;
|
|
1146
|
-
if (Array.isArray(f) && f.length > 0) {
|
|
1147
|
-
const lifted = toAnalystFindings(f, { analystId: "report-findings", area: "report" });
|
|
1148
|
-
if (lifted.length > 0) return lifted;
|
|
1149
|
-
}
|
|
1150
|
-
}
|
|
1151
|
-
return toAnalystFindings(ctx.findings ?? [], { analystId: "loop-context", area: "seed" });
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
// src/improvement/profile-surface.ts
|
|
1155
|
-
import { canonicalJson } from "@tangle-network/agent-eval";
|
|
1156
|
-
import {
|
|
1157
|
-
agentProfileSchema
|
|
1158
|
-
} from "@tangle-network/agent-interface";
|
|
1159
|
-
function prepareProfileSurface(profile, surface, skills, profileComponents) {
|
|
1160
|
-
switch (surface) {
|
|
1161
|
-
case "prompt":
|
|
1162
|
-
return {
|
|
1163
|
-
surface: profile.prompt?.systemPrompt ?? "",
|
|
1164
|
-
value: profile.prompt?.systemPrompt ?? ""
|
|
1165
|
-
};
|
|
1166
|
-
case "skills": {
|
|
1167
|
-
const value = inlineSkill(profile, skills).content;
|
|
1168
|
-
return { surface: value, value };
|
|
1169
|
-
}
|
|
1170
|
-
case "tools": {
|
|
1171
|
-
const value = profile.tools ?? {};
|
|
1172
|
-
return { surface: canonicalJson(value), value };
|
|
1173
|
-
}
|
|
1174
|
-
case "mcp": {
|
|
1175
|
-
const value = profile.mcp ?? {};
|
|
1176
|
-
return { surface: canonicalJson(value), value };
|
|
1177
|
-
}
|
|
1178
|
-
case "hooks": {
|
|
1179
|
-
const value = profile.hooks ?? {};
|
|
1180
|
-
return { surface: canonicalJson(value), value };
|
|
1181
|
-
}
|
|
1182
|
-
case "subagents": {
|
|
1183
|
-
const value = profile.subagents ?? {};
|
|
1184
|
-
return { surface: canonicalJson(value), value };
|
|
1185
|
-
}
|
|
1186
|
-
case "agent-profile": {
|
|
1187
|
-
if (profileComponents) {
|
|
1188
|
-
const value = profileComponents.read(profile);
|
|
1189
|
-
return {
|
|
1190
|
-
surface: componentSurface(value, "profileComponents.read"),
|
|
1191
|
-
value
|
|
1192
|
-
};
|
|
1193
|
-
}
|
|
1194
|
-
return { surface: canonicalJson(profile), value: profile };
|
|
1195
|
-
}
|
|
1196
|
-
case "memory": {
|
|
1197
|
-
const value = profileInstructions(profile);
|
|
1198
|
-
return { surface: value, value };
|
|
1199
|
-
}
|
|
1200
|
-
case "rollout-policy": {
|
|
1201
|
-
const policy = structuralRolloutPolicyFromProfile(profile);
|
|
1202
|
-
return {
|
|
1203
|
-
surface: policy ? serializeRolloutPolicy(policy) : "",
|
|
1204
|
-
value: policy ?? null
|
|
1205
|
-
};
|
|
1206
|
-
}
|
|
1207
|
-
case "code":
|
|
1208
|
-
throw new ConfigError(
|
|
1209
|
-
"improve(): code requires the isolated baseline created from opts.code.repoRoot"
|
|
1210
|
-
);
|
|
1211
|
-
}
|
|
1212
|
-
}
|
|
1213
|
-
function isCodeSurface(surface) {
|
|
1214
|
-
return typeof surface === "object" && surface !== null && surface.kind === "code";
|
|
1215
|
-
}
|
|
1216
|
-
function isComponentSurface(surface) {
|
|
1217
|
-
return typeof surface === "object" && surface !== null && surface.kind === "components";
|
|
1218
|
-
}
|
|
1219
|
-
function componentSurface(components, source) {
|
|
1220
|
-
const entries = validateComponents(components, source);
|
|
1221
|
-
return immutableCandidateValue({
|
|
1222
|
-
kind: "components",
|
|
1223
|
-
components: Object.fromEntries(entries)
|
|
1224
|
-
});
|
|
1225
|
-
}
|
|
1226
|
-
function validateComponents(components, source) {
|
|
1227
|
-
if (typeof components !== "object" || components === null || Array.isArray(components)) {
|
|
1228
|
-
throw new ConfigError(`improve(): ${source} must return a component record`);
|
|
1229
|
-
}
|
|
1230
|
-
const entries = Object.entries(components);
|
|
1231
|
-
if (entries.length === 0) {
|
|
1232
|
-
throw new ConfigError(`improve(): ${source} must return at least one component`);
|
|
1233
|
-
}
|
|
1234
|
-
for (const [name, value] of entries) {
|
|
1235
|
-
if (!name || name.trim() !== name || typeof value !== "string") {
|
|
1236
|
-
throw new ConfigError(
|
|
1237
|
-
`improve(): ${source} must return trimmed component names with string values`
|
|
1238
|
-
);
|
|
1239
|
-
}
|
|
1240
|
-
}
|
|
1241
|
-
return entries;
|
|
1242
|
-
}
|
|
1243
|
-
function parseWinnerJson(winner, surface) {
|
|
1244
|
-
try {
|
|
1245
|
-
return JSON.parse(winner);
|
|
1246
|
-
} catch (cause) {
|
|
1247
|
-
throw new ConfigError(
|
|
1248
|
-
`improve(): the '${surface}' candidate is not valid JSON, so it cannot form a profile candidate: ${cause.message}`
|
|
1249
|
-
);
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
function assertCandidateSurfaceKind(surface, baseline, winner) {
|
|
1253
|
-
if (surface === "code") {
|
|
1254
|
-
if (isCodeSurface(winner)) return;
|
|
1255
|
-
throw new ConfigError(
|
|
1256
|
-
`improve(): the '${surface}' candidate returned an incompatible surface value`
|
|
1257
|
-
);
|
|
1258
|
-
}
|
|
1259
|
-
if (typeof baseline === "string") {
|
|
1260
|
-
if (typeof winner === "string") return;
|
|
1261
|
-
throw new ConfigError(
|
|
1262
|
-
`improve(): the '${surface}' candidate changed from a text surface to an incompatible surface value`
|
|
1263
|
-
);
|
|
1264
|
-
}
|
|
1265
|
-
if (!isComponentSurface(baseline) || !isComponentSurface(winner)) {
|
|
1266
|
-
throw new ConfigError(
|
|
1267
|
-
`improve(): the '${surface}' candidate returned an incompatible surface value`
|
|
1268
|
-
);
|
|
1269
|
-
}
|
|
1270
|
-
validateComponents(winner.components, `the '${surface}' candidate`);
|
|
1271
|
-
const baselineNames = Object.keys(baseline.components).sort();
|
|
1272
|
-
const winnerNames = Object.keys(winner.components).sort();
|
|
1273
|
-
if (baselineNames.length !== winnerNames.length || baselineNames.some((name, index) => name !== winnerNames[index])) {
|
|
1274
|
-
throw new ConfigError(
|
|
1275
|
-
`improve(): the '${surface}' candidate must preserve the exact component names`
|
|
1276
|
-
);
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1279
|
-
function materializeImprovementProfileCandidate(profile, surface, winner, skills, profileComponents) {
|
|
1280
|
-
let candidate;
|
|
1281
|
-
if (isComponentSurface(winner)) {
|
|
1282
|
-
if (surface !== "agent-profile" || !profileComponents) {
|
|
1283
|
-
throw new ConfigError(
|
|
1284
|
-
`improve(): the '${surface}' candidate has no profile component mapping`
|
|
1285
|
-
);
|
|
1286
|
-
}
|
|
1287
|
-
const winnerComponents = immutableCandidateValue({ ...winner.components });
|
|
1288
|
-
const applied = profileComponents.apply(profile, winnerComponents);
|
|
1289
|
-
const validated = validateProfileCandidate(applied, surface);
|
|
1290
|
-
const materializedComponents = Object.fromEntries(
|
|
1291
|
-
validateComponents(profileComponents.read(validated), "profileComponents.read after apply")
|
|
1292
|
-
);
|
|
1293
|
-
const names = Object.keys(winnerComponents);
|
|
1294
|
-
if (names.length !== Object.keys(materializedComponents).length || names.some((name) => materializedComponents[name] !== winnerComponents[name])) {
|
|
1295
|
-
throw new ConfigError(
|
|
1296
|
-
"improve(): profileComponents.apply must round-trip every winning component exactly"
|
|
1297
|
-
);
|
|
1298
|
-
}
|
|
1299
|
-
return validated;
|
|
1300
|
-
}
|
|
1301
|
-
if (typeof winner !== "string") {
|
|
1302
|
-
throw new ConfigError(`improve(): the '${surface}' candidate cannot form an AgentProfile`);
|
|
1303
|
-
}
|
|
1304
|
-
switch (surface) {
|
|
1305
|
-
case "prompt":
|
|
1306
|
-
candidate = { ...profile, prompt: { ...profile.prompt, systemPrompt: winner } };
|
|
1307
|
-
break;
|
|
1308
|
-
case "skills": {
|
|
1309
|
-
const selectedSkill = inlineSkill(profile, skills);
|
|
1310
|
-
candidate = {
|
|
1311
|
-
...profile,
|
|
1312
|
-
resources: {
|
|
1313
|
-
...profile.resources,
|
|
1314
|
-
skills: profile.resources?.skills?.map(
|
|
1315
|
-
(resource) => resource === selectedSkill ? { ...resource, content: winner } : resource
|
|
1316
|
-
)
|
|
1317
|
-
}
|
|
1318
|
-
};
|
|
1319
|
-
break;
|
|
1320
|
-
}
|
|
1321
|
-
case "tools":
|
|
1322
|
-
candidate = { ...profile, tools: parseWinnerJson(winner, surface) };
|
|
1323
|
-
break;
|
|
1324
|
-
case "mcp":
|
|
1325
|
-
candidate = { ...profile, mcp: parseWinnerJson(winner, surface) };
|
|
1326
|
-
break;
|
|
1327
|
-
case "hooks":
|
|
1328
|
-
candidate = { ...profile, hooks: parseWinnerJson(winner, surface) };
|
|
1329
|
-
break;
|
|
1330
|
-
case "subagents":
|
|
1331
|
-
candidate = { ...profile, subagents: parseWinnerJson(winner, surface) };
|
|
1332
|
-
break;
|
|
1333
|
-
case "agent-profile":
|
|
1334
|
-
candidate = parseWinnerJson(winner, surface);
|
|
1335
|
-
break;
|
|
1336
|
-
case "memory":
|
|
1337
|
-
candidate = {
|
|
1338
|
-
...profile,
|
|
1339
|
-
resources: {
|
|
1340
|
-
...profile.resources,
|
|
1341
|
-
instructions: replaceProfileInstructions(profile, winner)
|
|
1342
|
-
}
|
|
1343
|
-
};
|
|
1344
|
-
break;
|
|
1345
|
-
case "rollout-policy": {
|
|
1346
|
-
const policy = normalizeRolloutPolicy(parseWinnerJson(winner, surface));
|
|
1347
|
-
if (!policy) {
|
|
1348
|
-
throw new ConfigError(
|
|
1349
|
-
`improve(): the shipped 'rollout-policy' winner is not a valid StructuralRolloutPolicy (integer k >= 1, repairRounds >= 0, testgen >= 0), so it cannot be applied: ${winner}`
|
|
1350
|
-
);
|
|
1351
|
-
}
|
|
1352
|
-
candidate = applyRolloutPolicyToProfile(profile, policy);
|
|
1353
|
-
break;
|
|
1354
|
-
}
|
|
1355
|
-
}
|
|
1356
|
-
return validateProfileCandidate(candidate, surface);
|
|
1357
|
-
}
|
|
1358
|
-
function validateProfileCandidate(candidate, surface) {
|
|
1359
|
-
const parsed = agentProfileSchema.safeParse(candidate);
|
|
1360
|
-
if (!parsed.success) {
|
|
1361
|
-
throw new ConfigError(
|
|
1362
|
-
`improve(): the '${surface}' candidate does not produce a valid AgentProfile: ${parsed.error.message}`
|
|
1363
|
-
);
|
|
1364
|
-
}
|
|
1365
|
-
return immutableCandidateValue(parsed.data);
|
|
1366
|
-
}
|
|
1367
|
-
function createProfileCandidateMaterializer(profile, surface, baselineSurface, skills, profileComponents) {
|
|
1368
|
-
const baselineDigest = canonicalCandidateDigest(baselineSurface);
|
|
1369
|
-
if (profileComponents) {
|
|
1370
|
-
const reappliedBaseline = materializeImprovementProfileCandidate(
|
|
1371
|
-
profile,
|
|
1372
|
-
surface,
|
|
1373
|
-
baselineSurface,
|
|
1374
|
-
skills,
|
|
1375
|
-
profileComponents
|
|
1376
|
-
);
|
|
1377
|
-
if (canonicalCandidateDigest(reappliedBaseline) !== canonicalCandidateDigest(profile)) {
|
|
1378
|
-
throw new ConfigError(
|
|
1379
|
-
"improve(): profileComponents.apply(profile, profileComponents.read(profile)) must reproduce the complete baseline profile exactly"
|
|
1380
|
-
);
|
|
1381
|
-
}
|
|
1382
|
-
}
|
|
1383
|
-
const candidates = /* @__PURE__ */ new Map([[baselineDigest, profile]]);
|
|
1384
|
-
return (candidateSurface) => {
|
|
1385
|
-
assertCandidateSurfaceKind(surface, baselineSurface, candidateSurface);
|
|
1386
|
-
const digest = canonicalCandidateDigest(candidateSurface);
|
|
1387
|
-
const existing = candidates.get(digest);
|
|
1388
|
-
if (existing) return existing;
|
|
1389
|
-
const candidate = materializeImprovementProfileCandidate(
|
|
1390
|
-
profile,
|
|
1391
|
-
surface,
|
|
1392
|
-
immutableCandidateValue(candidateSurface),
|
|
1393
|
-
skills,
|
|
1394
|
-
profileComponents
|
|
1395
|
-
);
|
|
1396
|
-
candidates.set(digest, candidate);
|
|
1397
|
-
return candidate;
|
|
1398
|
-
};
|
|
1399
|
-
}
|
|
1400
|
-
function inlineSkill(profile, options) {
|
|
1401
|
-
const resourceName = options?.resourceName.trim();
|
|
1402
|
-
if (!resourceName) {
|
|
1403
|
-
throw new ConfigError(
|
|
1404
|
-
"improve(): surface 'skills' requires opts.skills.resourceName for one inline profile skill"
|
|
1405
|
-
);
|
|
1406
|
-
}
|
|
1407
|
-
assertFailClosedResources(profile, "skills");
|
|
1408
|
-
const matches = (profile.resources?.skills ?? []).filter(
|
|
1409
|
-
(resource) => resource.name === resourceName
|
|
1410
|
-
);
|
|
1411
|
-
if (matches.length !== 1 || matches[0]?.kind !== "inline") {
|
|
1412
|
-
throw new ConfigError(
|
|
1413
|
-
`improve(): skill '${resourceName}' must identify exactly one inline profile resource`
|
|
1414
|
-
);
|
|
1415
|
-
}
|
|
1416
|
-
return matches[0];
|
|
1417
|
-
}
|
|
1418
|
-
function profileInstructions(profile) {
|
|
1419
|
-
assertFailClosedResources(profile, "memory");
|
|
1420
|
-
const instructions = profile.resources?.instructions;
|
|
1421
|
-
if (instructions === void 0) return "";
|
|
1422
|
-
if (typeof instructions === "string") return instructions;
|
|
1423
|
-
if (instructions.kind === "inline") return instructions.content;
|
|
1424
|
-
throw new ConfigError(
|
|
1425
|
-
"improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact"
|
|
1426
|
-
);
|
|
1427
|
-
}
|
|
1428
|
-
function replaceProfileInstructions(profile, content) {
|
|
1429
|
-
const instructions = profile.resources?.instructions;
|
|
1430
|
-
if (typeof instructions !== "object") return content;
|
|
1431
|
-
if (instructions.kind !== "inline") {
|
|
1432
|
-
throw new ConfigError(
|
|
1433
|
-
"improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact"
|
|
1434
|
-
);
|
|
1435
|
-
}
|
|
1436
|
-
return { ...instructions, content };
|
|
1437
|
-
}
|
|
1438
|
-
function assertFailClosedResources(profile, surface) {
|
|
1439
|
-
if (profile.resources?.failOnError !== true) {
|
|
1440
|
-
throw new ConfigError(
|
|
1441
|
-
`improve(): surface '${surface}' requires profile.resources.failOnError: true`
|
|
1442
|
-
);
|
|
1443
|
-
}
|
|
1444
|
-
}
|
|
1445
|
-
|
|
1446
|
-
// src/improvement/code-execution.ts
|
|
1447
|
-
var distilledNotesMaxChars = 1500;
|
|
1448
|
-
var distilledErrorMaxChars = 500;
|
|
1449
|
-
function generationFailureDistiller(staticFindings) {
|
|
1450
|
-
const CAP = 12;
|
|
1451
|
-
return async (input) => {
|
|
1452
|
-
const failures = [];
|
|
1453
|
-
for (const candidate of input.candidates) {
|
|
1454
|
-
for (const rawCell of candidate.campaign.cells) {
|
|
1455
|
-
const cell = rawCell;
|
|
1456
|
-
const scenario = String(cell.scenarioId ?? "unknown");
|
|
1457
|
-
const error = typeof cell.error === "string" ? cell.error : void 0;
|
|
1458
|
-
const judgeScores = cell.judgeScores && typeof cell.judgeScores === "object" ? Object.values(
|
|
1459
|
-
cell.judgeScores
|
|
1460
|
-
) : [];
|
|
1461
|
-
const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((sum, judge) => sum + (judge.composite ?? 0), 0) / judgeScores.length;
|
|
1462
|
-
if (!error && composite >= 0.999) continue;
|
|
1463
|
-
const notes = judgeScores.map((judge) => judge.notes).filter((note) => typeof note === "string" && note.length > 0).join("; ").slice(0, distilledNotesMaxChars);
|
|
1464
|
-
const claim = notes || (error ? `Scenario ${scenario} failed: ${error.slice(0, distilledErrorMaxChars)}` : "");
|
|
1465
|
-
failures.push({
|
|
1466
|
-
scenario,
|
|
1467
|
-
composite: Number(composite.toFixed(3)),
|
|
1468
|
-
notes,
|
|
1469
|
-
...claim ? { claim } : {},
|
|
1470
|
-
...error ? { error: error.slice(0, distilledErrorMaxChars) } : {}
|
|
1471
|
-
});
|
|
1472
|
-
}
|
|
1473
|
-
}
|
|
1474
|
-
if (failures.length === 0) return staticFindings;
|
|
1475
|
-
failures.sort((left, right) => left.composite - right.composite);
|
|
1476
|
-
return failures.slice(0, CAP).map(
|
|
1477
|
-
(failure) => makeFinding3({
|
|
1478
|
-
analyst_id: "generation-failure-distiller",
|
|
1479
|
-
severity: failure.error !== void 0 || failure.composite < 0.5 ? "high" : "medium",
|
|
1480
|
-
area: "generation-failure",
|
|
1481
|
-
confidence: 1,
|
|
1482
|
-
subject: failure.scenario,
|
|
1483
|
-
claim: `Scenario ${failure.scenario} scored composite ${failure.composite}${failure.notes ? `: ${failure.notes}` : ""}${failure.error ? ` (error: ${failure.error})` : ""}`,
|
|
1484
|
-
evidence_refs: [],
|
|
1485
|
-
metadata: {
|
|
1486
|
-
scenario: failure.scenario,
|
|
1487
|
-
composite: failure.composite,
|
|
1488
|
-
...failure.notes ? { notes: failure.notes } : {},
|
|
1489
|
-
...failure.error !== void 0 ? { error: failure.error } : {}
|
|
1490
|
-
}
|
|
1491
|
-
})
|
|
1492
|
-
);
|
|
1493
|
-
};
|
|
1494
|
-
}
|
|
1495
|
-
function defaultDistillerFor(opts, findings) {
|
|
1496
|
-
const durableRun = opts.runDir !== void 0 && !opts.runDir.startsWith("mem://");
|
|
1497
|
-
const useRawTraces = opts.rawTraceContext ?? durableRun;
|
|
1498
|
-
if (useRawTraces) return rawTraceDistiller({ fallbackFindings: findings });
|
|
1499
|
-
return generationFailureDistiller(findings);
|
|
1500
|
-
}
|
|
1501
|
-
async function discardPreparedBaseline(worktree, baselineWorktree, cause) {
|
|
1502
|
-
return rethrowAfterCleanup(
|
|
1503
|
-
cause,
|
|
1504
|
-
() => worktree.discard(baselineWorktree),
|
|
1505
|
-
"improve(): code preparation failed"
|
|
1506
|
-
);
|
|
1507
|
-
}
|
|
1508
|
-
async function prepareCodeRun(code) {
|
|
1509
|
-
const baseRef = code.baseRef ?? "main";
|
|
1510
|
-
const worktree = code.worktree ?? gitWorktreeAdapter({
|
|
1511
|
-
repoRoot: code.repoRoot,
|
|
1512
|
-
...code.worktreeDir ? { worktreeDir: code.worktreeDir } : {}
|
|
1513
|
-
});
|
|
1514
|
-
const baselineWorktree = await worktree.create({ baseRef, label: "incumbent-baseline" });
|
|
1515
|
-
try {
|
|
1516
|
-
const baseline = await worktree.finalize(baselineWorktree, "Incumbent code checkout");
|
|
1517
|
-
let baselineDiscarded = false;
|
|
1518
|
-
const generator = code.generator ?? agenticGenerator({
|
|
1519
|
-
...code.harness ? { harness: code.harness } : {},
|
|
1520
|
-
...code.verify ? { verify: code.verify } : {},
|
|
1521
|
-
...code.timeoutMs ? { timeoutMs: code.timeoutMs } : {}
|
|
1522
|
-
});
|
|
1523
|
-
const managed = improvementDriver({ worktree, generator, baseRef });
|
|
1524
|
-
return {
|
|
1525
|
-
baseline,
|
|
1526
|
-
proposer: managed,
|
|
1527
|
-
async cleanup(retainedWinner) {
|
|
1528
|
-
const errors = [];
|
|
1529
|
-
const retainedWorktreeRef = isCodeSurface(retainedWinner) ? retainedWinner.worktreeRef : void 0;
|
|
1530
|
-
try {
|
|
1531
|
-
await managed?.cleanup(retainedWorktreeRef ? [retainedWorktreeRef] : []);
|
|
1532
|
-
} catch (cause) {
|
|
1533
|
-
errors.push(cause);
|
|
1534
|
-
}
|
|
1535
|
-
if (!baselineDiscarded && retainedWorktreeRef !== baseline.worktreeRef) {
|
|
1536
|
-
try {
|
|
1537
|
-
await worktree.discard(baselineWorktree);
|
|
1538
|
-
baselineDiscarded = true;
|
|
1539
|
-
} catch (cause) {
|
|
1540
|
-
errors.push(cause);
|
|
1541
|
-
}
|
|
1542
|
-
}
|
|
1543
|
-
if (errors.length > 0) {
|
|
1544
|
-
throw new AggregateError(errors, "improve(): failed to clean code improvement worktrees");
|
|
1545
|
-
}
|
|
1546
|
-
}
|
|
1547
|
-
};
|
|
1548
|
-
} catch (cause) {
|
|
1549
|
-
return discardPreparedBaseline(worktree, baselineWorktree, cause);
|
|
1550
|
-
}
|
|
1551
|
-
}
|
|
1552
|
-
function idempotentDispose(dispose) {
|
|
1553
|
-
let disposed = false;
|
|
1554
|
-
let inFlight;
|
|
1555
|
-
return async () => {
|
|
1556
|
-
if (disposed) return;
|
|
1557
|
-
if (inFlight) return inFlight;
|
|
1558
|
-
inFlight = (async () => {
|
|
1559
|
-
await dispose();
|
|
1560
|
-
disposed = true;
|
|
1561
|
-
})();
|
|
1562
|
-
try {
|
|
1563
|
-
await inFlight;
|
|
1564
|
-
} finally {
|
|
1565
|
-
inFlight = void 0;
|
|
1566
|
-
}
|
|
1567
|
-
};
|
|
1568
|
-
}
|
|
1569
|
-
async function runCodeImprovement(opts) {
|
|
1570
|
-
const {
|
|
1571
|
-
gate = "holdout",
|
|
1572
|
-
findings: inputFindings = [],
|
|
1573
|
-
rawTraceContext: _rawTraceContext,
|
|
1574
|
-
code,
|
|
1575
|
-
promotionGate,
|
|
1576
|
-
analyzeGeneration,
|
|
1577
|
-
surface: _surface,
|
|
1578
|
-
...sharedOptions
|
|
1579
|
-
} = opts;
|
|
1580
|
-
const findings = [...inputFindings];
|
|
1581
|
-
const preparedCode = await prepareCodeRun(code);
|
|
1582
|
-
const budget = gate === "none" ? { ...sharedOptions.budget, generations: 0 } : { ...sharedOptions.budget };
|
|
1583
|
-
let raw;
|
|
1584
|
-
try {
|
|
1585
|
-
raw = await selfImprove({
|
|
1586
|
-
...sharedOptions,
|
|
1587
|
-
baselineSurface: preparedCode.baseline,
|
|
1588
|
-
proposer: preparedCode.proposer,
|
|
1589
|
-
budget,
|
|
1590
|
-
findings,
|
|
1591
|
-
...promotionGate !== void 0 ? { gate: promotionGate } : {},
|
|
1592
|
-
...analyzeGeneration === null ? {} : {
|
|
1593
|
-
analyzeGeneration: analyzeGeneration ?? defaultDistillerFor(opts, findings)
|
|
1594
|
-
}
|
|
1595
|
-
});
|
|
1596
|
-
} catch (cause) {
|
|
1597
|
-
return rethrowAfterCleanup(
|
|
1598
|
-
cause,
|
|
1599
|
-
() => preparedCode.cleanup(),
|
|
1600
|
-
"improve(): code improvement failed"
|
|
1601
|
-
);
|
|
1602
|
-
}
|
|
1603
|
-
const winnerSurface = raw.winner.surface;
|
|
1604
|
-
assertCandidateSurfaceKind("code", preparedCode.baseline, winnerSurface);
|
|
1605
|
-
try {
|
|
1606
|
-
await preparedCode.cleanup(winnerSurface);
|
|
1607
|
-
} catch (cleanupCause) {
|
|
1608
|
-
try {
|
|
1609
|
-
await preparedCode.cleanup();
|
|
1610
|
-
} catch (finalCleanupCause) {
|
|
1611
|
-
throw new AggregateError(
|
|
1612
|
-
[cleanupCause, finalCleanupCause],
|
|
1613
|
-
"improve(): code result cleanup failed, including the final all-worktree retry"
|
|
1614
|
-
);
|
|
1615
|
-
}
|
|
1616
|
-
throw new AggregateError(
|
|
1617
|
-
[cleanupCause],
|
|
1618
|
-
"improve(): code result cleanup failed; the final all-worktree retry succeeded"
|
|
1619
|
-
);
|
|
1620
|
-
}
|
|
1621
|
-
const dispose = idempotentDispose(async () => preparedCode.cleanup());
|
|
1622
|
-
const candidate = immutableCandidateValue({
|
|
1623
|
-
surface: "code",
|
|
1624
|
-
value: winnerSurface
|
|
1625
|
-
});
|
|
1626
|
-
return {
|
|
1627
|
-
mode: "code",
|
|
1628
|
-
candidate,
|
|
1629
|
-
decision: raw.gateDecision,
|
|
1630
|
-
...raw.lift !== void 0 ? { lift: raw.lift } : {},
|
|
1631
|
-
cost: copyImproveCost(raw.cost),
|
|
1632
|
-
durationMs: raw.durationMs,
|
|
1633
|
-
lineage: Object.freeze({
|
|
1634
|
-
invocationId: raw.provenance.runId,
|
|
1635
|
-
runId: raw.provenance.runId,
|
|
1636
|
-
developmentSplitDigest: raw.provenance.evidence.search.splitDigest
|
|
1637
|
-
}),
|
|
1638
|
-
generationsExplored: raw.generationsExplored,
|
|
1639
|
-
raw,
|
|
1640
|
-
dispose
|
|
1641
|
-
};
|
|
1642
|
-
}
|
|
1643
|
-
|
|
1644
|
-
// src/improvement/method-execution.ts
|
|
1645
|
-
import { randomUUID } from "crypto";
|
|
1646
|
-
import {
|
|
1647
|
-
compareOptimizationMethods
|
|
1648
|
-
} from "@tangle-network/agent-eval/campaign";
|
|
1649
|
-
import {
|
|
1650
|
-
sha256DigestSchema
|
|
1651
|
-
} from "@tangle-network/agent-interface";
|
|
1652
|
-
|
|
1653
|
-
// src/improvement/method-controls.ts
|
|
1654
|
-
var methodRuntimeControls = /* @__PURE__ */ Symbol("agent-runtime.improvement.method-runtime-controls");
|
|
1655
|
-
function withMethodRuntimeControls(method, controls) {
|
|
1656
|
-
return Object.freeze({
|
|
1657
|
-
...method,
|
|
1658
|
-
[methodRuntimeControls]: Object.freeze({ ...controls })
|
|
1659
|
-
});
|
|
1660
|
-
}
|
|
1661
|
-
function methodRuntimeControlsOf(method) {
|
|
1662
|
-
return method[methodRuntimeControls];
|
|
1663
|
-
}
|
|
1664
|
-
|
|
1665
|
-
// src/improvement/method-cost.ts
|
|
1666
|
-
var EVALUATION_TAG = "runtimeEvaluationRef";
|
|
1667
|
-
var INVOCATION_TAG = "runtimeInvocationId";
|
|
1668
|
-
function methodInputWithScopedCost(input, scope, costAttribution = "invocation") {
|
|
1669
|
-
return Object.freeze({
|
|
1670
|
-
...input,
|
|
1671
|
-
costLedger: scopedCostLedger(input.costLedger, scope, costAttribution)
|
|
1672
|
-
});
|
|
1673
|
-
}
|
|
1674
|
-
function methodInvocationCostLedger(ledger, scope) {
|
|
1675
|
-
return scopedCostLedger(ledger, scope, "invocation");
|
|
1676
|
-
}
|
|
1677
|
-
function assertMethodCostRecorded(methodName, result, compatibleLedger, invocationLedger, costCeiling, costAttribution = "invocation") {
|
|
1678
|
-
const { cost } = result;
|
|
1679
|
-
const observed = costAttribution === "optimizer-run" && result.provenance?.runId ? compatibleLedger.summary({ tags: { optimizerRun: result.provenance.runId } }) : invocationLedger.summary();
|
|
1680
|
-
if (costCeiling !== void 0 && !cost.accountingComplete) {
|
|
1681
|
-
throw new ConfigError(
|
|
1682
|
-
`improve(): method '${methodName}' returned incomplete cost accounting under costCeiling; refusing final scoring`
|
|
1683
|
-
);
|
|
1684
|
-
}
|
|
1685
|
-
if (costCeiling !== void 0 && exceeds(cost.totalCostUsd, costCeiling)) {
|
|
1686
|
-
throw new ConfigError(
|
|
1687
|
-
`improve(): method '${methodName}' reported cost $${cost.totalCostUsd} above costCeiling $${costCeiling}; refusing final scoring`
|
|
1688
|
-
);
|
|
1689
|
-
}
|
|
1690
|
-
if (cost.accountingComplete && !observed.accountingComplete) {
|
|
1691
|
-
throw new ConfigError(
|
|
1692
|
-
`improve(): method '${methodName}' reported complete cost accounting but its shared cost receipts are incomplete`
|
|
1693
|
-
);
|
|
1694
|
-
}
|
|
1695
|
-
if (cost.accountingComplete && !approximatelyEqual(cost.totalCostUsd, observed.totalCostUsd)) {
|
|
1696
|
-
throw new ConfigError(
|
|
1697
|
-
`improve(): method '${methodName}' reported $${cost.totalCostUsd} but recorded $${observed.totalCostUsd} through input.costLedger`
|
|
1698
|
-
);
|
|
1699
|
-
}
|
|
1700
|
-
}
|
|
1701
|
-
function scopedCostLedger(parent, scope, costAttribution) {
|
|
1702
|
-
const evaluationTags = {
|
|
1703
|
-
[EVALUATION_TAG]: scope.evaluationRef
|
|
1704
|
-
};
|
|
1705
|
-
const invocationTags = {
|
|
1706
|
-
...evaluationTags,
|
|
1707
|
-
[INVOCATION_TAG]: scope.invocationId
|
|
1708
|
-
};
|
|
1709
|
-
const readTags = (filter) => costAttribution === "optimizer-run" && hasOptimizerRunFilter(filter) ? evaluationTags : invocationTags;
|
|
1710
|
-
return {
|
|
1711
|
-
costCeilingUsd: parent.costCeilingUsd,
|
|
1712
|
-
runPaidCall: (input) => parent.runPaidCall({
|
|
1713
|
-
...input,
|
|
1714
|
-
tags: { ...input.tags ?? {}, ...invocationTags }
|
|
1715
|
-
}),
|
|
1716
|
-
reconcile: (...args) => parent.reconcile(...args),
|
|
1717
|
-
list: (filter) => parent.list(withTags(filter, readTags(filter))),
|
|
1718
|
-
listPending: (filter) => parent.listPending?.(withTags(filter, readTags(filter))) ?? [],
|
|
1719
|
-
summary: (filter) => parent.summary(withTags(filter, readTags(filter))),
|
|
1720
|
-
waitForIdle: (options = {}) => parent.waitForIdle?.({
|
|
1721
|
-
...options,
|
|
1722
|
-
filter: withTags(options.filter, readTags(options.filter))
|
|
1723
|
-
}) ?? Promise.resolve(
|
|
1724
|
-
parent.summary(withTags(options.filter, readTags(options.filter))).pendingCalls === 0
|
|
1725
|
-
),
|
|
1726
|
-
markCompleted: (count) => parent.markCompleted(count),
|
|
1727
|
-
costPerCompletedTask: () => parent.costPerCompletedTask()
|
|
1728
|
-
};
|
|
1729
|
-
}
|
|
1730
|
-
function hasOptimizerRunFilter(filter) {
|
|
1731
|
-
return typeof filter?.tags?.optimizerRun === "string" && filter.tags.optimizerRun.length > 0;
|
|
1732
|
-
}
|
|
1733
|
-
function withTags(filter, tags) {
|
|
1734
|
-
return {
|
|
1735
|
-
...filter ?? {},
|
|
1736
|
-
tags: { ...filter?.tags ?? {}, ...tags }
|
|
1737
|
-
};
|
|
1738
|
-
}
|
|
1739
|
-
function approximatelyEqual(left, right) {
|
|
1740
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 8;
|
|
1741
|
-
return Math.abs(left - right) <= tolerance;
|
|
1742
|
-
}
|
|
1743
|
-
function exceeds(value, limit) {
|
|
1744
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(value), Math.abs(limit)) * 8;
|
|
1745
|
-
return value - limit > tolerance;
|
|
1746
|
-
}
|
|
1747
|
-
|
|
1748
|
-
// src/improvement/method-identity.ts
|
|
1749
|
-
import {
|
|
1750
|
-
campaignSplitDigest
|
|
1751
|
-
} from "@tangle-network/agent-eval/campaign";
|
|
1752
|
-
function buildMethodEvaluationIdentity(input) {
|
|
1753
|
-
const optimizationReps = input.optimizationRunOptions?.reps ?? 1;
|
|
1754
|
-
const developmentSplitDigest = canonicalCandidateDigest({
|
|
1755
|
-
train: campaignSplitDigest(input.trainScenarios, optimizationReps),
|
|
1756
|
-
selection: campaignSplitDigest(input.selectionScenarios, optimizationReps)
|
|
1757
|
-
});
|
|
1758
|
-
const judgeDescriptors = input.judges.map(judgeDescriptor);
|
|
1759
|
-
const evaluationRef = canonicalCandidateDigest({
|
|
1760
|
-
executionRef: input.executionRef,
|
|
1761
|
-
baselineProfileDigest: input.baselineProfileDigest,
|
|
1762
|
-
coordinate: profileCoordinate(input.surface, input.baselineSurface, input.skills),
|
|
1763
|
-
candidateValidation: input.validateCandidate ? Function.prototype.toString.call(input.validateCandidate) : null,
|
|
1764
|
-
developmentSplitDigest,
|
|
1765
|
-
findings: input.findings,
|
|
1766
|
-
judges: judgeDescriptors,
|
|
1767
|
-
run: {
|
|
1768
|
-
seed: input.seed ?? 42,
|
|
1769
|
-
finalReps: input.reps ?? 1,
|
|
1770
|
-
optimizationReps,
|
|
1771
|
-
costCeiling: input.costCeiling ?? null,
|
|
1772
|
-
resumable: input.optimizationRunOptions?.resumable ?? true,
|
|
1773
|
-
maxConcurrency: input.optimizationRunOptions?.maxConcurrency ?? 2,
|
|
1774
|
-
abortOnCellError: input.optimizationRunOptions?.abortOnCellError ?? false,
|
|
1775
|
-
dispatchTimeoutMs: input.optimizationRunOptions?.dispatchTimeoutMs ?? null,
|
|
1776
|
-
dispatchShutdownTimeoutMs: input.optimizationRunOptions?.dispatchShutdownTimeoutMs ?? 5e3,
|
|
1777
|
-
tracing: input.optimizationRunOptions?.tracing ?? "on",
|
|
1778
|
-
expectUsage: input.optimizationRunOptions?.expectUsage ?? "warn",
|
|
1779
|
-
captureSource: input.optimizationRunOptions?.captureSource ?? null,
|
|
1780
|
-
captureSourceVersionHash: input.optimizationRunOptions?.captureSourceVersionHash ?? null
|
|
1781
|
-
}
|
|
1782
|
-
});
|
|
1783
|
-
return {
|
|
1784
|
-
evaluationRef,
|
|
1785
|
-
developmentSplitDigest,
|
|
1786
|
-
judgeDescriptors
|
|
1787
|
-
};
|
|
1788
|
-
}
|
|
1789
|
-
function profileCoordinate(surface, baselineSurface, skills) {
|
|
1790
|
-
return {
|
|
1791
|
-
surface,
|
|
1792
|
-
resourceName: surface === "skills" ? skills?.resourceName.trim() ?? null : null,
|
|
1793
|
-
componentNames: typeof baselineSurface === "object" && baselineSurface !== null && baselineSurface.kind === "components" ? Object.keys(baselineSurface.components).sort() : []
|
|
1794
|
-
};
|
|
1795
|
-
}
|
|
1796
|
-
function judgeDescriptor(judge) {
|
|
1797
|
-
return {
|
|
1798
|
-
name: judge.name,
|
|
1799
|
-
dimensions: judge.dimensions.map((dimension) => ({ ...dimension })),
|
|
1800
|
-
declaredVersion: judge.judgeVersion ?? null,
|
|
1801
|
-
scoreImplementation: Function.prototype.toString.call(judge.score),
|
|
1802
|
-
appliesToImplementation: judge.appliesTo ? Function.prototype.toString.call(judge.appliesTo) : null
|
|
1803
|
-
};
|
|
1804
|
-
}
|
|
1805
|
-
|
|
1806
|
-
// src/improvement/method-execution.ts
|
|
1807
|
-
function resolveOptimizationMethod(source, context) {
|
|
1808
|
-
const method = typeof source === "function" ? source(context) : source;
|
|
1809
|
-
if (!method || typeof method !== "object" || typeof method.name !== "string" || method.name.trim() !== method.name || method.name.length === 0 || typeof method.optimize !== "function") {
|
|
1810
|
-
throw new ConfigError(
|
|
1811
|
-
"improve(): method must be a complete OptimizationMethod with a trimmed name and optimize(input)"
|
|
1812
|
-
);
|
|
1813
|
-
}
|
|
1814
|
-
return method;
|
|
1815
|
-
}
|
|
1816
|
-
function copyProvenance(provenance) {
|
|
1817
|
-
return immutableCandidateValue(provenance);
|
|
1818
|
-
}
|
|
1819
|
-
function validateExecutionRef(value) {
|
|
1820
|
-
const parsed = sha256DigestSchema.safeParse(value);
|
|
1821
|
-
if (!parsed.success) {
|
|
1822
|
-
throw new ConfigError("improve(): executionRef must be a lowercase sha256:<64 hex> digest");
|
|
1823
|
-
}
|
|
1824
|
-
return parsed.data;
|
|
1825
|
-
}
|
|
1826
|
-
async function runMethodImprovement(profile, opts) {
|
|
1827
|
-
const {
|
|
1828
|
-
surface = "prompt",
|
|
1829
|
-
executionRef: inputExecutionRef,
|
|
1830
|
-
method: methodSource,
|
|
1831
|
-
agent,
|
|
1832
|
-
validateCandidate,
|
|
1833
|
-
findings: inputFindings = [],
|
|
1834
|
-
skills,
|
|
1835
|
-
profileComponents,
|
|
1836
|
-
optimizationRunOptions,
|
|
1837
|
-
minimumLift = 0,
|
|
1838
|
-
...comparisonOptions
|
|
1839
|
-
} = opts;
|
|
1840
|
-
if (!Number.isFinite(minimumLift) || minimumLift < 0) {
|
|
1841
|
-
throw new ConfigError(
|
|
1842
|
-
"improve(): minimumLift must be a finite number greater than or equal to 0"
|
|
1843
|
-
);
|
|
1844
|
-
}
|
|
1845
|
-
if (profileComponents && surface !== "agent-profile") {
|
|
1846
|
-
throw new ConfigError("improve(): profileComponents is valid only with surface 'agent-profile'");
|
|
1847
|
-
}
|
|
1848
|
-
const executionRef = validateExecutionRef(inputExecutionRef);
|
|
1849
|
-
const findings = [...inputFindings];
|
|
1850
|
-
const preparedSurface = prepareProfileSurface(profile, surface, skills, profileComponents);
|
|
1851
|
-
const baselineSurface = preparedSurface.surface;
|
|
1852
|
-
const baselineValue = immutableCandidateValue(preparedSurface.value);
|
|
1853
|
-
const baselineProfileDigest = canonicalCandidateDigest(profile);
|
|
1854
|
-
const identity = buildMethodEvaluationIdentity({
|
|
1855
|
-
executionRef,
|
|
1856
|
-
baselineProfileDigest,
|
|
1857
|
-
baselineSurface,
|
|
1858
|
-
surface,
|
|
1859
|
-
skills,
|
|
1860
|
-
validateCandidate,
|
|
1861
|
-
findings,
|
|
1862
|
-
trainScenarios: comparisonOptions.trainScenarios,
|
|
1863
|
-
selectionScenarios: comparisonOptions.selectionScenarios,
|
|
1864
|
-
judges: comparisonOptions.judges,
|
|
1865
|
-
seed: comparisonOptions.seed,
|
|
1866
|
-
reps: comparisonOptions.reps,
|
|
1867
|
-
costCeiling: comparisonOptions.costCeiling,
|
|
1868
|
-
optimizationRunOptions
|
|
1869
|
-
});
|
|
1870
|
-
const { evaluationRef, developmentSplitDigest } = identity;
|
|
1871
|
-
const dispatchRef = `improve:${evaluationRef}`;
|
|
1872
|
-
const identifiedJudges = comparisonOptions.judges.map(
|
|
1873
|
-
(judge, index) => Object.freeze({
|
|
1874
|
-
...judge,
|
|
1875
|
-
judgeVersion: canonicalCandidateDigest({
|
|
1876
|
-
evaluationRef,
|
|
1877
|
-
descriptor: identity.judgeDescriptors[index]
|
|
1878
|
-
})
|
|
1879
|
-
})
|
|
1880
|
-
);
|
|
1881
|
-
const rawMaterializeProfile = createProfileCandidateMaterializer(
|
|
1882
|
-
profile,
|
|
1883
|
-
surface,
|
|
1884
|
-
baselineSurface,
|
|
1885
|
-
skills,
|
|
1886
|
-
profileComponents
|
|
1887
|
-
);
|
|
1888
|
-
const runtimeInvocationId = `runtime-optimization:${randomUUID()}`;
|
|
1889
|
-
const method = resolveOptimizationMethod(methodSource, {
|
|
1890
|
-
profile,
|
|
1891
|
-
evaluationRef,
|
|
1892
|
-
surface,
|
|
1893
|
-
baselineSurface,
|
|
1894
|
-
baselineValue,
|
|
1895
|
-
findings
|
|
1896
|
-
});
|
|
1897
|
-
const methodControls = methodRuntimeControlsOf(method);
|
|
1898
|
-
const baselineSurfaceDigest = canonicalCandidateDigest(baselineSurface);
|
|
1899
|
-
const validatedCandidates = /* @__PURE__ */ new Set();
|
|
1900
|
-
const materializeProfile2 = (candidateSurface) => {
|
|
1901
|
-
const candidate2 = rawMaterializeProfile(candidateSurface);
|
|
1902
|
-
const candidateDigest = canonicalCandidateDigest(candidateSurface);
|
|
1903
|
-
if (!validatedCandidates.has(candidateDigest)) {
|
|
1904
|
-
const prepared = prepareProfileSurface(candidate2, surface, skills, profileComponents);
|
|
1905
|
-
const validationInput = Object.freeze({
|
|
1906
|
-
profile: candidate2,
|
|
1907
|
-
surface,
|
|
1908
|
-
candidateSurface: immutableCandidateValue(candidateSurface),
|
|
1909
|
-
value: immutableCandidateValue(prepared.value),
|
|
1910
|
-
isBaseline: candidateDigest === baselineSurfaceDigest
|
|
1911
|
-
});
|
|
1912
|
-
methodControls?.validateCandidate(validationInput);
|
|
1913
|
-
validateCandidate?.(validationInput);
|
|
1914
|
-
validatedCandidates.add(candidateDigest);
|
|
1915
|
-
}
|
|
1916
|
-
return candidate2;
|
|
1917
|
-
};
|
|
1918
|
-
materializeProfile2(baselineSurface);
|
|
1919
|
-
const measuredMethod = {
|
|
1920
|
-
...method,
|
|
1921
|
-
async optimize(input) {
|
|
1922
|
-
const costScope = {
|
|
1923
|
-
evaluationRef,
|
|
1924
|
-
invocationId: runtimeInvocationId
|
|
1925
|
-
};
|
|
1926
|
-
const scopedInput = methodInputWithScopedCost(
|
|
1927
|
-
input,
|
|
1928
|
-
costScope,
|
|
1929
|
-
methodControls?.costAttribution
|
|
1930
|
-
);
|
|
1931
|
-
const invocationLedger = methodInvocationCostLedger(input.costLedger, costScope);
|
|
1932
|
-
const result = await method.optimize(scopedInput);
|
|
1933
|
-
assertMethodCostRecorded(
|
|
1934
|
-
method.name,
|
|
1935
|
-
result,
|
|
1936
|
-
scopedInput.costLedger,
|
|
1937
|
-
invocationLedger,
|
|
1938
|
-
comparisonOptions.costCeiling,
|
|
1939
|
-
methodControls?.costAttribution
|
|
1940
|
-
);
|
|
1941
|
-
materializeProfile2(result.winnerSurface);
|
|
1942
|
-
return result;
|
|
1943
|
-
}
|
|
1944
|
-
};
|
|
1945
|
-
const startedAt = Date.now();
|
|
1946
|
-
const raw = await compareOptimizationMethods({
|
|
1947
|
-
...comparisonOptions,
|
|
1948
|
-
judges: identifiedJudges,
|
|
1949
|
-
dispatchRef,
|
|
1950
|
-
optimizationRunOptions: {
|
|
1951
|
-
...optimizationRunOptions ?? {},
|
|
1952
|
-
dispatchRef
|
|
1953
|
-
},
|
|
1954
|
-
methods: [measuredMethod],
|
|
1955
|
-
baselineSurface,
|
|
1956
|
-
dispatchWithSurface: (candidateSurface, scenario, ctx) => agent(materializeProfile2(candidateSurface), scenario, ctx)
|
|
1957
|
-
});
|
|
1958
|
-
if (comparisonOptions.costCeiling !== void 0 && raw.totalCost.totalCostUsd > comparisonOptions.costCeiling) {
|
|
1959
|
-
throw new ConfigError(
|
|
1960
|
-
`improve(): reported total cost $${raw.totalCost.totalCostUsd} exceeds costCeiling $${comparisonOptions.costCeiling}`
|
|
1961
|
-
);
|
|
1962
|
-
}
|
|
1963
|
-
const score = raw.best;
|
|
1964
|
-
const winnerSurface = immutableCandidateValue(score.winnerSurface);
|
|
1965
|
-
assertCandidateSurfaceKind(surface, baselineSurface, winnerSurface);
|
|
1966
|
-
const candidateProfile = materializeProfile2(winnerSurface);
|
|
1967
|
-
const candidate = Object.freeze({
|
|
1968
|
-
surface,
|
|
1969
|
-
value: winnerSurface,
|
|
1970
|
-
profile: candidateProfile
|
|
1971
|
-
});
|
|
1972
|
-
const cost = copyImproveCost(raw.totalCost);
|
|
1973
|
-
return {
|
|
1974
|
-
mode: "method",
|
|
1975
|
-
method: method.name,
|
|
1976
|
-
...score.provenance ? { provenance: copyProvenance(score.provenance) } : {},
|
|
1977
|
-
candidate,
|
|
1978
|
-
decision: cost.accountingComplete && score.liftCi.low > minimumLift ? "ship" : "hold",
|
|
1979
|
-
lift: score.lift,
|
|
1980
|
-
liftInterval: { ...score.liftCi },
|
|
1981
|
-
cost,
|
|
1982
|
-
durationMs: Date.now() - startedAt,
|
|
1983
|
-
lineage: Object.freeze({
|
|
1984
|
-
invocationId: runtimeInvocationId,
|
|
1985
|
-
runId: score.provenance?.runId ?? runtimeInvocationId,
|
|
1986
|
-
developmentSplitDigest,
|
|
1987
|
-
executionRef,
|
|
1988
|
-
baselineProfileDigest
|
|
1989
|
-
}),
|
|
1990
|
-
raw,
|
|
1991
|
-
async dispose() {
|
|
1992
|
-
}
|
|
1993
|
-
};
|
|
1994
|
-
}
|
|
1995
|
-
|
|
1996
|
-
// src/improvement/improve.ts
|
|
1997
|
-
async function improve(profileOrCode, opts) {
|
|
1998
|
-
if (opts === void 0) {
|
|
1999
|
-
const code = profileOrCode;
|
|
2000
|
-
if (!code || code.surface !== "code") {
|
|
2001
|
-
throw new ConfigError("improve(): the one-argument form requires { surface: 'code', ... }");
|
|
2002
|
-
}
|
|
2003
|
-
return runCodeImprovement(code);
|
|
2004
|
-
}
|
|
2005
|
-
if (opts.surface === "code") {
|
|
2006
|
-
throw new ConfigError("improve(): code takes one argument: improve({ surface: 'code', ... })");
|
|
2007
|
-
}
|
|
2008
|
-
const parsedProfile = agentProfileSchema2.safeParse(profileOrCode);
|
|
2009
|
-
if (!parsedProfile.success) {
|
|
2010
|
-
throw new ConfigError(
|
|
2011
|
-
`improve(): input is not a valid AgentProfile: ${parsedProfile.error.message}`
|
|
2012
|
-
);
|
|
2013
|
-
}
|
|
2014
|
-
return runMethodImprovement(immutableCandidateValue(parsedProfile.data), opts);
|
|
2015
|
-
}
|
|
2016
|
-
|
|
2017
|
-
// src/intelligence/improvement-surfaces.ts
|
|
2018
|
-
import {
|
|
2019
|
-
agentProfileDiffSchema,
|
|
2020
|
-
agentProfileSchema as agentProfileSchema3,
|
|
2021
|
-
defineAgentProfileDiff
|
|
2022
|
-
} from "@tangle-network/agent-interface";
|
|
2023
|
-
var changedSurfaceOrder = [
|
|
2024
|
-
"prompt",
|
|
2025
|
-
"skills",
|
|
2026
|
-
"tools",
|
|
2027
|
-
"mcp",
|
|
2028
|
-
"hooks",
|
|
2029
|
-
"subagents",
|
|
2030
|
-
"agent-profile",
|
|
2031
|
-
"memory",
|
|
2032
|
-
"code",
|
|
2033
|
-
"knowledge"
|
|
2034
|
-
];
|
|
2035
|
-
var AGENT_IMPROVEMENT_PROFILE_SURFACES = [
|
|
2036
|
-
"prompt",
|
|
2037
|
-
"skills",
|
|
2038
|
-
"tools",
|
|
2039
|
-
"mcp",
|
|
2040
|
-
"hooks",
|
|
2041
|
-
"subagents"
|
|
2042
|
-
];
|
|
2043
|
-
function deriveChangedSurfaces(baselineBundle, candidateBundle) {
|
|
2044
|
-
if (baselineBundle.knowledge || candidateBundle.knowledge) {
|
|
2045
|
-
assertKnowledgeCandidatePair(baselineBundle, candidateBundle);
|
|
2046
|
-
}
|
|
2047
|
-
assertCodeCandidatePair(baselineBundle, candidateBundle);
|
|
2048
|
-
const baseline = improvementSurfaceValues(baselineBundle);
|
|
2049
|
-
const candidate = improvementSurfaceValues(candidateBundle);
|
|
2050
|
-
const changed = changedSurfaceOrder.filter(
|
|
2051
|
-
(surface) => canonicalCandidateDigest(baseline[surface]) !== canonicalCandidateDigest(candidate[surface])
|
|
2052
|
-
);
|
|
2053
|
-
if (changed.length === 0) throw new Error("candidate experiment does not change an agent surface");
|
|
2054
|
-
return changed;
|
|
2055
|
-
}
|
|
2056
|
-
function assertAgentImprovementActivationTargets(surfaces, experiment, intent, targets) {
|
|
2057
|
-
const expected = new Set(surfaces);
|
|
2058
|
-
const actual = new Set(targets.map((target) => target.surface));
|
|
2059
|
-
const sourceArm = intent === "activate-candidate" ? "baseline" : "candidate";
|
|
2060
|
-
if (targets.some((target) => !target.identity.trim()) || targets.some(
|
|
2061
|
-
(target) => target.expectedBaseDigest !== agentImprovementTargetDigest(experiment, sourceArm, target.surface)
|
|
2062
|
-
) || targets.length !== surfaces.length || expected.size !== actual.size || [...expected].some((surface) => !actual.has(surface))) {
|
|
2063
|
-
throw new Error("candidate activation targets must cover exactly the changed surfaces");
|
|
2064
|
-
}
|
|
2065
|
-
}
|
|
2066
|
-
function buildAgentImprovementActivationTargets(surfaces, experiment, intent, identities) {
|
|
2067
|
-
const sourceArm = intent === "activate-candidate" ? "baseline" : "candidate";
|
|
2068
|
-
const targets = identities.map((target) => ({
|
|
2069
|
-
...target,
|
|
2070
|
-
expectedBaseDigest: agentImprovementTargetDigest(experiment, sourceArm, target.surface)
|
|
2071
|
-
}));
|
|
2072
|
-
assertAgentImprovementActivationTargets(surfaces, experiment, intent, targets);
|
|
2073
|
-
return targets;
|
|
2074
|
-
}
|
|
2075
|
-
function agentImprovementTargetDigest(experiment, arm, surface) {
|
|
2076
|
-
if (surface === "knowledge") {
|
|
2077
|
-
const knowledge = assertKnowledgeCandidatePair(experiment.baseline, experiment.candidate);
|
|
2078
|
-
return arm === "baseline" ? knowledge.candidate.baseHash : knowledge.candidate.candidateHash;
|
|
2079
|
-
}
|
|
2080
|
-
if (surface === "code") assertCodeCandidatePair(experiment.baseline, experiment.candidate);
|
|
2081
|
-
return canonicalCandidateDigest(improvementSurfaceValues(experiment[arm])[surface]);
|
|
2082
|
-
}
|
|
2083
|
-
function agentImprovementTargetInput(bundle, surface) {
|
|
2084
|
-
return improvementSurfaceValues(bundle)[surface];
|
|
2085
|
-
}
|
|
2086
|
-
function isAgentImprovementProfileSurface(surface) {
|
|
2087
|
-
return AGENT_IMPROVEMENT_PROFILE_SURFACES.includes(surface);
|
|
2088
|
-
}
|
|
2089
|
-
function agentImprovementProfileSurfaceInput(profile, surface) {
|
|
2090
|
-
const parsed = parseExactAgentProfile(
|
|
2091
|
-
omitUndefinedObjectFields(profile, "agent improvement profile"),
|
|
2092
|
-
"agent improvement profile"
|
|
2093
|
-
);
|
|
2094
|
-
return immutableCandidateValue(profileSurfaceInput(parsed, surface));
|
|
2095
|
-
}
|
|
2096
|
-
function profileSurfaceInput(profile, surface) {
|
|
2097
|
-
switch (surface) {
|
|
2098
|
-
case "prompt":
|
|
2099
|
-
return { prompt: profile.prompt ?? null };
|
|
2100
|
-
case "skills":
|
|
2101
|
-
return profile.resources?.skills ?? null;
|
|
2102
|
-
case "tools":
|
|
2103
|
-
return {
|
|
2104
|
-
tools: profile.tools ?? null,
|
|
2105
|
-
resources: profile.resources?.tools ?? null
|
|
2106
|
-
};
|
|
2107
|
-
case "mcp":
|
|
2108
|
-
return profile.mcp ?? null;
|
|
2109
|
-
case "hooks":
|
|
2110
|
-
return profile.hooks ?? null;
|
|
2111
|
-
case "subagents":
|
|
2112
|
-
return {
|
|
2113
|
-
subagents: profile.subagents ?? null,
|
|
2114
|
-
resources: profile.resources?.agents ?? null
|
|
2115
|
-
};
|
|
2116
|
-
}
|
|
2117
|
-
}
|
|
2118
|
-
function agentImprovementProfileSurfaceDigest(profile, surface) {
|
|
2119
|
-
return canonicalCandidateDigest(agentImprovementProfileSurfaceInput(profile, surface));
|
|
2120
|
-
}
|
|
2121
|
-
function agentImprovementTargetProfileDiffs(target, options) {
|
|
2122
|
-
const { remove, set } = improvementSurfaceReplacement(target);
|
|
2123
|
-
const common = {
|
|
2124
|
-
kind: "agent-profile-diff",
|
|
2125
|
-
...options.source ? { source: options.source } : {},
|
|
2126
|
-
metadata: {
|
|
2127
|
-
...options.metadata ?? {},
|
|
2128
|
-
surface: target.surface
|
|
2129
|
-
}
|
|
2130
|
-
};
|
|
2131
|
-
const reset = agentProfileDiffSchema.parse(
|
|
2132
|
-
defineAgentProfileDiff({
|
|
2133
|
-
...common,
|
|
2134
|
-
id: `${options.id}:${target.surface}:reset`,
|
|
2135
|
-
title: `Replace active ${target.surface}`,
|
|
2136
|
-
remove
|
|
2137
|
-
})
|
|
2138
|
-
);
|
|
2139
|
-
if (!set) return [reset];
|
|
2140
|
-
const replacement = agentProfileDiffSchema.parse(
|
|
2141
|
-
defineAgentProfileDiff({
|
|
2142
|
-
...common,
|
|
2143
|
-
id: `${options.id}:${target.surface}:set`,
|
|
2144
|
-
title: `Activate measured ${target.surface}`,
|
|
2145
|
-
set
|
|
2146
|
-
})
|
|
2147
|
-
);
|
|
2148
|
-
return [reset, replacement];
|
|
2149
|
-
}
|
|
2150
|
-
function improvementSurfaceReplacement(target) {
|
|
2151
|
-
const value = target.desiredInput;
|
|
2152
|
-
switch (target.surface) {
|
|
2153
|
-
case "prompt": {
|
|
2154
|
-
const prompt = exactObject(value, ["prompt"], "prompt activation input").prompt;
|
|
2155
|
-
assertDefined(prompt, "prompt activation input.prompt");
|
|
2156
|
-
return {
|
|
2157
|
-
remove: { prompt: true },
|
|
2158
|
-
...prompt === null ? {} : { set: parseProfileSet({ prompt }) }
|
|
2159
|
-
};
|
|
2160
|
-
}
|
|
2161
|
-
case "skills":
|
|
2162
|
-
assertDefined(value, "skills activation input");
|
|
2163
|
-
return {
|
|
2164
|
-
remove: { resources: { skills: true } },
|
|
2165
|
-
...value === null ? {} : { set: parseProfileSet({ resources: { skills: value } }) }
|
|
2166
|
-
};
|
|
2167
|
-
case "tools": {
|
|
2168
|
-
const parsed = exactObject(value, ["tools", "resources"], "tools activation input");
|
|
2169
|
-
assertDefined(parsed.tools, "tools activation input.tools");
|
|
2170
|
-
assertDefined(parsed.resources, "tools activation input.resources");
|
|
2171
|
-
const set = {
|
|
2172
|
-
...parsed.tools === null ? {} : { tools: parsed.tools },
|
|
2173
|
-
...parsed.resources === null ? {} : { resources: { tools: parsed.resources } }
|
|
2174
|
-
};
|
|
2175
|
-
return {
|
|
2176
|
-
remove: { tools: true, resources: { tools: true } },
|
|
2177
|
-
...Object.keys(set).length === 0 ? {} : { set: parseProfileSet(set) }
|
|
2178
|
-
};
|
|
2179
|
-
}
|
|
2180
|
-
case "mcp":
|
|
2181
|
-
assertDefined(value, "mcp activation input");
|
|
2182
|
-
return {
|
|
2183
|
-
remove: { mcp: true },
|
|
2184
|
-
...value === null ? {} : { set: parseProfileSet({ mcp: value }) }
|
|
2185
|
-
};
|
|
2186
|
-
case "hooks":
|
|
2187
|
-
assertDefined(value, "hooks activation input");
|
|
2188
|
-
return {
|
|
2189
|
-
remove: { hooks: true },
|
|
2190
|
-
...value === null ? {} : { set: parseProfileSet({ hooks: value }) }
|
|
2191
|
-
};
|
|
2192
|
-
case "subagents": {
|
|
2193
|
-
const parsed = exactObject(value, ["subagents", "resources"], "subagents activation input");
|
|
2194
|
-
assertDefined(parsed.subagents, "subagents activation input.subagents");
|
|
2195
|
-
assertDefined(parsed.resources, "subagents activation input.resources");
|
|
2196
|
-
const set = {
|
|
2197
|
-
...parsed.subagents === null ? {} : { subagents: parsed.subagents },
|
|
2198
|
-
...parsed.resources === null ? {} : { resources: { agents: parsed.resources } }
|
|
2199
|
-
};
|
|
2200
|
-
return {
|
|
2201
|
-
remove: { subagents: true, resources: { agents: true } },
|
|
2202
|
-
...Object.keys(set).length === 0 ? {} : { set: parseProfileSet(set) }
|
|
2203
|
-
};
|
|
2204
|
-
}
|
|
2205
|
-
}
|
|
2206
|
-
}
|
|
2207
|
-
function assertDefined(value, label) {
|
|
2208
|
-
if (value === void 0) throw new Error(`${label} must not be undefined`);
|
|
2209
|
-
}
|
|
2210
|
-
function exactObject(value, keys, label) {
|
|
2211
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
2212
|
-
throw new Error(`${label} must be an object`);
|
|
2213
|
-
}
|
|
2214
|
-
const record = value;
|
|
2215
|
-
const actual = Object.keys(record).sort();
|
|
2216
|
-
const expected = [...keys].sort();
|
|
2217
|
-
if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) {
|
|
2218
|
-
throw new Error(`${label} must contain exactly: ${expected.join(", ")}`);
|
|
2219
|
-
}
|
|
2220
|
-
return record;
|
|
2221
|
-
}
|
|
2222
|
-
function parseProfileSet(value) {
|
|
2223
|
-
return agentProfileSchema3.parse(value);
|
|
2224
|
-
}
|
|
2225
|
-
function assertKnowledgeCandidatePair(baselineBundle, candidateBundle) {
|
|
2226
|
-
const baseline = baselineBundle.knowledge;
|
|
2227
|
-
const candidate = candidateBundle.knowledge;
|
|
2228
|
-
if (!baseline && !candidate) {
|
|
2229
|
-
throw new Error("knowledge candidate pair is not present");
|
|
2230
|
-
}
|
|
2231
|
-
if (!baseline || !candidate || canonicalCandidateDigest({ candidate: baseline.candidate, evaluation: baseline.evaluation }) !== canonicalCandidateDigest({ candidate: candidate.candidate, evaluation: candidate.evaluation })) {
|
|
2232
|
-
throw new Error(
|
|
2233
|
-
"knowledge experiment arms must share one measured candidate and evaluation identity"
|
|
2234
|
-
);
|
|
2235
|
-
}
|
|
2236
|
-
return candidate;
|
|
2237
|
-
}
|
|
2238
|
-
function assertCodeCandidatePair(baselineBundle, candidateBundle) {
|
|
2239
|
-
const baseline = baselineBundle.code;
|
|
2240
|
-
const candidate = candidateBundle.code;
|
|
2241
|
-
if (canonicalCandidateDigest(baseline) === canonicalCandidateDigest(candidate) || candidate.kind !== "git-patch") {
|
|
2242
|
-
return;
|
|
2243
|
-
}
|
|
2244
|
-
const baselineTree = baseline.kind === "no-op" ? baseline.baseTree : baseline.kind === "git-patch" ? baseline.candidateTree : void 0;
|
|
2245
|
-
const baselineRepository = baseline.kind === "disabled" ? void 0 : baseline.repository;
|
|
2246
|
-
if (baselineTree !== candidate.baseTree || !baselineRepository || canonicalCandidateDigest(baselineRepository) !== canonicalCandidateDigest(candidate.repository)) {
|
|
2247
|
-
throw new Error(
|
|
2248
|
-
"code candidate must be based on the exact repository tree measured by the baseline arm"
|
|
2249
|
-
);
|
|
2250
|
-
}
|
|
2251
|
-
}
|
|
2252
|
-
function improvementSurfaceValues(bundle) {
|
|
2253
|
-
const profile = agentCandidateProfileAsAgentProfile(bundle.profile);
|
|
2254
|
-
return {
|
|
2255
|
-
prompt: agentImprovementProfileSurfaceInput(profile, "prompt"),
|
|
2256
|
-
skills: agentImprovementProfileSurfaceInput(profile, "skills"),
|
|
2257
|
-
tools: agentImprovementProfileSurfaceInput(profile, "tools"),
|
|
2258
|
-
mcp: agentImprovementProfileSurfaceInput(profile, "mcp"),
|
|
2259
|
-
hooks: agentImprovementProfileSurfaceInput(profile, "hooks"),
|
|
2260
|
-
subagents: agentImprovementProfileSurfaceInput(profile, "subagents"),
|
|
2261
|
-
"agent-profile": { profile: opaqueProfileSlice(profile), execution: bundle.execution },
|
|
2262
|
-
memory: {
|
|
2263
|
-
instructions: profile.resources?.instructions ?? null,
|
|
2264
|
-
executionPolicy: bundle.memory
|
|
2265
|
-
},
|
|
2266
|
-
code: bundle.code,
|
|
2267
|
-
knowledge: bundle.knowledge ?? null
|
|
2268
|
-
};
|
|
2269
|
-
}
|
|
2270
|
-
function opaqueProfileSlice(profile) {
|
|
2271
|
-
const {
|
|
2272
|
-
prompt: _prompt,
|
|
2273
|
-
tools: _tools,
|
|
2274
|
-
mcp: _mcp,
|
|
2275
|
-
hooks: _hooks,
|
|
2276
|
-
subagents: _subagents,
|
|
2277
|
-
resources,
|
|
2278
|
-
...opaqueProfile
|
|
2279
|
-
} = profile;
|
|
2280
|
-
const {
|
|
2281
|
-
instructions: _instructions,
|
|
2282
|
-
skills: _skills,
|
|
2283
|
-
tools: _resourceTools,
|
|
2284
|
-
agents: _agents,
|
|
2285
|
-
...opaqueResources
|
|
2286
|
-
} = resources ?? {};
|
|
2287
|
-
return {
|
|
2288
|
-
...opaqueProfile,
|
|
2289
|
-
...Object.keys(opaqueResources).length > 0 ? { resources: opaqueResources } : {}
|
|
2290
|
-
};
|
|
2291
|
-
}
|
|
2292
|
-
|
|
2293
|
-
// src/intelligence/optimization-receipt.ts
|
|
2294
|
-
import {
|
|
2295
|
-
agentProfileModelHintsSchema
|
|
2296
|
-
} from "@tangle-network/agent-interface";
|
|
2297
|
-
var optimizationReceiptMetadataKey = "optimizationReceipt";
|
|
2298
|
-
function createOptimizationActivationReceipt(improvement) {
|
|
2299
|
-
const provenance = improvement.provenance;
|
|
2300
|
-
if (!provenance) return void 0;
|
|
2301
|
-
const optimizerModel = provenance.optimizerModel;
|
|
2302
|
-
const candidateModel = improvement.candidate.profile.model;
|
|
2303
|
-
return canonicalCandidateDocument({
|
|
2304
|
-
kind: "optimization-activation-receipt",
|
|
2305
|
-
method: improvement.method,
|
|
2306
|
-
source: provenance.source,
|
|
2307
|
-
...provenance.bridge ? { bridge: provenance.bridge } : {},
|
|
2308
|
-
...provenance.modules ? { modules: provenance.modules } : {},
|
|
2309
|
-
...provenance.python ? { python: provenance.python } : {},
|
|
2310
|
-
...candidateModel || optimizerModel ? {
|
|
2311
|
-
models: {
|
|
2312
|
-
...candidateModel ? { candidate: candidateModel } : {},
|
|
2313
|
-
...optimizerModel ? { optimizer: optimizerModel } : {}
|
|
2314
|
-
}
|
|
2315
|
-
} : {},
|
|
2316
|
-
usage: {
|
|
2317
|
-
optimizerEvaluations: provenance.evaluationCount,
|
|
2318
|
-
...provenance.tokenUsage ? { optimizerTokens: provenance.tokenUsage } : {}
|
|
2319
|
-
},
|
|
2320
|
-
cost: {
|
|
2321
|
-
optimization: receiptCost(improvement.raw.optimizationCost),
|
|
2322
|
-
finalTest: receiptCost(improvement.raw.testCost),
|
|
2323
|
-
total: receiptCost(improvement.raw.totalCost)
|
|
2324
|
-
},
|
|
2325
|
-
invocation: {
|
|
2326
|
-
runtimeInvocationId: improvement.lineage.invocationId,
|
|
2327
|
-
optimizerRunId: provenance.runId,
|
|
2328
|
-
...provenance.compatibleRunId ? { compatibleOptimizerRunId: provenance.compatibleRunId } : {},
|
|
2329
|
-
resumed: provenance.resumed,
|
|
2330
|
-
artifactDir: provenance.artifactDir
|
|
2331
|
-
},
|
|
2332
|
-
developmentDataDigest: improvement.lineage.developmentSplitDigest
|
|
2333
|
-
}).value;
|
|
2334
|
-
}
|
|
2335
|
-
function attachOptimizationActivationReceipt(metadata, receipt) {
|
|
2336
|
-
assertNoCallerOptimizationReceipt(metadata);
|
|
2337
|
-
return immutableCandidateValue({
|
|
2338
|
-
...metadata ?? {},
|
|
2339
|
-
[optimizationReceiptMetadataKey]: receipt
|
|
2340
|
-
});
|
|
2341
|
-
}
|
|
2342
|
-
function assertNoCallerOptimizationReceipt(metadata) {
|
|
2343
|
-
if (metadata && Object.hasOwn(metadata, optimizationReceiptMetadataKey)) {
|
|
2344
|
-
throw new Error(`candidate metadata reserves '${optimizationReceiptMetadataKey}' for Runtime`);
|
|
2345
|
-
}
|
|
2346
|
-
}
|
|
2347
|
-
function optimizationActivationReceiptFromMetadata(metadata) {
|
|
2348
|
-
const value = metadata?.[optimizationReceiptMetadataKey];
|
|
2349
|
-
if (value === void 0) return void 0;
|
|
2350
|
-
return immutableCandidateValue(parseOptimizationActivationReceipt(value));
|
|
2351
|
-
}
|
|
2352
|
-
function parseOptimizationActivationReceipt(value) {
|
|
2353
|
-
if (!isRecord(value) || value.kind !== "optimization-activation-receipt") {
|
|
2354
|
-
throw new Error("optimization receipt must be an optimization-activation-receipt");
|
|
2355
|
-
}
|
|
2356
|
-
if (!isNonEmptyString(value.method) || !isPackageSource(value.source) || value.bridge !== void 0 && !isPackageSource(value.bridge) || !isModules(value.modules) || !isPythonRuntime(value.python) || !isModels(value.models) || !isUsage(value.usage) || !isCost(value.cost) || !isInvocation(value.invocation) || !isSha256Digest(value.developmentDataDigest) || !isSha256Digest(value.digest)) {
|
|
2357
|
-
throw new Error("optimization receipt contains invalid evidence");
|
|
2358
|
-
}
|
|
2359
|
-
const receipt = value;
|
|
2360
|
-
if (canonicalCandidateDigest(omitTopLevelDigest(receipt)) !== receipt.digest) {
|
|
2361
|
-
throw new Error("optimization receipt digest does not match its evidence");
|
|
2362
|
-
}
|
|
2363
|
-
return receipt;
|
|
2364
|
-
}
|
|
2365
|
-
function isPackageSource(value) {
|
|
2366
|
-
if (!isRecord(value) || value.kind !== "package" || value.evidence !== "observed" && value.evidence !== "declared" || !isNonEmptyString(value.package) || !isNonEmptyString(value.version)) {
|
|
2367
|
-
return false;
|
|
2368
|
-
}
|
|
2369
|
-
return isOptionalNonEmptyString(value.sourceUrl) && isOptionalNonEmptyString(value.revision) && (value.sourceSha256 === void 0 || typeof value.sourceSha256 === "string" && /^[0-9a-f]{64}$/.test(value.sourceSha256));
|
|
2370
|
-
}
|
|
2371
|
-
function isModules(value) {
|
|
2372
|
-
return value === void 0 || Array.isArray(value) && value.every(
|
|
2373
|
-
(module) => isRecord(module) && isNonEmptyString(module.module) && typeof module.sourceSha256 === "string" && /^[0-9a-f]{64}$/.test(module.sourceSha256)
|
|
2374
|
-
);
|
|
2375
|
-
}
|
|
2376
|
-
function isPythonRuntime(value) {
|
|
2377
|
-
return value === void 0 || isRecord(value) && isNonEmptyString(value.implementation) && isNonEmptyString(value.version);
|
|
2378
|
-
}
|
|
2379
|
-
function isModels(value) {
|
|
2380
|
-
return value === void 0 || isRecord(value) && (value.candidate === void 0 || agentProfileModelHintsSchema.safeParse(value.candidate).success) && isOptionalNonEmptyString(value.optimizer) && (value.candidate !== void 0 || value.optimizer !== void 0);
|
|
2381
|
-
}
|
|
2382
|
-
function isUsage(value) {
|
|
2383
|
-
if (!isRecord(value) || !isNonNegativeInteger(value.optimizerEvaluations)) return false;
|
|
2384
|
-
if (value.optimizerTokens === void 0) return true;
|
|
2385
|
-
const tokens = value.optimizerTokens;
|
|
2386
|
-
if (!isRecord(tokens)) return false;
|
|
2387
|
-
const inputTokens = tokens.inputTokens;
|
|
2388
|
-
const outputTokens = tokens.outputTokens;
|
|
2389
|
-
const totalTokens = tokens.totalTokens;
|
|
2390
|
-
const calls = tokens.calls;
|
|
2391
|
-
if (!isNonNegativeInteger(inputTokens) || !isNonNegativeInteger(outputTokens) || !isNonNegativeInteger(totalTokens) || !isNonNegativeInteger(calls)) {
|
|
2392
|
-
return false;
|
|
2393
|
-
}
|
|
2394
|
-
if (!isOptionalNonNegativeInteger(tokens.cachedInputTokens) || !isOptionalNonNegativeInteger(tokens.cacheWriteInputTokens) || !isOptionalNonNegativeInteger(tokens.reasoningTokens)) {
|
|
2395
|
-
return false;
|
|
2396
|
-
}
|
|
2397
|
-
const cachedInputTokens = typeof tokens.cachedInputTokens === "number" ? tokens.cachedInputTokens : 0;
|
|
2398
|
-
const cacheWriteInputTokens = typeof tokens.cacheWriteInputTokens === "number" ? tokens.cacheWriteInputTokens : 0;
|
|
2399
|
-
const reasoningTokens = typeof tokens.reasoningTokens === "number" ? tokens.reasoningTokens : 0;
|
|
2400
|
-
return totalTokens === inputTokens + outputTokens && cachedInputTokens + cacheWriteInputTokens <= inputTokens && reasoningTokens <= outputTokens && (totalTokens === 0 || calls > 0);
|
|
2401
|
-
}
|
|
2402
|
-
function isCost(value) {
|
|
2403
|
-
if (!isRecord(value) || !isCostPart(value.optimization) || !isCostPart(value.finalTest) || !isCostPart(value.total)) {
|
|
2404
|
-
return false;
|
|
2405
|
-
}
|
|
2406
|
-
const optimization = value.optimization;
|
|
2407
|
-
const finalTest = value.finalTest;
|
|
2408
|
-
const total = value.total;
|
|
2409
|
-
return approximatelyEqual2(total.totalUsd, optimization.totalUsd + finalTest.totalUsd) && total.accountingComplete === (optimization.accountingComplete && finalTest.accountingComplete);
|
|
2410
|
-
}
|
|
2411
|
-
function isCostPart(value) {
|
|
2412
|
-
return isRecord(value) && typeof value.totalUsd === "number" && Number.isFinite(value.totalUsd) && value.totalUsd >= 0 && typeof value.accountingComplete === "boolean" && Array.isArray(value.incompleteReasons) && value.incompleteReasons.every((reason) => typeof reason === "string");
|
|
2413
|
-
}
|
|
2414
|
-
function receiptCost(value) {
|
|
2415
|
-
return {
|
|
2416
|
-
totalUsd: value.totalCostUsd,
|
|
2417
|
-
accountingComplete: value.accountingComplete,
|
|
2418
|
-
incompleteReasons: [...value.incompleteReasons]
|
|
2419
|
-
};
|
|
2420
|
-
}
|
|
2421
|
-
function approximatelyEqual2(left, right) {
|
|
2422
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 8;
|
|
2423
|
-
return Math.abs(left - right) <= tolerance;
|
|
2424
|
-
}
|
|
2425
|
-
function isInvocation(value) {
|
|
2426
|
-
return isRecord(value) && isNonEmptyString(value.runtimeInvocationId) && isNonEmptyString(value.optimizerRunId) && isOptionalNonEmptyString(value.compatibleOptimizerRunId) && typeof value.resumed === "boolean" && isNonEmptyString(value.artifactDir);
|
|
2427
|
-
}
|
|
2428
|
-
function isRecord(value) {
|
|
2429
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
2430
|
-
}
|
|
2431
|
-
function isNonEmptyString(value) {
|
|
2432
|
-
return typeof value === "string" && value.trim().length > 0;
|
|
2433
|
-
}
|
|
2434
|
-
function isOptionalNonEmptyString(value) {
|
|
2435
|
-
return value === void 0 || isNonEmptyString(value);
|
|
2436
|
-
}
|
|
2437
|
-
function isNonNegativeInteger(value) {
|
|
2438
|
-
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
2439
|
-
}
|
|
2440
|
-
function isOptionalNonNegativeInteger(value) {
|
|
2441
|
-
return value === void 0 || isNonNegativeInteger(value);
|
|
2442
|
-
}
|
|
2443
|
-
function isSha256Digest(value) {
|
|
2444
|
-
return typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value);
|
|
2445
|
-
}
|
|
2446
|
-
|
|
2447
|
-
// src/intelligence/improvement-cycle.ts
|
|
2448
|
-
import { assertNoJudgeVerdict } from "@tangle-network/agent-eval/analyst";
|
|
2449
|
-
import {
|
|
2450
|
-
measuredComparisonFromCandidateExperiment,
|
|
2451
|
-
runCandidateExperiment,
|
|
2452
|
-
sealCandidateExperiment,
|
|
2453
|
-
verifyCandidateExperiment,
|
|
2454
|
-
verifyCandidateExperimentComparison
|
|
2455
|
-
} from "@tangle-network/agent-eval/contract";
|
|
2456
|
-
import {
|
|
2457
|
-
agentCandidateMaterializationReceiptSchema,
|
|
2458
|
-
agentCandidateRunReceiptSchema,
|
|
2459
|
-
agentImprovementActivationSchema,
|
|
2460
|
-
agentImprovementProposalSchema,
|
|
2461
|
-
agentImprovementReviewSchema,
|
|
2462
|
-
candidateExecutionEvidenceSchema
|
|
2463
|
-
} from "@tangle-network/agent-interface";
|
|
2464
|
-
import { materializeCandidateProfile } from "@tangle-network/agent-profile-materialize";
|
|
2465
|
-
var AgentCandidateExperimentCellExecutionError = class extends Error {
|
|
2466
|
-
finalization;
|
|
2467
|
-
constructor(finalization) {
|
|
2468
|
-
super(`candidate experiment cell failed: ${finalization.reason}`);
|
|
2469
|
-
this.name = "AgentCandidateExperimentCellExecutionError";
|
|
2470
|
-
this.finalization = finalization;
|
|
2471
|
-
}
|
|
2472
|
-
};
|
|
2473
|
-
function sealAgentImprovementExperiment(material, improvement) {
|
|
2474
|
-
assertRuntimeOwnedExperimentFieldsAbsent(material);
|
|
2475
|
-
const candidateLineage = {
|
|
2476
|
-
source: "optimizer",
|
|
2477
|
-
parentDigests: [material.baseline.digest],
|
|
2478
|
-
runIds: [improvement.lineage.runId],
|
|
2479
|
-
developmentSplitDigest: improvement.lineage.developmentSplitDigest
|
|
2480
|
-
};
|
|
2481
|
-
return sealCandidateExperiment({ ...material, candidateLineage });
|
|
2482
|
-
}
|
|
2483
|
-
function assertRuntimeOwnedExperimentFieldsAbsent(material) {
|
|
2484
|
-
if (material === null || typeof material !== "object" || Array.isArray(material)) {
|
|
2485
|
-
throw new Error("agent improvement experiment material must be an object");
|
|
2486
|
-
}
|
|
2487
|
-
const supplied = ["candidateLineage", "digest"].filter((field) => Object.hasOwn(material, field));
|
|
2488
|
-
if (supplied.length > 0) {
|
|
2489
|
-
throw new Error(
|
|
2490
|
-
`agent improvement experiment material must not supply Runtime-owned fields: ${supplied.join(", ")}`
|
|
2491
|
-
);
|
|
2492
|
-
}
|
|
2493
|
-
}
|
|
2494
|
-
async function runAgentCandidateExperiment(options) {
|
|
2495
|
-
const experiment = verifyCandidateExperiment(options.experiment);
|
|
2496
|
-
const measurements = await runCandidateExperiment({
|
|
2497
|
-
experiment,
|
|
2498
|
-
...options.maxConcurrency === void 0 ? {} : { maxConcurrency: options.maxConcurrency },
|
|
2499
|
-
...options.signal ? { signal: options.signal } : {},
|
|
2500
|
-
execute: async (input) => {
|
|
2501
|
-
const placement = await options.placeCell(input);
|
|
2502
|
-
return await executeAgentCandidateExperimentCell({ ...input, ...placement });
|
|
2503
|
-
}
|
|
2504
|
-
});
|
|
2505
|
-
const evaluation = createAgentImprovementMeasuredComparison({
|
|
2506
|
-
experiment,
|
|
2507
|
-
measurements,
|
|
2508
|
-
runId: options.runId,
|
|
2509
|
-
...options.candidate ? { candidate: options.candidate } : {},
|
|
2510
|
-
...options.generationsExplored === void 0 ? {} : { generationsExplored: options.generationsExplored },
|
|
2511
|
-
...options.searchDurationMs === void 0 ? {} : { searchDurationMs: options.searchDurationMs },
|
|
2512
|
-
...options.searchCostUsd === void 0 ? {} : { searchCostUsd: options.searchCostUsd },
|
|
2513
|
-
...options.metadata ? { metadata: options.metadata } : {}
|
|
2514
|
-
});
|
|
2515
|
-
return { experiment, measurements, evaluation };
|
|
2516
|
-
}
|
|
2517
|
-
async function executeAgentCandidateExperimentCell(options) {
|
|
2518
|
-
const experiment = verifyCandidateExperiment(options.experiment);
|
|
2519
|
-
const bundle = experiment[options.arm];
|
|
2520
|
-
assertExactExperimentInput(options, experiment, bundle);
|
|
2521
|
-
const attempt = options.attempt ?? 1;
|
|
2522
|
-
if (attempt > options.task.attempt.maxAttempts) {
|
|
2523
|
-
throw new Error("candidate experiment attempt exceeds the signed task policy");
|
|
2524
|
-
}
|
|
2525
|
-
const runCell = canonicalCandidateDocument({
|
|
2526
|
-
kind: "agent-candidate-run-cell",
|
|
2527
|
-
experimentDigest: experiment.digest,
|
|
2528
|
-
arm: options.arm,
|
|
2529
|
-
bundleDigest: bundle.digest,
|
|
2530
|
-
suiteDigest: options.benchmarkCell.suiteDigest,
|
|
2531
|
-
taskDigest: options.task.digest,
|
|
2532
|
-
taskIndex: options.benchmarkCell.taskIndex,
|
|
2533
|
-
repetition: options.benchmarkCell.repetition,
|
|
2534
|
-
seed: options.seed,
|
|
2535
|
-
attempt
|
|
2536
|
-
}).value;
|
|
2537
|
-
const verified = await verifyAgentCandidateBundle(bundle, options.ports);
|
|
2538
|
-
const prepared = await prepareAgentCandidateExecution(
|
|
2539
|
-
verified,
|
|
2540
|
-
{
|
|
2541
|
-
executionId: options.executionId,
|
|
2542
|
-
runCell,
|
|
2543
|
-
benchmarkSuite: experiment.benchmark.suite,
|
|
2544
|
-
task: options.task,
|
|
2545
|
-
executionRoots: options.executionRoots,
|
|
2546
|
-
stagingRoots: options.stagingRoots
|
|
2547
|
-
},
|
|
2548
|
-
options.ports,
|
|
2549
|
-
options.preparation
|
|
2550
|
-
);
|
|
2551
|
-
const finalization = await executePreparedAgentCandidate(prepared, options.execution);
|
|
2552
|
-
if (!finalization.succeeded) {
|
|
2553
|
-
throw new AgentCandidateExperimentCellExecutionError(finalization);
|
|
2554
|
-
}
|
|
2555
|
-
const evidence = canonicalCandidateDocument({
|
|
2556
|
-
kind: "agent-candidate-execution-evidence",
|
|
2557
|
-
materializationReceipt: prepared.materializationReceipt.value,
|
|
2558
|
-
receipt: finalization.receipt.value
|
|
2559
|
-
}).value;
|
|
2560
|
-
return verifyCandidateExecutionEvidence(evidence, {
|
|
2561
|
-
experiment,
|
|
2562
|
-
arm: options.arm,
|
|
2563
|
-
benchmarkCell: options.benchmarkCell,
|
|
2564
|
-
seed: options.seed,
|
|
2565
|
-
attempt,
|
|
2566
|
-
resolvedResources: verifiedResourceTextByDigest(verified)
|
|
2567
|
-
});
|
|
2568
|
-
}
|
|
2569
|
-
function createAgentImprovementMeasuredComparison(options) {
|
|
2570
|
-
return verifyCandidateExperimentComparison(measuredComparisonFromCandidateExperiment(options));
|
|
2571
|
-
}
|
|
2572
|
-
async function proposeAgentImprovement(options) {
|
|
2573
|
-
assertNoCallerOptimizationReceipt(options.metadata);
|
|
2574
|
-
const analysis = await runAnalystLoop({ ...options.analysis, runId: options.runId });
|
|
2575
|
-
const findings = assertNoJudgeVerdict(
|
|
2576
|
-
analysis.analystResult.findings,
|
|
2577
|
-
"proposeAgentImprovement findings"
|
|
2578
|
-
);
|
|
2579
|
-
const improvementInput = {
|
|
2580
|
-
...options.improvement,
|
|
2581
|
-
findings: [...options.improvement.findings ?? [], ...findings]
|
|
2582
|
-
};
|
|
2583
|
-
const improvement = improvementInput.surface === "code" ? await improve(improvementInput) : await improve(options.profile, improvementInput);
|
|
2584
|
-
try {
|
|
2585
|
-
if (improvement.decision !== "ship") {
|
|
2586
|
-
throw new Error("agent improvement search did not produce a promotable candidate");
|
|
2587
|
-
}
|
|
2588
|
-
const optimizationReceipt = improvement.mode === "method" ? createOptimizationActivationReceipt(improvement) : void 0;
|
|
2589
|
-
const experiment = sealAgentImprovementExperiment(
|
|
2590
|
-
await options.buildExperiment({ analysis, improvement }),
|
|
2591
|
-
improvement
|
|
2592
|
-
);
|
|
2593
|
-
assertCandidateProfileBinding(options.profile, experiment.baseline.profile);
|
|
2594
|
-
assertImprovementCandidateBinding(improvement, experiment);
|
|
2595
|
-
const measured = await runAgentCandidateExperiment({
|
|
2596
|
-
experiment,
|
|
2597
|
-
runId: options.runId,
|
|
2598
|
-
placeCell: options.placeCell,
|
|
2599
|
-
...options.maxConcurrency === void 0 ? {} : { maxConcurrency: options.maxConcurrency },
|
|
2600
|
-
...options.signal ? { signal: options.signal } : {},
|
|
2601
|
-
...options.candidate ? { candidate: options.candidate } : {},
|
|
2602
|
-
...optimizationReceipt ? {
|
|
2603
|
-
metadata: attachOptimizationActivationReceipt(options.metadata, optimizationReceipt)
|
|
2604
|
-
} : options.metadata ? { metadata: options.metadata } : {},
|
|
2605
|
-
...improvement.generationsExplored === void 0 ? {} : { generationsExplored: improvement.generationsExplored },
|
|
2606
|
-
searchDurationMs: improvement.durationMs,
|
|
2607
|
-
searchCostUsd: improvement.cost.totalCostUsd
|
|
2608
|
-
});
|
|
2609
|
-
const proposal = createAgentImprovementProposal({
|
|
2610
|
-
runId: options.runId,
|
|
2611
|
-
findings,
|
|
2612
|
-
evaluation: measured.evaluation,
|
|
2613
|
-
...options.now ? { now: options.now } : {}
|
|
2614
|
-
});
|
|
2615
|
-
return {
|
|
2616
|
-
analysis,
|
|
2617
|
-
improvement,
|
|
2618
|
-
experiment,
|
|
2619
|
-
measurements: measured.measurements,
|
|
2620
|
-
proposal
|
|
2621
|
-
};
|
|
2622
|
-
} catch (cause) {
|
|
2623
|
-
return rethrowAfterCleanup(cause, () => improvement.dispose(), "proposeAgentImprovement failed");
|
|
2624
|
-
}
|
|
2625
|
-
}
|
|
2626
|
-
function assertImprovementCandidateBinding(improvement, experiment) {
|
|
2627
|
-
const candidate = improvement.candidate;
|
|
2628
|
-
if (candidate.surface !== "code") {
|
|
2629
|
-
try {
|
|
2630
|
-
assertCandidateProfileBinding(candidate.profile, experiment.candidate.profile);
|
|
2631
|
-
} catch (cause) {
|
|
2632
|
-
throw new Error("candidate experiment does not contain the improvement winner", { cause });
|
|
2633
|
-
}
|
|
2634
|
-
return;
|
|
2635
|
-
}
|
|
2636
|
-
const surface = candidate.value;
|
|
2637
|
-
const code = experiment.candidate.code;
|
|
2638
|
-
if (typeof surface !== "object" || surface === null || surface.kind !== "code" || code.kind !== "git-patch" || code.baseCommit !== surface.baseCommit || code.baseTree !== surface.baseTree || code.candidateTree !== surface.candidateTree || code.patch.artifact.sha256 !== surface.patch.sha256 || code.patch.artifact.byteLength !== surface.patch.byteLength) {
|
|
2639
|
-
throw new Error("candidate experiment does not contain the improvement winner");
|
|
2640
|
-
}
|
|
2641
|
-
if (canonicalCandidateDigest(experiment.baseline.profile) !== canonicalCandidateDigest(experiment.candidate.profile)) {
|
|
2642
|
-
throw new Error("code improvement candidate changed the agent profile");
|
|
2643
|
-
}
|
|
2644
|
-
}
|
|
2645
|
-
function createAgentImprovementProposal(options) {
|
|
2646
|
-
const findings = assertNoJudgeVerdict(
|
|
2647
|
-
[...options.findings],
|
|
2648
|
-
"createAgentImprovementProposal findings"
|
|
2649
|
-
);
|
|
2650
|
-
const evaluation = verifyCandidateExperimentComparison(options.evaluation);
|
|
2651
|
-
optimizationActivationReceiptFromMetadata(evaluation.metadata);
|
|
2652
|
-
if (evaluation.decision.outcome !== "ship") {
|
|
2653
|
-
throw new Error("agent improvement proposal requires a passing experiment");
|
|
2654
|
-
}
|
|
2655
|
-
if (options.runId !== evaluation.provenance.runId) {
|
|
2656
|
-
throw new Error("proposal runId does not match its measured experiment");
|
|
2657
|
-
}
|
|
2658
|
-
const changedSurfaces = deriveChangedSurfaces(
|
|
2659
|
-
evaluation.experiment.baseline,
|
|
2660
|
-
evaluation.experiment.candidate
|
|
2661
|
-
);
|
|
2662
|
-
return agentImprovementProposalSchema.parse(
|
|
2663
|
-
canonicalCandidateDocument({
|
|
2664
|
-
kind: "agent-improvement-proposal",
|
|
2665
|
-
runId: options.runId,
|
|
2666
|
-
changedSurfaces,
|
|
2667
|
-
proposedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
|
|
2668
|
-
findings: [...findings],
|
|
2669
|
-
evaluation
|
|
2670
|
-
}).value
|
|
2671
|
-
);
|
|
2672
|
-
}
|
|
2673
|
-
function reviewAgentImprovementProposal(inputProposal, input) {
|
|
2674
|
-
const proposal = verifyAgentImprovementProposal(inputProposal);
|
|
2675
|
-
if (!input.reviewedBy.trim()) throw new Error("candidate review requires reviewedBy");
|
|
2676
|
-
if (!input.reason.trim()) throw new Error("candidate review requires a reason");
|
|
2677
|
-
if (input.decision === "approve" && proposal.evaluation.decision.outcome !== "ship") {
|
|
2678
|
-
throw new Error("candidate cannot be approved without a passing experiment");
|
|
2679
|
-
}
|
|
2680
|
-
const reviewedAt = (input.now ?? (() => /* @__PURE__ */ new Date()))().toISOString();
|
|
2681
|
-
if (Date.parse(reviewedAt) < Date.parse(proposal.proposedAt)) {
|
|
2682
|
-
throw new Error("candidate review cannot predate its proposal");
|
|
2683
|
-
}
|
|
2684
|
-
return agentImprovementReviewSchema.parse(
|
|
2685
|
-
canonicalCandidateDocument({
|
|
2686
|
-
kind: "agent-improvement-review",
|
|
2687
|
-
proposalDigest: proposal.digest,
|
|
2688
|
-
decision: input.decision,
|
|
2689
|
-
reviewedBy: input.reviewedBy,
|
|
2690
|
-
reviewedAt,
|
|
2691
|
-
reason: input.reason,
|
|
2692
|
-
...input.feedback === void 0 ? {} : { feedback: input.feedback }
|
|
2693
|
-
}).value
|
|
2694
|
-
);
|
|
2695
|
-
}
|
|
2696
|
-
function createAgentImprovementActivation(inputProposal, inputReview, options) {
|
|
2697
|
-
const proposal = verifyAgentImprovementProposal(inputProposal);
|
|
2698
|
-
const review = verifyAgentImprovementReview(inputReview);
|
|
2699
|
-
if (review.decision !== "approve" || review.proposalDigest !== proposal.digest) {
|
|
2700
|
-
throw new Error("candidate activation requires an approval for the exact proposal");
|
|
2701
|
-
}
|
|
2702
|
-
if (!options.fundingOwner.trim() || !options.authorizedBy.trim()) {
|
|
2703
|
-
throw new Error("candidate activation authority must be non-empty");
|
|
2704
|
-
}
|
|
2705
|
-
const experiment = proposal.evaluation.experiment;
|
|
2706
|
-
const authorizedAt = (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString();
|
|
2707
|
-
if (Date.parse(authorizedAt) < Date.parse(review.reviewedAt)) {
|
|
2708
|
-
throw new Error("candidate activation cannot predate its approval");
|
|
2709
|
-
}
|
|
2710
|
-
const targets = buildAgentImprovementActivationTargets(
|
|
2711
|
-
proposal.changedSurfaces,
|
|
2712
|
-
experiment,
|
|
2713
|
-
options.intent,
|
|
2714
|
-
options.targets
|
|
2715
|
-
);
|
|
2716
|
-
return agentImprovementActivationSchema.parse(
|
|
2717
|
-
canonicalCandidateDocument({
|
|
2718
|
-
kind: "agent-improvement-activation",
|
|
2719
|
-
proposalDigest: proposal.digest,
|
|
2720
|
-
reviewDigest: review.digest,
|
|
2721
|
-
experimentDigest: experiment.digest,
|
|
2722
|
-
candidateBundleDigest: experiment.candidate.digest,
|
|
2723
|
-
intent: options.intent,
|
|
2724
|
-
targets,
|
|
2725
|
-
fundingOwner: options.fundingOwner,
|
|
2726
|
-
authorizedBy: options.authorizedBy,
|
|
2727
|
-
authorizedAt,
|
|
2728
|
-
expiresAt: options.expiresAt
|
|
2729
|
-
}).value
|
|
2730
|
-
);
|
|
2731
|
-
}
|
|
2732
|
-
function verifyAgentImprovementProposal(input) {
|
|
2733
|
-
const proposal = verifyCanonicalCandidateDocument(
|
|
2734
|
-
agentImprovementProposalSchema.parse(input),
|
|
2735
|
-
"agent improvement proposal"
|
|
2736
|
-
);
|
|
2737
|
-
const evaluation = verifyCandidateExperimentComparison(proposal.evaluation);
|
|
2738
|
-
optimizationActivationReceiptFromMetadata(evaluation.metadata);
|
|
2739
|
-
if (evaluation.decision.outcome !== "ship") {
|
|
2740
|
-
throw new Error("agent improvement proposal does not contain a passing experiment");
|
|
2741
|
-
}
|
|
2742
|
-
if (proposal.runId !== evaluation.provenance.runId) {
|
|
2743
|
-
throw new Error("proposal runId does not match its measured experiment");
|
|
2744
|
-
}
|
|
2745
|
-
const changedSurfaces = deriveChangedSurfaces(
|
|
2746
|
-
evaluation.experiment.baseline,
|
|
2747
|
-
evaluation.experiment.candidate
|
|
2748
|
-
);
|
|
2749
|
-
if (!sameOrderedValues(proposal.changedSurfaces, changedSurfaces)) {
|
|
2750
|
-
throw new Error("proposal changed surfaces do not match its exact experiment");
|
|
2751
|
-
}
|
|
2752
|
-
assertNoJudgeDerivedProposalFindings(proposal.findings);
|
|
2753
|
-
return proposal;
|
|
2754
|
-
}
|
|
2755
|
-
function verifyAgentImprovementReview(input) {
|
|
2756
|
-
return verifyCanonicalCandidateDocument(
|
|
2757
|
-
agentImprovementReviewSchema.parse(input),
|
|
2758
|
-
"agent improvement review"
|
|
2759
|
-
);
|
|
2760
|
-
}
|
|
2761
|
-
function verifyAgentImprovementActivation(input) {
|
|
2762
|
-
const proposal = verifyAgentImprovementProposal(input.proposal);
|
|
2763
|
-
const review = verifyAgentImprovementReview(input.review);
|
|
2764
|
-
const activation = verifyCanonicalCandidateDocument(
|
|
2765
|
-
agentImprovementActivationSchema.parse(input.activation),
|
|
2766
|
-
"agent improvement activation"
|
|
2767
|
-
);
|
|
2768
|
-
const experiment = proposal.evaluation.experiment;
|
|
2769
|
-
if (review.decision !== "approve" || review.proposalDigest !== proposal.digest || activation.proposalDigest !== proposal.digest || activation.reviewDigest !== review.digest || activation.experimentDigest !== experiment.digest || activation.candidateBundleDigest !== experiment.candidate.digest || Date.parse(review.reviewedAt) < Date.parse(proposal.proposedAt) || Date.parse(activation.authorizedAt) < Date.parse(review.reviewedAt)) {
|
|
2770
|
-
throw new Error("candidate activation does not bind the measured and approved candidate");
|
|
2771
|
-
}
|
|
2772
|
-
assertAgentImprovementActivationTargets(
|
|
2773
|
-
proposal.changedSurfaces,
|
|
2774
|
-
experiment,
|
|
2775
|
-
activation.intent,
|
|
2776
|
-
activation.targets
|
|
2777
|
-
);
|
|
2778
|
-
return activation;
|
|
2779
|
-
}
|
|
2780
|
-
function verifyCandidateExecutionEvidence(input, options) {
|
|
2781
|
-
const experiment = verifyCandidateExperiment(options.experiment);
|
|
2782
|
-
const bundle = experiment[options.arm];
|
|
2783
|
-
const task = experiment.benchmark.tasks[options.benchmarkCell.taskIndex];
|
|
2784
|
-
const index = options.benchmarkCell.taskIndex * experiment.benchmark.suite.reps + options.benchmarkCell.repetition;
|
|
2785
|
-
if (!task || options.benchmarkCell.suiteDigest !== experiment.benchmark.suite.digest || options.seed !== experiment.benchmark.suite.seeds[index]) {
|
|
2786
|
-
throw new Error("candidate execution evidence points outside its signed experiment");
|
|
2787
|
-
}
|
|
2788
|
-
const evidence = verifyCanonicalCandidateDocument(
|
|
2789
|
-
candidateExecutionEvidenceSchema.parse(input),
|
|
2790
|
-
"candidate execution evidence"
|
|
2791
|
-
);
|
|
2792
|
-
const materialization = verifyCanonicalCandidateDocument(
|
|
2793
|
-
agentCandidateMaterializationReceiptSchema.parse(evidence.materializationReceipt),
|
|
2794
|
-
"candidate materialization receipt"
|
|
2795
|
-
);
|
|
2796
|
-
const receipt = verifyCanonicalCandidateDocument(
|
|
2797
|
-
agentCandidateRunReceiptSchema.parse(evidence.receipt),
|
|
2798
|
-
"candidate run receipt"
|
|
2799
|
-
);
|
|
2800
|
-
const plan = materialization.executionPlan;
|
|
2801
|
-
const cell = plan.material.runCell;
|
|
2802
|
-
const attempt = options.attempt ?? 1;
|
|
2803
|
-
if (cell.experimentDigest !== experiment.digest || cell.arm !== options.arm || cell.bundleDigest !== bundle.digest || cell.suiteDigest !== experiment.benchmark.suite.digest || cell.taskDigest !== task.digest || cell.taskIndex !== options.benchmarkCell.taskIndex || cell.repetition !== options.benchmarkCell.repetition || cell.seed !== options.seed || cell.attempt !== attempt || canonicalCandidateDigest(omitTopLevelDigest(cell)) !== cell.digest) {
|
|
2804
|
-
throw new Error("candidate execution receipt substituted its signed experiment cell");
|
|
2805
|
-
}
|
|
2806
|
-
assertCapturedInput(
|
|
2807
|
-
materialization.benchmark.suite,
|
|
2808
|
-
experiment.benchmark.suite,
|
|
2809
|
-
"benchmark suite"
|
|
2810
|
-
);
|
|
2811
|
-
assertCapturedInput(materialization.benchmark.task, task, "benchmark task");
|
|
2812
|
-
assertEvidenceMaterialDigest(plan, "candidate execution plan");
|
|
2813
|
-
assertEvidenceMaterialDigest(
|
|
2814
|
-
materialization.profileActivation.profilePlan,
|
|
2815
|
-
"candidate profile plan"
|
|
2816
|
-
);
|
|
2817
|
-
const expectedProfilePlan = materializeCandidateProfile(
|
|
2818
|
-
bundle.profile,
|
|
2819
|
-
candidateMaterializerHarness(materialization.harness),
|
|
2820
|
-
{ resolvedResources: options.resolvedResources }
|
|
2821
|
-
);
|
|
2822
|
-
const activation = parseAgentCandidateProfileActivation(
|
|
2823
|
-
materialization.profileActivation,
|
|
2824
|
-
materialization.profileActivation.profilePlan.digest
|
|
2825
|
-
);
|
|
2826
|
-
const regeneratedActivation = createAgentCandidateProfileActivation(
|
|
2827
|
-
expectedProfilePlan,
|
|
2828
|
-
materialization.profileActivation.profilePlan
|
|
2829
|
-
);
|
|
2830
|
-
if (activation.digest !== regeneratedActivation.digest) {
|
|
2831
|
-
throw new Error("candidate profile activation does not match the experiment bundle");
|
|
2832
|
-
}
|
|
2833
|
-
if (materialization.bundleDigest !== bundle.digest || receipt.bundleDigest !== bundle.digest || receipt.runCellDigest !== cell.digest || receipt.materializationReceiptDigest !== materialization.digest || receipt.executionPlanDigest !== plan.digest) {
|
|
2834
|
-
throw new Error("candidate execution evidence does not bind one exact Runtime run");
|
|
2835
|
-
}
|
|
2836
|
-
assertEvidenceMaterialDigest(receipt.modelSettlement, "candidate model settlement");
|
|
2837
|
-
assertEvidenceMaterialDigest(receipt.taskOutcome, "candidate task outcome");
|
|
2838
|
-
assertEvidenceMaterialDigest(receipt.benchmarkResult, "candidate benchmark result");
|
|
2839
|
-
return immutableCandidateValue(evidence);
|
|
2840
|
-
}
|
|
2841
|
-
function assertExactExperimentInput(input, experiment, bundle) {
|
|
2842
|
-
const task = experiment.benchmark.tasks[input.benchmarkCell.taskIndex];
|
|
2843
|
-
const index = input.benchmarkCell.taskIndex * experiment.benchmark.suite.reps + input.benchmarkCell.repetition;
|
|
2844
|
-
if (input.experiment.digest !== experiment.digest || input.bundle.digest !== bundle.digest || !task || input.task.digest !== task.digest || input.benchmarkCell.suiteDigest !== experiment.benchmark.suite.digest || input.seed !== experiment.benchmark.suite.seeds[index]) {
|
|
2845
|
-
throw new Error("Runtime received a substituted candidate experiment cell");
|
|
2846
|
-
}
|
|
2847
|
-
}
|
|
2848
|
-
function assertCapturedInput(captured, expected, label) {
|
|
2849
|
-
const bytes = canonicalCandidateBytes(omitTopLevelDigest(expected));
|
|
2850
|
-
if (captured.digest !== expected.digest || captured.material.sha256 !== expected.digest || captured.material.byteLength !== bytes.byteLength) {
|
|
2851
|
-
throw new Error(`candidate materialization substituted its ${label}`);
|
|
2852
|
-
}
|
|
2853
|
-
}
|
|
2854
|
-
function assertEvidenceMaterialDigest(evidence, label) {
|
|
2855
|
-
const bytes = canonicalCandidateBytes(evidence.material);
|
|
2856
|
-
if (canonicalCandidateDigest(evidence.material) !== evidence.digest || evidence.artifact.sha256 !== evidence.digest || evidence.artifact.byteLength !== bytes.byteLength) {
|
|
2857
|
-
throw new Error(`${label} digest does not match its canonical material`);
|
|
2858
|
-
}
|
|
2859
|
-
}
|
|
2860
|
-
function sameOrderedValues(left, right) {
|
|
2861
|
-
return left.length === right.length && left.every((value, index) => value === right[index]);
|
|
2862
|
-
}
|
|
2863
|
-
function assertNoJudgeDerivedProposalFindings(findings) {
|
|
2864
|
-
const leaked = findings.filter((finding) => finding.derived_from_judge === true);
|
|
2865
|
-
if (leaked.length === 0) return;
|
|
2866
|
-
const identifiers = leaked.map(
|
|
2867
|
-
(finding) => typeof finding.finding_id === "string" ? finding.finding_id : "<unknown>"
|
|
2868
|
-
);
|
|
2869
|
-
throw new Error(
|
|
2870
|
-
`agent improvement proposal findings: judge-derived findings cannot steer an improvement: [${identifiers.join(", ")}]`
|
|
2871
|
-
);
|
|
2872
|
-
}
|
|
2873
|
-
|
|
2874
|
-
export {
|
|
2875
|
-
AGENTIC_PROFILE_RESOURCE_ROOT,
|
|
2876
|
-
agenticGenerator,
|
|
2877
|
-
defaultBuildPrompt,
|
|
2878
|
-
rawTraceEvidenceProblem,
|
|
2879
|
-
requiresRawTraceEvidence,
|
|
2880
|
-
commandVerifier,
|
|
2881
|
-
summarizeFindings,
|
|
2882
|
-
worktreeChangedPaths,
|
|
2883
|
-
LIFTED_FINDING_ANALYST_ID,
|
|
2884
|
-
isAnalystFinding,
|
|
2885
|
-
toAnalystFindings,
|
|
2886
|
-
ROLLOUT_POLICY_EXTENSION,
|
|
2887
|
-
parseRolloutPolicy,
|
|
2888
|
-
normalizeRolloutPolicy,
|
|
2889
|
-
serializeRolloutPolicy,
|
|
2890
|
-
structuralRolloutPolicyFromProfile,
|
|
2891
|
-
applyRolloutPolicyToProfile,
|
|
2892
|
-
rawTraceDistiller,
|
|
2893
|
-
withMethodRuntimeControls,
|
|
2894
|
-
improve,
|
|
2895
|
-
AGENT_IMPROVEMENT_PROFILE_SURFACES,
|
|
2896
|
-
buildAgentImprovementActivationTargets,
|
|
2897
|
-
agentImprovementTargetDigest,
|
|
2898
|
-
agentImprovementTargetInput,
|
|
2899
|
-
isAgentImprovementProfileSurface,
|
|
2900
|
-
agentImprovementProfileSurfaceInput,
|
|
2901
|
-
agentImprovementProfileSurfaceDigest,
|
|
2902
|
-
agentImprovementTargetProfileDiffs,
|
|
2903
|
-
createOptimizationActivationReceipt,
|
|
2904
|
-
optimizationActivationReceiptFromMetadata,
|
|
2905
|
-
AgentCandidateExperimentCellExecutionError,
|
|
2906
|
-
runAgentCandidateExperiment,
|
|
2907
|
-
executeAgentCandidateExperimentCell,
|
|
2908
|
-
createAgentImprovementMeasuredComparison,
|
|
2909
|
-
proposeAgentImprovement,
|
|
2910
|
-
createAgentImprovementProposal,
|
|
2911
|
-
reviewAgentImprovementProposal,
|
|
2912
|
-
createAgentImprovementActivation,
|
|
2913
|
-
verifyAgentImprovementProposal,
|
|
2914
|
-
verifyAgentImprovementReview,
|
|
2915
|
-
verifyAgentImprovementActivation,
|
|
2916
|
-
verifyCandidateExecutionEvidence
|
|
2917
|
-
};
|
|
2918
|
-
//# sourceMappingURL=chunk-UHEZW5BU.js.map
|