@tangle-network/agent-eval 0.107.0 → 0.108.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +1 -1
- package/dist/adapters/langchain.d.ts +1 -1
- package/dist/adapters/otel.d.ts +1 -1
- package/dist/benchmarks/index.d.ts +3 -1
- package/dist/benchmarks/index.js +49 -3
- package/dist/campaign/index.d.ts +10 -7
- package/dist/campaign/index.js +72 -3407
- package/dist/campaign/index.js.map +1 -1
- package/dist/chunk-6SKVFBTR.js +4306 -0
- package/dist/chunk-6SKVFBTR.js.map +1 -0
- package/dist/{chunk-G4DLZAV5.js → chunk-OVPVM4JC.js} +1044 -643
- package/dist/chunk-OVPVM4JC.js.map +1 -0
- package/dist/chunk-T6W5ADLG.js +766 -0
- package/dist/chunk-T6W5ADLG.js.map +1 -0
- package/dist/contract/index.d.ts +16 -6
- package/dist/contract/index.js +10 -11
- package/dist/contract/index.js.map +1 -1
- package/dist/{gepa-BmqTrdtg.d.ts → gepa-B3x5Ulcv.d.ts} +3 -40
- package/dist/hosted/index.d.ts +1 -1
- package/dist/index-pPtfoIJO.d.ts +423 -0
- package/dist/index.d.ts +6 -5
- package/dist/index.js +8 -7
- package/dist/index.js.map +1 -1
- package/dist/multishot/index.d.ts +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{pre-registration-x3f0VpxT.d.ts → pre-registration-BUhVPzE7.d.ts} +1 -1
- package/dist/{provenance-CIjRcI6m.d.ts → provenance-DdDhf6cg.d.ts} +3 -2
- package/dist/rl.d.ts +1 -1
- package/dist/rl.js +6 -6
- package/dist/storage-Dw_f7WMt.d.ts +39 -0
- package/dist/{types-CNZ0tHET.d.ts → types-BdIv5dvA.d.ts} +1 -1
- package/docs/improvement-glossary.md +5 -1
- package/package.json +1 -1
- package/dist/chunk-6PF6LXRY.js +0 -861
- package/dist/chunk-6PF6LXRY.js.map +0 -1
- package/dist/chunk-6QDKWHLS.js +0 -223
- package/dist/chunk-6QDKWHLS.js.map +0 -1
- package/dist/chunk-G4DLZAV5.js.map +0 -1
- package/dist/chunk-V75NN2ZR.js +0 -421
- package/dist/chunk-V75NN2ZR.js.map +0 -1
- package/dist/index-C2CC_dry.d.ts +0 -159
package/dist/campaign/index.js
CHANGED
|
@@ -1,32 +1,70 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
2
|
+
FsLabeledScenarioStore,
|
|
3
|
+
LabeledScenarioStoreError,
|
|
4
|
+
Lineage,
|
|
5
|
+
ProfileMatrixError,
|
|
6
|
+
SkillPatchParseError,
|
|
7
|
+
WorktreeAdapterError,
|
|
8
|
+
aceProposer,
|
|
9
|
+
applySkillPatch,
|
|
10
|
+
buildAnalystSurfaceDispatch,
|
|
11
|
+
callbackGovernor,
|
|
12
|
+
compareProposers,
|
|
13
|
+
discoverEvalFixtures,
|
|
14
|
+
extractFapoAttributionSignals,
|
|
15
|
+
failureModeRecallJudge,
|
|
16
|
+
fapoEscalationEntry,
|
|
17
|
+
fapoProposer,
|
|
18
|
+
fsLineageStore,
|
|
19
|
+
gepaParetoEntry,
|
|
20
|
+
gepaReflectionEntry,
|
|
21
|
+
gitWorktreeAdapter,
|
|
22
|
+
haloProposer,
|
|
23
|
+
heuristicGovernor,
|
|
24
|
+
lineageNodeId,
|
|
18
25
|
llmJudge,
|
|
19
|
-
|
|
20
|
-
|
|
26
|
+
loadEvalFixture,
|
|
27
|
+
loadEvalFixtureScenarios,
|
|
28
|
+
makePlaybackDispatch,
|
|
29
|
+
memLineageStore,
|
|
30
|
+
memoryCurationProposer,
|
|
31
|
+
neutralizationGate,
|
|
32
|
+
neutralizeText,
|
|
33
|
+
parameterSweepProposer,
|
|
34
|
+
parseSkillPatchResponse,
|
|
35
|
+
patchEditCount,
|
|
36
|
+
planEvalFixtureRun,
|
|
37
|
+
policyEditProposer,
|
|
38
|
+
renderScoreboardMarkdown,
|
|
39
|
+
resolveWorktreePath,
|
|
40
|
+
runLineage,
|
|
41
|
+
runLineageLoop,
|
|
42
|
+
runProfileMatrix,
|
|
43
|
+
runSkillOpt,
|
|
44
|
+
scoreDiscrimination,
|
|
45
|
+
scoreUserStory,
|
|
46
|
+
scoreboardSummary,
|
|
47
|
+
selectDiscriminative,
|
|
48
|
+
sequentialDecide,
|
|
49
|
+
sequentialPairedGate,
|
|
50
|
+
skillOptEntry,
|
|
51
|
+
skillOptProposer,
|
|
52
|
+
traceAnalystProposer,
|
|
53
|
+
userStoryScoreboard
|
|
54
|
+
} from "../chunk-6SKVFBTR.js";
|
|
21
55
|
import {
|
|
56
|
+
buildEvidenceVector,
|
|
22
57
|
buildLoopProvenanceRecord,
|
|
23
58
|
campaignBreakdown,
|
|
24
59
|
campaignMeanComposite,
|
|
60
|
+
composeGate,
|
|
25
61
|
countSentenceEdits,
|
|
62
|
+
defaultProductionGate,
|
|
26
63
|
defaultRenderDiff,
|
|
27
64
|
detectScale,
|
|
28
65
|
dimensionRegressions,
|
|
29
66
|
emitLoopProvenance,
|
|
67
|
+
evolutionaryProposer,
|
|
30
68
|
extractH2Sections,
|
|
31
69
|
gepaProposer,
|
|
32
70
|
heldOutGate,
|
|
@@ -36,3414 +74,41 @@ import {
|
|
|
36
74
|
loopProvenanceSpans,
|
|
37
75
|
openAutoPr,
|
|
38
76
|
pairHoldout,
|
|
77
|
+
paretoPolicy,
|
|
78
|
+
paretoSignificanceGate,
|
|
79
|
+
powerPreflight,
|
|
39
80
|
provenanceRecordPath,
|
|
40
81
|
provenanceSpansPath,
|
|
41
|
-
|
|
82
|
+
runEval,
|
|
42
83
|
runImprovementLoop,
|
|
43
84
|
runOptimization,
|
|
44
85
|
surfaceContentHash,
|
|
45
86
|
surfaceHash
|
|
46
|
-
} from "../chunk-
|
|
87
|
+
} from "../chunk-OVPVM4JC.js";
|
|
47
88
|
import {
|
|
48
|
-
assertRealBackend,
|
|
49
|
-
contentHash,
|
|
50
89
|
fsCampaignStorage,
|
|
51
90
|
inMemoryCampaignStorage,
|
|
52
91
|
planCampaignRun,
|
|
53
92
|
resolveRunDir,
|
|
54
93
|
runCampaign,
|
|
55
|
-
summarizeBackendIntegrity,
|
|
56
94
|
tangleTracesRoot
|
|
57
95
|
} from "../chunk-3PFZBGMR.js";
|
|
58
|
-
import
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
} from "../chunk-VI2UW6B6.js";
|
|
62
|
-
import {
|
|
63
|
-
admitPolicyEdit,
|
|
64
|
-
applyPolicyEditToSurface,
|
|
65
|
-
isPolicyEdit,
|
|
66
|
-
policyEditsFromFindings
|
|
67
|
-
} from "../chunk-LOJ2QVCE.js";
|
|
68
|
-
import {
|
|
69
|
-
AnalystRegistry,
|
|
70
|
-
DEFAULT_TRACE_ANALYST_KINDS,
|
|
71
|
-
createTraceAnalystKind
|
|
72
|
-
} from "../chunk-2OGPXHOB.js";
|
|
96
|
+
import "../chunk-VI2UW6B6.js";
|
|
97
|
+
import "../chunk-LOJ2QVCE.js";
|
|
98
|
+
import "../chunk-2OGPXHOB.js";
|
|
73
99
|
import "../chunk-N22ZO7FV.js";
|
|
74
|
-
import
|
|
75
|
-
callLlm
|
|
76
|
-
} from "../chunk-FUCQVFMU.js";
|
|
100
|
+
import "../chunk-FUCQVFMU.js";
|
|
77
101
|
import "../chunk-45EEMHTC.js";
|
|
78
|
-
import
|
|
79
|
-
|
|
80
|
-
mulberry32,
|
|
81
|
-
pairedBootstrap
|
|
82
|
-
} from "../chunk-XMSYF4A7.js";
|
|
83
|
-
import {
|
|
84
|
-
analyzeTraces
|
|
85
|
-
} from "../chunk-RPDDVKI7.js";
|
|
102
|
+
import "../chunk-XMSYF4A7.js";
|
|
103
|
+
import "../chunk-RPDDVKI7.js";
|
|
86
104
|
import "../chunk-GGE4NNQT.js";
|
|
87
|
-
import
|
|
88
|
-
OtlpFileTraceStore
|
|
89
|
-
} from "../chunk-LNQEP766.js";
|
|
105
|
+
import "../chunk-LNQEP766.js";
|
|
90
106
|
import "../chunk-PC4UYEBM.js";
|
|
91
|
-
import
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
import {
|
|
96
|
-
buildAgentProfileCell
|
|
97
|
-
} from "../chunk-XJYR7XFV.js";
|
|
98
|
-
import {
|
|
99
|
-
canonicalize
|
|
100
|
-
} from "../chunk-VSMTAMNK.js";
|
|
101
|
-
import {
|
|
102
|
-
AgentEvalError
|
|
103
|
-
} from "../chunk-ONWEPEDO.js";
|
|
107
|
+
import "../chunk-VK6HBGAE.js";
|
|
108
|
+
import "../chunk-XJYR7XFV.js";
|
|
109
|
+
import "../chunk-VSMTAMNK.js";
|
|
110
|
+
import "../chunk-ONWEPEDO.js";
|
|
104
111
|
import "../chunk-PZ5AY32C.js";
|
|
105
|
-
|
|
106
|
-
// src/campaign/lineage.ts
|
|
107
|
-
import { createHash } from "crypto";
|
|
108
|
-
import { appendFile, mkdir, readFile, writeFile } from "fs/promises";
|
|
109
|
-
import { dirname } from "path";
|
|
110
|
-
function lineageNodeId(input) {
|
|
111
|
-
const parents = [...input.parentIds].sort().join(",");
|
|
112
|
-
const payload = `${parents}|${input.track}|${input.surface}|${input.proposer}`;
|
|
113
|
-
return createHash("sha256").update(payload).digest("hex").slice(0, 16);
|
|
114
|
-
}
|
|
115
|
-
var Lineage = class _Lineage {
|
|
116
|
-
byId = /* @__PURE__ */ new Map();
|
|
117
|
-
childIds = /* @__PURE__ */ new Map();
|
|
118
|
-
nextSeq = 0;
|
|
119
|
-
constructor(nodes) {
|
|
120
|
-
if (!nodes) return;
|
|
121
|
-
for (const node of [...nodes].sort((a, b) => a.seq - b.seq)) {
|
|
122
|
-
this.index(node);
|
|
123
|
-
this.nextSeq = Math.max(this.nextSeq, node.seq + 1);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
/** Append a node. Derives `id` (via {@link lineageNodeId}), assigns the next
|
|
127
|
-
* `seq`, and derives `generation` as `max(parent.generation) + 1` (root = 0)
|
|
128
|
-
* when omitted. Throws on an unknown parent. Idempotent: re-adding an identical
|
|
129
|
-
* node returns the existing one.
|
|
130
|
-
*
|
|
131
|
-
* Acyclicity is guaranteed by construction, not by a runtime check: every
|
|
132
|
-
* parent must already exist (a child can never point at a not-yet-added node),
|
|
133
|
-
* ids are immutable content hashes (an existing node can never gain new
|
|
134
|
-
* parents), and the store is append-only — so no back-edge can form. (Traversal
|
|
135
|
-
* is still cycle-safe against hand-corrupted deserialized input via visited
|
|
136
|
-
* sets in {@link ancestors}/{@link descendants}.) */
|
|
137
|
-
addNode(input) {
|
|
138
|
-
for (const parentId of input.parentIds) {
|
|
139
|
-
if (!this.byId.has(parentId)) {
|
|
140
|
-
throw new Error(`Lineage.addNode: unknown parent '${parentId}'`);
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
const id = lineageNodeId({
|
|
144
|
-
parentIds: input.parentIds,
|
|
145
|
-
track: input.track,
|
|
146
|
-
surface: input.surface,
|
|
147
|
-
proposer: input.proposer
|
|
148
|
-
});
|
|
149
|
-
const existing = this.byId.get(id);
|
|
150
|
-
if (existing) return existing;
|
|
151
|
-
const generation = input.generation ?? (input.parentIds.length === 0 ? 0 : Math.max(...input.parentIds.map((p) => this.byId.get(p).generation)) + 1);
|
|
152
|
-
const node = {
|
|
153
|
-
id,
|
|
154
|
-
parentIds: [...input.parentIds],
|
|
155
|
-
track: input.track,
|
|
156
|
-
surface: input.surface,
|
|
157
|
-
score: input.score,
|
|
158
|
-
proposer: input.proposer,
|
|
159
|
-
generation,
|
|
160
|
-
seq: this.nextSeq++,
|
|
161
|
-
...input.vision !== void 0 ? { vision: input.vision } : {},
|
|
162
|
-
...input.scoreVector !== void 0 ? { scoreVector: [...input.scoreVector] } : {},
|
|
163
|
-
...input.rationale !== void 0 ? { rationale: input.rationale } : {},
|
|
164
|
-
...input.gate !== void 0 ? { gate: input.gate } : {}
|
|
165
|
-
};
|
|
166
|
-
this.index(node);
|
|
167
|
-
return node;
|
|
168
|
-
}
|
|
169
|
-
/** Collapse 2+ parents into a single node (a merge/"collapse"). A merge is an
|
|
170
|
-
* ordinary multi-parent node; this is a guarded convenience. */
|
|
171
|
-
merge(input) {
|
|
172
|
-
if (input.parentIds.length < 2) {
|
|
173
|
-
throw new Error(`Lineage.merge: a merge needs >= 2 parents, got ${input.parentIds.length}`);
|
|
174
|
-
}
|
|
175
|
-
return this.addNode({ ...input, proposer: input.proposer ?? "merge" });
|
|
176
|
-
}
|
|
177
|
-
get(id) {
|
|
178
|
-
return this.byId.get(id);
|
|
179
|
-
}
|
|
180
|
-
has(id) {
|
|
181
|
-
return this.byId.has(id);
|
|
182
|
-
}
|
|
183
|
-
/** All nodes, in insertion (`seq`) order. */
|
|
184
|
-
all() {
|
|
185
|
-
return [...this.byId.values()].sort(bySeq);
|
|
186
|
-
}
|
|
187
|
-
roots() {
|
|
188
|
-
return this.all().filter((n) => n.parentIds.length === 0);
|
|
189
|
-
}
|
|
190
|
-
parents(id) {
|
|
191
|
-
const node = this.byId.get(id);
|
|
192
|
-
if (!node) return [];
|
|
193
|
-
return node.parentIds.map((p) => this.byId.get(p)).filter((n) => n !== void 0).sort(bySeq);
|
|
194
|
-
}
|
|
195
|
-
children(id) {
|
|
196
|
-
return (this.childIds.get(id) ?? []).map((c) => this.byId.get(c)).sort(bySeq);
|
|
197
|
-
}
|
|
198
|
-
/** Transitive ancestors (excludes `id`). */
|
|
199
|
-
ancestors(id) {
|
|
200
|
-
const out = /* @__PURE__ */ new Set();
|
|
201
|
-
const stack = [...this.byId.get(id)?.parentIds ?? []];
|
|
202
|
-
while (stack.length > 0) {
|
|
203
|
-
const cur = stack.pop();
|
|
204
|
-
if (out.has(cur)) continue;
|
|
205
|
-
out.add(cur);
|
|
206
|
-
const node = this.byId.get(cur);
|
|
207
|
-
if (node) stack.push(...node.parentIds);
|
|
208
|
-
}
|
|
209
|
-
return out;
|
|
210
|
-
}
|
|
211
|
-
/** Transitive descendants (excludes `id`). */
|
|
212
|
-
descendants(id) {
|
|
213
|
-
const out = /* @__PURE__ */ new Set();
|
|
214
|
-
const stack = [...this.childIds.get(id) ?? []];
|
|
215
|
-
while (stack.length > 0) {
|
|
216
|
-
const cur = stack.pop();
|
|
217
|
-
if (out.has(cur)) continue;
|
|
218
|
-
out.add(cur);
|
|
219
|
-
stack.push(...this.childIds.get(cur) ?? []);
|
|
220
|
-
}
|
|
221
|
-
return out;
|
|
222
|
-
}
|
|
223
|
-
/** Nodes with no children (leaf/branch tips). */
|
|
224
|
-
tips() {
|
|
225
|
-
return this.all().filter((n) => (this.childIds.get(n.id) ?? []).length === 0);
|
|
226
|
-
}
|
|
227
|
-
/** Distinct track ids, in first-seen (`seq`) order. */
|
|
228
|
-
tracks() {
|
|
229
|
-
const seen = /* @__PURE__ */ new Set();
|
|
230
|
-
const out = [];
|
|
231
|
-
for (const node of this.all()) {
|
|
232
|
-
if (!seen.has(node.track)) {
|
|
233
|
-
seen.add(node.track);
|
|
234
|
-
out.push(node.track);
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
return out;
|
|
238
|
-
}
|
|
239
|
-
trackNodes(track) {
|
|
240
|
-
return this.all().filter((n) => n.track === track);
|
|
241
|
-
}
|
|
242
|
-
/** The highest-`score` tip of a track (ties broken by lowest `seq`). */
|
|
243
|
-
trackTip(track) {
|
|
244
|
-
return pickBest(this.tips().filter((n) => n.track === track));
|
|
245
|
-
}
|
|
246
|
-
/** The highest-`score` node overall (ties broken by lowest `seq`). */
|
|
247
|
-
best() {
|
|
248
|
-
return pickBest(this.all());
|
|
249
|
-
}
|
|
250
|
-
/** The Pareto-non-dominated set among TIPS. Uses `scoreVector` when every
|
|
251
|
-
* compared tip carries one, else the scalar `score`. A dominates B iff A is
|
|
252
|
-
* >= B on every component and > B on at least one. */
|
|
253
|
-
frontier() {
|
|
254
|
-
const tips = this.tips();
|
|
255
|
-
const useVector = tips.length > 0 && tips.every((n) => n.scoreVector !== void 0);
|
|
256
|
-
const vecOf = (n) => useVector ? n.scoreVector : [n.score];
|
|
257
|
-
return tips.filter((a) => !tips.some((b) => b.id !== a.id && dominates(vecOf(b), vecOf(a)))).sort(bySeq);
|
|
258
|
-
}
|
|
259
|
-
toGraph() {
|
|
260
|
-
const nodes = this.all();
|
|
261
|
-
const edges = [];
|
|
262
|
-
for (const node of nodes) {
|
|
263
|
-
for (const parentId of node.parentIds) {
|
|
264
|
-
edges.push({ from: parentId, to: node.id });
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
return { nodes, edges };
|
|
268
|
-
}
|
|
269
|
-
/** One JSON node per line, in `seq` order. */
|
|
270
|
-
toJSONL() {
|
|
271
|
-
return this.all().map((n) => JSON.stringify(n)).join("\n");
|
|
272
|
-
}
|
|
273
|
-
static fromJSONL(text) {
|
|
274
|
-
const nodes = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
|
|
275
|
-
return new _Lineage(nodes);
|
|
276
|
-
}
|
|
277
|
-
index(node) {
|
|
278
|
-
this.byId.set(node.id, node);
|
|
279
|
-
for (const parentId of node.parentIds) {
|
|
280
|
-
const list = this.childIds.get(parentId);
|
|
281
|
-
if (list) {
|
|
282
|
-
if (!list.includes(node.id)) list.push(node.id);
|
|
283
|
-
} else {
|
|
284
|
-
this.childIds.set(parentId, [node.id]);
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
};
|
|
289
|
-
function bySeq(a, b) {
|
|
290
|
-
return a.seq - b.seq;
|
|
291
|
-
}
|
|
292
|
-
function pickBest(nodes) {
|
|
293
|
-
let best;
|
|
294
|
-
for (const node of nodes) {
|
|
295
|
-
if (!best || node.score > best.score || node.score === best.score && node.seq < best.seq) {
|
|
296
|
-
best = node;
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
return best;
|
|
300
|
-
}
|
|
301
|
-
function dominates(a, b) {
|
|
302
|
-
if (a.length !== b.length) return false;
|
|
303
|
-
let strictlyBetter = false;
|
|
304
|
-
for (let i = 0; i < a.length; i += 1) {
|
|
305
|
-
if (a[i] < b[i]) return false;
|
|
306
|
-
if (a[i] > b[i]) strictlyBetter = true;
|
|
307
|
-
}
|
|
308
|
-
return strictlyBetter;
|
|
309
|
-
}
|
|
310
|
-
function fsLineageStore(path) {
|
|
311
|
-
const ensureDir = () => mkdir(dirname(path), { recursive: true });
|
|
312
|
-
return {
|
|
313
|
-
async load() {
|
|
314
|
-
try {
|
|
315
|
-
return Lineage.fromJSONL(await readFile(path, "utf8"));
|
|
316
|
-
} catch (err) {
|
|
317
|
-
if (err.code === "ENOENT") return new Lineage();
|
|
318
|
-
throw err;
|
|
319
|
-
}
|
|
320
|
-
},
|
|
321
|
-
async append(node) {
|
|
322
|
-
await ensureDir();
|
|
323
|
-
await appendFile(path, `${JSON.stringify(node)}
|
|
324
|
-
`, "utf8");
|
|
325
|
-
},
|
|
326
|
-
async save(lineage) {
|
|
327
|
-
await ensureDir();
|
|
328
|
-
const jsonl = lineage.toJSONL();
|
|
329
|
-
await writeFile(path, jsonl.length > 0 ? `${jsonl}
|
|
330
|
-
` : "", "utf8");
|
|
331
|
-
}
|
|
332
|
-
};
|
|
333
|
-
}
|
|
334
|
-
function memLineageStore() {
|
|
335
|
-
const nodes = [];
|
|
336
|
-
return {
|
|
337
|
-
async load() {
|
|
338
|
-
return new Lineage(nodes);
|
|
339
|
-
},
|
|
340
|
-
async append(node) {
|
|
341
|
-
nodes.push(node);
|
|
342
|
-
},
|
|
343
|
-
async save(lineage) {
|
|
344
|
-
nodes.length = 0;
|
|
345
|
-
nodes.push(...lineage.all());
|
|
346
|
-
}
|
|
347
|
-
};
|
|
348
|
-
}
|
|
349
|
-
function heuristicGovernor(opts = {}) {
|
|
350
|
-
const maxTracks = opts.maxTracks ?? 3;
|
|
351
|
-
const plateauSteps = opts.plateauSteps ?? 2;
|
|
352
|
-
const mergeFrontierAt = opts.mergeFrontierAt ?? 2;
|
|
353
|
-
return {
|
|
354
|
-
decide(ctx) {
|
|
355
|
-
const { lineage } = ctx;
|
|
356
|
-
if (ctx.budgetRemaining <= 0) return { op: "stop" };
|
|
357
|
-
const pruned = new Set(ctx.prunedTracks);
|
|
358
|
-
const liveTracks = lineage.tracks().filter((t) => !pruned.has(t));
|
|
359
|
-
if (liveTracks.length === 0) return { op: "stop" };
|
|
360
|
-
const frontierByTrack = /* @__PURE__ */ new Map();
|
|
361
|
-
for (const tip of lineage.frontier()) {
|
|
362
|
-
if (pruned.has(tip.track)) continue;
|
|
363
|
-
if (!frontierByTrack.has(tip.track)) frontierByTrack.set(tip.track, tip);
|
|
364
|
-
}
|
|
365
|
-
if (frontierByTrack.size >= mergeFrontierAt) {
|
|
366
|
-
const tips = [...frontierByTrack.values()].sort(bySeq);
|
|
367
|
-
const target = pickBest(tips);
|
|
368
|
-
return { op: "merge", parentIds: tips.map((t) => t.id), track: target.track };
|
|
369
|
-
}
|
|
370
|
-
for (const track of liveTracks) {
|
|
371
|
-
if (isPlateaued(lineage.trackNodes(track), plateauSteps)) {
|
|
372
|
-
if (liveTracks.length > 1 || liveTracks.length < maxTracks) {
|
|
373
|
-
return { op: "prune", track };
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
const leader = pickBest(liveTracks.map((t) => lineage.trackTip(t)).filter(Boolean));
|
|
378
|
-
if (leader && liveTracks.length < maxTracks && isPlateaued(lineage.trackNodes(leader.track), plateauSteps)) {
|
|
379
|
-
return {
|
|
380
|
-
op: "branch",
|
|
381
|
-
fromNodeId: leader.id,
|
|
382
|
-
track: `${leader.track}+${lineage.tracks().length}`,
|
|
383
|
-
proposer: "gepa"
|
|
384
|
-
};
|
|
385
|
-
}
|
|
386
|
-
if (leader) return { op: "extend", track: leader.track };
|
|
387
|
-
return { op: "stop" };
|
|
388
|
-
}
|
|
389
|
-
};
|
|
390
|
-
}
|
|
391
|
-
function callbackGovernor(decide) {
|
|
392
|
-
return { decide };
|
|
393
|
-
}
|
|
394
|
-
function isPlateaued(trackNodes, window) {
|
|
395
|
-
if (trackNodes.length <= window) return false;
|
|
396
|
-
const ordered = [...trackNodes].sort(bySeq);
|
|
397
|
-
const head = ordered.slice(0, ordered.length - window);
|
|
398
|
-
const tail = ordered.slice(ordered.length - window);
|
|
399
|
-
const bestBefore = Math.max(...head.map((n) => n.score));
|
|
400
|
-
const bestRecent = Math.max(...tail.map((n) => n.score));
|
|
401
|
-
return bestRecent <= bestBefore;
|
|
402
|
-
}
|
|
403
|
-
async function runLineage(opts) {
|
|
404
|
-
const store = opts.store ?? memLineageStore();
|
|
405
|
-
const lineage = await store.load();
|
|
406
|
-
const log = opts.log ?? (() => {
|
|
407
|
-
});
|
|
408
|
-
const pruned = /* @__PURE__ */ new Set();
|
|
409
|
-
const trackProposer = /* @__PURE__ */ new Map();
|
|
410
|
-
const persist = async (node) => {
|
|
411
|
-
await store.append(node);
|
|
412
|
-
};
|
|
413
|
-
for (const seed of opts.seeds) {
|
|
414
|
-
const node = lineage.addNode({
|
|
415
|
-
parentIds: [],
|
|
416
|
-
track: seed.track,
|
|
417
|
-
surface: seed.surface,
|
|
418
|
-
score: seed.score,
|
|
419
|
-
proposer: seed.proposer,
|
|
420
|
-
...seed.vision !== void 0 ? { vision: seed.vision } : {},
|
|
421
|
-
...seed.scoreVector !== void 0 ? { scoreVector: seed.scoreVector } : {}
|
|
422
|
-
});
|
|
423
|
-
trackProposer.set(seed.track, seed.proposer);
|
|
424
|
-
await persist(node);
|
|
425
|
-
}
|
|
426
|
-
let steps = 0;
|
|
427
|
-
while (steps < opts.budget.maxSteps) {
|
|
428
|
-
const op = await opts.governor.decide({
|
|
429
|
-
lineage,
|
|
430
|
-
step: steps,
|
|
431
|
-
budgetRemaining: opts.budget.maxSteps - steps,
|
|
432
|
-
prunedTracks: [...pruned]
|
|
433
|
-
});
|
|
434
|
-
if (op.op === "stop") {
|
|
435
|
-
log("lineage: governor stop", { steps });
|
|
436
|
-
break;
|
|
437
|
-
}
|
|
438
|
-
if (op.op === "prune") {
|
|
439
|
-
pruned.add(op.track);
|
|
440
|
-
log("lineage: prune", { track: op.track });
|
|
441
|
-
steps += 1;
|
|
442
|
-
continue;
|
|
443
|
-
}
|
|
444
|
-
if (op.op === "merge") {
|
|
445
|
-
const parents = op.parentIds.map((id) => lineage.get(id)).filter((n) => n !== void 0);
|
|
446
|
-
if (parents.length < 2) {
|
|
447
|
-
log("lineage: merge skipped (fewer than 2 known parents)", { op });
|
|
448
|
-
steps += 1;
|
|
449
|
-
continue;
|
|
450
|
-
}
|
|
451
|
-
const result2 = await opts.merge({ parents, track: op.track });
|
|
452
|
-
const node2 = lineage.merge({
|
|
453
|
-
parentIds: parents.map((p) => p.id),
|
|
454
|
-
track: op.track,
|
|
455
|
-
surface: result2.surface,
|
|
456
|
-
score: result2.score,
|
|
457
|
-
...parents[0].vision !== void 0 ? { vision: parents[0].vision } : {},
|
|
458
|
-
...result2.scoreVector !== void 0 ? { scoreVector: result2.scoreVector } : {},
|
|
459
|
-
...result2.rationale !== void 0 ? { rationale: result2.rationale } : {}
|
|
460
|
-
});
|
|
461
|
-
await persist(node2);
|
|
462
|
-
steps += 1;
|
|
463
|
-
continue;
|
|
464
|
-
}
|
|
465
|
-
if (op.op === "branch") {
|
|
466
|
-
if (pruned.has(op.track)) {
|
|
467
|
-
log("lineage: branch skipped (target track pruned)", { track: op.track });
|
|
468
|
-
steps += 1;
|
|
469
|
-
continue;
|
|
470
|
-
}
|
|
471
|
-
const from = lineage.get(op.fromNodeId);
|
|
472
|
-
if (!from) {
|
|
473
|
-
log("lineage: branch skipped (unknown fromNodeId)", { op });
|
|
474
|
-
steps += 1;
|
|
475
|
-
continue;
|
|
476
|
-
}
|
|
477
|
-
trackProposer.set(op.track, op.proposer);
|
|
478
|
-
const result2 = await opts.step({ track: op.track, proposer: op.proposer, tip: from });
|
|
479
|
-
const node2 = lineage.addNode({
|
|
480
|
-
parentIds: [from.id],
|
|
481
|
-
track: op.track,
|
|
482
|
-
surface: result2.surface,
|
|
483
|
-
score: result2.score,
|
|
484
|
-
proposer: op.proposer,
|
|
485
|
-
...op.vision !== void 0 ? { vision: op.vision } : {},
|
|
486
|
-
...result2.scoreVector !== void 0 ? { scoreVector: result2.scoreVector } : {},
|
|
487
|
-
...result2.rationale !== void 0 ? { rationale: result2.rationale } : {},
|
|
488
|
-
...result2.gate !== void 0 ? { gate: result2.gate } : {}
|
|
489
|
-
});
|
|
490
|
-
await persist(node2);
|
|
491
|
-
steps += 1;
|
|
492
|
-
continue;
|
|
493
|
-
}
|
|
494
|
-
if (pruned.has(op.track)) {
|
|
495
|
-
log("lineage: extend skipped (track pruned)", { track: op.track });
|
|
496
|
-
steps += 1;
|
|
497
|
-
continue;
|
|
498
|
-
}
|
|
499
|
-
const tip = lineage.trackTip(op.track);
|
|
500
|
-
if (!tip) {
|
|
501
|
-
log("lineage: extend skipped (no tip for track)", { track: op.track });
|
|
502
|
-
steps += 1;
|
|
503
|
-
continue;
|
|
504
|
-
}
|
|
505
|
-
const proposer = trackProposer.get(op.track) ?? tip.proposer;
|
|
506
|
-
const result = await opts.step({ track: op.track, proposer, tip });
|
|
507
|
-
const node = lineage.addNode({
|
|
508
|
-
parentIds: [tip.id],
|
|
509
|
-
track: op.track,
|
|
510
|
-
surface: result.surface,
|
|
511
|
-
score: result.score,
|
|
512
|
-
proposer,
|
|
513
|
-
...tip.vision !== void 0 ? { vision: tip.vision } : {},
|
|
514
|
-
...result.scoreVector !== void 0 ? { scoreVector: result.scoreVector } : {},
|
|
515
|
-
...result.rationale !== void 0 ? { rationale: result.rationale } : {},
|
|
516
|
-
...result.gate !== void 0 ? { gate: result.gate } : {}
|
|
517
|
-
});
|
|
518
|
-
await persist(node);
|
|
519
|
-
steps += 1;
|
|
520
|
-
}
|
|
521
|
-
return { lineage, best: lineage.best(), steps };
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
// src/campaign/analyst-surface.ts
|
|
525
|
-
function surfaceToText(surface) {
|
|
526
|
-
if (typeof surface === "string") return surface;
|
|
527
|
-
throw new Error(
|
|
528
|
-
`buildAnalystSurfaceDispatch: the analyst surface must be a string actorDescription, got a ${surface.kind}-tier surface (${surface.worktreeRef}). The analyst prompt is prompt-tier.`
|
|
529
|
-
);
|
|
530
|
-
}
|
|
531
|
-
function buildAnalystSurfaceDispatch(opts) {
|
|
532
|
-
const analyze = opts.analyze ?? analyzeTraces;
|
|
533
|
-
return async (surface, scenario, _ctx) => {
|
|
534
|
-
const actorDescription = surfaceToText(surface);
|
|
535
|
-
const res = await analyze(
|
|
536
|
-
{ question: scenario.question },
|
|
537
|
-
{ ...opts.analystOptions, actorDescription, source: scenario.source }
|
|
538
|
-
);
|
|
539
|
-
return {
|
|
540
|
-
answer: res.answer,
|
|
541
|
-
findings: res.findings,
|
|
542
|
-
actorPromptVersion: res.actorPromptVersion
|
|
543
|
-
};
|
|
544
|
-
};
|
|
545
|
-
}
|
|
546
|
-
function failureModeRecallJudge(opts = {}) {
|
|
547
|
-
const recallWeight = opts.recallWeight ?? 0.5;
|
|
548
|
-
return {
|
|
549
|
-
name: "failure-mode-recall",
|
|
550
|
-
dimensions: [
|
|
551
|
-
{ key: "recall", description: "fraction of ground-truth failure modes the analyst surfaced" },
|
|
552
|
-
{
|
|
553
|
-
key: "precision",
|
|
554
|
-
description: "1 \u2212 share of findings that named a failure/tool/error absent from this corpus"
|
|
555
|
-
}
|
|
556
|
-
],
|
|
557
|
-
appliesTo: (s) => s.kind === "analyst-surface",
|
|
558
|
-
score({ artifact, scenario }) {
|
|
559
|
-
const modes = scenario.expectedFailureModes;
|
|
560
|
-
if (modes.length === 0) {
|
|
561
|
-
throw new Error(
|
|
562
|
-
`failureModeRecallJudge: scenario '${scenario.id}' has no expectedFailureModes \u2014 refusing to score (a vacuous 1.0 would corrupt the comparison)`
|
|
563
|
-
);
|
|
564
|
-
}
|
|
565
|
-
const hay = artifact.findings.join("\n").toLowerCase();
|
|
566
|
-
const matched = modes.filter((m) => m.cues.some((c) => hay.includes(c.toLowerCase())));
|
|
567
|
-
const recall = matched.length / modes.length;
|
|
568
|
-
const forbidden = (scenario.forbiddenCues ?? []).map((c) => c.toLowerCase());
|
|
569
|
-
let precision = 1;
|
|
570
|
-
let hallucinated = 0;
|
|
571
|
-
if (forbidden.length > 0) {
|
|
572
|
-
const denom = Math.max(1, artifact.findings.length);
|
|
573
|
-
hallucinated = artifact.findings.filter(
|
|
574
|
-
(f) => forbidden.some((c) => f.toLowerCase().includes(c))
|
|
575
|
-
).length;
|
|
576
|
-
precision = 1 - hallucinated / denom;
|
|
577
|
-
}
|
|
578
|
-
const composite = forbidden.length > 0 ? recallWeight * recall + (1 - recallWeight) * precision : recall;
|
|
579
|
-
const missed = modes.filter((m) => !matched.includes(m)).map((m) => m.id);
|
|
580
|
-
const notes = `matched ${matched.length}/${modes.length} failure modes` + (missed.length ? `; missed [${missed.join(", ")}]` : "") + (hallucinated ? `; ${hallucinated} out-of-corpus finding(s)` : "");
|
|
581
|
-
return { dimensions: { recall, precision }, composite, notes };
|
|
582
|
-
}
|
|
583
|
-
};
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
// src/campaign/fixtures.ts
|
|
587
|
-
import { createHash as createHash2 } from "crypto";
|
|
588
|
-
import { existsSync, readdirSync, readFileSync, statSync } from "fs";
|
|
589
|
-
import { isAbsolute, join, relative, resolve } from "path";
|
|
590
|
-
function discoverEvalFixtures(evalsDir) {
|
|
591
|
-
const root = resolve(evalsDir);
|
|
592
|
-
if (!existsSync(root)) throw new Error(`discoverEvalFixtures: evalsDir not found: ${root}`);
|
|
593
|
-
const fixtures = [];
|
|
594
|
-
const walk = (dir, base = "") => {
|
|
595
|
-
for (const entry of readdirSync(dir).sort()) {
|
|
596
|
-
if (entry.startsWith(".") || entry === "node_modules") continue;
|
|
597
|
-
const fullPath = join(dir, entry);
|
|
598
|
-
if (!statSync(fullPath).isDirectory()) continue;
|
|
599
|
-
const name = base ? `${base}/${entry}` : entry;
|
|
600
|
-
if (existsWithExactCase(fullPath, "PROMPT.md")) {
|
|
601
|
-
fixtures.push(name);
|
|
602
|
-
} else {
|
|
603
|
-
walk(fullPath, name);
|
|
604
|
-
}
|
|
605
|
-
}
|
|
606
|
-
};
|
|
607
|
-
walk(root);
|
|
608
|
-
return fixtures;
|
|
609
|
-
}
|
|
610
|
-
function loadEvalFixture(evalsDir, name, options = {}) {
|
|
611
|
-
const validation = options.validation ?? "vitest";
|
|
612
|
-
const root = resolve(evalsDir);
|
|
613
|
-
const fixturePath = resolve(root, name);
|
|
614
|
-
assertInside(root, fixturePath, name);
|
|
615
|
-
if (!existsSync(fixturePath) || !statSync(fixturePath).isDirectory()) {
|
|
616
|
-
throw new Error(`loadEvalFixture: fixture not found: ${name}`);
|
|
617
|
-
}
|
|
618
|
-
if (!existsWithExactCase(fixturePath, "PROMPT.md")) {
|
|
619
|
-
throw new Error(`loadEvalFixture: ${name} is missing exact-case PROMPT.md`);
|
|
620
|
-
}
|
|
621
|
-
const promptPath = join(fixturePath, "PROMPT.md");
|
|
622
|
-
const evalPath = resolveEvalPath(fixturePath);
|
|
623
|
-
const packageJsonPath = existsWithExactCase(fixturePath, "package.json") ? join(fixturePath, "package.json") : void 0;
|
|
624
|
-
if (validation !== "none") {
|
|
625
|
-
if (!evalPath)
|
|
626
|
-
throw new Error(`loadEvalFixture: ${name} is missing exact-case EVAL.ts or EVAL.tsx`);
|
|
627
|
-
if (!packageJsonPath)
|
|
628
|
-
throw new Error(`loadEvalFixture: ${name} is missing exact-case package.json`);
|
|
629
|
-
assertModulePackage(packageJsonPath, name);
|
|
630
|
-
}
|
|
631
|
-
const files = collectFixtureFiles(fixturePath);
|
|
632
|
-
return {
|
|
633
|
-
name,
|
|
634
|
-
path: fixturePath,
|
|
635
|
-
promptPath,
|
|
636
|
-
evalPath,
|
|
637
|
-
packageJsonPath,
|
|
638
|
-
prompt: readFileSync(promptPath, "utf8"),
|
|
639
|
-
files,
|
|
640
|
-
fingerprint: contentHash({
|
|
641
|
-
files,
|
|
642
|
-
config: options.fingerprintConfig ?? null
|
|
643
|
-
})
|
|
644
|
-
};
|
|
645
|
-
}
|
|
646
|
-
function loadEvalFixtureScenarios(evalsDir, options = {}) {
|
|
647
|
-
const names = options.names ?? discoverEvalFixtures(evalsDir);
|
|
648
|
-
return names.map((name) => {
|
|
649
|
-
const fixture = loadEvalFixture(evalsDir, name, options);
|
|
650
|
-
return {
|
|
651
|
-
id: fixture.name,
|
|
652
|
-
kind: "eval-fixture",
|
|
653
|
-
tags: ["eval-fixture"],
|
|
654
|
-
fixtureName: fixture.name,
|
|
655
|
-
fixturePath: fixture.path,
|
|
656
|
-
promptPath: fixture.promptPath,
|
|
657
|
-
evalPath: fixture.evalPath,
|
|
658
|
-
packageJsonPath: fixture.packageJsonPath,
|
|
659
|
-
prompt: fixture.prompt,
|
|
660
|
-
fingerprint: fixture.fingerprint
|
|
661
|
-
};
|
|
662
|
-
});
|
|
663
|
-
}
|
|
664
|
-
function planEvalFixtureRun(options) {
|
|
665
|
-
const scenarios = loadEvalFixtureScenarios(options.evalsDir, {
|
|
666
|
-
names: options.names,
|
|
667
|
-
validation: options.validation,
|
|
668
|
-
fingerprintConfig: options.fingerprintConfig
|
|
669
|
-
});
|
|
670
|
-
const plan = planCampaignRun({
|
|
671
|
-
scenarios,
|
|
672
|
-
dispatchRef: options.dispatchRef ?? "eval-fixture-dispatch",
|
|
673
|
-
judges: options.judges,
|
|
674
|
-
seed: options.seed,
|
|
675
|
-
reps: options.reps,
|
|
676
|
-
resumable: options.resumable,
|
|
677
|
-
runDir: options.runDir,
|
|
678
|
-
storage: options.storage
|
|
679
|
-
});
|
|
680
|
-
return {
|
|
681
|
-
...plan,
|
|
682
|
-
fixtures: scenarios.map((scenario) => ({
|
|
683
|
-
fixtureName: scenario.fixtureName,
|
|
684
|
-
fixturePath: scenario.fixturePath,
|
|
685
|
-
fingerprint: scenario.fingerprint
|
|
686
|
-
}))
|
|
687
|
-
};
|
|
688
|
-
}
|
|
689
|
-
function existsWithExactCase(dirPath, fileName) {
|
|
690
|
-
try {
|
|
691
|
-
return readdirSync(dirPath).includes(fileName);
|
|
692
|
-
} catch {
|
|
693
|
-
return false;
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
function resolveEvalPath(fixturePath) {
|
|
697
|
-
if (existsWithExactCase(fixturePath, "EVAL.ts")) return join(fixturePath, "EVAL.ts");
|
|
698
|
-
if (existsWithExactCase(fixturePath, "EVAL.tsx")) return join(fixturePath, "EVAL.tsx");
|
|
699
|
-
return void 0;
|
|
700
|
-
}
|
|
701
|
-
function assertModulePackage(packageJsonPath, name) {
|
|
702
|
-
let parsed;
|
|
703
|
-
try {
|
|
704
|
-
parsed = JSON.parse(readFileSync(packageJsonPath, "utf8"));
|
|
705
|
-
} catch (err) {
|
|
706
|
-
throw new Error(
|
|
707
|
-
`loadEvalFixture: ${name} package.json is invalid JSON: ${err instanceof Error ? err.message : String(err)}`
|
|
708
|
-
);
|
|
709
|
-
}
|
|
710
|
-
if (typeof parsed !== "object" || parsed === null || parsed.type !== "module") {
|
|
711
|
-
throw new Error(`loadEvalFixture: ${name} package.json must set "type": "module"`);
|
|
712
|
-
}
|
|
713
|
-
}
|
|
714
|
-
function collectFixtureFiles(fixturePath, base = "") {
|
|
715
|
-
const files = [];
|
|
716
|
-
for (const entry of readdirSync(join(fixturePath, base)).sort()) {
|
|
717
|
-
if (entry === "node_modules" || entry === ".git") continue;
|
|
718
|
-
const relativePath = base ? `${base}/${entry}` : entry;
|
|
719
|
-
const fullPath = join(fixturePath, relativePath);
|
|
720
|
-
const stat = statSync(fullPath);
|
|
721
|
-
if (stat.isDirectory()) {
|
|
722
|
-
files.push(...collectFixtureFiles(fixturePath, relativePath));
|
|
723
|
-
continue;
|
|
724
|
-
}
|
|
725
|
-
const bytes = readFileSync(fullPath);
|
|
726
|
-
files.push({
|
|
727
|
-
path: relativePath,
|
|
728
|
-
sha256: createHash2("sha256").update(bytes).digest("hex"),
|
|
729
|
-
bytes: bytes.byteLength
|
|
730
|
-
});
|
|
731
|
-
}
|
|
732
|
-
return files;
|
|
733
|
-
}
|
|
734
|
-
function assertInside(root, target, label) {
|
|
735
|
-
const rel = relative(root, target);
|
|
736
|
-
if (rel === "" || !rel.startsWith("..") && !isAbsolute(rel)) return;
|
|
737
|
-
throw new Error(`loadEvalFixture: fixture path escapes evalsDir: ${label}`);
|
|
738
|
-
}
|
|
739
|
-
|
|
740
|
-
// src/campaign/gates/neutralization-gate.ts
|
|
741
|
-
function mean(xs) {
|
|
742
|
-
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
743
|
-
}
|
|
744
|
-
function pairedLift(arm, baseline, scenarioIds) {
|
|
745
|
-
const paired = pairHoldout(arm, baseline, scenarioIds, (s) => s.composite);
|
|
746
|
-
const deltas = paired.after.map((a, i) => a - (paired.before[i] ?? 0));
|
|
747
|
-
return { lift: mean(deltas), n: deltas.length };
|
|
748
|
-
}
|
|
749
|
-
function neutralizationGate(options) {
|
|
750
|
-
const maxDecorativeFraction = options.maxDecorativeFraction ?? 0.5;
|
|
751
|
-
return {
|
|
752
|
-
name: "neutralizationGate",
|
|
753
|
-
async decide(ctx) {
|
|
754
|
-
if (!ctx.baselineJudgeScores) {
|
|
755
|
-
throw new Error(
|
|
756
|
-
"neutralizationGate: ctx.baselineJudgeScores is required \u2014 the placebo control measures lift OVER baseline."
|
|
757
|
-
);
|
|
758
|
-
}
|
|
759
|
-
if (!ctx.neutralizedJudgeScores) {
|
|
760
|
-
throw new Error(
|
|
761
|
-
"neutralizationGate: ctx.neutralizedJudgeScores is required. It is populated by runImprovementLoop only when a `neutralize` function is supplied \u2014 composing this gate without that wiring would pass an unproven candidate."
|
|
762
|
-
);
|
|
763
|
-
}
|
|
764
|
-
const scenarioIds = new Set(options.scenarios.map((s) => s.id));
|
|
765
|
-
const cand = pairedLift(
|
|
766
|
-
ctx.judgeScores,
|
|
767
|
-
ctx.baselineJudgeScores,
|
|
768
|
-
scenarioIds
|
|
769
|
-
);
|
|
770
|
-
const neut = pairedLift(
|
|
771
|
-
ctx.neutralizedJudgeScores,
|
|
772
|
-
ctx.baselineJudgeScores,
|
|
773
|
-
scenarioIds
|
|
774
|
-
);
|
|
775
|
-
if (cand.lift <= 0) {
|
|
776
|
-
return {
|
|
777
|
-
decision: "hold",
|
|
778
|
-
reasons: [
|
|
779
|
-
`neutralization: candidate held-out lift ${cand.lift.toFixed(3)} \u2264 0 \u2014 no positive lift to attribute to content`
|
|
780
|
-
],
|
|
781
|
-
contributingGates: [
|
|
782
|
-
{
|
|
783
|
-
name: "neutralizationGate",
|
|
784
|
-
passed: false,
|
|
785
|
-
detail: { candidateLift: cand.lift, neutralizedLift: neut.lift, n: cand.n }
|
|
786
|
-
}
|
|
787
|
-
],
|
|
788
|
-
delta: cand.lift
|
|
789
|
-
};
|
|
790
|
-
}
|
|
791
|
-
const decorativeFraction = neut.lift / cand.lift;
|
|
792
|
-
const passed = decorativeFraction < maxDecorativeFraction;
|
|
793
|
-
const pct = (decorativeFraction * 100).toFixed(0);
|
|
794
|
-
return {
|
|
795
|
-
decision: passed ? "ship" : "hold",
|
|
796
|
-
reasons: passed ? [
|
|
797
|
-
`neutralization: content is causal \u2014 blanked variant reproduces ${pct}% of the lift (< ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
|
|
798
|
-
] : [
|
|
799
|
-
`neutralization: lift is DECORATIVE \u2014 blanking the content (footprint-matched) reproduces ${pct}% of the lift (\u2265 ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
|
|
800
|
-
],
|
|
801
|
-
contributingGates: [
|
|
802
|
-
{
|
|
803
|
-
name: "neutralizationGate",
|
|
804
|
-
passed,
|
|
805
|
-
detail: {
|
|
806
|
-
candidateLift: cand.lift,
|
|
807
|
-
neutralizedLift: neut.lift,
|
|
808
|
-
decorativeFraction,
|
|
809
|
-
maxDecorativeFraction,
|
|
810
|
-
n: cand.n
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
|
-
],
|
|
814
|
-
delta: cand.lift
|
|
815
|
-
};
|
|
816
|
-
}
|
|
817
|
-
};
|
|
818
|
-
}
|
|
819
|
-
|
|
820
|
-
// src/campaign/gates/sequential.ts
|
|
821
|
-
import { createHash as createHash3 } from "crypto";
|
|
822
|
-
function verifyManifestSync(m) {
|
|
823
|
-
if (m.algo !== void 0 && m.algo !== "sha256-content") {
|
|
824
|
-
throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
|
|
825
|
-
}
|
|
826
|
-
const { contentHash: contentHash2, algo: _algo, ...rest } = m;
|
|
827
|
-
void _algo;
|
|
828
|
-
const bytes = JSON.stringify(canonicalize(rest));
|
|
829
|
-
const hash = createHash3("sha256").update(bytes, "utf8").digest("hex");
|
|
830
|
-
return hash === contentHash2;
|
|
831
|
-
}
|
|
832
|
-
function resolveConfig(opts) {
|
|
833
|
-
const m = opts.preRegistration;
|
|
834
|
-
let alpha;
|
|
835
|
-
let maxN;
|
|
836
|
-
let direction;
|
|
837
|
-
let minEffect;
|
|
838
|
-
if (m) {
|
|
839
|
-
if (!verifyManifestSync(m)) {
|
|
840
|
-
throw new Error(
|
|
841
|
-
`sequentialPairedGate: pre-registration manifest '${m.id}' content hash mismatch (tampered)`
|
|
842
|
-
);
|
|
843
|
-
}
|
|
844
|
-
if (opts.alpha !== void 0 && opts.alpha !== m.alpha) {
|
|
845
|
-
throw new Error(
|
|
846
|
-
`sequentialPairedGate: alpha ${opts.alpha} conflicts with pre-registered alpha ${m.alpha} \u2014 the registered statistic is the only one anytime validity covers`
|
|
847
|
-
);
|
|
848
|
-
}
|
|
849
|
-
if (opts.maxN !== void 0 && opts.maxN !== m.preRegisteredN) {
|
|
850
|
-
throw new Error(
|
|
851
|
-
`sequentialPairedGate: maxN ${opts.maxN} conflicts with pre-registered N ${m.preRegisteredN}`
|
|
852
|
-
);
|
|
853
|
-
}
|
|
854
|
-
alpha = m.alpha;
|
|
855
|
-
maxN = m.preRegisteredN;
|
|
856
|
-
direction = m.direction;
|
|
857
|
-
minEffect = m.minEffect;
|
|
858
|
-
} else {
|
|
859
|
-
alpha = opts.alpha ?? 0.05;
|
|
860
|
-
if (opts.maxN === void 0) {
|
|
861
|
-
throw new Error(
|
|
862
|
-
"sequentialPairedGate: maxN is required (or bind a preRegistration manifest whose preRegisteredN is the budget) \u2014 an unbounded stream has no pre-registered budget"
|
|
863
|
-
);
|
|
864
|
-
}
|
|
865
|
-
maxN = opts.maxN;
|
|
866
|
-
direction = "increase";
|
|
867
|
-
minEffect = 0;
|
|
868
|
-
}
|
|
869
|
-
const scale = opts.scale ?? 1;
|
|
870
|
-
if (!Number.isFinite(scale) || scale <= 0) {
|
|
871
|
-
throw new Error(`sequentialPairedGate: scale must be > 0, got ${scale}`);
|
|
872
|
-
}
|
|
873
|
-
if (!Number.isInteger(maxN) || maxN < 1) {
|
|
874
|
-
throw new Error(`sequentialPairedGate: maxN must be a positive integer, got ${maxN}`);
|
|
875
|
-
}
|
|
876
|
-
const minN = opts.minN ?? 5;
|
|
877
|
-
if (!Number.isInteger(minN) || minN < 1 || minN > maxN) {
|
|
878
|
-
throw new Error(
|
|
879
|
-
`sequentialPairedGate: minN must be an integer in [1, maxN=${maxN}], got ${minN}`
|
|
880
|
-
);
|
|
881
|
-
}
|
|
882
|
-
if (!Number.isFinite(minEffect) || minEffect < 0 || minEffect >= scale) {
|
|
883
|
-
throw new Error(
|
|
884
|
-
`sequentialPairedGate: minEffect must be in [0, scale=${scale}), got ${minEffect}`
|
|
885
|
-
);
|
|
886
|
-
}
|
|
887
|
-
const nullMean = 0.5 + minEffect / (2 * scale);
|
|
888
|
-
return {
|
|
889
|
-
alpha,
|
|
890
|
-
minN,
|
|
891
|
-
maxN,
|
|
892
|
-
maxBet: opts.maxBet ?? 0.5,
|
|
893
|
-
scale,
|
|
894
|
-
shuffleSeed: opts.shuffleSeed ?? 1337,
|
|
895
|
-
direction,
|
|
896
|
-
nullMean,
|
|
897
|
-
minEffect
|
|
898
|
-
};
|
|
899
|
-
}
|
|
900
|
-
function makeStream(cfg) {
|
|
901
|
-
const proc = eProcess({ alpha: cfg.alpha, maxBet: cfg.maxBet, nullMean: cfg.nullMean });
|
|
902
|
-
const threshold = 1 / cfg.alpha;
|
|
903
|
-
let terminal;
|
|
904
|
-
return {
|
|
905
|
-
observe(delta) {
|
|
906
|
-
if (terminal === "undecided-at-maxN") {
|
|
907
|
-
throw new Error(
|
|
908
|
-
`sequentialPairedGate: pre-registered maxN=${cfg.maxN} exhausted \u2014 extending the stream after seeing the result reopens optional stopping; start a NEW pre-registered test`
|
|
909
|
-
);
|
|
910
|
-
}
|
|
911
|
-
if (!Number.isFinite(delta) || Math.abs(delta) > cfg.scale) {
|
|
912
|
-
throw new Error(
|
|
913
|
-
`sequentialPairedGate: delta ${delta} outside \xB1scale=${cfg.scale} \u2014 pass the judge's native scale explicitly (detectScale helps pick 1 vs 100)`
|
|
914
|
-
);
|
|
915
|
-
}
|
|
916
|
-
const d = cfg.direction === "decrease" ? -delta : delta;
|
|
917
|
-
const step = proc.update((d / cfg.scale + 1) / 2);
|
|
918
|
-
if (terminal === void 0 && step.n >= cfg.minN && step.wealth >= threshold) {
|
|
919
|
-
terminal = "promote";
|
|
920
|
-
}
|
|
921
|
-
if (terminal === "promote") {
|
|
922
|
-
return {
|
|
923
|
-
decision: "promote",
|
|
924
|
-
eValue: step.wealth,
|
|
925
|
-
n: step.n,
|
|
926
|
-
reason: `e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} at n=${step.n} (minN=${cfg.minN}): the paired improvement exceeds ${cfg.minEffect} at anytime-valid level \u03B1=${cfg.alpha}`
|
|
927
|
-
};
|
|
928
|
-
}
|
|
929
|
-
if (step.n >= cfg.maxN) {
|
|
930
|
-
terminal = "undecided-at-maxN";
|
|
931
|
-
return {
|
|
932
|
-
decision: "undecided-at-maxN",
|
|
933
|
-
eValue: step.wealth,
|
|
934
|
-
n: step.n,
|
|
935
|
-
reason: `undecided at pre-registered maxN=${cfg.maxN} (e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)}). This is NOT evidence of no effect \u2014 the effect may be real but smaller than this budget can detect; re-register with a larger N to test that`
|
|
936
|
-
};
|
|
937
|
-
}
|
|
938
|
-
return {
|
|
939
|
-
decision: "continue",
|
|
940
|
-
eValue: step.wealth,
|
|
941
|
-
n: step.n,
|
|
942
|
-
reason: `e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)} at n=${step.n}/${cfg.maxN} \u2014 keep observing`
|
|
943
|
-
};
|
|
944
|
-
},
|
|
945
|
-
state() {
|
|
946
|
-
return { ...proc.state(), decision: terminal ?? "continue" };
|
|
947
|
-
}
|
|
948
|
-
};
|
|
949
|
-
}
|
|
950
|
-
function seededShuffle(items, seed) {
|
|
951
|
-
const rng = mulberry32(seed);
|
|
952
|
-
for (let i = items.length - 1; i > 0; i--) {
|
|
953
|
-
const j = Math.floor(rng() * (i + 1));
|
|
954
|
-
const tmp = items[i];
|
|
955
|
-
items[i] = items[j];
|
|
956
|
-
items[j] = tmp;
|
|
957
|
-
}
|
|
958
|
-
return items;
|
|
959
|
-
}
|
|
960
|
-
function sequentialPairedGate(options) {
|
|
961
|
-
const cfg = resolveConfig(options);
|
|
962
|
-
const name = options.name ?? "sequentialPairedGate";
|
|
963
|
-
const manifest = options.preRegistration;
|
|
964
|
-
const observeStream = makeStream(cfg);
|
|
965
|
-
return {
|
|
966
|
-
name,
|
|
967
|
-
observe(delta) {
|
|
968
|
-
return observeStream.observe(delta);
|
|
969
|
-
},
|
|
970
|
-
state() {
|
|
971
|
-
return observeStream.state();
|
|
972
|
-
},
|
|
973
|
-
async decide(ctx) {
|
|
974
|
-
if (!ctx.baselineJudgeScores) {
|
|
975
|
-
throw new Error(
|
|
976
|
-
`${name}: ctx.baselineJudgeScores is required \u2014 falling back to the candidate's own scores would compare the candidate against itself (delta 0, silent no-op)`
|
|
977
|
-
);
|
|
978
|
-
}
|
|
979
|
-
const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
|
|
980
|
-
const paired = pairHoldout(
|
|
981
|
-
ctx.judgeScores,
|
|
982
|
-
ctx.baselineJudgeScores,
|
|
983
|
-
scenarioIds,
|
|
984
|
-
(s) => s.composite
|
|
985
|
-
);
|
|
986
|
-
const deltas = paired.after.map((a, i) => a - paired.before[i]);
|
|
987
|
-
seededShuffle(deltas, cfg.shuffleSeed);
|
|
988
|
-
const stream = makeStream(cfg);
|
|
989
|
-
let last;
|
|
990
|
-
for (const d of deltas) {
|
|
991
|
-
last = stream.observe(d);
|
|
992
|
-
if (last.decision !== "continue") break;
|
|
993
|
-
}
|
|
994
|
-
const detail = {
|
|
995
|
-
...stream.state(),
|
|
996
|
-
minN: cfg.minN,
|
|
997
|
-
maxN: cfg.maxN,
|
|
998
|
-
scale: cfg.scale,
|
|
999
|
-
shuffleSeed: cfg.shuffleSeed,
|
|
1000
|
-
direction: cfg.direction,
|
|
1001
|
-
minEffect: cfg.minEffect,
|
|
1002
|
-
pairedN: deltas.length,
|
|
1003
|
-
preRegisteredId: manifest?.id,
|
|
1004
|
-
metric: manifest?.metric
|
|
1005
|
-
};
|
|
1006
|
-
const meanDelta = deltas.length === 0 ? void 0 : deltas.reduce((s, d) => s + d, 0) / deltas.length;
|
|
1007
|
-
if (last === void 0) {
|
|
1008
|
-
return {
|
|
1009
|
-
decision: "need_more_work",
|
|
1010
|
-
reasons: [`${name}: no paired holdout observations \u2014 nothing to test`],
|
|
1011
|
-
contributingGates: [{ name, passed: false, detail }]
|
|
1012
|
-
};
|
|
1013
|
-
}
|
|
1014
|
-
const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
|
|
1015
|
-
return {
|
|
1016
|
-
decision,
|
|
1017
|
-
reasons: [`${name}: ${last.reason}`],
|
|
1018
|
-
contributingGates: [{ name, passed: decision === "ship", detail }],
|
|
1019
|
-
delta: meanDelta
|
|
1020
|
-
};
|
|
1021
|
-
}
|
|
1022
|
-
};
|
|
1023
|
-
}
|
|
1024
|
-
function sequentialDecide(options = {}) {
|
|
1025
|
-
const alpha = options.alpha ?? 0.05;
|
|
1026
|
-
const minN = options.minN ?? 5;
|
|
1027
|
-
const scale = options.scale ?? 1;
|
|
1028
|
-
if (!Number.isFinite(scale) || scale <= 0) {
|
|
1029
|
-
throw new Error(`sequentialDecide: scale must be > 0, got ${scale}`);
|
|
1030
|
-
}
|
|
1031
|
-
const proc = eProcess({ alpha, maxBet: options.maxBet ?? 0.5, nullMean: 0.5 });
|
|
1032
|
-
const threshold = 1 / alpha;
|
|
1033
|
-
let processedGenerations = 0;
|
|
1034
|
-
let reference;
|
|
1035
|
-
let stopped;
|
|
1036
|
-
const topCandidate = (record) => {
|
|
1037
|
-
if (record.candidates.length === 0) {
|
|
1038
|
-
throw new Error(
|
|
1039
|
-
`sequentialDecide: generation ${record.generationIndex} has no candidates \u2014 cannot extract a top candidate`
|
|
1040
|
-
);
|
|
1041
|
-
}
|
|
1042
|
-
return record.candidates.reduce((best, c) => c.composite > best.composite ? c : best);
|
|
1043
|
-
};
|
|
1044
|
-
const decide = ({ history }) => {
|
|
1045
|
-
if (stopped) return stopped;
|
|
1046
|
-
if (history.length === 0) return { stop: false };
|
|
1047
|
-
if (reference === void 0) {
|
|
1048
|
-
reference = new Map(
|
|
1049
|
-
topCandidate(history[0]).scenarios.map((s) => [s.scenarioId, s.composite])
|
|
1050
|
-
);
|
|
1051
|
-
}
|
|
1052
|
-
for (let g = Math.max(1, processedGenerations); g < history.length; g++) {
|
|
1053
|
-
const top = topCandidate(history[g]);
|
|
1054
|
-
const byScenario = new Map(top.scenarios.map((s) => [s.scenarioId, s.composite]));
|
|
1055
|
-
for (const [scenarioId, refComposite] of reference) {
|
|
1056
|
-
const candComposite = byScenario.get(scenarioId);
|
|
1057
|
-
if (candComposite === void 0) {
|
|
1058
|
-
throw new Error(
|
|
1059
|
-
`sequentialDecide: generation ${history[g].generationIndex} top candidate is missing scenario '${scenarioId}' \u2014 generations must score the same scenario set to pair`
|
|
1060
|
-
);
|
|
1061
|
-
}
|
|
1062
|
-
const delta = candComposite - refComposite;
|
|
1063
|
-
if (!Number.isFinite(delta) || Math.abs(delta) > scale) {
|
|
1064
|
-
throw new Error(
|
|
1065
|
-
`sequentialDecide: paired delta ${delta} outside \xB1scale=${scale} on scenario '${scenarioId}' \u2014 pass the composite scale explicitly`
|
|
1066
|
-
);
|
|
1067
|
-
}
|
|
1068
|
-
const step = proc.update((delta / scale + 1) / 2);
|
|
1069
|
-
if (step.n >= minN && step.wealth >= threshold) {
|
|
1070
|
-
stopped = {
|
|
1071
|
-
stop: true,
|
|
1072
|
-
reason: `sequential e-process decided at generation ${history[g].generationIndex}: e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} after n=${step.n} paired deltas vs the generation-0 incumbent \u2014 the improvement is real at \u03B1=${alpha}; stop exploring and promote via the gate`
|
|
1073
|
-
};
|
|
1074
|
-
processedGenerations = history.length;
|
|
1075
|
-
return stopped;
|
|
1076
|
-
}
|
|
1077
|
-
}
|
|
1078
|
-
}
|
|
1079
|
-
processedGenerations = history.length;
|
|
1080
|
-
return { stop: false };
|
|
1081
|
-
};
|
|
1082
|
-
decide.state = () => proc.state();
|
|
1083
|
-
return decide;
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
// src/campaign/labeled-store/fs-adapter.ts
|
|
1087
|
-
import { createHash as createHash4 } from "crypto";
|
|
1088
|
-
import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
1089
|
-
import { join as join2 } from "path";
|
|
1090
|
-
var LabeledScenarioStoreError = class extends Error {
|
|
1091
|
-
constructor(code, message) {
|
|
1092
|
-
super(message);
|
|
1093
|
-
this.code = code;
|
|
1094
|
-
this.name = "LabeledScenarioStoreError";
|
|
1095
|
-
}
|
|
1096
|
-
code;
|
|
1097
|
-
};
|
|
1098
|
-
var FsLabeledScenarioStore = class {
|
|
1099
|
-
constructor(options) {
|
|
1100
|
-
this.options = options;
|
|
1101
|
-
if (!existsSync2(options.root)) mkdirSync(options.root, { recursive: true });
|
|
1102
|
-
this.now = options.now ?? Date.now;
|
|
1103
|
-
}
|
|
1104
|
-
options;
|
|
1105
|
-
now;
|
|
1106
|
-
rateLimits = /* @__PURE__ */ new Map();
|
|
1107
|
-
async observe(write) {
|
|
1108
|
-
this.assertProvenance(write);
|
|
1109
|
-
this.assertRateLimit(write);
|
|
1110
|
-
const record = this.toRecord(write);
|
|
1111
|
-
const path = this.pathForSource(write.source);
|
|
1112
|
-
const line = `${JSON.stringify(record)}
|
|
1113
|
-
`;
|
|
1114
|
-
appendLine(path, line);
|
|
1115
|
-
}
|
|
1116
|
-
async sample(args) {
|
|
1117
|
-
if (!args.split) {
|
|
1118
|
-
throw new LabeledScenarioStoreError(
|
|
1119
|
-
"split_required",
|
|
1120
|
-
"sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
|
|
1121
|
-
);
|
|
1122
|
-
}
|
|
1123
|
-
if (!args.capturedBefore) {
|
|
1124
|
-
throw new LabeledScenarioStoreError(
|
|
1125
|
-
"capturedBefore_required",
|
|
1126
|
-
"sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
|
|
1127
|
-
);
|
|
1128
|
-
}
|
|
1129
|
-
const all = [];
|
|
1130
|
-
for (const source of ALL_SOURCES) {
|
|
1131
|
-
if (args.split === "train" && source === "production-trace") {
|
|
1132
|
-
const explicit = sourceFilterContains(args.filter?.source, "production-trace");
|
|
1133
|
-
if (!explicit) continue;
|
|
1134
|
-
}
|
|
1135
|
-
const path = this.pathForSource(source);
|
|
1136
|
-
if (!existsSync2(path)) continue;
|
|
1137
|
-
const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
|
|
1138
|
-
for (const line of lines) {
|
|
1139
|
-
let record;
|
|
1140
|
-
try {
|
|
1141
|
-
record = JSON.parse(line);
|
|
1142
|
-
} catch {
|
|
1143
|
-
continue;
|
|
1144
|
-
}
|
|
1145
|
-
if (!matchesFilter(record, args, source)) continue;
|
|
1146
|
-
all.push(record);
|
|
1147
|
-
}
|
|
1148
|
-
}
|
|
1149
|
-
all.sort((a, b) => {
|
|
1150
|
-
if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
|
|
1151
|
-
return a.recordHash.localeCompare(b.recordHash);
|
|
1152
|
-
});
|
|
1153
|
-
return all.slice(0, args.count);
|
|
1154
|
-
}
|
|
1155
|
-
async size() {
|
|
1156
|
-
const bySource = {};
|
|
1157
|
-
const byTrust = {
|
|
1158
|
-
unverified: 0,
|
|
1159
|
-
"verified-signal": 0,
|
|
1160
|
-
"human-rated": 0
|
|
1161
|
-
};
|
|
1162
|
-
let total = 0;
|
|
1163
|
-
for (const source of ALL_SOURCES) {
|
|
1164
|
-
const path = this.pathForSource(source);
|
|
1165
|
-
if (!existsSync2(path)) {
|
|
1166
|
-
bySource[source] = 0;
|
|
1167
|
-
continue;
|
|
1168
|
-
}
|
|
1169
|
-
const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
|
|
1170
|
-
bySource[source] = lines.length;
|
|
1171
|
-
total += lines.length;
|
|
1172
|
-
for (const line of lines) {
|
|
1173
|
-
let trust = "unverified";
|
|
1174
|
-
try {
|
|
1175
|
-
trust = JSON.parse(line).labelTrust ?? "unverified";
|
|
1176
|
-
} catch {
|
|
1177
|
-
}
|
|
1178
|
-
byTrust[trust] += 1;
|
|
1179
|
-
}
|
|
1180
|
-
}
|
|
1181
|
-
return { train: total, test: total, bySource, byTrust };
|
|
1182
|
-
}
|
|
1183
|
-
assertProvenance(write) {
|
|
1184
|
-
if (!write.source) {
|
|
1185
|
-
throw new LabeledScenarioStoreError(
|
|
1186
|
-
"missing_source",
|
|
1187
|
-
"LabeledScenarioWrite requires `source`"
|
|
1188
|
-
);
|
|
1189
|
-
}
|
|
1190
|
-
if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
|
|
1191
|
-
throw new LabeledScenarioStoreError(
|
|
1192
|
-
"missing_source_version",
|
|
1193
|
-
"LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
|
|
1194
|
-
);
|
|
1195
|
-
}
|
|
1196
|
-
if (!write.capturedAt) {
|
|
1197
|
-
throw new LabeledScenarioStoreError(
|
|
1198
|
-
"missing_captured_at",
|
|
1199
|
-
"LabeledScenarioWrite requires `capturedAt` ISO timestamp"
|
|
1200
|
-
);
|
|
1201
|
-
}
|
|
1202
|
-
if (!write.redactionStatus) {
|
|
1203
|
-
throw new LabeledScenarioStoreError(
|
|
1204
|
-
"missing_redaction_status",
|
|
1205
|
-
"LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
|
|
1206
|
-
);
|
|
1207
|
-
}
|
|
1208
|
-
if (!ALL_SOURCES.includes(write.source)) {
|
|
1209
|
-
throw new LabeledScenarioStoreError(
|
|
1210
|
-
"unknown_source",
|
|
1211
|
-
`LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
|
|
1212
|
-
);
|
|
1213
|
-
}
|
|
1214
|
-
}
|
|
1215
|
-
assertRateLimit(write) {
|
|
1216
|
-
const cap = this.options.maxWritesPerMinutePerBucket;
|
|
1217
|
-
if (!cap || !write.rateLimitBucket) return;
|
|
1218
|
-
const now = this.now();
|
|
1219
|
-
const windowMs = 6e4;
|
|
1220
|
-
let state = this.rateLimits.get(write.rateLimitBucket);
|
|
1221
|
-
if (!state || now - state.windowStartMs >= windowMs) {
|
|
1222
|
-
state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
|
|
1223
|
-
this.rateLimits.set(write.rateLimitBucket, state);
|
|
1224
|
-
}
|
|
1225
|
-
if (state.count >= cap) {
|
|
1226
|
-
throw new LabeledScenarioStoreError(
|
|
1227
|
-
"rate_limit_exceeded",
|
|
1228
|
-
`LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
|
|
1229
|
-
);
|
|
1230
|
-
}
|
|
1231
|
-
state.count += 1;
|
|
1232
|
-
}
|
|
1233
|
-
toRecord(write) {
|
|
1234
|
-
const recordHash = sha256(
|
|
1235
|
-
JSON.stringify({
|
|
1236
|
-
id: write.scenario.id,
|
|
1237
|
-
src: write.source,
|
|
1238
|
-
at: write.capturedAt,
|
|
1239
|
-
ver: write.sourceVersionHash
|
|
1240
|
-
})
|
|
1241
|
-
);
|
|
1242
|
-
return {
|
|
1243
|
-
...write,
|
|
1244
|
-
recordHash,
|
|
1245
|
-
split: "train"
|
|
1246
|
-
};
|
|
1247
|
-
}
|
|
1248
|
-
pathForSource(source) {
|
|
1249
|
-
return join2(this.options.root, `${source}.jsonl`);
|
|
1250
|
-
}
|
|
1251
|
-
};
|
|
1252
|
-
var ALL_SOURCES = [
|
|
1253
|
-
"production-trace",
|
|
1254
|
-
"eval-run",
|
|
1255
|
-
"manual",
|
|
1256
|
-
"red-team",
|
|
1257
|
-
"synthetic"
|
|
1258
|
-
];
|
|
1259
|
-
function sourceFilterContains(filter, needle) {
|
|
1260
|
-
if (!filter) return false;
|
|
1261
|
-
if (Array.isArray(filter)) return filter.includes(needle);
|
|
1262
|
-
return filter === needle;
|
|
1263
|
-
}
|
|
1264
|
-
function matchesFilter(record, args, source) {
|
|
1265
|
-
if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
|
|
1266
|
-
if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
|
|
1267
|
-
const f = args.filter;
|
|
1268
|
-
if (!f) return true;
|
|
1269
|
-
if (f.kind && record.scenario.kind !== f.kind) return false;
|
|
1270
|
-
if (f.source) {
|
|
1271
|
-
const sources = Array.isArray(f.source) ? f.source : [f.source];
|
|
1272
|
-
if (!sources.includes(source)) return false;
|
|
1273
|
-
}
|
|
1274
|
-
if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
|
|
1275
|
-
const composites = Object.values(record.judgeScores).map((s) => s.composite);
|
|
1276
|
-
const max = composites.length === 0 ? 0 : Math.max(...composites);
|
|
1277
|
-
if (f.minComposite !== void 0 && max < f.minComposite) return false;
|
|
1278
|
-
if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
|
|
1279
|
-
}
|
|
1280
|
-
if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
|
|
1281
|
-
return false;
|
|
1282
|
-
}
|
|
1283
|
-
return true;
|
|
1284
|
-
}
|
|
1285
|
-
function sha256(input) {
|
|
1286
|
-
return createHash4("sha256").update(input).digest("hex").slice(0, 16);
|
|
1287
|
-
}
|
|
1288
|
-
function appendLine(path, line) {
|
|
1289
|
-
if (existsSync2(path)) {
|
|
1290
|
-
const existing = readFileSync2(path, "utf8");
|
|
1291
|
-
writeFileSync(path, existing + line);
|
|
1292
|
-
} else {
|
|
1293
|
-
writeFileSync(path, line);
|
|
1294
|
-
}
|
|
1295
|
-
}
|
|
1296
|
-
|
|
1297
|
-
// src/campaign/neutralize.ts
|
|
1298
|
-
var FILLER = "#";
|
|
1299
|
-
function neutralizeText(content) {
|
|
1300
|
-
return content.replace(/\S/g, FILLER);
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
// src/campaign/proposers/fapo.ts
|
|
1304
|
-
var FAPO_LEVELS = ["prompt", "parameter", "structural"];
|
|
1305
|
-
var MAX_FINDING_DEPTH = 16;
|
|
1306
|
-
var UNSAFE_JSON_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
|
|
1307
|
-
function fapoProposer(opts) {
|
|
1308
|
-
const proposers = levelProposers(opts);
|
|
1309
|
-
const allowed = allowedLevels(opts, proposers);
|
|
1310
|
-
const plateauWindow = opts.plateauWindow ?? 3;
|
|
1311
|
-
const minDistinctStrategies = opts.minDistinctStrategies ?? 3;
|
|
1312
|
-
const minImprovement = opts.minImprovement ?? 0;
|
|
1313
|
-
const proposalsPerCycle = opts.proposalsPerCycle ?? 1;
|
|
1314
|
-
if (allowed.length === 0) {
|
|
1315
|
-
throw new Error("fapoProposer: at least one allowed level must have a proposer");
|
|
1316
|
-
}
|
|
1317
|
-
if (plateauWindow < 1) throw new Error("fapoProposer: plateauWindow must be >= 1");
|
|
1318
|
-
if (minDistinctStrategies < 1) {
|
|
1319
|
-
throw new Error("fapoProposer: minDistinctStrategies must be >= 1");
|
|
1320
|
-
}
|
|
1321
|
-
if (proposalsPerCycle < 1) throw new Error("fapoProposer: proposalsPerCycle must be >= 1");
|
|
1322
|
-
return {
|
|
1323
|
-
kind: "fapo",
|
|
1324
|
-
async propose(ctx) {
|
|
1325
|
-
const state = policyState(ctx.history, ctx.findings, {
|
|
1326
|
-
plateauWindow,
|
|
1327
|
-
minDistinctStrategies,
|
|
1328
|
-
minImprovement
|
|
1329
|
-
});
|
|
1330
|
-
const decision = chooseLevel({
|
|
1331
|
-
allowed,
|
|
1332
|
-
available: allowed.filter((level) => !state.exhausted[level]),
|
|
1333
|
-
proposers,
|
|
1334
|
-
state,
|
|
1335
|
-
promptFirst: opts.promptFirst ?? true,
|
|
1336
|
-
parameterBeforeStructural: opts.parameterBeforeStructural ?? true
|
|
1337
|
-
});
|
|
1338
|
-
if (!decision) return [];
|
|
1339
|
-
const proposer = proposers[decision.level];
|
|
1340
|
-
if (!proposer) {
|
|
1341
|
-
throw new Error(`fapoProposer: selected ${decision.level} but no proposer is configured`);
|
|
1342
|
-
}
|
|
1343
|
-
const requested = Math.min(ctx.populationSize, proposalsPerCycle);
|
|
1344
|
-
const raw = await proposer.propose({ ...ctx, populationSize: requested });
|
|
1345
|
-
const wrapped = raw.map((candidate) => normalizeProposal(candidate)).map((candidate) => wrapProposal(candidate, decision.level, decision.reason));
|
|
1346
|
-
const reviewed = [];
|
|
1347
|
-
for (const candidate of wrapped) {
|
|
1348
|
-
const review = opts.reviewCandidate ? await opts.reviewCandidate({
|
|
1349
|
-
level: decision.level,
|
|
1350
|
-
candidate,
|
|
1351
|
-
context: ctx,
|
|
1352
|
-
reason: decision.reason
|
|
1353
|
-
}) : { verdict: "pass" };
|
|
1354
|
-
if (review.verdict === "fail") continue;
|
|
1355
|
-
const warn = review.verdict === "warn" ? renderReviewWarnings(review) : "";
|
|
1356
|
-
reviewed.push(
|
|
1357
|
-
warn ? { ...candidate, rationale: `${candidate.rationale}
|
|
1358
|
-
${warn}` } : candidate
|
|
1359
|
-
);
|
|
1360
|
-
}
|
|
1361
|
-
if (wrapped.length > 0 && reviewed.length === 0) {
|
|
1362
|
-
throw new Error(
|
|
1363
|
-
`fapoProposer: reviewer blocked every ${decision.level} candidate; refusing to evaluate an unreviewed variant`
|
|
1364
|
-
);
|
|
1365
|
-
}
|
|
1366
|
-
return reviewed;
|
|
1367
|
-
},
|
|
1368
|
-
decide({ history }) {
|
|
1369
|
-
const last = history[history.length - 1];
|
|
1370
|
-
if (last && last.candidates.length === 0) {
|
|
1371
|
-
return { stop: true, reason: "FAPO produced no scoped candidate in the prior generation" };
|
|
1372
|
-
}
|
|
1373
|
-
const state = policyState(history, [], {
|
|
1374
|
-
plateauWindow,
|
|
1375
|
-
minDistinctStrategies,
|
|
1376
|
-
minImprovement
|
|
1377
|
-
});
|
|
1378
|
-
const everyAllowedLevelExhausted = allowed.every((level) => state.exhausted[level]);
|
|
1379
|
-
return everyAllowedLevelExhausted ? { stop: true, reason: `all FAPO levels exhausted (${allowed.join(", ")})` } : { stop: false };
|
|
1380
|
-
}
|
|
1381
|
-
};
|
|
1382
|
-
}
|
|
1383
|
-
function levelProposers(opts) {
|
|
1384
|
-
return {
|
|
1385
|
-
...opts.proposers ?? {},
|
|
1386
|
-
...opts.promptProposer ? { prompt: opts.promptProposer } : {},
|
|
1387
|
-
...opts.parameterProposer ? { parameter: opts.parameterProposer } : {},
|
|
1388
|
-
...opts.structuralProposer ? { structural: opts.structuralProposer } : {}
|
|
1389
|
-
};
|
|
1390
|
-
}
|
|
1391
|
-
function allowedLevels(opts, proposers) {
|
|
1392
|
-
const explicit = opts.scope?.allowedLevels;
|
|
1393
|
-
const forbidden = new Set(opts.scope?.forbiddenLevels ?? []);
|
|
1394
|
-
return FAPO_LEVELS.filter((level) => {
|
|
1395
|
-
if (!proposers[level]) return false;
|
|
1396
|
-
if (forbidden.has(level)) return false;
|
|
1397
|
-
return explicit ? explicit.includes(level) : true;
|
|
1398
|
-
});
|
|
1399
|
-
}
|
|
1400
|
-
function policyState(history, findings, opts) {
|
|
1401
|
-
const attempts = extractAttempts(history);
|
|
1402
|
-
return {
|
|
1403
|
-
attempts,
|
|
1404
|
-
exhausted: {
|
|
1405
|
-
prompt: isLevelExhausted("prompt", attempts, opts),
|
|
1406
|
-
parameter: isLevelExhausted("parameter", attempts, opts),
|
|
1407
|
-
structural: isLevelExhausted("structural", attempts, opts)
|
|
1408
|
-
},
|
|
1409
|
-
signals: extractFapoAttributionSignals(findings)
|
|
1410
|
-
};
|
|
1411
|
-
}
|
|
1412
|
-
function extractAttempts(history) {
|
|
1413
|
-
const attempts = [];
|
|
1414
|
-
for (const generation of history) {
|
|
1415
|
-
for (const candidate of generation.candidates) {
|
|
1416
|
-
const level = candidateLevel(candidate);
|
|
1417
|
-
if (!level) continue;
|
|
1418
|
-
attempts.push({
|
|
1419
|
-
level,
|
|
1420
|
-
composite: candidate.composite,
|
|
1421
|
-
strategy: candidateStrategy(candidate)
|
|
1422
|
-
});
|
|
1423
|
-
}
|
|
1424
|
-
}
|
|
1425
|
-
return attempts;
|
|
1426
|
-
}
|
|
1427
|
-
function candidateLevel(candidate) {
|
|
1428
|
-
const label = candidate.label ?? "";
|
|
1429
|
-
const rationale = candidate.rationale ?? "";
|
|
1430
|
-
return parseLevel(label) ?? parseLevel(rationale);
|
|
1431
|
-
}
|
|
1432
|
-
function parseLevel(text) {
|
|
1433
|
-
const match = /\bfapo:(prompt|parameter|structural)\b/.exec(text);
|
|
1434
|
-
return match ? match[1] : null;
|
|
1435
|
-
}
|
|
1436
|
-
function candidateStrategy(candidate) {
|
|
1437
|
-
const label = candidate.label ?? candidate.rationale ?? candidate.surfaceHash;
|
|
1438
|
-
return label.replace(/\bfapo:(prompt|parameter|structural):?/g, "").trim() || candidate.surfaceHash;
|
|
1439
|
-
}
|
|
1440
|
-
function isLevelExhausted(level, attempts, opts) {
|
|
1441
|
-
const own = attempts.filter((attempt) => attempt.level === level);
|
|
1442
|
-
if (own.length < opts.plateauWindow) return false;
|
|
1443
|
-
const distinctStrategies = new Set(own.map((attempt) => attempt.strategy)).size;
|
|
1444
|
-
if (distinctStrategies < opts.minDistinctStrategies) return false;
|
|
1445
|
-
let best = Number.NEGATIVE_INFINITY;
|
|
1446
|
-
let nonImproving = 0;
|
|
1447
|
-
for (const attempt of own) {
|
|
1448
|
-
if (attempt.composite > best + opts.minImprovement) {
|
|
1449
|
-
best = attempt.composite;
|
|
1450
|
-
nonImproving = 0;
|
|
1451
|
-
} else {
|
|
1452
|
-
nonImproving += 1;
|
|
1453
|
-
}
|
|
1454
|
-
}
|
|
1455
|
-
return nonImproving >= opts.plateauWindow;
|
|
1456
|
-
}
|
|
1457
|
-
function chooseLevel(args) {
|
|
1458
|
-
const { available, state } = args;
|
|
1459
|
-
if (available.length === 0) return null;
|
|
1460
|
-
const triedPrompt = state.attempts.some((attempt) => attempt.level === "prompt");
|
|
1461
|
-
if (args.promptFirst && available.includes("prompt") && !triedPrompt) {
|
|
1462
|
-
return {
|
|
1463
|
-
level: "prompt",
|
|
1464
|
-
reason: "prompt-first policy: no prompt-level variant has been tried yet"
|
|
1465
|
-
};
|
|
1466
|
-
}
|
|
1467
|
-
const supported = available.filter((level) => state.signals.counts[level] > 0);
|
|
1468
|
-
if (supported.includes("prompt")) {
|
|
1469
|
-
return {
|
|
1470
|
-
level: "prompt",
|
|
1471
|
-
reason: `attribution has ${state.signals.counts.prompt} prompt-addressable failure(s)`
|
|
1472
|
-
};
|
|
1473
|
-
}
|
|
1474
|
-
const strongest = supported.sort(
|
|
1475
|
-
(a, b) => state.signals.counts[b] - state.signals.counts[a] || levelRank(a) - levelRank(b)
|
|
1476
|
-
)[0];
|
|
1477
|
-
if (strongest) {
|
|
1478
|
-
if (strongest === "structural" && args.parameterBeforeStructural && available.includes("parameter") && args.proposers.parameter) {
|
|
1479
|
-
return {
|
|
1480
|
-
level: "parameter",
|
|
1481
|
-
reason: "attribution indicates a non-prompt bottleneck; trying parameter/config edits before structural edits"
|
|
1482
|
-
};
|
|
1483
|
-
}
|
|
1484
|
-
return {
|
|
1485
|
-
level: strongest,
|
|
1486
|
-
reason: `attribution has ${state.signals.counts[strongest]} ${strongest}-addressable failure(s)`
|
|
1487
|
-
};
|
|
1488
|
-
}
|
|
1489
|
-
if (state.attempts.length === 0) {
|
|
1490
|
-
const first = available.slice().sort((a, b) => levelRank(a) - levelRank(b))[0];
|
|
1491
|
-
return { level: first, reason: "no attribution yet; starting at the cheapest allowed level" };
|
|
1492
|
-
}
|
|
1493
|
-
return null;
|
|
1494
|
-
}
|
|
1495
|
-
function levelRank(level) {
|
|
1496
|
-
return level === "prompt" ? 0 : level === "parameter" ? 1 : 2;
|
|
1497
|
-
}
|
|
1498
|
-
function normalizeProposal(value) {
|
|
1499
|
-
return isProposedCandidate(value) ? value : { surface: value, label: "candidate", rationale: "bare candidate from level proposer" };
|
|
1500
|
-
}
|
|
1501
|
-
function wrapProposal(candidate, level, reason) {
|
|
1502
|
-
const label = candidate.label ? `fapo:${level}:${candidate.label}` : `fapo:${level}`;
|
|
1503
|
-
return {
|
|
1504
|
-
...candidate,
|
|
1505
|
-
label,
|
|
1506
|
-
rationale: [
|
|
1507
|
-
`fapo:${level} selected by reviewed escalation policy`,
|
|
1508
|
-
`Reason: ${reason}`,
|
|
1509
|
-
candidate.rationale ? `Level proposer rationale: ${candidate.rationale}` : ""
|
|
1510
|
-
].filter(Boolean).join("\n")
|
|
1511
|
-
};
|
|
1512
|
-
}
|
|
1513
|
-
function renderReviewWarnings(review) {
|
|
1514
|
-
const issues = review.issues?.filter((issue) => issue.severity === "warn") ?? [];
|
|
1515
|
-
if (issues.length === 0) return "";
|
|
1516
|
-
return `Reviewer warnings:
|
|
1517
|
-
${issues.map((issue) => `- ${issue.checkName}: ${issue.description}`).join("\n")}`;
|
|
1518
|
-
}
|
|
1519
|
-
function extractFapoAttributionSignals(findings) {
|
|
1520
|
-
const signals = {
|
|
1521
|
-
counts: { prompt: 0, parameter: 0, structural: 0 },
|
|
1522
|
-
clusters: []
|
|
1523
|
-
};
|
|
1524
|
-
const seen = /* @__PURE__ */ new WeakSet();
|
|
1525
|
-
for (const finding of findings) {
|
|
1526
|
-
collectFinding(signals, finding, seen, 0);
|
|
1527
|
-
}
|
|
1528
|
-
return signals;
|
|
1529
|
-
}
|
|
1530
|
-
function collectFinding(signals, finding, seen, depth) {
|
|
1531
|
-
if (depth > MAX_FINDING_DEPTH) return;
|
|
1532
|
-
if (!finding) return;
|
|
1533
|
-
if (typeof finding === "string") {
|
|
1534
|
-
const level = inferLevelFromText(finding);
|
|
1535
|
-
if (level) addCluster(signals, { label: finding, level, count: 1 });
|
|
1536
|
-
return;
|
|
1537
|
-
}
|
|
1538
|
-
if (Array.isArray(finding)) {
|
|
1539
|
-
if (seen.has(finding)) return;
|
|
1540
|
-
seen.add(finding);
|
|
1541
|
-
for (const item of finding) collectFinding(signals, item, seen, depth + 1);
|
|
1542
|
-
return;
|
|
1543
|
-
}
|
|
1544
|
-
if (typeof finding !== "object") return;
|
|
1545
|
-
if (seen.has(finding)) return;
|
|
1546
|
-
seen.add(finding);
|
|
1547
|
-
const obj = finding;
|
|
1548
|
-
collectLevelPartition(signals, obj.level_partition ?? obj.levelPartition, seen, depth + 1);
|
|
1549
|
-
collectCounts(signals, obj);
|
|
1550
|
-
collectClusters(signals, obj.clusters, true, seen, depth + 1);
|
|
1551
|
-
const explicit = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel);
|
|
1552
|
-
const text = textFields(obj);
|
|
1553
|
-
const inferred = explicit ?? inferLevelFromText(text);
|
|
1554
|
-
if (inferred) {
|
|
1555
|
-
addCluster(signals, {
|
|
1556
|
-
label: text || inferred,
|
|
1557
|
-
level: inferred,
|
|
1558
|
-
count: numberField(obj.count) ?? 1,
|
|
1559
|
-
confidence: confidenceField(obj.confidence),
|
|
1560
|
-
suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
|
|
1561
|
-
caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
|
|
1562
|
-
});
|
|
1563
|
-
}
|
|
1564
|
-
}
|
|
1565
|
-
function collectLevelPartition(signals, raw, seen, depth) {
|
|
1566
|
-
if (!raw || typeof raw !== "object") return;
|
|
1567
|
-
if (seen.has(raw)) return;
|
|
1568
|
-
seen.add(raw);
|
|
1569
|
-
const partition = raw;
|
|
1570
|
-
for (const level of FAPO_LEVELS) {
|
|
1571
|
-
const bucket = partition[level];
|
|
1572
|
-
if (!bucket || typeof bucket !== "object") continue;
|
|
1573
|
-
const obj = bucket;
|
|
1574
|
-
const count = numberField(obj.count) ?? 0;
|
|
1575
|
-
if (count > 0) signals.counts[level] += count;
|
|
1576
|
-
collectClusters(signals, obj.clusters, false, seen, depth + 1);
|
|
1577
|
-
}
|
|
1578
|
-
}
|
|
1579
|
-
function collectCounts(signals, obj) {
|
|
1580
|
-
const prompt = numberField(obj.prompt_addressable ?? obj.promptAddressable);
|
|
1581
|
-
const structural = numberField(obj.structural_addressable ?? obj.structuralAddressable);
|
|
1582
|
-
const tool = numberField(obj.tool_addressable ?? obj.toolAddressable);
|
|
1583
|
-
if (prompt) signals.counts.prompt += prompt;
|
|
1584
|
-
if (structural) signals.counts.structural += structural;
|
|
1585
|
-
if (tool) signals.counts.structural += tool;
|
|
1586
|
-
}
|
|
1587
|
-
function collectClusters(signals, raw, countClusters, seen, depth) {
|
|
1588
|
-
if (!Array.isArray(raw)) return;
|
|
1589
|
-
if (seen.has(raw)) return;
|
|
1590
|
-
seen.add(raw);
|
|
1591
|
-
for (const item of raw) {
|
|
1592
|
-
if (countClusters) {
|
|
1593
|
-
collectFinding(signals, item, seen, depth + 1);
|
|
1594
|
-
continue;
|
|
1595
|
-
}
|
|
1596
|
-
const cluster = parseCluster(item);
|
|
1597
|
-
if (cluster) signals.clusters.push(cluster);
|
|
1598
|
-
}
|
|
1599
|
-
}
|
|
1600
|
-
function addCluster(signals, cluster) {
|
|
1601
|
-
signals.counts[cluster.level] += Math.max(1, cluster.count);
|
|
1602
|
-
signals.clusters.push(cluster);
|
|
1603
|
-
}
|
|
1604
|
-
function parseCluster(raw) {
|
|
1605
|
-
if (!raw || typeof raw !== "object") return null;
|
|
1606
|
-
const obj = raw;
|
|
1607
|
-
const text = textFields(obj);
|
|
1608
|
-
const level = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel) ?? inferLevelFromText(text);
|
|
1609
|
-
if (!level) return null;
|
|
1610
|
-
return {
|
|
1611
|
-
label: text || level,
|
|
1612
|
-
level,
|
|
1613
|
-
count: numberField(obj.count) ?? 1,
|
|
1614
|
-
confidence: confidenceField(obj.confidence),
|
|
1615
|
-
suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
|
|
1616
|
-
caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
|
|
1617
|
-
};
|
|
1618
|
-
}
|
|
1619
|
-
function parseFindingLevel(raw) {
|
|
1620
|
-
if (typeof raw !== "string") return null;
|
|
1621
|
-
const lower = raw.toLowerCase();
|
|
1622
|
-
if (lower === "prompt" || lower === "parameter" || lower === "structural") return lower;
|
|
1623
|
-
if (lower === "chain" || lower === "tool" || lower === "code") return "structural";
|
|
1624
|
-
if (lower === "config" || lower === "params") return "parameter";
|
|
1625
|
-
return null;
|
|
1626
|
-
}
|
|
1627
|
-
function inferLevelFromText(text) {
|
|
1628
|
-
const lower = text.toLowerCase();
|
|
1629
|
-
if (/\b(retrieval_k|temperature|top_p|max_tokens|max_completion_tokens|reasoning_effort|config|parameter)\b/.test(
|
|
1630
|
-
lower
|
|
1631
|
-
)) {
|
|
1632
|
-
return "parameter";
|
|
1633
|
-
}
|
|
1634
|
-
if (/\b(retriev\w*|search\w*|bm25|hop|evidence|cascade|tool|node|chain|state|structur\w*|topology|import)\b/.test(
|
|
1635
|
-
lower
|
|
1636
|
-
)) {
|
|
1637
|
-
return "structural";
|
|
1638
|
-
}
|
|
1639
|
-
if (/\b(format\w*|verbose|brevity|abstain\w*|reasoning|instruction|prompt|output|answer)\b/.test(
|
|
1640
|
-
lower
|
|
1641
|
-
)) {
|
|
1642
|
-
return "prompt";
|
|
1643
|
-
}
|
|
1644
|
-
return null;
|
|
1645
|
-
}
|
|
1646
|
-
function textFields(obj) {
|
|
1647
|
-
return [
|
|
1648
|
-
obj.label,
|
|
1649
|
-
obj.claim,
|
|
1650
|
-
obj.recommended_action,
|
|
1651
|
-
obj.suggested_fix,
|
|
1652
|
-
obj.suggestedFix,
|
|
1653
|
-
obj.message,
|
|
1654
|
-
obj.text,
|
|
1655
|
-
obj.heuristic,
|
|
1656
|
-
obj.area
|
|
1657
|
-
].filter((value) => typeof value === "string" && value.trim().length > 0).join(" ");
|
|
1658
|
-
}
|
|
1659
|
-
function numberField(value) {
|
|
1660
|
-
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
1661
|
-
}
|
|
1662
|
-
function stringField(value) {
|
|
1663
|
-
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
1664
|
-
}
|
|
1665
|
-
function confidenceField(value) {
|
|
1666
|
-
return value === "high" || value === "medium" || value === "low" ? value : void 0;
|
|
1667
|
-
}
|
|
1668
|
-
function stringArrayField(value) {
|
|
1669
|
-
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : void 0;
|
|
1670
|
-
}
|
|
1671
|
-
function parameterSweepProposer(opts) {
|
|
1672
|
-
if (opts.candidates.length === 0) {
|
|
1673
|
-
throw new Error("parameterSweepProposer: candidates must not be empty");
|
|
1674
|
-
}
|
|
1675
|
-
return {
|
|
1676
|
-
kind: "parameter-sweep",
|
|
1677
|
-
async propose(ctx) {
|
|
1678
|
-
if (typeof ctx.currentSurface !== "string") {
|
|
1679
|
-
throw new Error(
|
|
1680
|
-
"parameterSweepProposer: currentSurface must be a JSON string config surface"
|
|
1681
|
-
);
|
|
1682
|
-
}
|
|
1683
|
-
const parse = opts.parse ?? parseJsonObject;
|
|
1684
|
-
const stringify = opts.stringify ?? ((config) => JSON.stringify(config, null, 2));
|
|
1685
|
-
const current = parse(ctx.currentSurface);
|
|
1686
|
-
const currentCanonical = canonicalJson(current);
|
|
1687
|
-
const tried = triedLabels(ctx.history);
|
|
1688
|
-
const out = [];
|
|
1689
|
-
for (const candidate of opts.candidates) {
|
|
1690
|
-
if (tried.has(candidate.label)) continue;
|
|
1691
|
-
const next = applyParameterCandidate(current, candidate);
|
|
1692
|
-
const surface = stringify(next);
|
|
1693
|
-
if (surface === ctx.currentSurface || canonicalJson(next) === currentCanonical) continue;
|
|
1694
|
-
out.push({ surface, label: candidate.label, rationale: candidate.rationale });
|
|
1695
|
-
if (out.length >= ctx.populationSize) break;
|
|
1696
|
-
}
|
|
1697
|
-
return out;
|
|
1698
|
-
}
|
|
1699
|
-
};
|
|
1700
|
-
}
|
|
1701
|
-
function parseJsonObject(surface) {
|
|
1702
|
-
let parsed;
|
|
1703
|
-
try {
|
|
1704
|
-
parsed = JSON.parse(surface);
|
|
1705
|
-
} catch (error) {
|
|
1706
|
-
throw new Error(
|
|
1707
|
-
`parameterSweepProposer: currentSurface must be valid JSON object config (${error instanceof Error ? error.message : String(error)})`
|
|
1708
|
-
);
|
|
1709
|
-
}
|
|
1710
|
-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
1711
|
-
throw new Error("parameterSweepProposer: JSON surface must parse to an object");
|
|
1712
|
-
}
|
|
1713
|
-
return parsed;
|
|
1714
|
-
}
|
|
1715
|
-
function triedLabels(history) {
|
|
1716
|
-
const tried = /* @__PURE__ */ new Set();
|
|
1717
|
-
for (const generation of history) {
|
|
1718
|
-
for (const candidate of generation.candidates) {
|
|
1719
|
-
if (!candidate.label) continue;
|
|
1720
|
-
tried.add(candidate.label);
|
|
1721
|
-
tried.add(candidateStrategy(candidate));
|
|
1722
|
-
}
|
|
1723
|
-
}
|
|
1724
|
-
return tried;
|
|
1725
|
-
}
|
|
1726
|
-
function applyParameterCandidate(current, candidate) {
|
|
1727
|
-
const next = cloneJsonObject(current);
|
|
1728
|
-
if (candidate.patch) deepMerge(next, candidate.patch);
|
|
1729
|
-
for (const change of candidate.changes ?? []) {
|
|
1730
|
-
setPath(
|
|
1731
|
-
next,
|
|
1732
|
-
typeof change.path === "string" ? change.path.split(".") : [...change.path],
|
|
1733
|
-
change.value
|
|
1734
|
-
);
|
|
1735
|
-
}
|
|
1736
|
-
return next;
|
|
1737
|
-
}
|
|
1738
|
-
function cloneJsonObject(value) {
|
|
1739
|
-
return JSON.parse(JSON.stringify(value));
|
|
1740
|
-
}
|
|
1741
|
-
function deepMerge(target, patch) {
|
|
1742
|
-
for (const [key, value] of Object.entries(patch)) {
|
|
1743
|
-
assertSafeJsonKey(key);
|
|
1744
|
-
if (isPlainObject(value) && isPlainObject(target[key])) {
|
|
1745
|
-
deepMerge(target[key], value);
|
|
1746
|
-
} else {
|
|
1747
|
-
target[key] = value;
|
|
1748
|
-
}
|
|
1749
|
-
}
|
|
1750
|
-
}
|
|
1751
|
-
function setPath(target, path, value) {
|
|
1752
|
-
if (path.length === 0) throw new Error("parameterSweepProposer: change path must not be empty");
|
|
1753
|
-
for (const part of path) assertSafeJsonKey(part);
|
|
1754
|
-
let cursor = target;
|
|
1755
|
-
for (const part of path.slice(0, -1)) {
|
|
1756
|
-
const existing = cursor[part];
|
|
1757
|
-
if (!isPlainObject(existing)) {
|
|
1758
|
-
cursor[part] = {};
|
|
1759
|
-
}
|
|
1760
|
-
cursor = cursor[part];
|
|
1761
|
-
}
|
|
1762
|
-
cursor[path[path.length - 1]] = value;
|
|
1763
|
-
}
|
|
1764
|
-
function assertSafeJsonKey(key) {
|
|
1765
|
-
if (!key.trim()) throw new Error("parameterSweepProposer: change path contains an empty key");
|
|
1766
|
-
if (UNSAFE_JSON_KEYS.has(key)) {
|
|
1767
|
-
throw new Error(`parameterSweepProposer: unsafe JSON key "${key}" is not allowed`);
|
|
1768
|
-
}
|
|
1769
|
-
}
|
|
1770
|
-
function canonicalJson(value) {
|
|
1771
|
-
if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
|
|
1772
|
-
if (isPlainObject(value)) {
|
|
1773
|
-
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
|
|
1774
|
-
}
|
|
1775
|
-
return JSON.stringify(value);
|
|
1776
|
-
}
|
|
1777
|
-
function isPlainObject(value) {
|
|
1778
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1779
|
-
const proto = Object.getPrototypeOf(value);
|
|
1780
|
-
return proto === Object.prototype || proto === null;
|
|
1781
|
-
}
|
|
1782
|
-
|
|
1783
|
-
// src/campaign/skill-patch.ts
|
|
1784
|
-
function applySkillPatch(surface, patch) {
|
|
1785
|
-
let lines = surface.split("\n");
|
|
1786
|
-
let applied = 0;
|
|
1787
|
-
const rejected = [];
|
|
1788
|
-
const findLine = (anchor) => lines.findIndex((l) => l.includes(anchor));
|
|
1789
|
-
for (const op of patch.ops) {
|
|
1790
|
-
if (op.op === "add") {
|
|
1791
|
-
if (typeof op.text !== "string" || op.text.trim() === "") {
|
|
1792
|
-
rejected.push({ op, reason: "empty add text" });
|
|
1793
|
-
continue;
|
|
1794
|
-
}
|
|
1795
|
-
const insert = op.text.split("\n");
|
|
1796
|
-
if (op.after === void 0 || op.after === "") {
|
|
1797
|
-
lines = [...lines, ...insert];
|
|
1798
|
-
applied++;
|
|
1799
|
-
continue;
|
|
1800
|
-
}
|
|
1801
|
-
const idx = findLine(op.after);
|
|
1802
|
-
if (idx === -1) {
|
|
1803
|
-
rejected.push({ op, reason: `add anchor not found: ${truncate(op.after)}` });
|
|
1804
|
-
continue;
|
|
1805
|
-
}
|
|
1806
|
-
lines = [...lines.slice(0, idx + 1), ...insert, ...lines.slice(idx + 1)];
|
|
1807
|
-
applied++;
|
|
1808
|
-
} else if (op.op === "delete") {
|
|
1809
|
-
const idx = findLine(op.anchor);
|
|
1810
|
-
if (idx === -1) {
|
|
1811
|
-
rejected.push({ op, reason: `delete anchor not found: ${truncate(op.anchor)}` });
|
|
1812
|
-
continue;
|
|
1813
|
-
}
|
|
1814
|
-
lines = [...lines.slice(0, idx), ...lines.slice(idx + 1)];
|
|
1815
|
-
applied++;
|
|
1816
|
-
} else {
|
|
1817
|
-
const idx = findLine(op.anchor);
|
|
1818
|
-
if (idx === -1) {
|
|
1819
|
-
rejected.push({ op, reason: `replace anchor not found: ${truncate(op.anchor)}` });
|
|
1820
|
-
continue;
|
|
1821
|
-
}
|
|
1822
|
-
if (typeof op.text !== "string") {
|
|
1823
|
-
rejected.push({ op, reason: "replace text missing" });
|
|
1824
|
-
continue;
|
|
1825
|
-
}
|
|
1826
|
-
lines = [...lines.slice(0, idx), ...op.text.split("\n"), ...lines.slice(idx + 1)];
|
|
1827
|
-
applied++;
|
|
1828
|
-
}
|
|
1829
|
-
}
|
|
1830
|
-
return { surface: lines.join("\n"), applied, rejected };
|
|
1831
|
-
}
|
|
1832
|
-
function patchEditCount(patch) {
|
|
1833
|
-
return patch.ops.length;
|
|
1834
|
-
}
|
|
1835
|
-
function truncate(s, max = 48) {
|
|
1836
|
-
return s.length <= max ? s : `${s.slice(0, max)}\u2026`;
|
|
1837
|
-
}
|
|
1838
|
-
|
|
1839
|
-
// src/campaign/proposers/skill-opt.ts
|
|
1840
|
-
var SKILLOPT_SYSTEM = 'You are a SkillOpt optimizer. You improve ONE skill document by proposing BOUNDED, anchored edits \u2014 never a full rewrite. Output ONLY a JSON object of shape {"patches":[{"label":string,"rationale":string,"ops":[op,...]}]} where each op is one of: {"op":"add","after":<exact substring of an existing line, or omit to append>,"text":<new line(s)>}, {"op":"delete","anchor":<exact substring of the line to remove>}, {"op":"replace","anchor":<exact substring of the line to replace>,"text":<replacement line(s)>}. Anchors MUST be verbatim substrings of lines that exist in the document. No prose outside JSON.';
|
|
1841
|
-
function skillOptProposer(opts) {
|
|
1842
|
-
const evidenceK = opts.evidenceK ?? 3;
|
|
1843
|
-
const defaultBudget = opts.editBudget ?? 3;
|
|
1844
|
-
async function proposePatches(args) {
|
|
1845
|
-
const userPrompt = buildPatchPrompt({
|
|
1846
|
-
target: opts.target,
|
|
1847
|
-
surface: args.surface,
|
|
1848
|
-
evidence: args.evidence,
|
|
1849
|
-
editBudget: args.editBudget,
|
|
1850
|
-
rejectedBuffer: args.rejectedBuffer,
|
|
1851
|
-
metaNote: args.metaNote,
|
|
1852
|
-
findingsNote: args.findingsNote,
|
|
1853
|
-
count: args.count
|
|
1854
|
-
});
|
|
1855
|
-
const result = await callLlm(
|
|
1856
|
-
{
|
|
1857
|
-
model: opts.model,
|
|
1858
|
-
messages: [
|
|
1859
|
-
{ role: "system", content: SKILLOPT_SYSTEM },
|
|
1860
|
-
{ role: "user", content: userPrompt }
|
|
1861
|
-
],
|
|
1862
|
-
jsonMode: true,
|
|
1863
|
-
temperature: opts.temperature ?? 0.6,
|
|
1864
|
-
maxTokens: opts.maxTokens ?? 4e3
|
|
1865
|
-
},
|
|
1866
|
-
opts.llm
|
|
1867
|
-
);
|
|
1868
|
-
return parseSkillPatchResponse(result.content, args.count, args.editBudget);
|
|
1869
|
-
}
|
|
1870
|
-
return {
|
|
1871
|
-
kind: "skill-opt",
|
|
1872
|
-
proposePatches,
|
|
1873
|
-
async propose(ctx) {
|
|
1874
|
-
if (typeof ctx.currentSurface !== "string") {
|
|
1875
|
-
throw new Error(
|
|
1876
|
-
"skillOptProposer: surface must be a string skill document (got a CodeSurface). SkillOpt patches text."
|
|
1877
|
-
);
|
|
1878
|
-
}
|
|
1879
|
-
const surface = ctx.currentSurface;
|
|
1880
|
-
const patches = await proposePatches({
|
|
1881
|
-
surface,
|
|
1882
|
-
evidence: evidenceFromHistory(ctx, evidenceK),
|
|
1883
|
-
editBudget: defaultBudget,
|
|
1884
|
-
rejectedBuffer: [],
|
|
1885
|
-
findingsNote: renderAnalystEvidence(ctx.findings, ctx.report) ?? void 0,
|
|
1886
|
-
count: ctx.populationSize,
|
|
1887
|
-
signal: ctx.signal
|
|
1888
|
-
});
|
|
1889
|
-
const out = [];
|
|
1890
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1891
|
-
for (const patch of patches) {
|
|
1892
|
-
const { surface: candidate, applied } = applySkillPatch(surface, patch);
|
|
1893
|
-
if (applied === 0 || candidate === surface || seen.has(candidate)) continue;
|
|
1894
|
-
seen.add(candidate);
|
|
1895
|
-
out.push({ surface: candidate, label: patch.label, rationale: patch.rationale });
|
|
1896
|
-
if (out.length >= ctx.populationSize) break;
|
|
1897
|
-
}
|
|
1898
|
-
return out;
|
|
1899
|
-
}
|
|
1900
|
-
};
|
|
1901
|
-
}
|
|
1902
|
-
function evidenceFromHistory(ctx, k) {
|
|
1903
|
-
const last = ctx.history.at(-1);
|
|
1904
|
-
if (!last || last.candidates.length === 0) return { weakScenarios: [], weakDimensions: [] };
|
|
1905
|
-
const best = [...last.candidates].sort((a, b) => b.composite - a.composite)[0];
|
|
1906
|
-
if (!best) return { weakScenarios: [], weakDimensions: [] };
|
|
1907
|
-
const weakScenarios = [...best.scenarios].sort((a, b) => a.composite - b.composite).slice(0, k);
|
|
1908
|
-
const weakDimensions = Object.entries(best.dimensions).sort((a, b) => a[1] - b[1]).slice(0, k).map(([dimension, score]) => ({ dimension, score }));
|
|
1909
|
-
return { weakScenarios, weakDimensions };
|
|
1910
|
-
}
|
|
1911
|
-
function buildPatchPrompt(args) {
|
|
1912
|
-
const lines = [
|
|
1913
|
-
`Skill document governs: ${args.target}.`,
|
|
1914
|
-
"",
|
|
1915
|
-
"Current skill document:",
|
|
1916
|
-
"```",
|
|
1917
|
-
args.surface,
|
|
1918
|
-
"```",
|
|
1919
|
-
"",
|
|
1920
|
-
`Propose ${args.count} candidate patch(es). Each patch is a SMALL bundle of`,
|
|
1921
|
-
`at most ${args.editBudget} op(s). Anchors must be verbatim substrings of`,
|
|
1922
|
-
"existing lines. Prefer adding a specific missing rule or sharpening a vague",
|
|
1923
|
-
"one over deleting; never rewrite the whole document."
|
|
1924
|
-
];
|
|
1925
|
-
if (args.evidence.weakScenarios.length > 0) {
|
|
1926
|
-
lines.push(
|
|
1927
|
-
"",
|
|
1928
|
-
"Weakest scenarios (patch to fix these):",
|
|
1929
|
-
...args.evidence.weakScenarios.map((s) => `- ${s.scenarioId} (${s.composite.toFixed(2)})`)
|
|
1930
|
-
);
|
|
1931
|
-
}
|
|
1932
|
-
if (args.evidence.weakDimensions.length > 0) {
|
|
1933
|
-
lines.push(
|
|
1934
|
-
"",
|
|
1935
|
-
"Weakest dimensions (what to improve):",
|
|
1936
|
-
...args.evidence.weakDimensions.map((d) => `- ${d.dimension} (${d.score.toFixed(2)})`)
|
|
1937
|
-
);
|
|
1938
|
-
}
|
|
1939
|
-
if (args.rejectedBuffer.length > 0) {
|
|
1940
|
-
lines.push(
|
|
1941
|
-
"",
|
|
1942
|
-
"Already tried and REJECTED (do not repeat or restate these edits):",
|
|
1943
|
-
...args.rejectedBuffer.map((e) => `- ${e.label}: ${e.rationale} \u2014 ${e.reason}`)
|
|
1944
|
-
);
|
|
1945
|
-
}
|
|
1946
|
-
if (args.findingsNote) {
|
|
1947
|
-
lines.push("", args.findingsNote);
|
|
1948
|
-
}
|
|
1949
|
-
if (args.metaNote) {
|
|
1950
|
-
lines.push("", `Strategy note from prior epochs: ${args.metaNote}`);
|
|
1951
|
-
}
|
|
1952
|
-
return lines.join("\n");
|
|
1953
|
-
}
|
|
1954
|
-
var SkillPatchParseError = class extends Error {
|
|
1955
|
-
constructor(message) {
|
|
1956
|
-
super(message);
|
|
1957
|
-
this.name = "SkillPatchParseError";
|
|
1958
|
-
}
|
|
1959
|
-
};
|
|
1960
|
-
function parseSkillPatchResponse(raw, maxPatches, editBudget) {
|
|
1961
|
-
let text = raw.trim();
|
|
1962
|
-
if (text.startsWith("```")) text = text.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
|
|
1963
|
-
const start = text.indexOf("{");
|
|
1964
|
-
const end = text.lastIndexOf("}");
|
|
1965
|
-
if (start < 0 || end <= start) {
|
|
1966
|
-
throw new SkillPatchParseError(
|
|
1967
|
-
`parseSkillPatchResponse: response was not valid JSON (no object found): ${snippet(raw)}`
|
|
1968
|
-
);
|
|
1969
|
-
}
|
|
1970
|
-
let parsed;
|
|
1971
|
-
try {
|
|
1972
|
-
parsed = JSON.parse(text.slice(start, end + 1));
|
|
1973
|
-
} catch (err) {
|
|
1974
|
-
throw new SkillPatchParseError(
|
|
1975
|
-
`parseSkillPatchResponse: response was not valid JSON (${err instanceof Error ? err.message : String(err)}): ${snippet(raw)}`
|
|
1976
|
-
);
|
|
1977
|
-
}
|
|
1978
|
-
const rawPatches = Array.isArray(parsed.patches) ? parsed.patches : [];
|
|
1979
|
-
const out = [];
|
|
1980
|
-
for (const rp of rawPatches) {
|
|
1981
|
-
if (typeof rp !== "object" || rp === null) continue;
|
|
1982
|
-
const obj = rp;
|
|
1983
|
-
const ops = Array.isArray(obj.ops) ? obj.ops.map(normalizeOp).filter(isOp) : [];
|
|
1984
|
-
if (ops.length === 0) continue;
|
|
1985
|
-
out.push({
|
|
1986
|
-
label: typeof obj.label === "string" ? obj.label : "patch",
|
|
1987
|
-
rationale: typeof obj.rationale === "string" ? obj.rationale : "",
|
|
1988
|
-
ops: ops.slice(0, editBudget)
|
|
1989
|
-
});
|
|
1990
|
-
if (out.length >= maxPatches) break;
|
|
1991
|
-
}
|
|
1992
|
-
return out;
|
|
1993
|
-
}
|
|
1994
|
-
function normalizeOp(raw) {
|
|
1995
|
-
if (typeof raw !== "object" || raw === null) return null;
|
|
1996
|
-
const o = raw;
|
|
1997
|
-
if (o.op === "add") {
|
|
1998
|
-
if (typeof o.text !== "string") return null;
|
|
1999
|
-
const op = { op: "add", text: o.text };
|
|
2000
|
-
if (typeof o.after === "string") op.after = o.after;
|
|
2001
|
-
return op;
|
|
2002
|
-
}
|
|
2003
|
-
if (o.op === "delete") {
|
|
2004
|
-
if (typeof o.anchor !== "string") return null;
|
|
2005
|
-
return { op: "delete", anchor: o.anchor };
|
|
2006
|
-
}
|
|
2007
|
-
if (o.op === "replace") {
|
|
2008
|
-
if (typeof o.anchor !== "string" || typeof o.text !== "string") return null;
|
|
2009
|
-
return { op: "replace", anchor: o.anchor, text: o.text };
|
|
2010
|
-
}
|
|
2011
|
-
return null;
|
|
2012
|
-
}
|
|
2013
|
-
function isOp(op) {
|
|
2014
|
-
return op !== null;
|
|
2015
|
-
}
|
|
2016
|
-
function snippet(s, max = 120) {
|
|
2017
|
-
const t = s.trim().replace(/\s+/g, " ");
|
|
2018
|
-
return t.length <= max ? t : `${t.slice(0, max)}\u2026`;
|
|
2019
|
-
}
|
|
2020
|
-
|
|
2021
|
-
// src/campaign/presets/run-skill-opt.ts
|
|
2022
|
-
async function runSkillOpt(opts) {
|
|
2023
|
-
if (opts.trainScenarios.length === 0) throw new Error("runSkillOpt: trainScenarios is empty");
|
|
2024
|
-
if (opts.holdoutScenarios.length === 0) throw new Error("runSkillOpt: holdoutScenarios is empty");
|
|
2025
|
-
if (!opts.judges || opts.judges.length === 0) {
|
|
2026
|
-
throw new Error(
|
|
2027
|
-
"runSkillOpt: at least one judge is required \u2014 scoring (and therefore acceptance) is meaningless without one, and would report a silent zero lift."
|
|
2028
|
-
);
|
|
2029
|
-
}
|
|
2030
|
-
const holdoutIds = new Set(opts.holdoutScenarios.map((s) => s.id));
|
|
2031
|
-
const overlap = opts.trainScenarios.filter((s) => holdoutIds.has(s.id)).map((s) => s.id);
|
|
2032
|
-
if (overlap.length > 0) {
|
|
2033
|
-
throw new Error(
|
|
2034
|
-
`runSkillOpt: trainScenarios and holdoutScenarios must be disjoint (overlap: [${overlap.join(
|
|
2035
|
-
", "
|
|
2036
|
-
)}]) \u2014 a shared scenario leaks the held-out acceptance axis into the proposal evidence.`
|
|
2037
|
-
);
|
|
2038
|
-
}
|
|
2039
|
-
const patchesPerEpoch = opts.patchesPerEpoch ?? 2;
|
|
2040
|
-
const initialBudget = opts.editBudget ?? 3;
|
|
2041
|
-
const minImprovement = opts.minImprovement ?? 0;
|
|
2042
|
-
if (minImprovement < 0) {
|
|
2043
|
-
throw new Error(
|
|
2044
|
-
"runSkillOpt: minImprovement must be >= 0 \u2014 a negative threshold would accept held-out regressions, breaking the monotonic-lift contract."
|
|
2045
|
-
);
|
|
2046
|
-
}
|
|
2047
|
-
const patience = opts.patience ?? opts.maxEpochs;
|
|
2048
|
-
const budgetAnneal = opts.budgetAnneal ?? true;
|
|
2049
|
-
const rejectedBufferSize = opts.rejectedBufferSize ?? 12;
|
|
2050
|
-
const slowMetaEvery = opts.slowMetaEvery ?? 2;
|
|
2051
|
-
let totalCostUsd = 0;
|
|
2052
|
-
const scoreHoldout = async (surface, tag) => {
|
|
2053
|
-
const campaign = await runScoringCampaign(opts, opts.holdoutScenarios, surface, tag);
|
|
2054
|
-
totalCostUsd += campaign.aggregates.totalCostUsd;
|
|
2055
|
-
return campaignMeanComposite(campaign);
|
|
2056
|
-
};
|
|
2057
|
-
const evidenceK = opts.evidenceK ?? 3;
|
|
2058
|
-
const trainEvidence = async (surface, tag) => {
|
|
2059
|
-
const campaign = await runScoringCampaign(opts, opts.trainScenarios, surface, tag);
|
|
2060
|
-
totalCostUsd += campaign.aggregates.totalCostUsd;
|
|
2061
|
-
return toEvidence(campaign, evidenceK);
|
|
2062
|
-
};
|
|
2063
|
-
let current = opts.baselineSurface;
|
|
2064
|
-
let currentEvidence = await trainEvidence(current, "baseline-train");
|
|
2065
|
-
const baselineHoldout = await scoreHoldout(current, "baseline-holdout");
|
|
2066
|
-
let currentHoldout = baselineHoldout;
|
|
2067
|
-
const buffer = [];
|
|
2068
|
-
const acceptedEdits = [];
|
|
2069
|
-
const rejectedAll = [];
|
|
2070
|
-
const history = [];
|
|
2071
|
-
let budget = initialBudget;
|
|
2072
|
-
let sinceAccept = 0;
|
|
2073
|
-
let metaNote;
|
|
2074
|
-
let epochsRun = 0;
|
|
2075
|
-
for (let epoch = 0; epoch < opts.maxEpochs; epoch++) {
|
|
2076
|
-
epochsRun++;
|
|
2077
|
-
const patches = await opts.proposer.proposePatches({
|
|
2078
|
-
surface: current,
|
|
2079
|
-
evidence: currentEvidence,
|
|
2080
|
-
editBudget: budget,
|
|
2081
|
-
rejectedBuffer: buffer,
|
|
2082
|
-
metaNote,
|
|
2083
|
-
count: patchesPerEpoch,
|
|
2084
|
-
signal: opts.signal ?? new AbortController().signal
|
|
2085
|
-
});
|
|
2086
|
-
let accepted = null;
|
|
2087
|
-
const rejectedThisEpoch = [];
|
|
2088
|
-
for (let i = 0; i < patches.length; i++) {
|
|
2089
|
-
const patch = patches[i];
|
|
2090
|
-
const { surface: candidate, applied } = applySkillPatch(current, patch);
|
|
2091
|
-
if (applied === 0 || candidate === current) {
|
|
2092
|
-
rejectedThisEpoch.push({
|
|
2093
|
-
label: patch.label,
|
|
2094
|
-
rationale: patch.rationale,
|
|
2095
|
-
reason: "no-op (unanchored or zero-change)"
|
|
2096
|
-
});
|
|
2097
|
-
continue;
|
|
2098
|
-
}
|
|
2099
|
-
const candidateHoldout = await scoreHoldout(candidate, `epoch-${epoch}-cand-${i}-holdout`);
|
|
2100
|
-
if (candidateHoldout > currentHoldout + minImprovement) {
|
|
2101
|
-
accepted = {
|
|
2102
|
-
epoch,
|
|
2103
|
-
label: patch.label,
|
|
2104
|
-
rationale: patch.rationale,
|
|
2105
|
-
holdoutDelta: candidateHoldout - currentHoldout
|
|
2106
|
-
};
|
|
2107
|
-
current = candidate;
|
|
2108
|
-
currentHoldout = candidateHoldout;
|
|
2109
|
-
currentEvidence = await trainEvidence(current, `epoch-${epoch}-train`);
|
|
2110
|
-
break;
|
|
2111
|
-
}
|
|
2112
|
-
rejectedThisEpoch.push({
|
|
2113
|
-
label: patch.label,
|
|
2114
|
-
rationale: patch.rationale,
|
|
2115
|
-
reason: `held-out ${candidateHoldout.toFixed(3)} \u2264 current ${currentHoldout.toFixed(3)}`
|
|
2116
|
-
});
|
|
2117
|
-
}
|
|
2118
|
-
if (accepted) {
|
|
2119
|
-
acceptedEdits.push(accepted);
|
|
2120
|
-
sinceAccept = 0;
|
|
2121
|
-
} else {
|
|
2122
|
-
sinceAccept++;
|
|
2123
|
-
if (budgetAnneal && sinceAccept >= 2 && budget > 1) budget--;
|
|
2124
|
-
}
|
|
2125
|
-
for (const r of rejectedThisEpoch) {
|
|
2126
|
-
buffer.push(r);
|
|
2127
|
-
rejectedAll.push(r);
|
|
2128
|
-
}
|
|
2129
|
-
while (buffer.length > rejectedBufferSize) buffer.shift();
|
|
2130
|
-
if (slowMetaEvery > 0 && (epoch + 1) % slowMetaEvery === 0) {
|
|
2131
|
-
metaNote = buildMetaNote(acceptedEdits, buffer);
|
|
2132
|
-
}
|
|
2133
|
-
history.push({
|
|
2134
|
-
epoch,
|
|
2135
|
-
editBudget: budget,
|
|
2136
|
-
proposed: patches.length,
|
|
2137
|
-
accepted,
|
|
2138
|
-
rejected: rejectedThisEpoch,
|
|
2139
|
-
holdoutComposite: currentHoldout
|
|
2140
|
-
});
|
|
2141
|
-
if (sinceAccept >= patience) break;
|
|
2142
|
-
}
|
|
2143
|
-
return {
|
|
2144
|
-
winnerSurface: current,
|
|
2145
|
-
baselineHoldoutComposite: baselineHoldout,
|
|
2146
|
-
winnerHoldoutComposite: currentHoldout,
|
|
2147
|
-
lift: currentHoldout - baselineHoldout,
|
|
2148
|
-
acceptedEdits,
|
|
2149
|
-
rejectedEdits: rejectedAll,
|
|
2150
|
-
epochsRun,
|
|
2151
|
-
history,
|
|
2152
|
-
totalCostUsd
|
|
2153
|
-
};
|
|
2154
|
-
}
|
|
2155
|
-
function runScoringCampaign(opts, scenarios, surface, tag) {
|
|
2156
|
-
return runCampaign({
|
|
2157
|
-
...opts,
|
|
2158
|
-
scenarios,
|
|
2159
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
2160
|
-
runDir: `${opts.runDir}/${tag}`
|
|
2161
|
-
});
|
|
2162
|
-
}
|
|
2163
|
-
function toEvidence(campaign, k) {
|
|
2164
|
-
const { dimensions, scenarios } = campaignBreakdown(campaign);
|
|
2165
|
-
const weakScenarios = [...scenarios].sort((a, b) => a.composite - b.composite).slice(0, k);
|
|
2166
|
-
const weakDimensions = Object.entries(dimensions).sort((a, b) => a[1] - b[1]).slice(0, k).map(([dimension, score]) => ({ dimension, score }));
|
|
2167
|
-
return { weakScenarios, weakDimensions };
|
|
2168
|
-
}
|
|
2169
|
-
function buildMetaNote(accepted, rejected) {
|
|
2170
|
-
const parts = [];
|
|
2171
|
-
if (accepted.length > 0) {
|
|
2172
|
-
parts.push(
|
|
2173
|
-
`Edits that improved held-out so far: ${accepted.map((a) => `"${a.label}" (+${a.holdoutDelta.toFixed(3)})`).join("; ")}. Build on these.`
|
|
2174
|
-
);
|
|
2175
|
-
}
|
|
2176
|
-
if (rejected.length > 0) {
|
|
2177
|
-
const labels = [...new Set(rejected.map((r) => r.label))].slice(0, 5);
|
|
2178
|
-
parts.push(`Dead ends to avoid: ${labels.join(", ")}. Try a different anchor or rule.`);
|
|
2179
|
-
}
|
|
2180
|
-
parts.push("Keep edits small and anchored to existing lines.");
|
|
2181
|
-
return parts.join(" ");
|
|
2182
|
-
}
|
|
2183
|
-
|
|
2184
|
-
// src/campaign/presets/compare-proposers.ts
|
|
2185
|
-
async function compareProposers(opts) {
|
|
2186
|
-
return compareProposerEntries(opts);
|
|
2187
|
-
}
|
|
2188
|
-
async function compareProposerEntries(opts) {
|
|
2189
|
-
if (opts.proposers.length === 0) throw new Error("compareProposers: no proposers to compare");
|
|
2190
|
-
const seed = opts.seed ?? 42;
|
|
2191
|
-
const resamples = opts.resamples ?? 2e3;
|
|
2192
|
-
const confidence = opts.confidence ?? 0.95;
|
|
2193
|
-
const scoreOnHoldout = async (surface, tag) => {
|
|
2194
|
-
const campaign = await runCampaign({
|
|
2195
|
-
...opts,
|
|
2196
|
-
scenarios: opts.holdoutScenarios,
|
|
2197
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
2198
|
-
runDir: `${opts.runDir}/${tag}`
|
|
2199
|
-
});
|
|
2200
|
-
const byScenario = {};
|
|
2201
|
-
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
|
|
2202
|
-
byScenario[scenarioId] = composite;
|
|
2203
|
-
}
|
|
2204
|
-
return byScenario;
|
|
2205
|
-
};
|
|
2206
|
-
const scenarioIds = [...new Set(opts.holdoutScenarios.map((s) => s.id))].sort();
|
|
2207
|
-
if (scenarioIds.length === 0) throw new Error("compareProposers: holdoutScenarios is empty");
|
|
2208
|
-
const align = (byScenario, label) => {
|
|
2209
|
-
const missing = scenarioIds.filter((id) => !(id in byScenario));
|
|
2210
|
-
if (missing.length > 0) {
|
|
2211
|
-
throw new Error(
|
|
2212
|
-
`compareProposers: ${label} produced no held-out score for scenario(s) [${missing.join(
|
|
2213
|
-
", "
|
|
2214
|
-
)}] \u2014 a cell errored or its judges returned nothing. Refusing to fabricate a 0 (it would corrupt the lift comparison). Fix the dispatch/judge or drop the scenario.`
|
|
2215
|
-
);
|
|
2216
|
-
}
|
|
2217
|
-
return scenarioIds.map((id) => byScenario[id]);
|
|
2218
|
-
};
|
|
2219
|
-
const baselineArr = align(
|
|
2220
|
-
await scoreOnHoldout(opts.baselineSurface, "compare-baseline"),
|
|
2221
|
-
"baseline"
|
|
2222
|
-
);
|
|
2223
|
-
const winners = [];
|
|
2224
|
-
for (const proposer of opts.proposers) {
|
|
2225
|
-
const out = await proposer.optimize();
|
|
2226
|
-
const byScenario = await scoreOnHoldout(out.winnerSurface, `compare-${slug(proposer.name)}`);
|
|
2227
|
-
winners.push({
|
|
2228
|
-
name: proposer.name,
|
|
2229
|
-
winnerSurface: out.winnerSurface,
|
|
2230
|
-
costUsd: out.costUsd,
|
|
2231
|
-
durationMs: out.durationMs,
|
|
2232
|
-
arr: align(byScenario, `proposer "${proposer.name}"`)
|
|
2233
|
-
});
|
|
2234
|
-
}
|
|
2235
|
-
const scores = winners.map((w) => {
|
|
2236
|
-
const boot = pairedBootstrap(baselineArr, w.arr, {
|
|
2237
|
-
seed,
|
|
2238
|
-
resamples,
|
|
2239
|
-
confidence,
|
|
2240
|
-
statistic: "mean"
|
|
2241
|
-
});
|
|
2242
|
-
const score = {
|
|
2243
|
-
name: w.name,
|
|
2244
|
-
baselineComposite: mean2(baselineArr),
|
|
2245
|
-
winnerComposite: mean2(w.arr),
|
|
2246
|
-
lift: boot.mean,
|
|
2247
|
-
liftCi: { low: boot.low, high: boot.high },
|
|
2248
|
-
costUsd: w.costUsd,
|
|
2249
|
-
winnerSurface: w.winnerSurface,
|
|
2250
|
-
rank: 0
|
|
2251
|
-
};
|
|
2252
|
-
if (w.durationMs !== void 0) score.durationMs = w.durationMs;
|
|
2253
|
-
return score;
|
|
2254
|
-
});
|
|
2255
|
-
scores.sort((a, b) => b.lift - a.lift || a.costUsd - b.costUsd);
|
|
2256
|
-
scores.forEach((s, i) => {
|
|
2257
|
-
s.rank = i + 1;
|
|
2258
|
-
});
|
|
2259
|
-
const best = scores[0];
|
|
2260
|
-
const byName = new Map(winners.map((w) => [w.name, w]));
|
|
2261
|
-
const bestArr = byName.get(best.name).arr;
|
|
2262
|
-
const pairwise = scores.slice(1).map((other) => {
|
|
2263
|
-
const otherArr = byName.get(other.name).arr;
|
|
2264
|
-
const boot = pairedBootstrap(otherArr, bestArr, {
|
|
2265
|
-
seed,
|
|
2266
|
-
resamples,
|
|
2267
|
-
confidence,
|
|
2268
|
-
statistic: "mean"
|
|
2269
|
-
});
|
|
2270
|
-
const favored = boot.low > 0 ? best.name : boot.high < 0 ? other.name : "tie";
|
|
2271
|
-
return {
|
|
2272
|
-
a: best.name,
|
|
2273
|
-
b: other.name,
|
|
2274
|
-
deltaMean: boot.mean,
|
|
2275
|
-
low: boot.low,
|
|
2276
|
-
high: boot.high,
|
|
2277
|
-
favored
|
|
2278
|
-
};
|
|
2279
|
-
});
|
|
2280
|
-
return { scores, best, pairwise, holdoutScenarioIds: scenarioIds };
|
|
2281
|
-
}
|
|
2282
|
-
function mean2(xs) {
|
|
2283
|
-
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
2284
|
-
}
|
|
2285
|
-
function slug(name) {
|
|
2286
|
-
return name.replace(/[^a-z0-9]+/gi, "-").toLowerCase();
|
|
2287
|
-
}
|
|
2288
|
-
function gepaReflectionEntry(config, name = "gepa-reflection") {
|
|
2289
|
-
return gepaEntry(config, false, name);
|
|
2290
|
-
}
|
|
2291
|
-
function gepaParetoEntry(config, name = "gepa-pareto") {
|
|
2292
|
-
return gepaEntry(config, true, name);
|
|
2293
|
-
}
|
|
2294
|
-
function gepaEntry(config, combineParents, name) {
|
|
2295
|
-
return {
|
|
2296
|
-
name,
|
|
2297
|
-
async optimize() {
|
|
2298
|
-
const started = Date.now();
|
|
2299
|
-
const proposer = gepaProposer({
|
|
2300
|
-
llm: config.llm,
|
|
2301
|
-
model: config.model,
|
|
2302
|
-
target: config.target,
|
|
2303
|
-
combineParents,
|
|
2304
|
-
...config.mutationPrimitives ? { mutationPrimitives: config.mutationPrimitives } : {}
|
|
2305
|
-
});
|
|
2306
|
-
const result = await runImprovementLoop({
|
|
2307
|
-
scenarios: config.trainScenarios,
|
|
2308
|
-
holdoutScenarios: config.holdoutScenarios,
|
|
2309
|
-
baselineSurface: config.baselineSurface,
|
|
2310
|
-
dispatchWithSurface: config.dispatchWithSurface,
|
|
2311
|
-
judges: config.judges,
|
|
2312
|
-
proposer,
|
|
2313
|
-
populationSize: config.populationSize ?? 2,
|
|
2314
|
-
maxGenerations: config.maxGenerations ?? 3,
|
|
2315
|
-
gate: defaultProductionGate({
|
|
2316
|
-
holdoutScenarios: config.holdoutScenarios,
|
|
2317
|
-
deltaThreshold: 0
|
|
2318
|
-
}),
|
|
2319
|
-
autoOnPromote: "none",
|
|
2320
|
-
runDir: `${config.runDir}/${slug(name)}-loop`,
|
|
2321
|
-
...config.seed !== void 0 ? { seed: config.seed } : {},
|
|
2322
|
-
// Flow findings to the proposer's propose(). These reach runOptimization
|
|
2323
|
-
// unchanged (runImprovementLoop extends RunOptimizationOptions and
|
|
2324
|
-
// forwards {...opts}); ctx.findings/report/analyzeGeneration are consumed there.
|
|
2325
|
-
...config.findings !== void 0 ? { findings: config.findings } : {},
|
|
2326
|
-
...config.analyzeGeneration ? { analyzeGeneration: config.analyzeGeneration } : {},
|
|
2327
|
-
...config.report !== void 0 ? { report: config.report } : {}
|
|
2328
|
-
});
|
|
2329
|
-
const costUsd = result.baselineCampaign.aggregates.totalCostUsd + result.generations.reduce(
|
|
2330
|
-
(sum, g) => sum + g.surfaces.reduce((s, sf) => s + sf.campaign.aggregates.totalCostUsd, 0),
|
|
2331
|
-
0
|
|
2332
|
-
);
|
|
2333
|
-
return { winnerSurface: result.winnerSurface, costUsd, durationMs: Date.now() - started };
|
|
2334
|
-
}
|
|
2335
|
-
};
|
|
2336
|
-
}
|
|
2337
|
-
function skillOptEntry(config, name = "skill-opt") {
|
|
2338
|
-
return {
|
|
2339
|
-
name,
|
|
2340
|
-
async optimize() {
|
|
2341
|
-
const started = Date.now();
|
|
2342
|
-
const proposer = skillOptProposer({
|
|
2343
|
-
llm: config.llm,
|
|
2344
|
-
model: config.model,
|
|
2345
|
-
target: config.target
|
|
2346
|
-
});
|
|
2347
|
-
const result = await runSkillOpt({
|
|
2348
|
-
baselineSurface: config.baselineSurface,
|
|
2349
|
-
dispatchWithSurface: config.dispatchWithSurface,
|
|
2350
|
-
judges: config.judges,
|
|
2351
|
-
proposer,
|
|
2352
|
-
trainScenarios: config.trainScenarios,
|
|
2353
|
-
holdoutScenarios: config.holdoutScenarios,
|
|
2354
|
-
maxEpochs: config.maxEpochs ?? 6,
|
|
2355
|
-
runDir: `${config.runDir}/${slug(name)}-loop`,
|
|
2356
|
-
...config.seed !== void 0 ? { seed: config.seed } : {}
|
|
2357
|
-
});
|
|
2358
|
-
return {
|
|
2359
|
-
winnerSurface: result.winnerSurface,
|
|
2360
|
-
costUsd: result.totalCostUsd,
|
|
2361
|
-
durationMs: Date.now() - started
|
|
2362
|
-
};
|
|
2363
|
-
}
|
|
2364
|
-
};
|
|
2365
|
-
}
|
|
2366
|
-
function fapoEscalationEntry(config, name = "fapo-escalation") {
|
|
2367
|
-
return {
|
|
2368
|
-
name,
|
|
2369
|
-
async optimize() {
|
|
2370
|
-
const started = Date.now();
|
|
2371
|
-
const promptProposer = config.promptProposer ?? gepaProposer({
|
|
2372
|
-
llm: config.llm,
|
|
2373
|
-
model: config.model,
|
|
2374
|
-
target: config.target,
|
|
2375
|
-
combineParents: true,
|
|
2376
|
-
...config.mutationPrimitives ? { mutationPrimitives: config.mutationPrimitives } : {}
|
|
2377
|
-
});
|
|
2378
|
-
const parameterProposer = config.parameterProposer ?? (config.parameterCandidates ? parameterSweepProposer({ candidates: config.parameterCandidates }) : void 0);
|
|
2379
|
-
const structuralProposer = config.structuralProposer;
|
|
2380
|
-
const proposer = fapoProposer({
|
|
2381
|
-
...config.fapo ?? {},
|
|
2382
|
-
promptProposer,
|
|
2383
|
-
...parameterProposer ? { parameterProposer } : {},
|
|
2384
|
-
...structuralProposer ? { structuralProposer } : {}
|
|
2385
|
-
});
|
|
2386
|
-
const result = await runImprovementLoop({
|
|
2387
|
-
scenarios: config.trainScenarios,
|
|
2388
|
-
holdoutScenarios: config.holdoutScenarios,
|
|
2389
|
-
baselineSurface: config.baselineSurface,
|
|
2390
|
-
dispatchWithSurface: config.dispatchWithSurface,
|
|
2391
|
-
judges: config.judges,
|
|
2392
|
-
proposer,
|
|
2393
|
-
populationSize: config.populationSize ?? 2,
|
|
2394
|
-
maxGenerations: config.maxGenerations ?? 3,
|
|
2395
|
-
gate: defaultProductionGate({
|
|
2396
|
-
holdoutScenarios: config.holdoutScenarios,
|
|
2397
|
-
deltaThreshold: 0
|
|
2398
|
-
}),
|
|
2399
|
-
autoOnPromote: "none",
|
|
2400
|
-
runDir: `${config.runDir}/${slug(name)}-loop`,
|
|
2401
|
-
...config.seed !== void 0 ? { seed: config.seed } : {},
|
|
2402
|
-
...config.findings !== void 0 ? { findings: config.findings } : {},
|
|
2403
|
-
...config.analyzeGeneration ? { analyzeGeneration: config.analyzeGeneration } : {},
|
|
2404
|
-
...config.report !== void 0 ? { report: config.report } : {}
|
|
2405
|
-
});
|
|
2406
|
-
const costUsd = result.baselineCampaign.aggregates.totalCostUsd + result.generations.reduce(
|
|
2407
|
-
(sum, g) => sum + g.surfaces.reduce((s, sf) => s + sf.campaign.aggregates.totalCostUsd, 0),
|
|
2408
|
-
0
|
|
2409
|
-
);
|
|
2410
|
-
return { winnerSurface: result.winnerSurface, costUsd, durationMs: Date.now() - started };
|
|
2411
|
-
}
|
|
2412
|
-
};
|
|
2413
|
-
}
|
|
2414
|
-
|
|
2415
|
-
// src/campaign/presets/playback.ts
|
|
2416
|
-
function makePlaybackDispatch(driver) {
|
|
2417
|
-
return async (profile, scenario, ctx) => {
|
|
2418
|
-
const events = await driver.run(scenario, { ...ctx, profile });
|
|
2419
|
-
return extractProducedState(events);
|
|
2420
|
-
};
|
|
2421
|
-
}
|
|
2422
|
-
async function scoreUserStory(story, state, checkCorrectness) {
|
|
2423
|
-
const verdict = await verifyCompletion(
|
|
2424
|
-
{ taskId: story.id, requirements: story.requirements },
|
|
2425
|
-
state,
|
|
2426
|
-
checkCorrectness
|
|
2427
|
-
);
|
|
2428
|
-
return { ...verdict, title: story.title };
|
|
2429
|
-
}
|
|
2430
|
-
function userStoryScoreboard(verdicts) {
|
|
2431
|
-
const rows = [];
|
|
2432
|
-
for (const v of verdicts) {
|
|
2433
|
-
for (const r of v.requirements) {
|
|
2434
|
-
rows.push({
|
|
2435
|
-
storyId: v.taskId,
|
|
2436
|
-
storyTitle: v.title,
|
|
2437
|
-
reqId: r.reqId,
|
|
2438
|
-
reqTitle: r.title,
|
|
2439
|
-
status: r.satisfied ? "PASS" : "FAIL",
|
|
2440
|
-
evidence: r.evidence
|
|
2441
|
-
});
|
|
2442
|
-
}
|
|
2443
|
-
}
|
|
2444
|
-
return rows;
|
|
2445
|
-
}
|
|
2446
|
-
function scoreboardSummary(rows) {
|
|
2447
|
-
const byStory = /* @__PURE__ */ new Map();
|
|
2448
|
-
let passed = 0;
|
|
2449
|
-
for (const r of rows) {
|
|
2450
|
-
const s = byStory.get(r.storyId) ?? { total: 0, passed: 0 };
|
|
2451
|
-
s.total++;
|
|
2452
|
-
if (r.status === "PASS") {
|
|
2453
|
-
s.passed++;
|
|
2454
|
-
passed++;
|
|
2455
|
-
}
|
|
2456
|
-
byStory.set(r.storyId, s);
|
|
2457
|
-
}
|
|
2458
|
-
let storiesFullyComplete = 0;
|
|
2459
|
-
for (const s of byStory.values()) if (s.total > 0 && s.passed === s.total) storiesFullyComplete++;
|
|
2460
|
-
return {
|
|
2461
|
-
stories: byStory.size,
|
|
2462
|
-
storiesFullyComplete,
|
|
2463
|
-
requirements: rows.length,
|
|
2464
|
-
passed,
|
|
2465
|
-
failed: rows.length - passed,
|
|
2466
|
-
passRate: rows.length === 0 ? 0 : passed / rows.length
|
|
2467
|
-
};
|
|
2468
|
-
}
|
|
2469
|
-
function escapeCell(s) {
|
|
2470
|
-
return s.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
|
|
2471
|
-
}
|
|
2472
|
-
function truncate2(s, max) {
|
|
2473
|
-
return s.length <= max ? s : `${s.slice(0, Math.max(0, max - 1))}\u2026`;
|
|
2474
|
-
}
|
|
2475
|
-
function renderScoreboardMarkdown(rows, opts = {}) {
|
|
2476
|
-
const maxEv = opts.maxEvidenceChars ?? 160;
|
|
2477
|
-
const sum = scoreboardSummary(rows);
|
|
2478
|
-
const pct = (n) => `${Math.round(n * 100)}%`;
|
|
2479
|
-
const ev = (e) => escapeCell(truncate2(e.join("; "), maxEv)) || "\u2014";
|
|
2480
|
-
const out = [`# ${opts.title ?? "Product-flow playback scoreboard"}`, ""];
|
|
2481
|
-
if (opts.meta) {
|
|
2482
|
-
for (const [k, v] of Object.entries(opts.meta)) out.push(`- **${k}:** ${v}`);
|
|
2483
|
-
out.push("");
|
|
2484
|
-
}
|
|
2485
|
-
out.push(
|
|
2486
|
-
`**${sum.storiesFullyComplete}/${sum.stories}** user stories fully shipped \xB7 **${sum.passed}/${sum.requirements}** requirements passing (${pct(sum.passRate)}) \xB7 **${sum.failed}** open`,
|
|
2487
|
-
""
|
|
2488
|
-
);
|
|
2489
|
-
const fails = rows.filter((r) => r.status === "FAIL");
|
|
2490
|
-
if (fails.length > 0) {
|
|
2491
|
-
out.push("## Open tickets", "", "| Story | Requirement | Evidence |", "| --- | --- | --- |");
|
|
2492
|
-
for (const r of fails) {
|
|
2493
|
-
out.push(`| ${escapeCell(r.storyTitle)} | ${escapeCell(r.reqTitle)} | ${ev(r.evidence)} |`);
|
|
2494
|
-
}
|
|
2495
|
-
out.push("");
|
|
2496
|
-
} else {
|
|
2497
|
-
out.push("_All requirements passing \u2014 no open tickets._", "");
|
|
2498
|
-
}
|
|
2499
|
-
out.push("## Per-story tick-off", "");
|
|
2500
|
-
for (const storyId of [...new Set(rows.map((r) => r.storyId))]) {
|
|
2501
|
-
const storyRows = rows.filter((r) => r.storyId === storyId);
|
|
2502
|
-
const passed = storyRows.filter((r) => r.status === "PASS").length;
|
|
2503
|
-
const mark = passed === storyRows.length ? "\u2705" : "\u26A0\uFE0F";
|
|
2504
|
-
out.push(
|
|
2505
|
-
`### ${escapeCell(storyRows[0].storyTitle)} \u2014 ${passed}/${storyRows.length} ${mark}`,
|
|
2506
|
-
"",
|
|
2507
|
-
"| Requirement | Status | Evidence |",
|
|
2508
|
-
"| --- | --- | --- |"
|
|
2509
|
-
);
|
|
2510
|
-
for (const r of storyRows) {
|
|
2511
|
-
out.push(
|
|
2512
|
-
`| ${escapeCell(r.reqTitle)} | ${r.status === "PASS" ? "\u2705 PASS" : "\u274C FAIL"} | ${ev(r.evidence)} |`
|
|
2513
|
-
);
|
|
2514
|
-
}
|
|
2515
|
-
out.push("");
|
|
2516
|
-
}
|
|
2517
|
-
return out.join("\n");
|
|
2518
|
-
}
|
|
2519
|
-
|
|
2520
|
-
// src/campaign/presets/run-lineage-loop.ts
|
|
2521
|
-
function toCandidate(p) {
|
|
2522
|
-
return isProposedCandidate(p) ? { surface: p.surface, ...p.rationale ? { rationale: p.rationale } : {} } : { surface: p };
|
|
2523
|
-
}
|
|
2524
|
-
async function runLineageLoop(opts) {
|
|
2525
|
-
const governor = opts.governor ?? heuristicGovernor();
|
|
2526
|
-
const populationSize = opts.populationSize ?? 4;
|
|
2527
|
-
if (populationSize < 1) {
|
|
2528
|
-
throw new Error("runLineageLoop: populationSize must be >= 1");
|
|
2529
|
-
}
|
|
2530
|
-
let proposer = opts.proposer;
|
|
2531
|
-
if (!proposer) {
|
|
2532
|
-
if (!opts.llm || !opts.model) {
|
|
2533
|
-
throw new Error(
|
|
2534
|
-
"runLineageLoop: a proposer is required \u2014 either inject `proposer`, or provide `llm` + `model` for the default gepaProposer."
|
|
2535
|
-
);
|
|
2536
|
-
}
|
|
2537
|
-
proposer = gepaProposer({
|
|
2538
|
-
llm: opts.llm,
|
|
2539
|
-
model: opts.model,
|
|
2540
|
-
target: opts.target ?? "agent surface",
|
|
2541
|
-
// GEPA combine-complementary-lessons is what the `merge` seam relies on.
|
|
2542
|
-
combineParents: true
|
|
2543
|
-
});
|
|
2544
|
-
}
|
|
2545
|
-
const scoringScenarios = opts.holdoutScenarios ?? opts.scenarios;
|
|
2546
|
-
let scoreSurface = opts.scoreSurface;
|
|
2547
|
-
if (!scoreSurface) {
|
|
2548
|
-
const dispatchWithSurface = opts.dispatchWithSurface;
|
|
2549
|
-
const runDir = opts.runDir;
|
|
2550
|
-
if (!dispatchWithSurface || !runDir) {
|
|
2551
|
-
throw new Error(
|
|
2552
|
-
"runLineageLoop: scoring is required \u2014 either inject `scoreSurface`, or provide `dispatchWithSurface` (the agent) + `runDir` for the default runCampaign scorer."
|
|
2553
|
-
);
|
|
2554
|
-
}
|
|
2555
|
-
if (scoringScenarios.length === 0) {
|
|
2556
|
-
throw new Error(
|
|
2557
|
-
"runLineageLoop: the default scorer needs at least one scenario (holdoutScenarios ?? scenarios is empty)."
|
|
2558
|
-
);
|
|
2559
|
-
}
|
|
2560
|
-
scoreSurface = async (surface) => {
|
|
2561
|
-
const campaign = await runCampaign({
|
|
2562
|
-
scenarios: scoringScenarios,
|
|
2563
|
-
dispatch: (scenario, ctx) => dispatchWithSurface(surface, scenario, ctx),
|
|
2564
|
-
dispatchRef: `lineage-loop:${surfaceHash(surface)}`,
|
|
2565
|
-
runDir: `${runDir}/score/${surfaceHash(surface)}`,
|
|
2566
|
-
...opts.judges ? { judges: opts.judges } : {},
|
|
2567
|
-
...opts.seed !== void 0 ? { seed: opts.seed } : {},
|
|
2568
|
-
...opts.reps !== void 0 ? { reps: opts.reps } : {},
|
|
2569
|
-
...opts.storage ? { storage: opts.storage } : {},
|
|
2570
|
-
...opts.tracing ? { tracing: opts.tracing } : {},
|
|
2571
|
-
...opts.expectUsage ? { expectUsage: opts.expectUsage } : {},
|
|
2572
|
-
...opts.maxConcurrency !== void 0 ? { maxConcurrency: opts.maxConcurrency } : {},
|
|
2573
|
-
...opts.dispatchTimeoutMs !== void 0 ? { dispatchTimeoutMs: opts.dispatchTimeoutMs } : {},
|
|
2574
|
-
...opts.now ? { now: opts.now } : {}
|
|
2575
|
-
});
|
|
2576
|
-
const score = campaignMeanComposite(campaign);
|
|
2577
|
-
const byId = new Map(
|
|
2578
|
-
campaignBreakdown(campaign).scenarios.map((s) => [s.scenarioId, s.composite])
|
|
2579
|
-
);
|
|
2580
|
-
const scoreVector = scoringScenarios.map((s) => byId.get(s.id) ?? 0);
|
|
2581
|
-
return { score, scoreVector };
|
|
2582
|
-
};
|
|
2583
|
-
}
|
|
2584
|
-
const objectivesOf = (node) => {
|
|
2585
|
-
const objectives = {};
|
|
2586
|
-
if (node.scoreVector && node.scoreVector.length > 0) {
|
|
2587
|
-
scoringScenarios.forEach((s, i) => {
|
|
2588
|
-
objectives[s.id] = node.scoreVector[i] ?? 0;
|
|
2589
|
-
});
|
|
2590
|
-
if (Object.keys(objectives).length > 0) return objectives;
|
|
2591
|
-
}
|
|
2592
|
-
objectives.composite = node.score;
|
|
2593
|
-
return objectives;
|
|
2594
|
-
};
|
|
2595
|
-
const scoredSeeds = [];
|
|
2596
|
-
for (const seed of opts.seeds) {
|
|
2597
|
-
const measured = await scoreSurface(seed.surface);
|
|
2598
|
-
scoredSeeds.push({
|
|
2599
|
-
surface: seed.surface,
|
|
2600
|
-
track: seed.track,
|
|
2601
|
-
proposer: seed.proposer,
|
|
2602
|
-
score: measured.score,
|
|
2603
|
-
...seed.vision !== void 0 ? { vision: seed.vision } : {},
|
|
2604
|
-
...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {}
|
|
2605
|
-
});
|
|
2606
|
-
}
|
|
2607
|
-
const step = async (args) => {
|
|
2608
|
-
const proposed = await proposer.propose({
|
|
2609
|
-
currentSurface: args.tip.surface,
|
|
2610
|
-
history: [],
|
|
2611
|
-
findings: [],
|
|
2612
|
-
populationSize,
|
|
2613
|
-
generation: 0,
|
|
2614
|
-
signal: new AbortController().signal,
|
|
2615
|
-
paretoParents: []
|
|
2616
|
-
});
|
|
2617
|
-
const pool = [
|
|
2618
|
-
{
|
|
2619
|
-
surface: args.tip.surface,
|
|
2620
|
-
score: args.tip.score,
|
|
2621
|
-
...args.tip.scoreVector !== void 0 ? { scoreVector: args.tip.scoreVector } : {},
|
|
2622
|
-
...args.tip.rationale !== void 0 ? { rationale: args.tip.rationale } : {}
|
|
2623
|
-
}
|
|
2624
|
-
];
|
|
2625
|
-
for (const p of proposed) {
|
|
2626
|
-
const { surface, rationale } = toCandidate(p);
|
|
2627
|
-
const measured = await scoreSurface(surface);
|
|
2628
|
-
pool.push({
|
|
2629
|
-
surface,
|
|
2630
|
-
score: measured.score,
|
|
2631
|
-
...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
|
|
2632
|
-
...rationale !== void 0 ? { rationale } : {}
|
|
2633
|
-
});
|
|
2634
|
-
}
|
|
2635
|
-
let best = pool[0];
|
|
2636
|
-
for (const entry of pool) {
|
|
2637
|
-
if (entry.score > best.score) best = entry;
|
|
2638
|
-
}
|
|
2639
|
-
return {
|
|
2640
|
-
surface: best.surface,
|
|
2641
|
-
score: best.score,
|
|
2642
|
-
...best.scoreVector !== void 0 ? { scoreVector: best.scoreVector } : {},
|
|
2643
|
-
...best.rationale !== void 0 ? { rationale: best.rationale } : {}
|
|
2644
|
-
};
|
|
2645
|
-
};
|
|
2646
|
-
const merge = async (args) => {
|
|
2647
|
-
const ordered = [...args.parents].sort((a, b) => b.score - a.score);
|
|
2648
|
-
const paretoParents = ordered.map((node) => ({
|
|
2649
|
-
surface: node.surface,
|
|
2650
|
-
surfaceHash: surfaceHash(node.surface),
|
|
2651
|
-
objectives: objectivesOf(node),
|
|
2652
|
-
composite: node.score,
|
|
2653
|
-
generation: node.generation
|
|
2654
|
-
}));
|
|
2655
|
-
const proposed = await proposer.propose({
|
|
2656
|
-
currentSurface: ordered[0].surface,
|
|
2657
|
-
history: [],
|
|
2658
|
-
findings: [],
|
|
2659
|
-
// A merge is a single crossover; combineParents fires because
|
|
2660
|
-
// paretoParents has > 1 string member.
|
|
2661
|
-
populationSize: 1,
|
|
2662
|
-
// generation >= 1 keeps the combine slot semantically a "merge" step.
|
|
2663
|
-
generation: 1,
|
|
2664
|
-
signal: new AbortController().signal,
|
|
2665
|
-
paretoParents
|
|
2666
|
-
});
|
|
2667
|
-
const first = proposed[0];
|
|
2668
|
-
const merged = first ? toCandidate(first) : { surface: ordered[0].surface };
|
|
2669
|
-
const measured = await scoreSurface(merged.surface);
|
|
2670
|
-
return {
|
|
2671
|
-
surface: merged.surface,
|
|
2672
|
-
score: measured.score,
|
|
2673
|
-
...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
|
|
2674
|
-
...merged.rationale !== void 0 ? { rationale: merged.rationale } : {}
|
|
2675
|
-
};
|
|
2676
|
-
};
|
|
2677
|
-
const result = await runLineage({
|
|
2678
|
-
seeds: scoredSeeds,
|
|
2679
|
-
step,
|
|
2680
|
-
merge,
|
|
2681
|
-
governor,
|
|
2682
|
-
budget: opts.budget,
|
|
2683
|
-
...opts.store ? { store: opts.store } : {},
|
|
2684
|
-
...opts.log ? { log: opts.log } : {}
|
|
2685
|
-
});
|
|
2686
|
-
return { lineage: result.lineage, best: result.best, steps: result.steps };
|
|
2687
|
-
}
|
|
2688
|
-
|
|
2689
|
-
// src/campaign/presets/run-profile-matrix.ts
|
|
2690
|
-
import { createHash as createHash5 } from "crypto";
|
|
2691
|
-
import { join as join3 } from "path";
|
|
2692
|
-
var ProfileMatrixError = class extends AgentEvalError {
|
|
2693
|
-
constructor(message) {
|
|
2694
|
-
super("profile_matrix", message);
|
|
2695
|
-
}
|
|
2696
|
-
};
|
|
2697
|
-
function sanitize(id) {
|
|
2698
|
-
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
2699
|
-
}
|
|
2700
|
-
function sha(input) {
|
|
2701
|
-
return createHash5("sha256").update(JSON.stringify(input)).digest("hex");
|
|
2702
|
-
}
|
|
2703
|
-
function mean3(xs) {
|
|
2704
|
-
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
2705
|
-
}
|
|
2706
|
-
function cellComposite(cell) {
|
|
2707
|
-
const composites = Object.values(cell.judgeScores).map((s) => s.composite);
|
|
2708
|
-
return composites.length === 0 ? 0 : mean3(composites);
|
|
2709
|
-
}
|
|
2710
|
-
function requireResolvedModel(cell, profileId) {
|
|
2711
|
-
const resolved = cell.resolvedModel?.trim();
|
|
2712
|
-
if (!resolved) {
|
|
2713
|
-
throw new ProfileMatrixError(
|
|
2714
|
-
`profile '${profileId}' declared the '${HARNESS_NATIVE_MODEL}' runtime-resolved model but its dispatch reported no resolved model for cell '${cell.cellId}' \u2014 report it via ctx.cost.observeModel(<id>) so the RunRecord pins the real model (never records '${HARNESS_NATIVE_MODEL}')`
|
|
2715
|
-
);
|
|
2716
|
-
}
|
|
2717
|
-
if (!modelHasSnapshot(resolved)) {
|
|
2718
|
-
throw new ProfileMatrixError(
|
|
2719
|
-
`profile '${profileId}' resolved to model '${resolved}' for cell '${cell.cellId}', which lacks a snapshot version \u2014 pin it (name@YYYY-MM-DD or name-YYYYMMDD) before reporting it via ctx.cost.observeModel`
|
|
2720
|
-
);
|
|
2721
|
-
}
|
|
2722
|
-
return resolved;
|
|
2723
|
-
}
|
|
2724
|
-
function buildRunRecord(args) {
|
|
2725
|
-
const { cell, profile, profileHash, configHash, experimentId, splitTag, commitSha, matrixId } = args;
|
|
2726
|
-
const profileId = agentProfileId(profile);
|
|
2727
|
-
const declaredModel = agentProfileModelId(profile);
|
|
2728
|
-
const model = declaredModel === HARNESS_NATIVE_MODEL ? requireResolvedModel(cell, profileId) : declaredModel;
|
|
2729
|
-
const composite = cellComposite(cell);
|
|
2730
|
-
const raw = { composite };
|
|
2731
|
-
const perJudge = {};
|
|
2732
|
-
const dimAccum = {};
|
|
2733
|
-
const notes = [];
|
|
2734
|
-
for (const [judgeName, js] of Object.entries(cell.judgeScores)) {
|
|
2735
|
-
perJudge[judgeName] = { ...js.dimensions };
|
|
2736
|
-
for (const [dim, value] of Object.entries(js.dimensions)) {
|
|
2737
|
-
raw[`${judgeName}.${dim}`] = value;
|
|
2738
|
-
dimAccum[dim] ??= [];
|
|
2739
|
-
dimAccum[dim].push(value);
|
|
2740
|
-
}
|
|
2741
|
-
if (js.notes) notes.push(`${judgeName}: ${js.notes}`);
|
|
2742
|
-
}
|
|
2743
|
-
const perDimMean = {};
|
|
2744
|
-
for (const [dim, values] of Object.entries(dimAccum)) perDimMean[dim] = mean3(values);
|
|
2745
|
-
let costUsd = cell.costUsd;
|
|
2746
|
-
let costEstimated = false;
|
|
2747
|
-
if (costUsd === 0 && cell.tokenUsage.output > 0 && isModelPriced(model)) {
|
|
2748
|
-
costUsd = estimateCost(cell.tokenUsage.input, cell.tokenUsage.output, model);
|
|
2749
|
-
costEstimated = costUsd > 0;
|
|
2750
|
-
}
|
|
2751
|
-
raw.cost_usd = costUsd;
|
|
2752
|
-
raw.cost_estimated = costEstimated ? 1 : 0;
|
|
2753
|
-
raw.tokens_input = cell.tokenUsage.input;
|
|
2754
|
-
raw.tokens_output = cell.tokenUsage.output;
|
|
2755
|
-
if (typeof cell.tokenUsage.cached === "number") raw.tokens_cached = cell.tokenUsage.cached;
|
|
2756
|
-
raw.latency_ms = cell.durationMs;
|
|
2757
|
-
if (costUsd > 0) {
|
|
2758
|
-
raw.tokens_per_dollar = (cell.tokenUsage.input + cell.tokenUsage.output) / costUsd;
|
|
2759
|
-
}
|
|
2760
|
-
if (composite > 0.01) raw.cost_per_quality = costUsd / composite;
|
|
2761
|
-
const outcome = splitTag === "holdout" ? { holdoutScore: composite, raw } : { searchScore: composite, raw };
|
|
2762
|
-
if (Object.keys(perJudge).length > 0) {
|
|
2763
|
-
outcome.judgeScores = {
|
|
2764
|
-
perJudge,
|
|
2765
|
-
perDimMean,
|
|
2766
|
-
composite,
|
|
2767
|
-
...notes.length > 0 ? { notes: notes.join(" | ") } : {}
|
|
2768
|
-
};
|
|
2769
|
-
}
|
|
2770
|
-
const record = {
|
|
2771
|
-
runId: `${matrixId}:${profileId}:${cell.cellId}`,
|
|
2772
|
-
experimentId,
|
|
2773
|
-
candidateId: profileId,
|
|
2774
|
-
seed: cell.seed,
|
|
2775
|
-
model,
|
|
2776
|
-
promptHash: profileHash,
|
|
2777
|
-
configHash,
|
|
2778
|
-
commitSha,
|
|
2779
|
-
wallMs: cell.durationMs,
|
|
2780
|
-
costUsd,
|
|
2781
|
-
tokenUsage: cell.tokenUsage,
|
|
2782
|
-
outcome,
|
|
2783
|
-
splitTag,
|
|
2784
|
-
scenarioId: cell.scenarioId,
|
|
2785
|
-
...args.agentProfileCell ? { agentProfile: args.agentProfileCell } : {},
|
|
2786
|
-
...cell.error ? { failureMode: cell.error } : {}
|
|
2787
|
-
};
|
|
2788
|
-
if (args.corpusText && args.scenario) {
|
|
2789
|
-
try {
|
|
2790
|
-
const text = args.corpusText(cell.artifact, args.scenario);
|
|
2791
|
-
if (text && typeof text.prompt === "string" && typeof text.completion === "string") {
|
|
2792
|
-
record.prompt = text.prompt;
|
|
2793
|
-
record.completion = text.completion;
|
|
2794
|
-
}
|
|
2795
|
-
} catch {
|
|
2796
|
-
}
|
|
2797
|
-
}
|
|
2798
|
-
return record;
|
|
2799
|
-
}
|
|
2800
|
-
async function runProfileMatrix(opts) {
|
|
2801
|
-
if (opts.profiles.length === 0) throw new ProfileMatrixError("profiles must not be empty");
|
|
2802
|
-
if (opts.scenarios.length === 0) throw new ProfileMatrixError("scenarios must not be empty");
|
|
2803
|
-
const splitTag = opts.splitTag ?? "search";
|
|
2804
|
-
const seed = opts.seed ?? 42;
|
|
2805
|
-
const validate = opts.validate ?? true;
|
|
2806
|
-
const integrityMode = opts.integrity ?? "assert";
|
|
2807
|
-
const profileIds = opts.profiles.map(agentProfileId);
|
|
2808
|
-
const experimentId = opts.experimentId ?? `pm_${sha({ profileIds, scenarios: opts.scenarios.map((s) => s.id) }).slice(0, 16)}`;
|
|
2809
|
-
const matrixId = `mtx_${sha({ experimentId, profileIds, seed, splitTag }).slice(0, 16)}`;
|
|
2810
|
-
const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
|
|
2811
|
-
for (const profile of opts.profiles) {
|
|
2812
|
-
const profileHash = agentProfileHash(profile);
|
|
2813
|
-
const profileId = agentProfileId(profile);
|
|
2814
|
-
const declaredModel = agentProfileModelId(profile);
|
|
2815
|
-
const model = declaredModel === HARNESS_NATIVE_MODEL ? `${HARNESS_NATIVE_MODEL}@runtime-resolved` : declaredModel;
|
|
2816
|
-
try {
|
|
2817
|
-
validateRunRecord({
|
|
2818
|
-
runId: `${matrixId}:${profileId}:probe`,
|
|
2819
|
-
experimentId,
|
|
2820
|
-
candidateId: profileId,
|
|
2821
|
-
seed,
|
|
2822
|
-
model,
|
|
2823
|
-
promptHash: profileHash,
|
|
2824
|
-
configHash: profileHash,
|
|
2825
|
-
commitSha: opts.commitSha,
|
|
2826
|
-
wallMs: 0,
|
|
2827
|
-
costUsd: 0,
|
|
2828
|
-
tokenUsage: { input: 0, output: 0 },
|
|
2829
|
-
outcome: splitTag === "holdout" ? { holdoutScore: 0, raw: {} } : { searchScore: 0, raw: {} },
|
|
2830
|
-
splitTag
|
|
2831
|
-
});
|
|
2832
|
-
} catch (err) {
|
|
2833
|
-
throw new ProfileMatrixError(
|
|
2834
|
-
`profile '${profileId}' is not recordable: ${err instanceof Error ? err.message : String(err)}`
|
|
2835
|
-
);
|
|
2836
|
-
}
|
|
2837
|
-
}
|
|
2838
|
-
const records = [];
|
|
2839
|
-
const campaigns = {};
|
|
2840
|
-
const byProfile = {};
|
|
2841
|
-
for (const profile of opts.profiles) {
|
|
2842
|
-
const profileHash = agentProfileHash(profile);
|
|
2843
|
-
const profileId = agentProfileId(profile);
|
|
2844
|
-
const declaredModel = agentProfileModelId(profile);
|
|
2845
|
-
const configHash = sha({
|
|
2846
|
-
profile: profileHash,
|
|
2847
|
-
judges: (opts.judges ?? []).map((j) => j.name),
|
|
2848
|
-
seed,
|
|
2849
|
-
splitTag
|
|
2850
|
-
});
|
|
2851
|
-
const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
|
|
2852
|
-
Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profileId)}` });
|
|
2853
|
-
const campaign = await runCampaign({
|
|
2854
|
-
scenarios: opts.scenarios,
|
|
2855
|
-
dispatch,
|
|
2856
|
-
judges: opts.judges,
|
|
2857
|
-
seed,
|
|
2858
|
-
reps: opts.reps,
|
|
2859
|
-
maxConcurrency: opts.maxConcurrency,
|
|
2860
|
-
costCeiling: opts.costCeiling,
|
|
2861
|
-
labeledStore: opts.labeledStore,
|
|
2862
|
-
captureSource: opts.captureSource,
|
|
2863
|
-
storage: opts.storage,
|
|
2864
|
-
now: opts.now,
|
|
2865
|
-
runDir: join3(opts.runDir, sanitize(profileId))
|
|
2866
|
-
});
|
|
2867
|
-
const axis = harnessAxisOf(profile);
|
|
2868
|
-
const buildCellIdentity = (cellModel) => buildAgentProfileCell({
|
|
2869
|
-
profileId,
|
|
2870
|
-
sourceProfile: { kind: "agent-interface-profile", hash: profileHash },
|
|
2871
|
-
model: cellModel,
|
|
2872
|
-
...axis ? { harness: { id: axis.harness } } : {}
|
|
2873
|
-
});
|
|
2874
|
-
const sharedCellIdentity = declaredModel === HARNESS_NATIVE_MODEL ? void 0 : await buildCellIdentity(declaredModel);
|
|
2875
|
-
const profileRecords = [];
|
|
2876
|
-
for (const cell of campaign.cells) {
|
|
2877
|
-
const agentProfileCell = sharedCellIdentity ?? await buildCellIdentity(requireResolvedModel(cell, profileId));
|
|
2878
|
-
const record = buildRunRecord({
|
|
2879
|
-
cell,
|
|
2880
|
-
profile,
|
|
2881
|
-
profileHash,
|
|
2882
|
-
configHash,
|
|
2883
|
-
experimentId,
|
|
2884
|
-
splitTag,
|
|
2885
|
-
commitSha: opts.commitSha,
|
|
2886
|
-
matrixId,
|
|
2887
|
-
agentProfileCell,
|
|
2888
|
-
scenario: scenarioById.get(cell.scenarioId),
|
|
2889
|
-
corpusText: opts.corpusText
|
|
2890
|
-
});
|
|
2891
|
-
if (validate) validateRunRecord(record);
|
|
2892
|
-
profileRecords.push(record);
|
|
2893
|
-
records.push(record);
|
|
2894
|
-
}
|
|
2895
|
-
const pricedTotalCostUsd = profileRecords.reduce((a, r) => a + r.costUsd, 0);
|
|
2896
|
-
campaigns[profileId] = {
|
|
2897
|
-
...campaign,
|
|
2898
|
-
aggregates: { ...campaign.aggregates, totalCostUsd: pricedTotalCostUsd }
|
|
2899
|
-
};
|
|
2900
|
-
byProfile[profileId] = {
|
|
2901
|
-
profileId,
|
|
2902
|
-
profileHash,
|
|
2903
|
-
// The declared model, unless it snapped to the sentinel — then the
|
|
2904
|
-
// resolved model the cells actually ran on (all cells of a profile share
|
|
2905
|
-
// one harness, so the first record's model is representative).
|
|
2906
|
-
model: declaredModel === HARNESS_NATIVE_MODEL ? profileRecords[0]?.model ?? declaredModel : declaredModel,
|
|
2907
|
-
records: profileRecords.length,
|
|
2908
|
-
meanComposite: mean3(profileRecords.map(compositeOf)),
|
|
2909
|
-
totalCostUsd: pricedTotalCostUsd,
|
|
2910
|
-
integrity: summarizeBackendIntegrity(profileRecords)
|
|
2911
|
-
};
|
|
2912
|
-
}
|
|
2913
|
-
const integrity = summarizeBackendIntegrity(records);
|
|
2914
|
-
if (integrityMode === "assert") {
|
|
2915
|
-
assertRealBackend(records, { allowMixed: opts.allowMixed ?? true });
|
|
2916
|
-
} else if (integrityMode === "warn" && integrity.verdict !== "real") {
|
|
2917
|
-
console.warn(
|
|
2918
|
-
`[runProfileMatrix] backend integrity: ${integrity.verdict} \u2014 ${integrity.diagnosis}`
|
|
2919
|
-
);
|
|
2920
|
-
}
|
|
2921
|
-
const byScenario = rollup(records, (r) => r.scenarioId);
|
|
2922
|
-
const byPersona = opts.personaOf ? rollupByPersona(records, opts.scenarios, opts.personaOf) : void 0;
|
|
2923
|
-
return { matrixId, experimentId, records, byProfile, byScenario, byPersona, integrity, campaigns };
|
|
2924
|
-
}
|
|
2925
|
-
function compositeOf(r) {
|
|
2926
|
-
return r.outcome.holdoutScore ?? r.outcome.searchScore ?? 0;
|
|
2927
|
-
}
|
|
2928
|
-
function rollup(records, keyOf) {
|
|
2929
|
-
const groups = /* @__PURE__ */ new Map();
|
|
2930
|
-
for (const r of records) {
|
|
2931
|
-
const key = keyOf(r);
|
|
2932
|
-
if (key === void 0) continue;
|
|
2933
|
-
const arr = groups.get(key) ?? [];
|
|
2934
|
-
arr.push(compositeOf(r));
|
|
2935
|
-
groups.set(key, arr);
|
|
2936
|
-
}
|
|
2937
|
-
const out = {};
|
|
2938
|
-
for (const [key, xs] of groups) out[key] = { meanComposite: mean3(xs), n: xs.length };
|
|
2939
|
-
return out;
|
|
2940
|
-
}
|
|
2941
|
-
function rollupByPersona(records, scenarios, personaOf) {
|
|
2942
|
-
const personaByScenarioId = /* @__PURE__ */ new Map();
|
|
2943
|
-
for (const s of scenarios) personaByScenarioId.set(s.id, personaOf(s));
|
|
2944
|
-
return rollup(records, (r) => r.scenarioId ? personaByScenarioId.get(r.scenarioId) : void 0);
|
|
2945
|
-
}
|
|
2946
|
-
|
|
2947
|
-
// src/campaign/proposers/_findings-text.ts
|
|
2948
|
-
function findingToLesson(f) {
|
|
2949
|
-
if (typeof f === "string") return f.trim() || null;
|
|
2950
|
-
if (f && typeof f === "object") {
|
|
2951
|
-
const o = f;
|
|
2952
|
-
const cand = o.recommended_action ?? o.claim ?? o.lesson ?? o.text ?? o.message;
|
|
2953
|
-
if (typeof cand === "string" && cand.trim()) return cand.trim();
|
|
2954
|
-
}
|
|
2955
|
-
return null;
|
|
2956
|
-
}
|
|
2957
|
-
function normKey(s) {
|
|
2958
|
-
return s.toLowerCase().replace(/\s+/g, " ").replace(/[.;:!?\s]+$/, "").trim();
|
|
2959
|
-
}
|
|
2960
|
-
function surfaceToText2(surface) {
|
|
2961
|
-
if (typeof surface === "string") return surface;
|
|
2962
|
-
throw new Error(
|
|
2963
|
-
`curator proposer: surface must be a string prompt, got a ${surface.kind}-tier surface (${surface.worktreeRef}) \u2014 curation is prompt-tier`
|
|
2964
|
-
);
|
|
2965
|
-
}
|
|
2966
|
-
function extractBlockBody(text, startMarker, endMarker) {
|
|
2967
|
-
const start = text.indexOf(startMarker);
|
|
2968
|
-
const end = text.indexOf(endMarker);
|
|
2969
|
-
if (start === -1 || end === -1 || end < start) return "";
|
|
2970
|
-
return text.slice(start + startMarker.length, end);
|
|
2971
|
-
}
|
|
2972
|
-
function stripBlock(text, startMarker, endMarker) {
|
|
2973
|
-
const start = text.indexOf(startMarker);
|
|
2974
|
-
const end = text.indexOf(endMarker);
|
|
2975
|
-
if (start === -1 || end === -1 || end < start) return text.trimEnd();
|
|
2976
|
-
return (text.slice(0, start) + text.slice(end + endMarker.length)).trimEnd();
|
|
2977
|
-
}
|
|
2978
|
-
|
|
2979
|
-
// src/campaign/proposers/ace.ts
|
|
2980
|
-
var BLOCK_START = "<!-- BEGIN ace-playbook (auto-managed by aceProposer) -->";
|
|
2981
|
-
var BLOCK_END = "<!-- END ace-playbook -->";
|
|
2982
|
-
var DEFAULT_HEADING = "## Playbook (accumulated lessons \u2014 append-only)";
|
|
2983
|
-
function parsePlaybook(surface) {
|
|
2984
|
-
const body = extractBlockBody(surface, BLOCK_START, BLOCK_END);
|
|
2985
|
-
const out = [];
|
|
2986
|
-
for (const raw of body.split("\n")) {
|
|
2987
|
-
const line = raw.trim();
|
|
2988
|
-
if (!line.startsWith("- ")) continue;
|
|
2989
|
-
const item = line.slice(2).trim();
|
|
2990
|
-
const tag = /^\[g(-?\d+)\]\s*(.*)$/.exec(item);
|
|
2991
|
-
if (tag) out.push({ gen: Number(tag[1]), text: tag[2].trim() });
|
|
2992
|
-
else out.push({ gen: -1, text: item });
|
|
2993
|
-
}
|
|
2994
|
-
return out;
|
|
2995
|
-
}
|
|
2996
|
-
function aceProposer(opts = {}) {
|
|
2997
|
-
const maxEntries = opts.maxEntries ?? 50;
|
|
2998
|
-
if (maxEntries < 1) throw new Error("aceProposer: maxEntries must be >= 1");
|
|
2999
|
-
const heading = opts.sectionHeading ?? DEFAULT_HEADING;
|
|
3000
|
-
return {
|
|
3001
|
-
kind: "ace",
|
|
3002
|
-
async propose(ctx) {
|
|
3003
|
-
const parent = surfaceToText2(ctx.currentSurface);
|
|
3004
|
-
const existing = parsePlaybook(parent);
|
|
3005
|
-
const seen = new Set(existing.map((b) => normKey(b.text)));
|
|
3006
|
-
const fresh = [];
|
|
3007
|
-
for (const f of ctx.findings ?? []) {
|
|
3008
|
-
const lesson = findingToLesson(f);
|
|
3009
|
-
if (!lesson) continue;
|
|
3010
|
-
const k = normKey(lesson);
|
|
3011
|
-
if (!k || seen.has(k)) continue;
|
|
3012
|
-
seen.add(k);
|
|
3013
|
-
fresh.push({ gen: ctx.generation, text: lesson });
|
|
3014
|
-
}
|
|
3015
|
-
if (fresh.length === 0) return [];
|
|
3016
|
-
const all = [...existing, ...fresh].slice(-maxEntries);
|
|
3017
|
-
const block = [
|
|
3018
|
-
BLOCK_START,
|
|
3019
|
-
heading,
|
|
3020
|
-
...all.map((b) => `- [g${b.gen}] ${b.text}`),
|
|
3021
|
-
BLOCK_END
|
|
3022
|
-
].join("\n");
|
|
3023
|
-
const base = stripBlock(parent, BLOCK_START, BLOCK_END);
|
|
3024
|
-
const surface = base ? `${base}
|
|
3025
|
-
|
|
3026
|
-
${block}` : block;
|
|
3027
|
-
return [
|
|
3028
|
-
{
|
|
3029
|
-
surface,
|
|
3030
|
-
label: `ace-playbook +${fresh.length}`,
|
|
3031
|
-
rationale: `appended ${fresh.length} new lesson(s) from gen ${ctx.generation} findings (playbook now ${all.length} bullet(s), append-only)`
|
|
3032
|
-
}
|
|
3033
|
-
];
|
|
3034
|
-
}
|
|
3035
|
-
};
|
|
3036
|
-
}
|
|
3037
|
-
|
|
3038
|
-
// src/campaign/proposers/halo.ts
|
|
3039
|
-
import { execFile } from "child_process";
|
|
3040
|
-
import { promisify } from "util";
|
|
3041
|
-
|
|
3042
|
-
// src/campaign/proposers/analysis-edit.ts
|
|
3043
|
-
import { mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
|
|
3044
|
-
import { tmpdir } from "os";
|
|
3045
|
-
import { join as join4 } from "path";
|
|
3046
|
-
var APPLY_SYSTEM = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
|
|
3047
|
-
function surfaceToPromptText(surface) {
|
|
3048
|
-
return typeof surface === "string" ? surface : JSON.stringify(surface);
|
|
3049
|
-
}
|
|
3050
|
-
function analysisEditProposer(opts) {
|
|
3051
|
-
return {
|
|
3052
|
-
kind: opts.kind,
|
|
3053
|
-
async propose(ctx) {
|
|
3054
|
-
const parent = surfaceToPromptText(ctx.currentSurface);
|
|
3055
|
-
const traces = await opts.resolveTraces(ctx) ?? "";
|
|
3056
|
-
if (!traces.trim()) throw new Error(opts.noTracesError);
|
|
3057
|
-
const dir = mkdtempSync(join4(tmpdir(), `${opts.kind}-proposer-`));
|
|
3058
|
-
const tracePath = join4(dir, "traces.jsonl");
|
|
3059
|
-
writeFileSync2(tracePath, traces.endsWith("\n") ? traces : `${traces}
|
|
3060
|
-
`);
|
|
3061
|
-
const report = await opts.analyze(tracePath, ctx);
|
|
3062
|
-
const applied = await callLlm(
|
|
3063
|
-
{
|
|
3064
|
-
model: opts.applyModel,
|
|
3065
|
-
messages: [
|
|
3066
|
-
{ role: "system", content: APPLY_SYSTEM },
|
|
3067
|
-
{
|
|
3068
|
-
role: "user",
|
|
3069
|
-
content: `CURRENT PROMPT:
|
|
3070
|
-
${parent}
|
|
3071
|
-
|
|
3072
|
-
TRACE-ANALYSIS REPORT:
|
|
3073
|
-
${report}
|
|
3074
|
-
|
|
3075
|
-
Return the full revised prompt.`
|
|
3076
|
-
}
|
|
3077
|
-
]
|
|
3078
|
-
},
|
|
3079
|
-
{ baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
|
|
3080
|
-
);
|
|
3081
|
-
const text = applied.content.trim();
|
|
3082
|
-
if (!text || text === parent) return [];
|
|
3083
|
-
return [{ surface: text, label: opts.label, rationale: opts.rationale(report) }];
|
|
3084
|
-
}
|
|
3085
|
-
};
|
|
3086
|
-
}
|
|
3087
|
-
|
|
3088
|
-
// src/campaign/proposers/halo.ts
|
|
3089
|
-
var execFileAsync = promisify(execFile);
|
|
3090
|
-
var DEFAULT_ANALYSIS_PROMPT = "Diagnose the failures in these agent execution traces \u2014 hallucinated tool calls, redundant tool arguments, refusal loops, and semantic-correctness errors \u2014 and suggest concrete, generalizable fixes to the agent instructions.";
|
|
3091
|
-
function haloProposer(opts) {
|
|
3092
|
-
const haloBin = opts.haloBin ?? "halo";
|
|
3093
|
-
const model = opts.model ?? "gpt-5.4-mini";
|
|
3094
|
-
return analysisEditProposer({
|
|
3095
|
-
kind: "halo",
|
|
3096
|
-
label: "halo",
|
|
3097
|
-
baseUrl: opts.baseUrl,
|
|
3098
|
-
apiKey: opts.apiKey,
|
|
3099
|
-
applyModel: opts.applyModel ?? model,
|
|
3100
|
-
fetchImpl: opts.fetchImpl,
|
|
3101
|
-
resolveTraces: opts.resolveTraces,
|
|
3102
|
-
noTracesError: "haloProposer: resolveTraces returned no OTLP traces \u2014 the halo engine has nothing to analyze",
|
|
3103
|
-
// HALO's real findings are preserved verbatim in the rationale (attribution).
|
|
3104
|
-
rationale: (findings) => `halo-engine findings:
|
|
3105
|
-
${findings.slice(0, 800)}`,
|
|
3106
|
-
analyze: async (tracePath, ctx) => {
|
|
3107
|
-
const args = [
|
|
3108
|
-
tracePath,
|
|
3109
|
-
"-p",
|
|
3110
|
-
opts.analysisPrompt ?? DEFAULT_ANALYSIS_PROMPT,
|
|
3111
|
-
"-m",
|
|
3112
|
-
model,
|
|
3113
|
-
...opts.maxDepth !== void 0 ? ["--max-depth", String(opts.maxDepth)] : [],
|
|
3114
|
-
...opts.maxTurns !== void 0 ? ["--max-turns", String(opts.maxTurns)] : []
|
|
3115
|
-
];
|
|
3116
|
-
let findings;
|
|
3117
|
-
try {
|
|
3118
|
-
const { stdout } = await execFileAsync(haloBin, args, {
|
|
3119
|
-
maxBuffer: 64 * 1024 * 1024,
|
|
3120
|
-
signal: ctx.signal,
|
|
3121
|
-
env: {
|
|
3122
|
-
...process.env,
|
|
3123
|
-
...opts.apiKey ? { OPENAI_API_KEY: opts.apiKey } : {},
|
|
3124
|
-
OPENAI_BASE_URL: opts.baseUrl
|
|
3125
|
-
}
|
|
3126
|
-
});
|
|
3127
|
-
findings = stdout.trim();
|
|
3128
|
-
} catch (e) {
|
|
3129
|
-
throw new Error(
|
|
3130
|
-
`haloProposer: halo-engine ('${haloBin}') failed \u2014 ${e instanceof Error ? e.message : String(e)}`
|
|
3131
|
-
);
|
|
3132
|
-
}
|
|
3133
|
-
if (!findings) throw new Error("haloProposer: halo-engine produced no findings");
|
|
3134
|
-
return findings;
|
|
3135
|
-
}
|
|
3136
|
-
});
|
|
3137
|
-
}
|
|
3138
|
-
|
|
3139
|
-
// src/campaign/proposers/memory.ts
|
|
3140
|
-
var BLOCK_START2 = "<!-- BEGIN curated-memory (auto-managed by memoryCurationProposer) -->";
|
|
3141
|
-
var BLOCK_END2 = "<!-- END curated-memory -->";
|
|
3142
|
-
var DEFAULT_HEADING2 = "## Learned from prior runs (curated memory)";
|
|
3143
|
-
var DISTILL_SYSTEM = 'You compress raw trace-analysis findings into crisp, generalizable agent guidance. Output ONLY a JSON array of strings, each one imperative lesson the agent should follow (e.g. "Always fetch a resource before mutating it"). No prose outside the JSON. Deduplicate; keep the most actionable and general; drop case-specific noise.';
|
|
3144
|
-
function extractExistingLessons(text) {
|
|
3145
|
-
return extractBlockBody(text, BLOCK_START2, BLOCK_END2).split("\n").map((l) => l.replace(/^\s*-\s+/, "").trim()).filter((l) => l && !l.startsWith("#"));
|
|
3146
|
-
}
|
|
3147
|
-
async function distillLessons(raw, distill) {
|
|
3148
|
-
const res = await callLlm(
|
|
3149
|
-
{
|
|
3150
|
-
model: distill.model,
|
|
3151
|
-
messages: [
|
|
3152
|
-
{ role: "system", content: DISTILL_SYSTEM },
|
|
3153
|
-
{ role: "user", content: `Findings:
|
|
3154
|
-
${raw.map((r) => `- ${r}`).join("\n")}` }
|
|
3155
|
-
]
|
|
3156
|
-
},
|
|
3157
|
-
{ baseUrl: distill.baseUrl, apiKey: distill.apiKey, fetch: distill.fetchImpl }
|
|
3158
|
-
);
|
|
3159
|
-
try {
|
|
3160
|
-
const parsed = JSON.parse(res.content.trim());
|
|
3161
|
-
if (Array.isArray(parsed)) {
|
|
3162
|
-
const lessons = parsed.filter(
|
|
3163
|
-
(x) => typeof x === "string" && x.trim().length > 0
|
|
3164
|
-
);
|
|
3165
|
-
if (lessons.length > 0) return lessons;
|
|
3166
|
-
}
|
|
3167
|
-
} catch {
|
|
3168
|
-
}
|
|
3169
|
-
return raw;
|
|
3170
|
-
}
|
|
3171
|
-
function memoryCurationProposer(opts = {}) {
|
|
3172
|
-
const maxEntries = opts.maxEntries ?? 12;
|
|
3173
|
-
const heading = opts.sectionHeading ?? DEFAULT_HEADING2;
|
|
3174
|
-
return {
|
|
3175
|
-
kind: "memory-curation",
|
|
3176
|
-
async propose(ctx) {
|
|
3177
|
-
const parent = surfaceToText2(ctx.currentSurface);
|
|
3178
|
-
const fresh = [];
|
|
3179
|
-
for (const f of ctx.findings ?? []) {
|
|
3180
|
-
const l = findingToLesson(f);
|
|
3181
|
-
if (l) fresh.push(l);
|
|
3182
|
-
}
|
|
3183
|
-
const carried = extractExistingLessons(parent);
|
|
3184
|
-
if (fresh.length === 0 && carried.length === 0) return [];
|
|
3185
|
-
const distilled = opts.distill && fresh.length > 0 ? await distillLessons(fresh, opts.distill) : fresh;
|
|
3186
|
-
const byKey = /* @__PURE__ */ new Map();
|
|
3187
|
-
for (const l of carried) {
|
|
3188
|
-
const k = normKey(l);
|
|
3189
|
-
if (k) byKey.set(k, { text: l, count: 1 });
|
|
3190
|
-
}
|
|
3191
|
-
for (const l of distilled) {
|
|
3192
|
-
const k = normKey(l);
|
|
3193
|
-
if (!k) continue;
|
|
3194
|
-
const e = byKey.get(k);
|
|
3195
|
-
if (e) e.count += 1;
|
|
3196
|
-
else byKey.set(k, { text: l, count: 1 });
|
|
3197
|
-
}
|
|
3198
|
-
const ranked = [...byKey.values()].sort((a, b) => b.count - a.count || a.text.localeCompare(b.text)).slice(0, maxEntries);
|
|
3199
|
-
if (ranked.length === 0) return [];
|
|
3200
|
-
const block = [BLOCK_START2, heading, ...ranked.map((e) => `- ${e.text}`), BLOCK_END2].join(
|
|
3201
|
-
"\n"
|
|
3202
|
-
);
|
|
3203
|
-
const next = `${stripBlock(parent, BLOCK_START2, BLOCK_END2)}
|
|
3204
|
-
|
|
3205
|
-
${block}
|
|
3206
|
-
`;
|
|
3207
|
-
if (next === parent) return [];
|
|
3208
|
-
return [
|
|
3209
|
-
{
|
|
3210
|
-
surface: next,
|
|
3211
|
-
label: "memory-curation",
|
|
3212
|
-
rationale: `curated ${ranked.length} lessons (from ${fresh.length} new finding(s) + ${carried.length} carried)`
|
|
3213
|
-
}
|
|
3214
|
-
];
|
|
3215
|
-
}
|
|
3216
|
-
};
|
|
3217
|
-
}
|
|
3218
|
-
|
|
3219
|
-
// src/campaign/proposers/policy-edit.ts
|
|
3220
|
-
function policyEditProposer(opts = {}) {
|
|
3221
|
-
return {
|
|
3222
|
-
kind: "policy-edit",
|
|
3223
|
-
async propose(ctx) {
|
|
3224
|
-
const edits = materializePolicyEdits(opts.edits ?? ctx.findings, opts.findingOptions);
|
|
3225
|
-
const limit = Math.max(
|
|
3226
|
-
0,
|
|
3227
|
-
Math.min(opts.maxCandidates ?? ctx.populationSize, ctx.populationSize)
|
|
3228
|
-
);
|
|
3229
|
-
const out = [];
|
|
3230
|
-
if (limit === 0) return out;
|
|
3231
|
-
for (const edit of edits) {
|
|
3232
|
-
const admission = admitPolicyEdit(edit, opts.admission);
|
|
3233
|
-
opts.onAdmission?.(admission);
|
|
3234
|
-
if (admission.decision !== "admit") continue;
|
|
3235
|
-
const surface = coerceCandidateSurface(applyPolicyEditToSurface(ctx.currentSurface, edit));
|
|
3236
|
-
if (sameSurface(ctx.currentSurface, surface)) continue;
|
|
3237
|
-
out.push({
|
|
3238
|
-
surface,
|
|
3239
|
-
label: `policy-edit:${edit.axis}`,
|
|
3240
|
-
rationale: `${edit.editId} expected ${edit.expectedGain.direction} ${edit.expectedGain.metric} by ${edit.expectedGain.amount}; source findings [${edit.source.findingIds.join(", ")}]`
|
|
3241
|
-
});
|
|
3242
|
-
if (out.length >= limit) break;
|
|
3243
|
-
}
|
|
3244
|
-
return out;
|
|
3245
|
-
}
|
|
3246
|
-
};
|
|
3247
|
-
}
|
|
3248
|
-
function materializePolicyEdits(inputs, findingOptions) {
|
|
3249
|
-
const edits = [];
|
|
3250
|
-
const findings = [];
|
|
3251
|
-
for (const input of inputs) {
|
|
3252
|
-
if (isPolicyEdit(input)) {
|
|
3253
|
-
edits.push(input);
|
|
3254
|
-
} else if (isAnalystFindingLike(input)) {
|
|
3255
|
-
findings.push(input);
|
|
3256
|
-
}
|
|
3257
|
-
}
|
|
3258
|
-
if (findings.length > 0) edits.push(...policyEditsFromFindings(findings, findingOptions));
|
|
3259
|
-
return edits;
|
|
3260
|
-
}
|
|
3261
|
-
function isAnalystFindingLike(input) {
|
|
3262
|
-
if (!input || typeof input !== "object") return false;
|
|
3263
|
-
const obj = input;
|
|
3264
|
-
return typeof obj.finding_id === "string" && typeof obj.analyst_id === "string" && typeof obj.claim === "string" && Array.isArray(obj.evidence_refs);
|
|
3265
|
-
}
|
|
3266
|
-
function coerceCandidateSurface(surface) {
|
|
3267
|
-
if (typeof surface === "string") return surface;
|
|
3268
|
-
if (surface && typeof surface === "object") {
|
|
3269
|
-
const obj = surface;
|
|
3270
|
-
if (obj.kind === "code" && typeof obj.worktreeRef === "string") {
|
|
3271
|
-
return surface;
|
|
3272
|
-
}
|
|
3273
|
-
return JSON.stringify(surface, null, 2);
|
|
3274
|
-
}
|
|
3275
|
-
throw new Error("policyEditProposer: policy edit produced an unsupported surface");
|
|
3276
|
-
}
|
|
3277
|
-
function sameSurface(a, b) {
|
|
3278
|
-
return JSON.stringify(a) === JSON.stringify(b);
|
|
3279
|
-
}
|
|
3280
|
-
|
|
3281
|
-
// src/campaign/proposers/trace-analyst.ts
|
|
3282
|
-
import { ai } from "@ax-llm/ax";
|
|
3283
|
-
function renderFindings(findings) {
|
|
3284
|
-
return findings.map((f, i) => {
|
|
3285
|
-
const action = f.recommended_action ? `
|
|
3286
|
-
FIX: ${f.recommended_action}` : "";
|
|
3287
|
-
const subject = f.subject ? ` (${f.subject})` : "";
|
|
3288
|
-
return `${i + 1}. [${f.severity}/${f.area}]${subject} ${f.claim}${action}`;
|
|
3289
|
-
}).join("\n");
|
|
3290
|
-
}
|
|
3291
|
-
function traceAnalystProposer(opts) {
|
|
3292
|
-
if (!opts.apiKey) throw new Error("traceAnalystProposer: apiKey is required");
|
|
3293
|
-
if (!opts.model) throw new Error("traceAnalystProposer: model is required");
|
|
3294
|
-
const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS;
|
|
3295
|
-
const produceFindings = opts.analyze ?? (async (path, c) => {
|
|
3296
|
-
const aiService = ai({
|
|
3297
|
-
name: opts.provider ?? "openai",
|
|
3298
|
-
apiKey: opts.apiKey,
|
|
3299
|
-
apiURL: opts.baseUrl,
|
|
3300
|
-
config: { model: opts.model }
|
|
3301
|
-
});
|
|
3302
|
-
const registry = new AnalystRegistry();
|
|
3303
|
-
for (const spec of kinds) {
|
|
3304
|
-
registry.register(createTraceAnalystKind(spec, { ai: aiService, model: opts.model }));
|
|
3305
|
-
}
|
|
3306
|
-
const result = await registry.run(
|
|
3307
|
-
`trace-analyst-gen-${c.generation}`,
|
|
3308
|
-
{ traceStore: new OtlpFileTraceStore({ path }) },
|
|
3309
|
-
{ signal: c.signal }
|
|
3310
|
-
);
|
|
3311
|
-
return result.findings;
|
|
3312
|
-
});
|
|
3313
|
-
return analysisEditProposer({
|
|
3314
|
-
kind: "trace-analyst",
|
|
3315
|
-
label: "trace-analyst",
|
|
3316
|
-
baseUrl: opts.baseUrl,
|
|
3317
|
-
apiKey: opts.apiKey,
|
|
3318
|
-
applyModel: opts.applyModel ?? opts.model,
|
|
3319
|
-
fetchImpl: opts.fetchImpl,
|
|
3320
|
-
resolveTraces: opts.resolveTraces,
|
|
3321
|
-
noTracesError: "traceAnalystProposer: resolveTraces returned no OTLP traces \u2014 the analyst has nothing to read",
|
|
3322
|
-
rationale: (report) => `trace-analyst findings:
|
|
3323
|
-
${report.slice(0, 800)}`,
|
|
3324
|
-
analyze: async (tracePath, ctx) => {
|
|
3325
|
-
let findings;
|
|
3326
|
-
try {
|
|
3327
|
-
findings = await produceFindings(tracePath, ctx);
|
|
3328
|
-
} catch (e) {
|
|
3329
|
-
throw new Error(
|
|
3330
|
-
`traceAnalystProposer: analyst engine failed \u2014 ${e instanceof Error ? e.message : String(e)}`
|
|
3331
|
-
);
|
|
3332
|
-
}
|
|
3333
|
-
if (findings.length === 0) {
|
|
3334
|
-
throw new Error("traceAnalystProposer: analyst engine produced no findings");
|
|
3335
|
-
}
|
|
3336
|
-
return renderFindings(findings);
|
|
3337
|
-
}
|
|
3338
|
-
});
|
|
3339
|
-
}
|
|
3340
|
-
|
|
3341
|
-
// src/campaign/scenario-selection.ts
|
|
3342
|
-
var DEFAULT_SATURATION_CEILING = 0.999;
|
|
3343
|
-
var VARIANCE_EPSILON = 1e-9;
|
|
3344
|
-
function moments(scores) {
|
|
3345
|
-
const n = scores.length;
|
|
3346
|
-
if (n === 0) return { meanScore: 0, variance: 0 };
|
|
3347
|
-
let sum = 0;
|
|
3348
|
-
for (const s of scores) sum += s;
|
|
3349
|
-
const meanScore = sum / n;
|
|
3350
|
-
let sqDev = 0;
|
|
3351
|
-
for (const s of scores) {
|
|
3352
|
-
const d = s - meanScore;
|
|
3353
|
-
sqDev += d * d;
|
|
3354
|
-
}
|
|
3355
|
-
return { meanScore, variance: sqDev / n };
|
|
3356
|
-
}
|
|
3357
|
-
function compareDiscrimination(a, b) {
|
|
3358
|
-
if (b.discrimination !== a.discrimination) return b.discrimination - a.discrimination;
|
|
3359
|
-
if (a.meanScore !== b.meanScore) return a.meanScore - b.meanScore;
|
|
3360
|
-
return a.scenarioId < b.scenarioId ? -1 : a.scenarioId > b.scenarioId ? 1 : 0;
|
|
3361
|
-
}
|
|
3362
|
-
function scoreDiscrimination(signals, opts) {
|
|
3363
|
-
const saturationCeiling = opts?.saturationCeiling ?? DEFAULT_SATURATION_CEILING;
|
|
3364
|
-
const scored = signals.map((signal) => {
|
|
3365
|
-
const { meanScore, variance } = moments(signal.scores);
|
|
3366
|
-
const tied = variance < VARIANCE_EPSILON && meanScore >= saturationCeiling;
|
|
3367
|
-
return {
|
|
3368
|
-
scenarioId: signal.scenarioId,
|
|
3369
|
-
discrimination: variance,
|
|
3370
|
-
meanScore,
|
|
3371
|
-
variance,
|
|
3372
|
-
tied
|
|
3373
|
-
};
|
|
3374
|
-
});
|
|
3375
|
-
return scored.sort(compareDiscrimination);
|
|
3376
|
-
}
|
|
3377
|
-
function selectDiscriminative(signals, k, opts) {
|
|
3378
|
-
if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`);
|
|
3379
|
-
const ranked = scoreDiscrimination(signals, opts);
|
|
3380
|
-
if (ranked.length <= k) return ranked.map((s) => s.scenarioId);
|
|
3381
|
-
const nonTied = ranked.filter((s) => !s.tied);
|
|
3382
|
-
if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId);
|
|
3383
|
-
const tied = ranked.filter((s) => s.tied);
|
|
3384
|
-
const fill = tied.slice(0, k - nonTied.length);
|
|
3385
|
-
return [...nonTied, ...fill].map((s) => s.scenarioId);
|
|
3386
|
-
}
|
|
3387
|
-
|
|
3388
|
-
// src/campaign/worktree/index.ts
|
|
3389
|
-
import { execFileSync } from "child_process";
|
|
3390
|
-
import { existsSync as existsSync3 } from "fs";
|
|
3391
|
-
import { basename, isAbsolute as isAbsolute2, join as join5 } from "path";
|
|
3392
|
-
var WorktreeAdapterError = class extends Error {
|
|
3393
|
-
constructor(message, cause) {
|
|
3394
|
-
super(message);
|
|
3395
|
-
this.cause = cause;
|
|
3396
|
-
this.name = "WorktreeAdapterError";
|
|
3397
|
-
}
|
|
3398
|
-
cause;
|
|
3399
|
-
};
|
|
3400
|
-
function defaultGit(args, cwd) {
|
|
3401
|
-
try {
|
|
3402
|
-
return execFileSync("git", args, { cwd, encoding: "utf8" }).trim();
|
|
3403
|
-
} catch (err) {
|
|
3404
|
-
const stderr = err && typeof err === "object" && "stderr" in err ? String(err.stderr) : "";
|
|
3405
|
-
throw new WorktreeAdapterError(`git ${args.join(" ")} failed: ${stderr || String(err)}`, err);
|
|
3406
|
-
}
|
|
3407
|
-
}
|
|
3408
|
-
function slug2(label) {
|
|
3409
|
-
return label.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "candidate";
|
|
3410
|
-
}
|
|
3411
|
-
function gitWorktreeAdapter(opts) {
|
|
3412
|
-
const git = opts.git ?? defaultGit;
|
|
3413
|
-
const worktreeDir = opts.worktreeDir ?? join5(opts.repoRoot, ".worktrees");
|
|
3414
|
-
const branchPrefix = opts.branchPrefix ?? "improve";
|
|
3415
|
-
return {
|
|
3416
|
-
async create({ baseRef, label }) {
|
|
3417
|
-
const id = `${slug2(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
|
|
3418
|
-
const branch = `${branchPrefix}/${id}`;
|
|
3419
|
-
const path = join5(worktreeDir, id);
|
|
3420
|
-
git(["worktree", "add", "-b", branch, path, baseRef], opts.repoRoot);
|
|
3421
|
-
return { path, branch, baseRef };
|
|
3422
|
-
},
|
|
3423
|
-
async finalize(worktree, summary) {
|
|
3424
|
-
const status = git(["status", "--porcelain"], worktree.path);
|
|
3425
|
-
if (status.length > 0) {
|
|
3426
|
-
git(["add", "-A"], worktree.path);
|
|
3427
|
-
git(["commit", "-m", summary], worktree.path);
|
|
3428
|
-
}
|
|
3429
|
-
return {
|
|
3430
|
-
kind: "code",
|
|
3431
|
-
worktreeRef: worktree.path,
|
|
3432
|
-
baseRef: worktree.baseRef,
|
|
3433
|
-
summary
|
|
3434
|
-
};
|
|
3435
|
-
},
|
|
3436
|
-
async discard(worktree) {
|
|
3437
|
-
git(["worktree", "remove", "--force", worktree.path], opts.repoRoot);
|
|
3438
|
-
git(["branch", "-D", worktree.branch], opts.repoRoot);
|
|
3439
|
-
}
|
|
3440
|
-
};
|
|
3441
|
-
}
|
|
3442
|
-
function resolveWorktreePath(surface, worktreeDir) {
|
|
3443
|
-
if (isAbsolute2(surface.worktreeRef) && existsSync3(surface.worktreeRef)) return surface.worktreeRef;
|
|
3444
|
-
if (worktreeDir) return join5(worktreeDir, basename(surface.worktreeRef));
|
|
3445
|
-
return surface.worktreeRef;
|
|
3446
|
-
}
|
|
3447
112
|
export {
|
|
3448
113
|
FsLabeledScenarioStore,
|
|
3449
114
|
LabeledScenarioStoreError,
|