@hmharness/evolution 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bench.d.ts +38 -0
- package/dist/bench.js +130 -0
- package/dist/evolve.d.ts +89 -0
- package/dist/evolve.js +479 -0
- package/dist/impact.d.ts +71 -0
- package/dist/impact.js +200 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +10 -0
- package/dist/insights.d.ts +17 -0
- package/dist/insights.js +59 -0
- package/dist/knowledge.d.ts +15 -0
- package/dist/knowledge.js +139 -0
- package/dist/memory.d.ts +21 -0
- package/dist/memory.js +98 -0
- package/dist/patches.d.ts +75 -0
- package/dist/patches.d.ts.bad-1788466569598 +75 -0
- package/dist/patches.js +238 -0
- package/dist/patches.js.bad-1788466569598 +238 -0
- package/dist/radar.d.ts +3 -0
- package/dist/radar.js +40 -0
- package/dist/skills.d.ts +54 -0
- package/dist/skills.js +321 -0
- package/dist/workflows.d.ts +28 -0
- package/dist/workflows.js +84 -0
- package/package.json +31 -0
package/dist/evolve.js
ADDED
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @hmharness/evolution - evolve
|
|
3
|
+
* The evolution loop, one cycle per invocation (schedule it however you
|
|
4
|
+
* like). Pipeline: mine insights -> propose skill drafts via a meta-model
|
|
5
|
+
* call -> bench A/B (baseline vs candidate injection) -> promote or reject
|
|
6
|
+
* -> append-only memory distillation -> everything logged to
|
|
7
|
+
* evolution/log.jsonl.
|
|
8
|
+
*
|
|
9
|
+
* Guardrails baked in (DGM/GDPevo/ICLR-misevolve lessons):
|
|
10
|
+
* - drafts are never injected into real sessions; only promotion changes
|
|
11
|
+
* behavior, and only after the bench shows no regression
|
|
12
|
+
* - the loop writes only under skills/ and memory/ - it cannot touch
|
|
13
|
+
* config, security settings, or code
|
|
14
|
+
* - memory is append-only (ACE: rewriting is how context gets lost)
|
|
15
|
+
*/
|
|
16
|
+
import { appendFile, mkdir, readFile } from 'node:fs/promises';
|
|
17
|
+
import { join } from 'node:path';
|
|
18
|
+
import { chat } from '@hmharness/kernel';
|
|
19
|
+
import { listCases, matchCase, seedCases } from "./bench.js";
|
|
20
|
+
import { deleteDraft, listCanary, listDrafts, listSkills, promoteSkill, rollbackSkill, skillsToPrompt, unpromoteSkill, writeDraft } from "./skills.js";
|
|
21
|
+
import { appendMemory, readNotes } from "./memory.js";
|
|
22
|
+
import { readInsights } from "./insights.js";
|
|
23
|
+
import { readBudget, recordParetoEntry, readParetoEntries, sampleAncestor, impactReport, decayUnusedSkills } from "./impact.js";
|
|
24
|
+
/** One bench run through the structured assertion (upgraded gate: exact/
|
|
25
|
+
* regex/none/any modes, not just substrings). Also returns the raw output
|
|
26
|
+
* so callers can cost-cap (verbose-but-passing candidates). */
|
|
27
|
+
async function runAndAssert(runCase, c, injection) {
|
|
28
|
+
const output = await runCase(c, injection);
|
|
29
|
+
return { pass: matchCase(output, c).pass, output };
|
|
30
|
+
}
|
|
31
|
+
/** Rough token estimate: chars/4 - good enough to catch a 3x bloat, never
|
|
32
|
+
* used as the only rejection reason (the pass-rate gate decides; cost-cap
|
|
33
|
+
* only vetoes candidates that pass by rambling). */
|
|
34
|
+
function estTokens(text) {
|
|
35
|
+
return Math.ceil(text.length / 4);
|
|
36
|
+
}
|
|
37
|
+
export async function runEvolution(opts) {
|
|
38
|
+
const { home, provider, runCase } = opts;
|
|
39
|
+
const maxProposals = opts.maxProposals ?? 2;
|
|
40
|
+
const say = opts.log ?? (() => undefined);
|
|
41
|
+
const report = {
|
|
42
|
+
time: new Date().toISOString(),
|
|
43
|
+
model: provider.model,
|
|
44
|
+
seededCases: [],
|
|
45
|
+
insightCount: 0,
|
|
46
|
+
noteCount: 0,
|
|
47
|
+
proposals: [],
|
|
48
|
+
outcomes: [],
|
|
49
|
+
memoryDistilled: null,
|
|
50
|
+
};
|
|
51
|
+
// P0 evolution budget gate (AZR "safety alarms" + cost control): a day's
|
|
52
|
+
// cycle count and a per-cycle token ceiling live in config; overspending
|
|
53
|
+
// skips the cycle instead of burning money unattended.
|
|
54
|
+
const budget = await readBudget(home);
|
|
55
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
56
|
+
if (budget.maxCyclesPerDay && budget.cyclesToday >= budget.maxCyclesPerDay) {
|
|
57
|
+
report.outcomes.push({ name: '(budget)', action: 'error', reason: `daily cycle limit reached (${budget.cyclesToday}/${budget.maxCyclesPerDay} today) - skipped` });
|
|
58
|
+
return report;
|
|
59
|
+
}
|
|
60
|
+
// 1. Seed bench cases on a fresh home so the gate always has a signal.
|
|
61
|
+
report.seededCases = await seedCases(home);
|
|
62
|
+
const cases = await listCases(home);
|
|
63
|
+
const train = cases.filter((c) => !c.holdout);
|
|
64
|
+
const holdout = cases.filter((c) => c.holdout);
|
|
65
|
+
if (train.length === 0) {
|
|
66
|
+
report.outcomes.push({ name: '(bench)', action: 'error', reason: 'no train bench cases available' });
|
|
67
|
+
return report;
|
|
68
|
+
}
|
|
69
|
+
// 2. Gather evolution signals.
|
|
70
|
+
const insights = await readInsights(home, 40);
|
|
71
|
+
const notes = await readNotes(home);
|
|
72
|
+
const active = await listSkills(home);
|
|
73
|
+
const drafts = await listDrafts(home);
|
|
74
|
+
const canary = await listCanary(home);
|
|
75
|
+
report.insightCount = insights.length;
|
|
76
|
+
report.noteCount = notes.length;
|
|
77
|
+
const insightIds = insights.map((i) => `${i.time.slice(0, 16)}:${i.session.slice(-6)}`);
|
|
78
|
+
const toolCounts = {};
|
|
79
|
+
for (const i of insights)
|
|
80
|
+
for (const t of i.toolsUsed)
|
|
81
|
+
toolCounts[t] = (toolCounts[t] ?? 0) + 1;
|
|
82
|
+
// Radar feed (ops keeper): the newest ecosystem brief as context for
|
|
83
|
+
// proposals - toolchain advice must know what shipped recently.
|
|
84
|
+
// Read-only, best-effort; no brief = no signal.
|
|
85
|
+
let radarBrief = null;
|
|
86
|
+
try {
|
|
87
|
+
const { latestRadarBrief } = await import("./radar.js");
|
|
88
|
+
radarBrief = await latestRadarBrief(home);
|
|
89
|
+
}
|
|
90
|
+
catch { /* radar feed absent = no ecosystem signal this cycle */ }
|
|
91
|
+
const signals = {
|
|
92
|
+
sessions: insights.length,
|
|
93
|
+
failures: insights.filter((i) => i.outcome !== 'ok').map((i) => ({ task: i.task, outcome: i.outcome })),
|
|
94
|
+
toolUsage: toolCounts,
|
|
95
|
+
activeSkills: active.map((s) => s.name),
|
|
96
|
+
canarySkills: canary.map((s) => s.name),
|
|
97
|
+
existingDrafts: drafts.map((s) => s.name),
|
|
98
|
+
recentNotes: notes.slice(-10).map((n) => n.text),
|
|
99
|
+
ecosystemNews: radarBrief ?? '(no recent ecosystem brief - run hmh ops scan)',
|
|
100
|
+
};
|
|
101
|
+
// 3. Baseline bench (train gates promotion; holdout re-verifies after).
|
|
102
|
+
// Per-case cost captured too - the dual-metric gate (P0 methodology):
|
|
103
|
+
// a candidate that passes only by RAMBLING (cost > baseline x cost-cap)
|
|
104
|
+
// gets vetoed even at a green pass rate.
|
|
105
|
+
say(`baseline bench: ${train.length} train + ${holdout.length} holdout cases`);
|
|
106
|
+
const baseResults = [];
|
|
107
|
+
const baseCost = {};
|
|
108
|
+
for (const c of train) {
|
|
109
|
+
try {
|
|
110
|
+
const r = await runAndAssert(runCase, c, skillsToPrompt(active));
|
|
111
|
+
baseResults.push({ name: c.name, pass: r.pass });
|
|
112
|
+
baseCost[c.name] = estTokens(r.output);
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
baseResults.push({ name: c.name, pass: false });
|
|
116
|
+
say(` case ${c.name} threw: ${String(err).slice(0, 100)}`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const baseRate = baseResults.filter((r) => r.pass).length / baseResults.length;
|
|
120
|
+
const holdoutBase = [];
|
|
121
|
+
for (const c of holdout) {
|
|
122
|
+
try {
|
|
123
|
+
holdoutBase.push({ name: c.name, pass: (await runAndAssert(runCase, c, skillsToPrompt(active))).pass });
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
holdoutBase.push({ name: c.name, pass: false });
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
const holdoutBaseRate = holdoutBase.length === 0 ? 1 : holdoutBase.filter((r) => r.pass).length / holdoutBase.length;
|
|
130
|
+
say(`baseline: train ${(baseRate * 100).toFixed(0)}%, holdout ${(holdoutBaseRate * 100).toFixed(0)}%`);
|
|
131
|
+
// 4. Proposals: preset (tests/UI) or meta-model call. The GEPA population
|
|
132
|
+
// loop: ONE random ancestor from the rejected-candidate pool (Pareto
|
|
133
|
+
// front) feeds the prompt so evolution varies around the archive, not
|
|
134
|
+
// around the single best; a complementary reject gets merged in (cross).
|
|
135
|
+
// AWM adds workflow-level induction from repeated task archetypes - a
|
|
136
|
+
// higher abstraction than per-mistake reflection (paper-verified).
|
|
137
|
+
const pool = await readParetoEntries(home, 60);
|
|
138
|
+
const { ancestor, mergeWith } = sampleAncestor(pool);
|
|
139
|
+
let proposals;
|
|
140
|
+
if (opts.presetProposals) {
|
|
141
|
+
proposals = opts.presetProposals;
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
proposals = await proposeSkills(provider, signals, say, ancestor ?? undefined, mergeWith ?? undefined);
|
|
145
|
+
if (proposals.length === 0) {
|
|
146
|
+
try {
|
|
147
|
+
const { workflowProposals } = await import("./workflows.js");
|
|
148
|
+
proposals = await workflowProposals(provider, home, say);
|
|
149
|
+
}
|
|
150
|
+
catch (err) {
|
|
151
|
+
say(` awm skipped: ${String(err).slice(0, 80)}`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
report.proposals = proposals;
|
|
156
|
+
// 5. A/B gate each proposal on the train set.
|
|
157
|
+
for (const p of proposals.slice(0, maxProposals)) {
|
|
158
|
+
say(`candidate "${p.name}": drafting + candidate bench`);
|
|
159
|
+
try {
|
|
160
|
+
// Write-channel anti-poisoning (Misevolve lesson): the behavior gate
|
|
161
|
+
// only sees bench output, so instructions the model merely IGNORES
|
|
162
|
+
// slip through. Screen drafted content for attempts to suppress tool
|
|
163
|
+
// use, rename toolchain identifiers, or bypass the approval gate.
|
|
164
|
+
const poison = screenForPoison(p.skill_md);
|
|
165
|
+
if (poison) {
|
|
166
|
+
report.outcomes.push({ name: p.name, action: 'rejected', reason: `poisoning screen: ${poison}` });
|
|
167
|
+
say(` rejected by poisoning screen (${poison.slice(0, 60)})`);
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
await writeDraft(home, p.name, p.skill_md);
|
|
171
|
+
const draftBlock = `## Draft skill under evaluation: ${p.name}\n\n${p.skill_md.slice(0, 4000)}`;
|
|
172
|
+
const candidateInjection = `${skillsToPrompt(active)}\n${draftBlock}`;
|
|
173
|
+
const candResults = [];
|
|
174
|
+
const candCost = {};
|
|
175
|
+
for (const c of train) {
|
|
176
|
+
try {
|
|
177
|
+
// two independent samples: a candidate passes only if it passes
|
|
178
|
+
// BOTH runs - a single lucky output must not clear the gate
|
|
179
|
+
const a = await runAndAssert(runCase, c, candidateInjection);
|
|
180
|
+
const b = a.pass ? await runAndAssert(runCase, c, candidateInjection) : { pass: false, output: '' };
|
|
181
|
+
candResults.push({ name: c.name, pass: a.pass && b.pass });
|
|
182
|
+
candCost[c.name] = Math.max(estTokens(a.output), estTokens(b.output));
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
candResults.push({ name: c.name, pass: false });
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
const candRate = candResults.filter((r) => r.pass).length / candResults.length;
|
|
189
|
+
const regression = baseResults.some((b) => b.pass && !candResults.find((c) => c.name === b.name)?.pass);
|
|
190
|
+
// dual-metric veto: a passing case that costs > cost-cap x its
|
|
191
|
+
// baseline counts as a cost regression (the candidate passed by
|
|
192
|
+
// rambling) - only enforced where the case declares a cost-cap
|
|
193
|
+
const costRegressions = train.filter((c) => {
|
|
194
|
+
const cap = c.costCap ?? 1.3; // default 1.3x for all cases
|
|
195
|
+
const base = baseCost[c.name] ?? 0;
|
|
196
|
+
const cand = candCost[c.name] ?? 0;
|
|
197
|
+
return base > 0 && cand > base * cap;
|
|
198
|
+
}).map((c) => c.name);
|
|
199
|
+
const summary = (rs) => rs.map((r) => `${r.name}:${r.pass ? 'pass' : 'FAIL'}`).join(' ');
|
|
200
|
+
if (regression || candRate < baseRate) {
|
|
201
|
+
await deleteDraft(home, p.name);
|
|
202
|
+
await recordParetoEntry(home, { name: p.name, parentInsights: insightIds, rejectedReason: regression ? 'bench regression' : `pass rate ${candRate.toFixed(2)} < baseline ${baseRate.toFixed(2)}`, scores: { train: candRate }, metaModel: provider.model, at: new Date().toISOString() });
|
|
203
|
+
report.outcomes.push({
|
|
204
|
+
name: p.name,
|
|
205
|
+
action: 'rejected',
|
|
206
|
+
reason: regression ? 'bench regression on a previously passing case' : `pass rate ${candRate} < baseline ${baseRate}`,
|
|
207
|
+
baseline: { passRate: baseRate, cases: summary(baseResults) },
|
|
208
|
+
candidate: { passRate: candRate, cases: summary(candResults) },
|
|
209
|
+
lineage: { parentInsights: insightIds, scores: { train: candRate }, metaModel: provider.model, decidedAt: new Date().toISOString() },
|
|
210
|
+
});
|
|
211
|
+
say(` rejected (${regression ? 'regression' : 'lower pass rate'})`);
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
if (costRegressions.length > 0) {
|
|
215
|
+
await deleteDraft(home, p.name);
|
|
216
|
+
await recordParetoEntry(home, { name: p.name, parentInsights: insightIds, rejectedReason: `cost regression on ${costRegressions.join(', ')}`, scores: { train: candRate }, metaModel: provider.model, at: new Date().toISOString() });
|
|
217
|
+
report.outcomes.push({
|
|
218
|
+
name: p.name,
|
|
219
|
+
action: 'rejected',
|
|
220
|
+
reason: `passed the bench but by rambling: output cost exceeded the baseline cap on ${costRegressions.join(', ')}`,
|
|
221
|
+
baseline: { passRate: baseRate, cases: summary(baseResults) },
|
|
222
|
+
candidate: { passRate: candRate, cases: summary(candResults) },
|
|
223
|
+
lineage: { parentInsights: insightIds, scores: { train: candRate }, metaModel: provider.model, decidedAt: new Date().toISOString() },
|
|
224
|
+
});
|
|
225
|
+
say(` rejected (cost regression on ${costRegressions.join(', ')})`);
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
// P0 canary promotion: passing the train+holdout gates earns a
|
|
229
|
+
// CANARY slot (injected into ~20% of sessions, watermarked as
|
|
230
|
+
// experimental), not immediate full-active. The impact loop
|
|
231
|
+
// (bench --impact) compares canary vs control sessions and promotes
|
|
232
|
+
// to active only on evidence - the objective-hacking defense:
|
|
233
|
+
// never trust only the metric the evolution system can see.
|
|
234
|
+
const { archivedPrevious } = await promoteSkill(home, p.name, { canary: true });
|
|
235
|
+
// Holdout re-verification (GDPevo anti-memorization): the gate saw the
|
|
236
|
+
// train cases; holdout cases check the skill generalizes. Regression
|
|
237
|
+
// here rolls the promotion back.
|
|
238
|
+
let holdoutRate = 1;
|
|
239
|
+
if (holdout.length > 0) {
|
|
240
|
+
const holdoutCand = [];
|
|
241
|
+
for (const c of holdout) {
|
|
242
|
+
try {
|
|
243
|
+
// same double-sample rule as the training gate
|
|
244
|
+
const a = await runAndAssert(runCase, c, candidateInjection);
|
|
245
|
+
const b = a.pass ? await runAndAssert(runCase, c, candidateInjection) : { pass: false, output: '' };
|
|
246
|
+
holdoutCand.push({ name: c.name, pass: a.pass && b.pass });
|
|
247
|
+
}
|
|
248
|
+
catch {
|
|
249
|
+
holdoutCand.push({ name: c.name, pass: false });
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
holdoutRate = holdoutCand.filter((r) => r.pass).length / holdoutCand.length;
|
|
253
|
+
if (holdoutRate < holdoutBaseRate) {
|
|
254
|
+
// Restore the previous version; when there is none (first-time
|
|
255
|
+
// promotion), demote the new skill back out of active and delete.
|
|
256
|
+
const restored = await rollbackSkill(home, p.name);
|
|
257
|
+
if (!restored) {
|
|
258
|
+
await unpromoteSkill(home, p.name);
|
|
259
|
+
await deleteDraft(home, p.name);
|
|
260
|
+
}
|
|
261
|
+
await recordParetoEntry(home, { name: p.name, parentInsights: insightIds, rejectedReason: `holdout regression (${holdoutRate.toFixed(2)} < ${holdoutBaseRate.toFixed(2)})`, scores: { train: candRate, holdout: holdoutRate }, metaModel: provider.model, at: new Date().toISOString() });
|
|
262
|
+
report.outcomes.push({
|
|
263
|
+
name: p.name,
|
|
264
|
+
action: 'rejected',
|
|
265
|
+
reason: `holdout regression after promotion (train ${candRate} vs ${baseRate}, holdout ${holdoutRate} vs ${holdoutBaseRate}) - rolled back`,
|
|
266
|
+
baseline: { passRate: baseRate, cases: summary(baseResults) },
|
|
267
|
+
candidate: { passRate: candRate, cases: summary(candResults) },
|
|
268
|
+
holdout: { baselineRate: holdoutBaseRate, candidateRate: holdoutRate },
|
|
269
|
+
lineage: { parentInsights: insightIds, scores: { train: candRate, holdout: holdoutRate }, metaModel: provider.model, decidedAt: new Date().toISOString() },
|
|
270
|
+
});
|
|
271
|
+
say(` rolled back (holdout regression: ${(holdoutRate * 100).toFixed(0)}% < ${(holdoutBaseRate * 100).toFixed(0)}%)`);
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
// A promotion without any holdout case is a WEAK gate (behavior-only
|
|
276
|
+
// signal); surfaced in the log so weakly-gated skills are auditable.
|
|
277
|
+
const weakGate = holdout.length === 0;
|
|
278
|
+
report.outcomes.push({
|
|
279
|
+
name: p.name,
|
|
280
|
+
action: 'promoted',
|
|
281
|
+
reason: `no regression (train ${(candRate * 100).toFixed(0)}% vs ${(baseRate * 100).toFixed(0)}%${holdout.length ? `, holdout ${(holdoutRate * 100).toFixed(0)}%` : ''}) - promoted to CANARY (20% sessions, impact-gated full promotion)${archivedPrevious ? '; previous version archived' : ''}${weakGate ? ' [WEAK GATE: no holdout cases defined]' : ''}`,
|
|
282
|
+
baseline: { passRate: baseRate, cases: summary(baseResults) },
|
|
283
|
+
candidate: { passRate: candRate, cases: summary(candResults) },
|
|
284
|
+
...(holdout.length ? { holdout: { baselineRate: holdoutBaseRate, candidateRate: holdoutRate } } : {}),
|
|
285
|
+
lineage: { parentInsights: insightIds, scores: { train: candRate, holdout: holdout.length ? holdoutRate : undefined }, metaModel: provider.model, decidedAt: new Date().toISOString() },
|
|
286
|
+
});
|
|
287
|
+
say(` promoted to canary${holdout.length ? ` (holdout ${(holdoutRate * 100).toFixed(0)}%)` : ' [weak gate: no holdout]'}`);
|
|
288
|
+
}
|
|
289
|
+
catch (err) {
|
|
290
|
+
report.outcomes.push({ name: p.name, action: 'error', reason: String(err).slice(0, 200) });
|
|
291
|
+
say(` error: ${String(err).slice(0, 120)}`);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
// 6. CODE-LEVEL evolution: propose + sandbox-bench + merge/revert patches.
|
|
295
|
+
// This is the DGM bridge - the agent can now modify its own tool code,
|
|
296
|
+
// gated by the same bench discipline as skill promotion.
|
|
297
|
+
try {
|
|
298
|
+
const { proposePatches, runPatchSandbox } = await import("./patches.js");
|
|
299
|
+
const repoRoot = process.cwd();
|
|
300
|
+
const patches = await proposePatches(provider, signals, readFile, repoRoot, say);
|
|
301
|
+
if (patches.length > 0) {
|
|
302
|
+
report.codePatches = patches.map((p) => ({ name: p.name, file: p.file, reason: p.reason }));
|
|
303
|
+
for (const patch of patches.slice(0, 1)) { // at most 1 patch per cycle
|
|
304
|
+
const outcome = await runPatchSandbox({
|
|
305
|
+
repoRoot,
|
|
306
|
+
patch,
|
|
307
|
+
baselineRate: baseRate,
|
|
308
|
+
runCase: async (c) => runCase(c, skillsToPrompt(active)),
|
|
309
|
+
benchCases: train,
|
|
310
|
+
log: say,
|
|
311
|
+
});
|
|
312
|
+
report.patchOutcomes = report.patchOutcomes ?? [];
|
|
313
|
+
report.patchOutcomes.push(outcome);
|
|
314
|
+
say(` code-patch ${outcome.action}: ${outcome.reason}`);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
catch (err) {
|
|
319
|
+
say(` code-evolution skipped: ${String(err).slice(0, 120)}`);
|
|
320
|
+
}
|
|
321
|
+
// 7. Append-only memory distillation.
|
|
322
|
+
if (notes.length >= 4) {
|
|
323
|
+
const distilled = await distillMemory(provider, notes.map((n) => n.text).slice(-20), say);
|
|
324
|
+
const distillPoison = distilled ? screenForPoison(distilled) : null;
|
|
325
|
+
if (distillPoison) {
|
|
326
|
+
say(`memory distillation rejected by poisoning screen (${distillPoison})`);
|
|
327
|
+
}
|
|
328
|
+
else if (distilled) {
|
|
329
|
+
await appendMemory(home, `(distilled) ${distilled}`);
|
|
330
|
+
report.memoryDistilled = distilled;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
// 7.5. P0 impact loop: graduate/retire canaries on evidence; P1 decay:
|
|
334
|
+
// quiet skills leave the injection set (never deleted). Both are
|
|
335
|
+
// best-effort - measurement must never fail the cycle.
|
|
336
|
+
try {
|
|
337
|
+
report.budget = { cyclesToday: budget.cyclesToday, maxCyclesPerDay: budget.maxCyclesPerDay };
|
|
338
|
+
const impact = await impactReport(home);
|
|
339
|
+
if (impact.rows.length > 0) {
|
|
340
|
+
report.impact = {
|
|
341
|
+
rows: impact.rows.map((r) => ({ skill: r.skill, exposed: `${r.exposed.sessions}s/${(r.exposed.okRate * 100).toFixed(0)}%`, control: `${r.control.sessions}s/${(r.control.okRate * 100).toFixed(0)}%`, verdict: r.verdict })),
|
|
342
|
+
applied: impact.applied,
|
|
343
|
+
};
|
|
344
|
+
for (const a of impact.applied)
|
|
345
|
+
say(` impact: ${a}`);
|
|
346
|
+
}
|
|
347
|
+
const decayed = await decayUnusedSkills(home);
|
|
348
|
+
if (decayed.length) {
|
|
349
|
+
report.decayed = decayed;
|
|
350
|
+
say(` decayed to dormant: ${decayed.join(', ')}`);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
catch (err) {
|
|
354
|
+
say(` impact/decay skipped: ${String(err).slice(0, 100)}`);
|
|
355
|
+
}
|
|
356
|
+
// 7. Durable evolution log.
|
|
357
|
+
const logDir = join(home, 'evolution');
|
|
358
|
+
await mkdir(logDir, { recursive: true });
|
|
359
|
+
await appendFile(join(logDir, 'log.jsonl'), JSON.stringify(report) + '\n', 'utf8');
|
|
360
|
+
return report;
|
|
361
|
+
}
|
|
362
|
+
async function proposeSkills(provider, signals, say, ancestor, mergeWith) {
|
|
363
|
+
const system = [
|
|
364
|
+
'You are the evolution module of hmharness, a self-evolving agent framework for HarmonyOS development.',
|
|
365
|
+
'Your job: read session signals and decide whether any repeatable procedure is worth crystallizing into a skill.',
|
|
366
|
+
'A skill is a markdown how-to document the agent reads on demand. Topics must be limited to: HarmonyOS toolchain usage (hdc/hvigorw/ohpm/DevEco), this framework\'s tools (list_dir/read_file/write_file/run_command/remember/harmony_*), and reusable task workflows observed in the signals.',
|
|
367
|
+
'Rules: name is kebab-case; description is one line; skill_md is at most 60 lines with concrete steps and example commands; do NOT propose skills about security config, approval policy, or anything outside the topics; if nothing is genuinely reusable, return an empty array.',
|
|
368
|
+
'Respond with ONLY a JSON array: [{"name":"...","description":"...","skill_md":"..."}] - no prose, no code fences.',
|
|
369
|
+
].join('\n');
|
|
370
|
+
// GEPA ancestor context: vary around a past rejection (its reason is the
|
|
371
|
+
// lesson), optionally crossing with a complementary one - steady-state
|
|
372
|
+
// genetic sampling instead of always restarting from scratch.
|
|
373
|
+
let lineage = '';
|
|
374
|
+
if (ancestor) {
|
|
375
|
+
lineage = `\nPast rejected candidate (sampled from the archive - learn from why it failed, propose a VARIATION that fixes it):\nname: ${ancestor.name}\nrejected because: ${ancestor.rejectedReason ?? 'unknown'}\n${ancestor.skillMd ? `its content (first 1500 chars):\n${ancestor.skillMd.slice(0, 1500)}\n` : ''}`;
|
|
376
|
+
if (mergeWith) {
|
|
377
|
+
lineage += `\nA second rejected candidate with a DIFFERENT failure mode - consider merging their complementary angles:\nname: ${mergeWith.name}\nrejected because: ${mergeWith.rejectedReason ?? 'unknown'}\n${mergeWith.skillMd ? `content (first 1000 chars):\n${mergeWith.skillMd.slice(0, 1000)}\n` : ''}`;
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
const user = `Session signals:\n${JSON.stringify(signals, null, 2)}${lineage}\n\nIf signals.ecosystemNews mentions recent OpenHarmony releases, prefer proposals that account for them over stale toolchain advice.\n\nPropose at most 2 skills (or []).`;
|
|
381
|
+
const r = await chat(provider, [
|
|
382
|
+
{ role: 'system', content: system },
|
|
383
|
+
{ role: 'user', content: user },
|
|
384
|
+
]);
|
|
385
|
+
const raw = r.message.content ?? '[]';
|
|
386
|
+
const parsed = parseJsonArray(raw);
|
|
387
|
+
const out = [];
|
|
388
|
+
for (const item of parsed) {
|
|
389
|
+
const o = item;
|
|
390
|
+
if (typeof o.name === 'string' && typeof o.skill_md === 'string' && o.name && o.skill_md) {
|
|
391
|
+
out.push({ name: o.name, description: typeof o.description === 'string' ? o.description : '', skill_md: o.skill_md });
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
say(`proposals: ${out.length ? out.map((p) => p.name).join(', ') : '(none)'}${ancestor ? ` (ancestor: ${ancestor.name})` : ''}`);
|
|
395
|
+
return out;
|
|
396
|
+
}
|
|
397
|
+
async function distillMemory(provider, recentNotes, say) {
|
|
398
|
+
const system = [
|
|
399
|
+
'You distill long-term memory notes for an agent. Given recent notes, write ONE new note (max 200 chars, Chinese or English matching the notes) that captures a repeated lesson or stable fact not yet obvious from any single note.',
|
|
400
|
+
'If the notes are too scattered to synthesize anything genuinely new, output exactly: NONE',
|
|
401
|
+
'Output only the note text, nothing else.',
|
|
402
|
+
].join('\n');
|
|
403
|
+
try {
|
|
404
|
+
const r = await chat(provider, [
|
|
405
|
+
{ role: 'system', content: system },
|
|
406
|
+
{ role: 'user', content: recentNotes.join('\n') },
|
|
407
|
+
]);
|
|
408
|
+
const text = (r.message.content ?? '').trim();
|
|
409
|
+
if (!text || text === 'NONE' || text.length > 300) {
|
|
410
|
+
say('memory distillation: none');
|
|
411
|
+
return null;
|
|
412
|
+
}
|
|
413
|
+
say(`memory distillation: ${text.slice(0, 80)}`);
|
|
414
|
+
return text;
|
|
415
|
+
}
|
|
416
|
+
catch {
|
|
417
|
+
return null;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
/** First balanced JSON array in the text; tolerates fences and prose around it. */
|
|
421
|
+
function parseJsonArray(text) {
|
|
422
|
+
const cleaned = text.replace(/```(?:json)?/g, '');
|
|
423
|
+
const start = cleaned.indexOf('[');
|
|
424
|
+
if (start < 0)
|
|
425
|
+
return [];
|
|
426
|
+
let depth = 0;
|
|
427
|
+
let inStr = false;
|
|
428
|
+
let esc = false;
|
|
429
|
+
for (let i = start; i < cleaned.length; i++) {
|
|
430
|
+
const ch = cleaned[i];
|
|
431
|
+
if (esc) {
|
|
432
|
+
esc = false;
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
if (ch === '\\') {
|
|
436
|
+
esc = true;
|
|
437
|
+
continue;
|
|
438
|
+
}
|
|
439
|
+
if (ch === '"')
|
|
440
|
+
inStr = !inStr;
|
|
441
|
+
if (inStr)
|
|
442
|
+
continue;
|
|
443
|
+
if (ch === '[' || ch === '{')
|
|
444
|
+
depth++;
|
|
445
|
+
else if (ch === ']' || ch === '}') {
|
|
446
|
+
depth--;
|
|
447
|
+
if (depth === 0 && ch === ']') {
|
|
448
|
+
try {
|
|
449
|
+
return JSON.parse(cleaned.slice(start, i + 1));
|
|
450
|
+
}
|
|
451
|
+
catch {
|
|
452
|
+
return [];
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
return [];
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Write-channel poisoning screen. Conservative pattern set; a hit rejects
|
|
461
|
+
* the draft outright (the bench cannot see instructions the model ignores).
|
|
462
|
+
* Deliberately窄: catches suppression/renaming/bypass directives, not
|
|
463
|
+
* ordinary technical content.
|
|
464
|
+
*/
|
|
465
|
+
const POISON_PATTERNS = [
|
|
466
|
+
{ re: /(不要|别|never|don'?t)\s*(调用|使用|执行|call|use|run)[^。\n]{0,20}(工具|tool)/i, why: 'suppresses tool usage' },
|
|
467
|
+
{ re: /(凭记忆|from memory|without (calling|using))[^。\n]{0,20}(回答|工具|tool)/i, why: 'answers from memory instead of tools' },
|
|
468
|
+
{ re: /(一律|always|永远|never)[^。\n]{0,30}(写成|写作|rename|称作|call it)\s*["'"]?[a-z_]+["'"]?/i, why: 'renames a toolchain identifier' },
|
|
469
|
+
{ re: /(跳过|绕过|忽略|bypass|skip|ignore)[^。\n]{0,20}(审批|批准|门禁|approval|confirm|guard|deny)/i, why: 'attempts to bypass the approval gate' },
|
|
470
|
+
{ re: /(自动批准|auto[- ]?approve|always approve)/i, why: 'attempts to force auto-approval' },
|
|
471
|
+
{ re: /(删除|disable|remove)[^。\n]{0,20}(护栏|防护|safety|deny|guard)/i, why: 'attempts to disable safety guards' },
|
|
472
|
+
];
|
|
473
|
+
export function screenForPoison(text) {
|
|
474
|
+
for (const p of POISON_PATTERNS) {
|
|
475
|
+
if (p.re.test(text))
|
|
476
|
+
return p.why;
|
|
477
|
+
}
|
|
478
|
+
return null;
|
|
479
|
+
}
|
package/dist/impact.d.ts
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
export interface EvolutionBudget {
|
|
2
|
+
maxCyclesPerDay?: number;
|
|
3
|
+
maxTokensPerCycle?: number;
|
|
4
|
+
}
|
|
5
|
+
/** How many evolve cycles already ran today (counts the log.jsonl). */
|
|
6
|
+
export declare function readBudget(home: string): Promise<EvolutionBudget & {
|
|
7
|
+
cyclesToday: number;
|
|
8
|
+
}>;
|
|
9
|
+
export interface ParetoEntry {
|
|
10
|
+
name: string;
|
|
11
|
+
parentInsights: string[];
|
|
12
|
+
rejectedReason?: string;
|
|
13
|
+
scores: {
|
|
14
|
+
train: number;
|
|
15
|
+
holdout?: number;
|
|
16
|
+
};
|
|
17
|
+
metaModel: string;
|
|
18
|
+
at: string;
|
|
19
|
+
/** skill_md snapshot for merge-crossing and ancestor re-proposal */
|
|
20
|
+
skillMd?: string;
|
|
21
|
+
}
|
|
22
|
+
/** Rejected proposals are never garbage - they are the population's
|
|
23
|
+
* diversity (GEPA's Pareto front). Kept under evolution/pareto/. */
|
|
24
|
+
export declare function recordParetoEntry(home: string, entry: ParetoEntry): Promise<void>;
|
|
25
|
+
export declare function readParetoEntries(home: string, limit?: number): Promise<ParetoEntry[]>;
|
|
26
|
+
/** GEPA's ancestor selection: ONE random entry from the pool feeds the
|
|
27
|
+
* next proposal prompt (steady-state genetic loop - the pool exists so
|
|
28
|
+
* evolution does not collapse onto the single global best). Two entries
|
|
29
|
+
* with complementary rejection reasons are returned for a Merge cross. */
|
|
30
|
+
export declare function sampleAncestor(entries: ParetoEntry[]): {
|
|
31
|
+
ancestor: ParetoEntry | null;
|
|
32
|
+
mergeWith: ParetoEntry | null;
|
|
33
|
+
};
|
|
34
|
+
/** Deterministic per-session canary assignment: a session gets the canary
|
|
35
|
+
* block if hash(sessionId) % 100 < 20 - the same session always resolves
|
|
36
|
+
* the same way (stable attribution), different sessions split ~20/80. */
|
|
37
|
+
export declare function sessionGetsCanary(sessionId: string): boolean;
|
|
38
|
+
/** The watermark every canary injection carries (Misevolve's mitigation:
|
|
39
|
+
* experimental knowledge must read as a REFERENCE to weigh, not a rule
|
|
40
|
+
* to obey - memory-as-rules is what decays safety alignment). */
|
|
41
|
+
export declare function canaryWatermark(names: string[]): string;
|
|
42
|
+
export interface ImpactRow {
|
|
43
|
+
skill: string;
|
|
44
|
+
window: 'canary-period';
|
|
45
|
+
exposed: {
|
|
46
|
+
sessions: number;
|
|
47
|
+
okRate: number;
|
|
48
|
+
};
|
|
49
|
+
control: {
|
|
50
|
+
sessions: number;
|
|
51
|
+
okRate: number;
|
|
52
|
+
};
|
|
53
|
+
verdict: 'insufficient-data' | 'keep' | 'promote' | 'retire';
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The impact loop: for each canary skill, compare sessions where it was
|
|
57
|
+
* injected (Insight.skillsInjected contains it) against sessions where it
|
|
58
|
+
* was not. Promote on evidence, retire on harm, and say so honestly when
|
|
59
|
+
* the data is too thin (never let a small sample auto-graduate anything).
|
|
60
|
+
* This is the attribution loop that makes "越用越聪明" falsifiable.
|
|
61
|
+
*/
|
|
62
|
+
export declare function impactReport(home: string): Promise<{
|
|
63
|
+
rows: ImpactRow[];
|
|
64
|
+
applied: string[];
|
|
65
|
+
}>;
|
|
66
|
+
/** Move active skills with zero injections in 30 days to skills/dormant/
|
|
67
|
+
* - not deleted (append-only red line), just out of the injection set and
|
|
68
|
+
* the prompt budget. The Voyager lesson (reversed): its ever-growing
|
|
69
|
+
* library was a selling point in the paper but is a retrieval-quality
|
|
70
|
+
* debt in production; decay is the missing lifecycle operator. */
|
|
71
|
+
export declare function decayUnusedSkills(home: string, days?: number): Promise<string[]>;
|