@hmharness/evolution 0.14.9 → 0.14.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/index.d.ts +1 -0
  2. package/dist/index.js +1 -0
  3. package/dist/reward-model.d.ts +62 -0
  4. package/dist/reward-model.js +184 -0
  5. package/package.json +1 -1
  6. package/dist/bench.d.ts.v2bak.v2bak +0 -44
  7. package/dist/bench.js.v2bak.v2bak +0 -156
  8. package/dist/candidates.d.ts.v2bak.v2bak +0 -114
  9. package/dist/candidates.js.v2bak.v2bak +0 -232
  10. package/dist/dataset.d.ts.v2bak.v2bak +0 -64
  11. package/dist/dataset.js.v2bak.v2bak +0 -184
  12. package/dist/evolve.d.ts.v2bak.v2bak +0 -98
  13. package/dist/evolve.js.v2bak.v2bak +0 -573
  14. package/dist/impact.d.ts.v2bak.v2bak +0 -77
  15. package/dist/impact.js.v2bak.v2bak +0 -214
  16. package/dist/index.d.ts.v2bak.v2bak +0 -16
  17. package/dist/index.js.v2bak.v2bak +0 -16
  18. package/dist/insights.d.ts.v2bak.v2bak +0 -18
  19. package/dist/insights.js.v2bak.v2bak +0 -80
  20. package/dist/knowledge.d.ts.v2bak.v2bak +0 -15
  21. package/dist/knowledge.js.v2bak.v2bak +0 -145
  22. package/dist/labels.d.ts.v2bak.v2bak +0 -20
  23. package/dist/labels.js.v2bak.v2bak +0 -57
  24. package/dist/memory.d.ts.v2bak.v2bak +0 -32
  25. package/dist/memory.js.v2bak.v2bak +0 -233
  26. package/dist/patches.d.ts.v2bak.v2bak +0 -83
  27. package/dist/patches.js.v2bak.v2bak +0 -253
  28. package/dist/radar.d.ts.v2bak.v2bak +0 -3
  29. package/dist/radar.js.v2bak.v2bak +0 -40
  30. package/dist/ranker.d.ts.v2bak.v2bak +0 -44
  31. package/dist/ranker.js.v2bak.v2bak +0 -55
  32. package/dist/readiness.d.ts.v2bak.v2bak +0 -15
  33. package/dist/readiness.js.v2bak.v2bak +0 -167
  34. package/dist/skillpayload.d.ts.v2bak.v2bak +0 -1
  35. package/dist/skillpayload.js.v2bak +0 -28
  36. package/dist/skillpayload.js.v2bak.v2bak.v2bak +0 -28
  37. package/dist/skills.d.ts.v2bak.v2bak +0 -54
  38. package/dist/skills.js.v2bak.v2bak +0 -321
  39. package/dist/workflows.d.ts.v2bak.v2bak +0 -28
  40. package/dist/workflows.js.v2bak.v2bak +0 -84
@@ -1,232 +0,0 @@
1
- /**
2
- * @hmharness/evolution - candidates (V2 M9: Evolution 泛化)
3
- * Blueprint M9: a typed candidate registry (prompt/skill/context/tool_policy/
4
- * model_router/workflow/harness), a Control/Treatment experiment runner,
5
- * statistical comparison, and gated promotion/rollback. See ADR-0003.
6
- *
7
- * 红线 (blueprint 14.3, enforced here, not by convention):
8
- * - 无实验报告(promote-eligible)不晋升 —— no baseline, no improvement claim
9
- * - 激活前必写 previous 指针 —— no rollback, no promotion
10
- * - origin=agent 的 prompt/harness 候选需人工标记才可激活 —— agents never
11
- * self-modify the production prompt
12
- * - 晋升判据是双比例检验,LLM self-report 无效力 —— M2 证据阶梯双保险
13
- */
14
- import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
15
- import { join } from 'node:path';
16
- import { screenForPoison } from "./evolve.js";
17
- export const CANDIDATE_TARGETS = [
18
- 'prompt', 'skill', 'context', 'tool_policy', 'model_router', 'workflow', 'harness',
19
- ];
20
- /* ---------------- registry ---------------- */
21
- function candDir(home) {
22
- return join(home, 'evolution', 'candidates');
23
- }
24
- export async function registerCandidate(home, cand) {
25
- if (!CANDIDATE_TARGETS.includes(cand.target))
26
- throw new Error(`unknown candidate target: ${cand.target}`);
27
- if (!cand.hypothesis.trim() || !cand.candidateVersion.trim())
28
- throw new Error('candidate needs a hypothesis and a candidateVersion');
29
- const poison = screenForPoison(cand.hypothesis + '\n' + (cand.payload ?? ''));
30
- if (poison)
31
- throw new Error(`candidate rejected by safety screen: ${poison}`);
32
- const full = {
33
- ...cand,
34
- id: cand.id ?? `cand_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`,
35
- createdAt: new Date().toISOString(),
36
- };
37
- await mkdir(candDir(home), { recursive: true });
38
- await writeFile(join(candDir(home), `${full.id}.json`), JSON.stringify(full, null, 2) + '\n', 'utf8');
39
- return full;
40
- }
41
- export async function listCandidates(home) {
42
- let files = [];
43
- try {
44
- files = (await readdir(candDir(home))).filter((f) => f.endsWith('.json'));
45
- }
46
- catch {
47
- return [];
48
- }
49
- const out = [];
50
- for (const f of files) {
51
- try {
52
- out.push(JSON.parse(await readFile(join(candDir(home), f), 'utf8')));
53
- }
54
- catch { /* skip torn */ }
55
- }
56
- return out.sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1));
57
- }
58
- export async function getCandidate(home, id) {
59
- try {
60
- return JSON.parse(await readFile(join(candDir(home), `${id}.json`), 'utf8'));
61
- }
62
- catch {
63
- return null;
64
- }
65
- }
66
- /* ---------------- statistics ---------------- */
67
- /** Standard normal CDF, Zelen & Severo approximation of A&S 26.2.17 (|err|<7.5e-8). */
68
- function normalCdf(z) {
69
- const t = 1 / (1 + 0.2316419 * Math.abs(z));
70
- const d = Math.exp(-z * z / 2) / Math.sqrt(2 * Math.PI);
71
- const poly = t * (0.319381530 + t * (-0.356563782 + t * (1.781477937 + t * (-1.821255978 + t * 1.330274429))));
72
- const upper = d * poly; // P(Z > |z|) tail
73
- return z >= 0 ? 1 - upper : upper;
74
- }
75
- /** Two-proportion two-sided test (treatment vs control pass rates). */
76
- export function twoProportionTest(control, treatment) {
77
- if (control.n <= 0 || treatment.n <= 0)
78
- return { z: 0, p: 1, diff: 0 };
79
- const pc = control.pass / control.n;
80
- const pt = treatment.pass / treatment.n;
81
- const pool = (control.pass + treatment.pass) / (control.n + treatment.n);
82
- const se = Math.sqrt(pool * (1 - pool) * (1 / control.n + 1 / treatment.n));
83
- const z = se > 0 ? (pt - pc) / se : 0;
84
- return { z, p: 2 * (1 - normalCdf(Math.abs(z))), diff: pt - pc };
85
- }
86
- /** Per-arm minimum before any verdict counts (aligned with impact.ts canary
87
- * threshold: >=8 sessions, and blueprint "one success is not significance"). */
88
- export const MIN_ARM_N = 8;
89
- export function verdictFor(control, treatment) {
90
- const { z, p, diff } = twoProportionTest(control, treatment);
91
- if (control.n < MIN_ARM_N || treatment.n < MIN_ARM_N) {
92
- return { verdict: 'needs-data', reason: `samples below ${MIN_ARM_N} per arm (control ${control.n}, treatment ${treatment.n})`, z, p, diff };
93
- }
94
- if (p < 0.05 && diff >= 0.10)
95
- return { verdict: 'promote-eligible', reason: `significant lift (p=${p.toFixed(4)}, diff=${(diff * 100).toFixed(1)}%)`, z, p, diff };
96
- if ((p < 0.05 && diff <= -0.05) || diff <= -0.10) {
97
- return { verdict: 'reject', reason: `regression (p=${p.toFixed(4)}, diff=${(diff * 100).toFixed(1)}%)`, z, p, diff };
98
- }
99
- return { verdict: 'needs-data', reason: `no significant difference (p=${p.toFixed(4)}, diff=${(diff * 100).toFixed(1)}%)`, z, p, diff };
100
- }
101
- export async function runCandidateExperiment(home, candidateId, opts) {
102
- const cand = await getCandidate(home, candidateId);
103
- if (!cand)
104
- throw new Error(`no candidate: ${candidateId}`);
105
- // holdout cases are excluded from promotion gates (bench.ts anti-memorization)
106
- const gate = opts.cases.filter((c) => !c.holdout).slice(0, Math.max(1, opts.maxCases ?? 12));
107
- let cPass = 0, cTok = 0, tPass = 0, tTok = 0;
108
- // per-case rows (day-56 observability: which case flipped is the whole
109
- // point of a diff - aggregates alone cannot answer it)
110
- const ctlRows = [];
111
- const trtRows = [];
112
- for (const c of gate) {
113
- const ctl = await opts.runCase(c, 'control');
114
- if (ctl.pass)
115
- cPass++;
116
- cTok += ctl.tokens;
117
- ctlRows.push({ name: c.name, pass: ctl.pass, tokens: ctl.tokens });
118
- const trt = await opts.runCase(c, 'treatment');
119
- if (trt.pass)
120
- tPass++;
121
- tTok += trt.tokens;
122
- trtRows.push({ name: c.name, pass: trt.pass, tokens: trt.tokens });
123
- }
124
- const v = verdictFor({ pass: cPass, n: gate.length }, { pass: tPass, n: gate.length });
125
- const report = {
126
- candidateId,
127
- ranAt: new Date().toISOString(),
128
- cases: gate.length,
129
- control: { pass: cPass, n: gate.length, passRate: gate.length ? cPass / gate.length : 0, tokens: cTok, results: ctlRows },
130
- treatment: { pass: tPass, n: gate.length, passRate: gate.length ? tPass / gate.length : 0, tokens: tTok, results: trtRows },
131
- ...v,
132
- };
133
- const dir = join(home, 'evolution', 'experiments', candidateId);
134
- await mkdir(dir, { recursive: true });
135
- await writeFile(join(dir, `${report.ranAt.replace(/[:.]/g, '-')}.json`), JSON.stringify(report, null, 2) + '\n', 'utf8');
136
- return report;
137
- }
138
- /** Latest experiment report for a candidate (promotion evidence). */
139
- export async function latestExperiment(home, candidateId) {
140
- const dir = join(home, 'evolution', 'experiments', candidateId);
141
- let files = [];
142
- try {
143
- files = (await readdir(dir)).filter((f) => f.endsWith('.json')).sort();
144
- }
145
- catch {
146
- return null;
147
- }
148
- if (files.length === 0)
149
- return null;
150
- try {
151
- return JSON.parse(await readFile(join(dir, files[files.length - 1]), 'utf8'));
152
- }
153
- catch {
154
- return null;
155
- }
156
- }
157
- const activeDir = (home) => join(home, 'evolution', 'active');
158
- export async function activeVersions(home) {
159
- const out = {};
160
- let files = [];
161
- try {
162
- files = await readdir(activeDir(home));
163
- }
164
- catch {
165
- return out;
166
- }
167
- for (const f of files.filter((f) => f.endsWith('.json') && !f.endsWith('.previous.json'))) {
168
- try {
169
- const v = JSON.parse(await readFile(join(activeDir(home), f), 'utf8'));
170
- out[v.target] = v;
171
- }
172
- catch { /* skip torn */ }
173
- }
174
- return out;
175
- }
176
- /** Blueprint gates, enforced: latest report must be promote-eligible, a
177
- * previous pointer is written BEFORE activation, agent-origin prompt/harness
178
- * candidates need approvedByHuman. Skill promotions ride the existing
179
- * promoteSkill canary channel. */
180
- export async function promoteCandidate(home, candidateId, opts = {}) {
181
- const cand = await getCandidate(home, candidateId);
182
- if (!cand)
183
- return { ok: false, error: `no candidate: ${candidateId}` };
184
- const report = await latestExperiment(home, candidateId);
185
- if (!report)
186
- return { ok: false, error: 'no experiment report - run the experiment first (no baseline, no promotion)' };
187
- if (report.verdict !== 'promote-eligible') {
188
- return { ok: false, error: `latest report verdict is ${report.verdict} (${report.reason}) - not promotable` };
189
- }
190
- if ((cand.target === 'prompt' || cand.target === 'harness') && cand.origin === 'agent' && !opts.approvedByHuman) {
191
- return { ok: false, error: 'agent-origin prompt/harness candidate requires approvedByHuman (production prompt is not agent-writable)' };
192
- }
193
- await mkdir(activeDir(home), { recursive: true });
194
- const cur = await activeVersions(home);
195
- const previous = cur[cand.target];
196
- if (previous) {
197
- await writeFile(join(activeDir(home), `${cand.target}.previous.json`), JSON.stringify(previous, null, 2) + '\n', 'utf8');
198
- }
199
- if (cand.target === 'skill') {
200
- // payload = skill name; the audited canary channel stays the only skill path
201
- const { promoteSkill } = await import("./skills.js");
202
- if (!cand.payload)
203
- return { ok: false, error: 'skill candidate has no payload (skill name)' };
204
- await promoteSkill(home, cand.payload, { canary: true });
205
- }
206
- const active = {
207
- target: cand.target,
208
- version: cand.candidateVersion,
209
- candidateId: cand.id,
210
- since: new Date().toISOString(),
211
- reportAt: report.ranAt,
212
- };
213
- await writeFile(join(activeDir(home), `${cand.target}.json`), JSON.stringify(active, null, 2) + '\n', 'utf8');
214
- return { ok: true, active };
215
- }
216
- export async function rollbackCandidate(home, target) {
217
- const dir = activeDir(home);
218
- const prevPath = join(dir, `${target}.previous.json`);
219
- let previous = null;
220
- try {
221
- previous = JSON.parse(await readFile(prevPath, 'utf8'));
222
- }
223
- catch { /* no previous */ }
224
- if (!previous)
225
- return { ok: false, error: `no previous pointer for ${target} - nothing to roll back to` };
226
- if (target === 'skill') {
227
- const { rollbackSkill } = await import("./skills.js");
228
- await rollbackSkill(home, previous.version);
229
- }
230
- await writeFile(join(dir, `${target}.json`), JSON.stringify(previous, null, 2) + '\n', 'utf8');
231
- return { ok: true };
232
- }
@@ -1,64 +0,0 @@
1
- export interface DatasetSample {
2
- runId: string;
3
- task: string;
4
- outcome: string;
5
- turns: number;
6
- toolUses: number;
7
- toolFailRate: number;
8
- toolsUsed: string[];
9
- model: string;
10
- /** evidence rank from the M2 ladder when the run was evaluated (0 = none) */
11
- evidenceRank: number;
12
- /** interpretable reward in [0,1]; llm-judged runs never reach 1.0 (M2 cap) */
13
- reward: number;
14
- /** evaluation notes / failure snippets attach here (bounded) */
15
- evidence?: string;
16
- split: 'train' | 'eval';
17
- }
18
- export interface DatasetManifest {
19
- version: string;
20
- createdAt: string;
21
- /** inclusive run-id scan window (first..last seen, lexicographic) */
22
- runWindow: {
23
- first: string | null;
24
- last: string | null;
25
- };
26
- filterFingerprint: string;
27
- splitSeed: number;
28
- splitRatio: number;
29
- counts: {
30
- scanned: number;
31
- kept: number;
32
- dropped: number;
33
- duplicates: number;
34
- train: number;
35
- eval: number;
36
- };
37
- rewardHistogram: Record<string, number>;
38
- }
39
- export interface DatasetBuildResult {
40
- manifest: DatasetManifest;
41
- dir: string;
42
- }
43
- /** Interpretable reward mapping (ADR-0004): ok runs start at 1.0 and lose
44
- * 0.1 per 20% tool-failure rate; non-ok runs cap at 0.3. Evidence rank
45
- * from the M2 ladder nudges confidence but never creates a free perfect. */
46
- export declare function rewardFor(outcome: string, toolFailRate: number, evidenceRank?: number): number;
47
- /** Stable fingerprint of the filter config - manifest must be reproducible. */
48
- export declare function filterFingerprint(opts: {
49
- keepOutcome?: string[];
50
- minTurns?: number;
51
- }): string;
52
- /** Deterministic hash-based split (same run always lands in the same half). */
53
- export declare function splitFor(runId: string, seed: number, evalRatio?: number): 'train' | 'eval';
54
- export interface BuildDatasetOptions {
55
- version?: string;
56
- /** outcomes to keep; default keeps everything labeled */
57
- keepOutcome?: string[];
58
- minTurns?: number;
59
- evalRatio?: number;
60
- splitSeed?: number;
61
- }
62
- export declare function buildDataset(home: string, opts?: BuildDatasetOptions): Promise<DatasetBuildResult>;
63
- export declare function listDatasets(home: string): Promise<DatasetManifest[]>;
64
- export declare function loadDataset(home: string, version: string, split?: 'train' | 'eval'): Promise<DatasetSample[]>;
@@ -1,184 +0,0 @@
1
- /**
2
- * @hmharness/evolution - dataset (V2 M10: Trajectory → Dataset Version)
3
- * Pure data engineering, zero model calls. Pipeline per blueprint:
4
- * Trajectory → Filter → Deduplicate → Label → Evidence Attach → Reward
5
- * → Dataset Version → Train/Eval Split
6
- * Source of truth: HMH_HOME/runs/<run-id>/ (M1 trajectory recorder's
7
- * summary.json + trajectory.jsonl). Output: versioned, redacted, split
8
- * under evolution/datasets/<version>/. See ADR-0004.
9
- */
10
- import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
11
- import { createHash } from 'node:crypto';
12
- import { join } from 'node:path';
13
- import { redactSecrets } from "./insights.js";
14
- const RUNS_DIR = 'runs';
15
- /** Interpretable reward mapping (ADR-0004): ok runs start at 1.0 and lose
16
- * 0.1 per 20% tool-failure rate; non-ok runs cap at 0.3. Evidence rank
17
- * from the M2 ladder nudges confidence but never creates a free perfect. */
18
- export function rewardFor(outcome, toolFailRate, evidenceRank = 0) {
19
- let r;
20
- if (outcome === 'ok') {
21
- r = 1.0 - Math.min(0.6, Math.round(toolFailRate * 10) / 10);
22
- }
23
- else {
24
- r = Math.max(0, 0.3 - Math.round(toolFailRate * 10) / 10);
25
- }
26
- // llm-judge ladder ranks (5+) can never mint a perfect sample
27
- if (evidenceRank >= 5)
28
- r = Math.min(r, 0.7);
29
- return Math.max(0, Math.min(1, r));
30
- }
31
- /** Stable fingerprint of the filter config - manifest must be reproducible. */
32
- export function filterFingerprint(opts) {
33
- return createHash('sha256').update(JSON.stringify({
34
- keepOutcome: opts.keepOutcome ?? ['ok', 'turn-budget', 'error'],
35
- minTurns: opts.minTurns ?? 1,
36
- redact: true,
37
- })).digest('hex').slice(0, 12);
38
- }
39
- /** Normalize both the recorder schema (outcome object + metrics block) and
40
- * the flat test/hand-written shape into one view. */
41
- function normalizeSummary(s) {
42
- const outcome = typeof s.outcome === 'string'
43
- ? s.outcome
44
- : (s.outcome?.success ? 'ok' : String(s.outcome?.reason ?? ''));
45
- return {
46
- task: String(s.task ?? ''),
47
- outcome,
48
- turns: Number(s.turns ?? s.metrics?.turns ?? 0),
49
- toolUses: Number(s.toolUses ?? s.metrics?.toolUses ?? 0),
50
- toolFailures: Number(s.toolFailures ?? s.metrics?.toolFailures ?? 0),
51
- toolsUsed: s.toolsUsed ?? [],
52
- model: String(s.model ?? ''),
53
- rank: Number(s.evidence?.rank ?? 0),
54
- detail: String(s.evidence?.detail ?? ''),
55
- };
56
- }
57
- async function readRunSummaries(home) {
58
- const root = join(home, RUNS_DIR);
59
- let ids = [];
60
- try {
61
- ids = (await readdir(root)).filter((d) => !d.startsWith('.'));
62
- }
63
- catch {
64
- return [];
65
- }
66
- ids.sort();
67
- const out = [];
68
- for (const id of ids) {
69
- try {
70
- const s = JSON.parse(await readFile(join(root, id, 'summary.json'), 'utf8'));
71
- out.push({ runId: id, summary: s });
72
- }
73
- catch { /* torn or absent summary - not a dataset candidate */ }
74
- }
75
- return out;
76
- }
77
- /** Deterministic hash-based split (same run always lands in the same half). */
78
- export function splitFor(runId, seed, evalRatio = 0.2) {
79
- const h = createHash('sha256').update(`${seed}:${runId}`).digest();
80
- const v = h.readUInt32BE(0) / 0xffffffff;
81
- return v < evalRatio ? 'eval' : 'train';
82
- }
83
- export async function buildDataset(home, opts = {}) {
84
- const keep = opts.keepOutcome ?? ['ok', 'turn-budget', 'error'];
85
- const minTurns = opts.minTurns ?? 1;
86
- const evalRatio = opts.evalRatio ?? 0.2;
87
- const seed = opts.splitSeed ?? 20260913;
88
- const runs = await readRunSummaries(home);
89
- const fp = filterFingerprint({ keepOutcome: keep, minTurns });
90
- const byFingerprint = new Map();
91
- let dropped = 0;
92
- let duplicates = 0;
93
- for (const { runId, summary } of runs) {
94
- const n = normalizeSummary(summary);
95
- if (!n.outcome || !keep.includes(n.outcome) || n.turns < minTurns) {
96
- dropped++;
97
- continue;
98
- }
99
- const fpRun = createHash('sha256').update(`${n.task}|${n.outcome}|${n.turns}|${n.toolsUsed.join(',')}`, 'utf8').digest('hex').slice(0, 16);
100
- if (byFingerprint.has(fpRun)) {
101
- duplicates++;
102
- continue;
103
- } // keep first (oldest) - newest is a rerun of the same shape
104
- byFingerprint.set(fpRun, { runId, summary });
105
- }
106
- const samples = [];
107
- const histogram = {};
108
- for (const { runId, summary } of byFingerprint.values()) {
109
- const n = normalizeSummary(summary);
110
- const failRate = n.toolUses > 0 ? n.toolFailures / n.toolUses : 0;
111
- const reward = rewardFor(n.outcome, failRate, n.rank);
112
- const bucket = reward.toFixed(1);
113
- histogram[bucket] = (histogram[bucket] ?? 0) + 1;
114
- samples.push({
115
- runId,
116
- task: redactSecrets(n.task),
117
- outcome: n.outcome,
118
- turns: n.turns,
119
- toolUses: n.toolUses,
120
- toolFailRate: Math.round(failRate * 100) / 100,
121
- toolsUsed: n.toolsUsed,
122
- model: n.model,
123
- evidenceRank: n.rank,
124
- reward,
125
- ...(n.detail ? { evidence: redactSecrets(n.detail).slice(0, 400) } : {}),
126
- split: splitFor(runId, seed, evalRatio),
127
- });
128
- }
129
- samples.sort((a, b) => (a.runId < b.runId ? -1 : 1));
130
- const version = opts.version ?? `v-${new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)}`;
131
- const dir = join(home, 'evolution', 'datasets', version);
132
- await mkdir(dir, { recursive: true });
133
- await writeFile(join(dir, 'samples.jsonl'), samples.map((s) => JSON.stringify(s)).join('\n') + (samples.length ? '\n' : ''), 'utf8');
134
- const manifest = {
135
- version,
136
- createdAt: new Date().toISOString(),
137
- runWindow: { first: runs[0]?.runId ?? null, last: runs[runs.length - 1]?.runId ?? null },
138
- filterFingerprint: fp,
139
- splitSeed: seed,
140
- splitRatio: evalRatio,
141
- counts: {
142
- scanned: runs.length,
143
- kept: samples.length,
144
- dropped,
145
- duplicates,
146
- train: samples.filter((s) => s.split === 'train').length,
147
- eval: samples.filter((s) => s.split === 'eval').length,
148
- },
149
- rewardHistogram: histogram,
150
- };
151
- await writeFile(join(dir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8');
152
- return { manifest, dir };
153
- }
154
- export async function listDatasets(home) {
155
- const root = join(home, 'evolution', 'datasets');
156
- let vers = [];
157
- try {
158
- vers = (await readdir(root)).filter((d) => d.startsWith('v-'));
159
- }
160
- catch {
161
- return [];
162
- }
163
- vers.sort();
164
- const out = [];
165
- for (const v of vers) {
166
- try {
167
- out.push(JSON.parse(await readFile(join(root, v, 'manifest.json'), 'utf8')));
168
- }
169
- catch { /* skip torn */ }
170
- }
171
- return out;
172
- }
173
- export async function loadDataset(home, version, split) {
174
- const file = join(home, 'evolution', 'datasets', version, 'samples.jsonl');
175
- let text;
176
- try {
177
- text = await readFile(file, 'utf8');
178
- }
179
- catch {
180
- return [];
181
- }
182
- return text.split('\n').filter(Boolean).map((l) => JSON.parse(l))
183
- .filter((s) => !split || s.split === split);
184
- }
@@ -1,98 +0,0 @@
1
- import { type ProviderConfig } from '@hmharness/kernel';
2
- import { type BenchCase } from './bench.ts';
3
- export interface SkillProposal {
4
- name: string;
5
- description: string;
6
- skill_md: string;
7
- }
8
- export interface ProposalOutcome {
9
- name: string;
10
- action: 'promoted' | 'rejected' | 'error';
11
- reason: string;
12
- baseline?: {
13
- passRate: number;
14
- cases: string;
15
- };
16
- candidate?: {
17
- passRate: number;
18
- cases: string;
19
- };
20
- holdout?: {
21
- baselineRate: number;
22
- candidateRate: number;
23
- };
24
- /** P0 lineage ledger: what fed this decision (provenance for impact
25
- * attribution - which insights produced which skill, with which scores).
26
- * Evolution-artifact genealogy is the GEPA/DGM archive lesson: decisions
27
- * without ancestry cannot be audited for objective hacking. */
28
- lineage?: {
29
- parentInsights: string[];
30
- scores: {
31
- train: number;
32
- holdout?: number;
33
- };
34
- metaModel: string;
35
- decidedAt: string;
36
- };
37
- }
38
- export interface EvolveReport {
39
- time: string;
40
- model: string;
41
- seededCases: string[];
42
- insightCount: number;
43
- noteCount: number;
44
- proposals: SkillProposal[];
45
- outcomes: ProposalOutcome[];
46
- memoryDistilled: string | null;
47
- /** code-level evolution: proposed patches (DGM bridge) */
48
- codePatches?: Array<{
49
- name: string;
50
- file: string;
51
- reason: string;
52
- }>;
53
- /** code-level outcomes: merged/reverted/error per patch */
54
- patchOutcomes?: Array<{
55
- name: string;
56
- action: string;
57
- reason: string;
58
- branch?: string;
59
- }>;
60
- /** P0 impact attribution: canary A/B comparison applied this cycle */
61
- impact?: {
62
- rows: Array<{
63
- skill: string;
64
- exposed: string;
65
- control: string;
66
- verdict: string;
67
- }>;
68
- applied: string[];
69
- };
70
- /** P1 lifecycle: skills moved to dormant this cycle */
71
- decayed?: string[];
72
- /** budget state at cycle start (observability) */
73
- budget?: {
74
- cyclesToday: number;
75
- maxCyclesPerDay?: number;
76
- };
77
- /** chars/4 estimate of this cycle's meta-call traffic; feeds the daily
78
- * token budget gate (readBudget sums today's entries) */
79
- estTokens?: number;
80
- }
81
- /** Runs one bench case with the given skills block injected. */
82
- export type CaseRunner = (c: BenchCase, skillsPrompt: string) => Promise<string>;
83
- /** Rough token estimate, language-aware: ASCII runs ~4 chars/token, CJK
84
- * ~1 char/token. A flat chars/4 undercounted Chinese 2-4x, letting verbose
85
- * zh candidates dodge the cost cap. Used on BOTH sides of every comparison
86
- * (baseline and candidate), never as the only rejection reason (the
87
- * pass-rate gate decides; cost-cap only vetoes pass-by-rambling). */
88
- export declare function estTokens(text: string): number;
89
- export declare function runEvolution(opts: {
90
- home: string;
91
- provider: ProviderConfig;
92
- runCase: CaseRunner;
93
- maxProposals?: number;
94
- /** Skip the meta-model call and evaluate these proposals directly (tests / future UIs). */
95
- presetProposals?: SkillProposal[];
96
- log?: (line: string) => void;
97
- }): Promise<EvolveReport>;
98
- export declare function screenForPoison(text: string): string | null;