@hmharness/evolution 0.14.9 → 0.14.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/reward-model.d.ts +62 -0
- package/dist/reward-model.js +184 -0
- package/package.json +1 -1
- package/dist/bench.d.ts.v2bak.v2bak +0 -44
- package/dist/bench.js.v2bak.v2bak +0 -156
- package/dist/candidates.d.ts.v2bak.v2bak +0 -114
- package/dist/candidates.js.v2bak.v2bak +0 -232
- package/dist/dataset.d.ts.v2bak.v2bak +0 -64
- package/dist/dataset.js.v2bak.v2bak +0 -184
- package/dist/evolve.d.ts.v2bak.v2bak +0 -98
- package/dist/evolve.js.v2bak.v2bak +0 -573
- package/dist/impact.d.ts.v2bak.v2bak +0 -77
- package/dist/impact.js.v2bak.v2bak +0 -214
- package/dist/index.d.ts.v2bak.v2bak +0 -16
- package/dist/index.js.v2bak.v2bak +0 -16
- package/dist/insights.d.ts.v2bak.v2bak +0 -18
- package/dist/insights.js.v2bak.v2bak +0 -80
- package/dist/knowledge.d.ts.v2bak.v2bak +0 -15
- package/dist/knowledge.js.v2bak.v2bak +0 -145
- package/dist/labels.d.ts.v2bak.v2bak +0 -20
- package/dist/labels.js.v2bak.v2bak +0 -57
- package/dist/memory.d.ts.v2bak.v2bak +0 -32
- package/dist/memory.js.v2bak.v2bak +0 -233
- package/dist/patches.d.ts.v2bak.v2bak +0 -83
- package/dist/patches.js.v2bak.v2bak +0 -253
- package/dist/radar.d.ts.v2bak.v2bak +0 -3
- package/dist/radar.js.v2bak.v2bak +0 -40
- package/dist/ranker.d.ts.v2bak.v2bak +0 -44
- package/dist/ranker.js.v2bak.v2bak +0 -55
- package/dist/readiness.d.ts.v2bak.v2bak +0 -15
- package/dist/readiness.js.v2bak.v2bak +0 -167
- package/dist/skillpayload.d.ts.v2bak.v2bak +0 -1
- package/dist/skillpayload.js.v2bak +0 -28
- package/dist/skillpayload.js.v2bak.v2bak.v2bak +0 -28
- package/dist/skills.d.ts.v2bak.v2bak +0 -54
- package/dist/skills.js.v2bak.v2bak +0 -321
- package/dist/workflows.d.ts.v2bak.v2bak +0 -28
- package/dist/workflows.js.v2bak.v2bak +0 -84
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
export const RANK_WEIGHTS = {
|
|
2
|
-
relevance: 0.30,
|
|
3
|
-
dependency: 0.20,
|
|
4
|
-
recency: 0.15,
|
|
5
|
-
importance: 0.15,
|
|
6
|
-
similarity: 0.10,
|
|
7
|
-
tokenCost: -0.10,
|
|
8
|
-
};
|
|
9
|
-
const clamp01 = (n) => Number.isFinite(n) ? Math.max(0, Math.min(1, n)) : 0;
|
|
10
|
-
/** Rank candidates: fixed weights, token cost normalized to the batch max,
|
|
11
|
-
* ties broken by cheaper-first. Returns a NEW sorted array. */
|
|
12
|
-
export function rankContext(candidates) {
|
|
13
|
-
const maxCost = Math.max(1, ...candidates.map((c) => c.tokenCost));
|
|
14
|
-
return candidates
|
|
15
|
-
.map((c) => {
|
|
16
|
-
const n = {
|
|
17
|
-
relevance: clamp01(c.relevance),
|
|
18
|
-
dependency: clamp01(c.dependency),
|
|
19
|
-
recency: clamp01(c.recency),
|
|
20
|
-
importance: clamp01(c.importance),
|
|
21
|
-
similarity: clamp01(c.similarity),
|
|
22
|
-
tokenCost: clamp01(c.tokenCost / maxCost),
|
|
23
|
-
};
|
|
24
|
-
const score = RANK_WEIGHTS.relevance * n.relevance +
|
|
25
|
-
RANK_WEIGHTS.dependency * n.dependency +
|
|
26
|
-
RANK_WEIGHTS.recency * n.recency +
|
|
27
|
-
RANK_WEIGHTS.importance * n.importance +
|
|
28
|
-
RANK_WEIGHTS.similarity * n.similarity +
|
|
29
|
-
RANK_WEIGHTS.tokenCost * n.tokenCost;
|
|
30
|
-
return { ...c, score: Math.round(score * 1000) / 1000 };
|
|
31
|
-
})
|
|
32
|
-
.sort((a, b) => b.score - a.score || a.tokenCost - b.tokenCost);
|
|
33
|
-
}
|
|
34
|
-
/** Pick the top-K candidates under a token budget (greedy, ranked order). */
|
|
35
|
-
export function packContext(candidates, budgetTokens) {
|
|
36
|
-
const ranked = rankContext(candidates);
|
|
37
|
-
const out = [];
|
|
38
|
-
let used = 0;
|
|
39
|
-
for (const c of ranked) {
|
|
40
|
-
if (used + c.tokenCost > budgetTokens)
|
|
41
|
-
continue;
|
|
42
|
-
out.push(c);
|
|
43
|
-
used += c.tokenCost;
|
|
44
|
-
}
|
|
45
|
-
return out;
|
|
46
|
-
}
|
|
47
|
-
export function classifyMemory(entry) {
|
|
48
|
-
if (/^\(distilled\)/.test(entry) || /api|sdk|版本|参数/.test(entry.slice(0, 80)))
|
|
49
|
-
return 'semantic';
|
|
50
|
-
if (/design|架构|决策|why|adr/i.test(entry.slice(0, 80)))
|
|
51
|
-
return 'project';
|
|
52
|
-
if (/self-note|工具|失败|重试|how to|怎么/.test(entry.slice(0, 80)))
|
|
53
|
-
return 'procedural';
|
|
54
|
-
return 'episodic';
|
|
55
|
-
}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
export interface ReadinessCondition {
|
|
2
|
-
id: string;
|
|
3
|
-
met: boolean;
|
|
4
|
-
current: number | string | boolean;
|
|
5
|
-
threshold: number | string;
|
|
6
|
-
evidence: string;
|
|
7
|
-
}
|
|
8
|
-
export interface ReadinessReport {
|
|
9
|
-
verdict: 'rl-eligible' | 'optimize-first';
|
|
10
|
-
conditions: ReadinessCondition[];
|
|
11
|
-
/** when optimize-first: which lever to pull first (weakest condition) */
|
|
12
|
-
recommendedLever?: 'data-collection' | 'bench-expansion' | 'reward-calibration' | 'eval-suite-stabilization' | 'version-provenance';
|
|
13
|
-
at: string;
|
|
14
|
-
}
|
|
15
|
-
export declare function rlReadiness(home: string): Promise<ReadinessReport>;
|
|
@@ -1,167 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @hmharness/evolution - readiness (V2 M11: RL 前置条件检查器)
|
|
3
|
-
* The blueprint's gate: RL is allowed ONLY when all six conditions hold.
|
|
4
|
-
* Every condition is measured from real on-disk evidence - nothing is
|
|
5
|
-
* estimated, nothing can be waved through. When the gate is closed the
|
|
6
|
-
* report says exactly which optimization to do instead. See ADR-0005.
|
|
7
|
-
*/
|
|
8
|
-
import { readdir, readFile, stat } from 'node:fs/promises';
|
|
9
|
-
import { join } from 'node:path';
|
|
10
|
-
import { listCases } from "./bench.js";
|
|
11
|
-
const QUALITY_MIN_REWARD = 0.5;
|
|
12
|
-
async function countRunTrajectories(home) {
|
|
13
|
-
const root = join(home, 'runs');
|
|
14
|
-
let ids = [];
|
|
15
|
-
try {
|
|
16
|
-
ids = (await readdir(root)).filter((d) => !d.startsWith('.'));
|
|
17
|
-
}
|
|
18
|
-
catch {
|
|
19
|
-
return { total: 0, highQuality: 0 };
|
|
20
|
-
}
|
|
21
|
-
let high = 0;
|
|
22
|
-
for (const id of ids) {
|
|
23
|
-
try {
|
|
24
|
-
const s = JSON.parse(await readFile(join(root, id, 'summary.json'), 'utf8'));
|
|
25
|
-
const outcome = typeof s.outcome === 'string' ? s.outcome : (s.outcome?.success ? 'ok' : String(s.outcome?.reason ?? ''));
|
|
26
|
-
const uses = Number(s.toolUses ?? s.metrics?.toolUses ?? 0);
|
|
27
|
-
const fails = Number(s.toolFailures ?? s.metrics?.toolFailures ?? 0);
|
|
28
|
-
const failRate = uses > 0 ? fails / uses : 0;
|
|
29
|
-
const reward = outcome === 'ok' ? 1 - Math.min(0.6, Math.round(failRate * 10) / 10) : 0.3;
|
|
30
|
-
if (outcome && (s.turns ?? s.metrics?.turns ?? 0) >= 1 && reward >= QUALITY_MIN_REWARD)
|
|
31
|
-
high++;
|
|
32
|
-
}
|
|
33
|
-
catch { /* torn */ }
|
|
34
|
-
}
|
|
35
|
-
return { total: ids.length, highQuality: high };
|
|
36
|
-
}
|
|
37
|
-
export async function rlReadiness(home) {
|
|
38
|
-
const conditions = [];
|
|
39
|
-
// 1. >= 1000 high-quality trajectories
|
|
40
|
-
const runs = await countRunTrajectories(home);
|
|
41
|
-
conditions.push({
|
|
42
|
-
id: 'high-quality-trajectories',
|
|
43
|
-
met: runs.highQuality >= 1000,
|
|
44
|
-
current: runs.highQuality,
|
|
45
|
-
threshold: 1000,
|
|
46
|
-
evidence: `runs/ scan: ${runs.total} trajectories, ${runs.highQuality} with outcome + turns>=1 + reward>=${QUALITY_MIN_REWARD}`,
|
|
47
|
-
});
|
|
48
|
-
// 2. >= 100 stable benchmark tasks
|
|
49
|
-
let cases = 0;
|
|
50
|
-
try {
|
|
51
|
-
cases = (await listCases(home)).length;
|
|
52
|
-
}
|
|
53
|
-
catch { /* none */ }
|
|
54
|
-
conditions.push({
|
|
55
|
-
id: 'stable-benchmark-tasks',
|
|
56
|
-
met: cases >= 100,
|
|
57
|
-
current: cases,
|
|
58
|
-
threshold: 100,
|
|
59
|
-
evidence: `bench/cases/ scan: ${cases} cases (holdout included; stability tracked by bench history)`,
|
|
60
|
-
});
|
|
61
|
-
// 3. reward vs human judgment correlation VERIFIED - no human-labeling
|
|
62
|
-
// channel exists yet, so this is honestly not met (never a guess)
|
|
63
|
-
let humanLabels = 0;
|
|
64
|
-
try {
|
|
65
|
-
const f = await stat(join(home, 'evolution', 'reward-human-labels.jsonl'));
|
|
66
|
-
if (f.isFile())
|
|
67
|
-
humanLabels = (await readFile(join(home, 'evolution', 'reward-human-labels.jsonl'), 'utf8')).split('\n').filter(Boolean).length;
|
|
68
|
-
}
|
|
69
|
-
catch { /* absent */ }
|
|
70
|
-
conditions.push({
|
|
71
|
-
id: 'reward-human-correlation',
|
|
72
|
-
met: humanLabels >= 100,
|
|
73
|
-
current: humanLabels,
|
|
74
|
-
threshold: 100,
|
|
75
|
-
evidence: 'evolution/reward-human-labels.jsonl (human-scored samples; dataset label field reserves the slot)',
|
|
76
|
-
});
|
|
77
|
-
// 4. evaluation regression suite STABLE - last two bench records, no regression
|
|
78
|
-
let suiteStable = false;
|
|
79
|
-
let suiteEvidence = 'no bench history records (evolution/benches/ empty)';
|
|
80
|
-
try {
|
|
81
|
-
const dir = join(home, 'evolution', 'benches');
|
|
82
|
-
const files = (await readdir(dir)).filter((f) => f.endsWith('.json')).sort();
|
|
83
|
-
if (files.length >= 2) {
|
|
84
|
-
const last = JSON.parse(await readFile(join(dir, files[files.length - 1]), 'utf8'));
|
|
85
|
-
const prev = JSON.parse(await readFile(join(dir, files[files.length - 2]), 'utf8'));
|
|
86
|
-
if (typeof last.passRate === 'number' && typeof prev.passRate === 'number') {
|
|
87
|
-
suiteStable = last.passRate >= prev.passRate - 0.05;
|
|
88
|
-
suiteEvidence = `last two bench passRates: ${(prev.passRate * 100).toFixed(0)}% -> ${(last.passRate * 100).toFixed(0)}% (tolerance -5%)`;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
else if (files.length === 1) {
|
|
92
|
-
suiteEvidence = 'only one bench record - stability needs at least two';
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
catch { /* absent */ }
|
|
96
|
-
conditions.push({
|
|
97
|
-
id: 'eval-regression-stable',
|
|
98
|
-
met: suiteStable,
|
|
99
|
-
current: suiteStable,
|
|
100
|
-
threshold: 'no >5% passRate regression across last two bench runs',
|
|
101
|
-
evidence: suiteEvidence,
|
|
102
|
-
});
|
|
103
|
-
// 5. model/skill/workflow version provenance
|
|
104
|
-
const provenance = {
|
|
105
|
-
model: true, // every run's summary carries model; rollout session_meta pins it
|
|
106
|
-
skills: false,
|
|
107
|
-
workflows: false,
|
|
108
|
-
};
|
|
109
|
-
try {
|
|
110
|
-
// skills layout has BOTH shapes in the wild: legacy flat `skills/<name>.md`
|
|
111
|
-
// and current `skills/active/<name>/` directories (plus `skills/drafts/*.md`).
|
|
112
|
-
// A top-level-only .md check silently fails every modern install.
|
|
113
|
-
const entries = await readdir(join(home, 'skills'), { withFileTypes: true });
|
|
114
|
-
const activeDir = entries.find((e) => e.isDirectory() && e.name === 'active');
|
|
115
|
-
const draftsDir = entries.find((e) => e.isDirectory() && e.name === 'drafts');
|
|
116
|
-
const flatMd = entries.some((e) => e.isFile() && e.name.endsWith('.md'));
|
|
117
|
-
let active = 0;
|
|
118
|
-
if (activeDir)
|
|
119
|
-
active = (await readdir(join(home, 'skills', 'active'))).length;
|
|
120
|
-
let draftMd = 0;
|
|
121
|
-
if (draftsDir)
|
|
122
|
-
draftMd = (await readdir(join(home, 'skills', 'drafts'))).filter((f) => f.endsWith('.md')).length;
|
|
123
|
-
provenance.skills = flatMd || active > 0 || draftMd > 0;
|
|
124
|
-
}
|
|
125
|
-
catch { /* none */ }
|
|
126
|
-
try {
|
|
127
|
-
const wf = await readdir(join(home, 'evolution', 'workflows'));
|
|
128
|
-
provenance.workflows = wf.length > 0;
|
|
129
|
-
}
|
|
130
|
-
catch { /* none */ }
|
|
131
|
-
conditions.push({
|
|
132
|
-
id: 'version-provenance',
|
|
133
|
-
met: provenance.skills && provenance.workflows,
|
|
134
|
-
current: provenance.skills && provenance.workflows,
|
|
135
|
-
threshold: 'skills + workflows under version control (model provenance ships with every run)',
|
|
136
|
-
evidence: `skills dir: ${provenance.skills ? 'versioned .md files' : 'empty'}; workflows dir: ${provenance.workflows ? 'present' : 'empty'}`,
|
|
137
|
-
});
|
|
138
|
-
// 6. offline evaluation + holdout set exist
|
|
139
|
-
let holdout = 0;
|
|
140
|
-
try {
|
|
141
|
-
holdout = (await listCases(home)).filter((c) => c.holdout).length;
|
|
142
|
-
}
|
|
143
|
-
catch { /* none */ }
|
|
144
|
-
conditions.push({
|
|
145
|
-
id: 'offline-eval-holdout',
|
|
146
|
-
met: holdout > 0 && cases > 0,
|
|
147
|
-
current: holdout,
|
|
148
|
-
threshold: '>0 holdout cases in a non-empty bench suite',
|
|
149
|
-
evidence: `bench/cases/: ${cases} total, ${holdout} holdout`,
|
|
150
|
-
});
|
|
151
|
-
const allMet = conditions.every((c) => c.met);
|
|
152
|
-
const weakest = conditions.find((c) => !c.met);
|
|
153
|
-
const leverMap = {
|
|
154
|
-
'high-quality-trajectories': 'data-collection',
|
|
155
|
-
'stable-benchmark-tasks': 'bench-expansion',
|
|
156
|
-
'reward-human-correlation': 'reward-calibration',
|
|
157
|
-
'eval-regression-stable': 'eval-suite-stabilization',
|
|
158
|
-
'version-provenance': 'version-provenance',
|
|
159
|
-
'offline-eval-holdout': 'bench-expansion',
|
|
160
|
-
};
|
|
161
|
-
return {
|
|
162
|
-
verdict: allMet ? 'rl-eligible' : 'optimize-first',
|
|
163
|
-
conditions,
|
|
164
|
-
...(allMet ? {} : { recommendedLever: weakest ? leverMap[weakest.id] : undefined }),
|
|
165
|
-
at: new Date().toISOString(),
|
|
166
|
-
};
|
|
167
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function loadSkillPayload(home: string, payload: string): Promise<string>;
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Resolve a treatment-arm skills prompt for a skill-target candidate: the
|
|
3
|
-
* payload names the skill; the arm must inject the skill's CONTENT (draft
|
|
4
|
-
* first, promoted second), never the bare name string. (Day-49 SELFFEED:
|
|
5
|
-
* the first real experiment injected "+34 tokens" of name - it measured the
|
|
6
|
-
* instrument, not the skill.)
|
|
7
|
-
*/
|
|
8
|
-
import { readFile } from 'node:fs/promises';
|
|
9
|
-
import { join } from 'node:path';
|
|
10
|
-
export async function loadSkillPayload(home, payload) {
|
|
11
|
-
const name = payload.trim();
|
|
12
|
-
if (!name)
|
|
13
|
-
return '';
|
|
14
|
-
const candidates = [
|
|
15
|
-
join(home, 'skills', 'drafts', `${name}.md`),
|
|
16
|
-
join(home, 'skills', `${name}.md`),
|
|
17
|
-
];
|
|
18
|
-
for (const f of candidates) {
|
|
19
|
-
try {
|
|
20
|
-
const md = await readFile(f, 'utf8');
|
|
21
|
-
if (md.trim())
|
|
22
|
-
return md.trim().slice(0, 4000);
|
|
23
|
-
}
|
|
24
|
-
catch { /* next */ }
|
|
25
|
-
}
|
|
26
|
-
// no file found: fall back to the raw string so the arm still runs
|
|
27
|
-
return name;
|
|
28
|
-
}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Resolve a treatment-arm skills prompt for a skill-target candidate: the
|
|
3
|
-
* payload names the skill; the arm must inject the skill's CONTENT (draft
|
|
4
|
-
* first, promoted second), never the bare name string. (Day-49 SELFFEED:
|
|
5
|
-
* the first real experiment injected "+34 tokens" of name - it measured the
|
|
6
|
-
* instrument, not the skill.)
|
|
7
|
-
*/
|
|
8
|
-
import { readFile } from 'node:fs/promises';
|
|
9
|
-
import { join } from 'node:path';
|
|
10
|
-
export async function loadSkillPayload(home, payload) {
|
|
11
|
-
const name = payload.trim();
|
|
12
|
-
if (!name)
|
|
13
|
-
return '';
|
|
14
|
-
const candidates = [
|
|
15
|
-
join(home, 'skills', 'drafts', `${name}.md`),
|
|
16
|
-
join(home, 'skills', `${name}.md`),
|
|
17
|
-
];
|
|
18
|
-
for (const f of candidates) {
|
|
19
|
-
try {
|
|
20
|
-
const md = await readFile(f, 'utf8');
|
|
21
|
-
if (md.trim())
|
|
22
|
-
return md.trim().slice(0, 4000);
|
|
23
|
-
}
|
|
24
|
-
catch { /* next */ }
|
|
25
|
-
}
|
|
26
|
-
// no file found: fall back to the raw string so the arm still runs
|
|
27
|
-
return name;
|
|
28
|
-
}
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
export type SkillState = 'draft' | 'canary' | 'active' | 'archived';
|
|
2
|
-
export interface SkillEntry {
|
|
3
|
-
name: string;
|
|
4
|
-
description: string;
|
|
5
|
-
file: string;
|
|
6
|
-
state: SkillState;
|
|
7
|
-
}
|
|
8
|
-
/** Skills the user pinned: evolution never moves, merges or decays them.
|
|
9
|
-
* Marked by a .pin file next to SKILL.md (cheap, visible, git-friendly). */
|
|
10
|
-
export declare function pinSkill(home: string, name: string): Promise<boolean>;
|
|
11
|
-
export declare function unpinSkill(home: string, name: string): Promise<boolean>;
|
|
12
|
-
export declare function isPinned(file: string): Promise<boolean>;
|
|
13
|
-
/**
|
|
14
|
-
* Install skills from a git URL or a local directory (`hmh skills add <src>`).
|
|
15
|
-
* Handles the three common repo layouts: SKILL.md at the root (single skill),
|
|
16
|
-
* skills/<name>/SKILL.md (multi-skill pack, e.g. greensock/gsap-skills), and
|
|
17
|
-
* <name>/SKILL.md one level down. Existing skill names are never overwritten.
|
|
18
|
-
*/
|
|
19
|
-
export declare function installSkills(src: string, home: string): Promise<{
|
|
20
|
-
installed: string[];
|
|
21
|
-
skipped: string[];
|
|
22
|
-
}>;
|
|
23
|
-
export declare function listSkills(home: string): Promise<SkillEntry[]>;
|
|
24
|
-
export declare function listDrafts(home: string): Promise<SkillEntry[]>;
|
|
25
|
-
/** Canary-state skills: promoted through the bench gates but still under
|
|
26
|
-
* impact evaluation - injected into a sample of sessions, watermarked. */
|
|
27
|
-
export declare function listCanary(home: string): Promise<SkillEntry[]>;
|
|
28
|
-
export declare function skillsToPrompt(entries: SkillEntry[]): string;
|
|
29
|
-
/** Write (or overwrite) a draft; drafts are cheap and reversible by deletion. */
|
|
30
|
-
export declare function writeDraft(home: string, name: string, skillMd: string): Promise<string>;
|
|
31
|
-
/**
|
|
32
|
-
* Promote a draft. Default target is `canary` (P0: bench-passing skills
|
|
33
|
-
* earn a canary slot first; full activation happens through the impact
|
|
34
|
-
* loop, not on the gate alone). `{canary: false}` promotes straight to
|
|
35
|
-
* active (used by the impact loop once evidence clears, and by `hmh skills
|
|
36
|
-
* promote`). If a same-name skill exists at the destination it is archived
|
|
37
|
-
* first (timestamped snapshot) so promotion is always reversible.
|
|
38
|
-
*/
|
|
39
|
-
export declare function promoteSkill(home: string, name: string, opts?: {
|
|
40
|
-
canary?: boolean;
|
|
41
|
-
}): Promise<{
|
|
42
|
-
file: string;
|
|
43
|
-
archivedPrevious: boolean;
|
|
44
|
-
}>;
|
|
45
|
-
/** Graduate a canary skill to full active (the impact loop's decision). */
|
|
46
|
-
export declare function promoteCanary(home: string, name: string): Promise<boolean>;
|
|
47
|
-
/** Retire a canary skill back to draft (impact loop's rejection path) -
|
|
48
|
-
* nothing is ever deleted (append-only red line). */
|
|
49
|
-
export declare function retireCanary(home: string, name: string): Promise<boolean>;
|
|
50
|
-
/** Restore the most recent archived snapshot of a skill back to active. */
|
|
51
|
-
export declare function rollbackSkill(home: string, name: string): Promise<boolean>;
|
|
52
|
-
/** Demote an active skill back to draft without deleting anything. */
|
|
53
|
-
export declare function unpromoteSkill(home: string, name: string): Promise<boolean>;
|
|
54
|
-
export declare function deleteDraft(home: string, name: string): Promise<boolean>;
|
|
@@ -1,321 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @hmharness/evolution - skills
|
|
3
|
-
* The skill library with a three-state lifecycle:
|
|
4
|
-
* skills/draft/<name>/ drafted by the evolution loop, never injected
|
|
5
|
-
* skills/active/<name>/ promoted skills, injected into the system prompt
|
|
6
|
-
* skills/archive/<ts>_<name>/ snapshots taken before each promotion
|
|
7
|
-
* Root-level skills/<name>/ from Phase 0 still counts as active (compat).
|
|
8
|
-
* Promotion is move-based and each promote snapshots the incumbent so a
|
|
9
|
-
* bench regression can roll back - the DGM/GDPevo lesson: no promotion
|
|
10
|
-
* without a gate, no gate without a rollback path.
|
|
11
|
-
*/
|
|
12
|
-
import { cp, mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
13
|
-
import { tmpdir } from 'node:os';
|
|
14
|
-
import { execFile } from 'node:child_process';
|
|
15
|
-
import { join } from 'node:path';
|
|
16
|
-
import { promisify } from 'node:util';
|
|
17
|
-
const execCb = promisify(execFile);
|
|
18
|
-
/** Skills the user pinned: evolution never moves, merges or decays them.
|
|
19
|
-
* Marked by a .pin file next to SKILL.md (cheap, visible, git-friendly). */
|
|
20
|
-
export async function pinSkill(home, name) {
|
|
21
|
-
for (const dir of [join(home, 'skills', 'active', sanitize(name)), join(home, 'skills', sanitize(name)), join(home, 'skills', 'canary', sanitize(name))]) {
|
|
22
|
-
try {
|
|
23
|
-
await writeFile(join(dir, '.pin'), String(new Date().toISOString()), 'utf8');
|
|
24
|
-
return true;
|
|
25
|
-
}
|
|
26
|
-
catch { /* try next location */ }
|
|
27
|
-
}
|
|
28
|
-
return false;
|
|
29
|
-
}
|
|
30
|
-
export async function unpinSkill(home, name) {
|
|
31
|
-
for (const dir of [join(home, 'skills', 'active', sanitize(name)), join(home, 'skills', sanitize(name)), join(home, 'skills', 'canary', sanitize(name))]) {
|
|
32
|
-
try {
|
|
33
|
-
await rm(join(dir, '.pin'), { force: true });
|
|
34
|
-
return true;
|
|
35
|
-
}
|
|
36
|
-
catch { /* try next */ }
|
|
37
|
-
}
|
|
38
|
-
return false;
|
|
39
|
-
}
|
|
40
|
-
export async function isPinned(file) {
|
|
41
|
-
try {
|
|
42
|
-
await readFile(join(dirname(file), '.pin'), 'utf8');
|
|
43
|
-
return true;
|
|
44
|
-
}
|
|
45
|
-
catch {
|
|
46
|
-
return false;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
function dirname(p) {
|
|
50
|
-
const m = /^(.*)[\\/][^\\/]+$/.exec(p);
|
|
51
|
-
return m?.[1] ?? p;
|
|
52
|
-
}
|
|
53
|
-
/**
|
|
54
|
-
* Install skills from a git URL or a local directory (`hmh skills add <src>`).
|
|
55
|
-
* Handles the three common repo layouts: SKILL.md at the root (single skill),
|
|
56
|
-
* skills/<name>/SKILL.md (multi-skill pack, e.g. greensock/gsap-skills), and
|
|
57
|
-
* <name>/SKILL.md one level down. Existing skill names are never overwritten.
|
|
58
|
-
*/
|
|
59
|
-
export async function installSkills(src, home) {
|
|
60
|
-
let rootDir;
|
|
61
|
-
let tmpDir = null;
|
|
62
|
-
const isUrl = /^https?:\/\/|git@/.test(src);
|
|
63
|
-
if (isUrl) {
|
|
64
|
-
tmpDir = join(tmpdir(), `hmh-skill-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`);
|
|
65
|
-
await execCb('git', ['clone', '--depth', '1', src, tmpDir], { timeout: 120_000, windowsHide: true });
|
|
66
|
-
rootDir = tmpDir;
|
|
67
|
-
}
|
|
68
|
-
else {
|
|
69
|
-
rootDir = src;
|
|
70
|
-
}
|
|
71
|
-
try {
|
|
72
|
-
// locate SKILL.md directories
|
|
73
|
-
const found = [];
|
|
74
|
-
if (await exists(join(rootDir, 'SKILL.md'))) {
|
|
75
|
-
found.push({ name: basename(rootDir), dir: rootDir });
|
|
76
|
-
}
|
|
77
|
-
for (const sub of ['skills', '.claude/skills', '.agents/skills']) {
|
|
78
|
-
const packDir = join(rootDir, sub);
|
|
79
|
-
for (const d of await dirs(packDir)) {
|
|
80
|
-
if (await exists(join(d.path, 'SKILL.md')))
|
|
81
|
-
found.push({ name: d.name, dir: d.path });
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
if (!found.length) {
|
|
85
|
-
for (const d of await dirs(rootDir)) {
|
|
86
|
-
if (['.git', 'node_modules', '.github', 'examples', 'assets'].includes(d.name))
|
|
87
|
-
continue;
|
|
88
|
-
if (await exists(join(d.path, 'SKILL.md')))
|
|
89
|
-
found.push({ name: d.name, dir: d.path });
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
const installed = [];
|
|
93
|
-
const skipped = [];
|
|
94
|
-
const dest = join(home, 'skills');
|
|
95
|
-
await mkdir(dest, { recursive: true });
|
|
96
|
-
const taken = new Set((await readdir(dest, { withFileTypes: true })).filter((d) => d.isDirectory()).map((d) => d.name));
|
|
97
|
-
for (const f of found) {
|
|
98
|
-
const name = f.name.replace(/[^a-zA-Z0-9-]/g, '-').toLowerCase();
|
|
99
|
-
if (taken.has(name) || (await listSkills(home)).some((s) => s.name === name)) {
|
|
100
|
-
skipped.push(name);
|
|
101
|
-
continue;
|
|
102
|
-
}
|
|
103
|
-
await cp(f.dir, join(dest, name), { recursive: true });
|
|
104
|
-
taken.add(name);
|
|
105
|
-
installed.push(name);
|
|
106
|
-
}
|
|
107
|
-
return { installed, skipped };
|
|
108
|
-
}
|
|
109
|
-
finally {
|
|
110
|
-
if (tmpDir)
|
|
111
|
-
await rm(tmpDir, { recursive: true, force: true }).catch(() => undefined);
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
async function exists(p) {
|
|
115
|
-
try {
|
|
116
|
-
await readFile(p, 'utf8');
|
|
117
|
-
return true;
|
|
118
|
-
}
|
|
119
|
-
catch {
|
|
120
|
-
return false;
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
async function dirs(p) {
|
|
124
|
-
try {
|
|
125
|
-
return (await readdir(p, { withFileTypes: true }))
|
|
126
|
-
.filter((d) => d.isDirectory())
|
|
127
|
-
.map((d) => ({ name: d.name, path: join(p, d.name) }));
|
|
128
|
-
}
|
|
129
|
-
catch {
|
|
130
|
-
return [];
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
function basename(p) {
|
|
134
|
-
const m = /[\\/]?([^\\/]+)[\\/]?$/.exec(p.replace(/[\\/]+$/, ''));
|
|
135
|
-
return m?.[1] ?? 'skill';
|
|
136
|
-
}
|
|
137
|
-
export async function listSkills(home) {
|
|
138
|
-
const root = join(home, 'skills');
|
|
139
|
-
// Phase 0 layout: skills/<name>/SKILL.md directly under root (excluding lifecycle dirs)
|
|
140
|
-
const legacy = await scan(join(root), ['draft', 'active', 'canary', 'archive'], 'active');
|
|
141
|
-
const active = await scan(join(root, 'active'), [], 'active');
|
|
142
|
-
return [...legacy, ...active].sort((a, b) => a.name.localeCompare(b.name));
|
|
143
|
-
}
|
|
144
|
-
export async function listDrafts(home) {
|
|
145
|
-
return scan(join(home, 'skills', 'draft'), [], 'draft');
|
|
146
|
-
}
|
|
147
|
-
/** Canary-state skills: promoted through the bench gates but still under
|
|
148
|
-
* impact evaluation - injected into a sample of sessions, watermarked. */
|
|
149
|
-
export async function listCanary(home) {
|
|
150
|
-
return scan(join(home, 'skills', 'canary'), [], 'canary');
|
|
151
|
-
}
|
|
152
|
-
async function scan(dir, exclude, state) {
|
|
153
|
-
let dirs;
|
|
154
|
-
try {
|
|
155
|
-
dirs = (await readdir(dir, { withFileTypes: true })).filter((d) => d.isDirectory() && !exclude.includes(d.name));
|
|
156
|
-
}
|
|
157
|
-
catch {
|
|
158
|
-
return [];
|
|
159
|
-
}
|
|
160
|
-
const entries = [];
|
|
161
|
-
for (const d of dirs) {
|
|
162
|
-
const file = join(dir, d.name, 'SKILL.md');
|
|
163
|
-
try {
|
|
164
|
-
entries.push({ name: d.name, description: parseDescription(await readFile(file, 'utf8')), file, state });
|
|
165
|
-
}
|
|
166
|
-
catch {
|
|
167
|
-
/* directory without SKILL.md - not a skill */
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
return entries;
|
|
171
|
-
}
|
|
172
|
-
export function skillsToPrompt(entries) {
|
|
173
|
-
if (entries.length === 0)
|
|
174
|
-
return '';
|
|
175
|
-
return entries.map((s) => `- ${s.name}: ${s.description || '(no description)'}`).join('\n');
|
|
176
|
-
}
|
|
177
|
-
/** Write (or overwrite) a draft; drafts are cheap and reversible by deletion. */
|
|
178
|
-
export async function writeDraft(home, name, skillMd) {
|
|
179
|
-
const dir = join(home, 'skills', 'draft', sanitize(name));
|
|
180
|
-
await mkdir(dir, { recursive: true });
|
|
181
|
-
const file = join(dir, 'SKILL.md');
|
|
182
|
-
await writeFile(file, skillMd, 'utf8');
|
|
183
|
-
return file;
|
|
184
|
-
}
|
|
185
|
-
/**
|
|
186
|
-
* Promote a draft. Default target is `canary` (P0: bench-passing skills
|
|
187
|
-
* earn a canary slot first; full activation happens through the impact
|
|
188
|
-
* loop, not on the gate alone). `{canary: false}` promotes straight to
|
|
189
|
-
* active (used by the impact loop once evidence clears, and by `hmh skills
|
|
190
|
-
* promote`). If a same-name skill exists at the destination it is archived
|
|
191
|
-
* first (timestamped snapshot) so promotion is always reversible.
|
|
192
|
-
*/
|
|
193
|
-
export async function promoteSkill(home, name, opts = {}) {
|
|
194
|
-
const safe = sanitize(name);
|
|
195
|
-
const draftDir = join(home, 'skills', 'draft', safe);
|
|
196
|
-
const activeDir = join(home, 'skills', 'active', safe);
|
|
197
|
-
const canaryDir = join(home, 'skills', 'canary', safe);
|
|
198
|
-
const legacyDir = join(home, 'skills', safe);
|
|
199
|
-
await mkdir(join(home, 'skills', 'active'), { recursive: true });
|
|
200
|
-
await mkdir(join(home, 'skills', 'archive'), { recursive: true });
|
|
201
|
-
if (opts.canary !== false)
|
|
202
|
-
await mkdir(join(home, 'skills', 'canary'), { recursive: true });
|
|
203
|
-
let archivedPrevious = false;
|
|
204
|
-
// archive incumbents at both destinations: active/legacy AND an existing
|
|
205
|
-
// canary slot (re-promoting a canary skill must not rename onto an
|
|
206
|
-
// occupied directory - that is EPERM on Windows)
|
|
207
|
-
for (const existing of [activeDir, legacyDir, join(home, 'skills', 'canary', safe)]) {
|
|
208
|
-
try {
|
|
209
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
210
|
-
await rename(existing, join(home, 'skills', 'archive', `${stamp}_${safe}`));
|
|
211
|
-
archivedPrevious = true;
|
|
212
|
-
}
|
|
213
|
-
catch {
|
|
214
|
-
/* nothing at this path */
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
await rename(draftDir, opts.canary === false ? activeDir : canaryDir);
|
|
218
|
-
return { file: join(opts.canary === false ? activeDir : canaryDir, 'SKILL.md'), archivedPrevious };
|
|
219
|
-
}
|
|
220
|
-
/** Graduate a canary skill to full active (the impact loop's decision). */
|
|
221
|
-
export async function promoteCanary(home, name) {
|
|
222
|
-
const safe = sanitize(name);
|
|
223
|
-
await mkdir(join(home, 'skills', 'active'), { recursive: true });
|
|
224
|
-
try {
|
|
225
|
-
await rename(join(home, 'skills', 'canary', safe), join(home, 'skills', 'active', safe));
|
|
226
|
-
return true;
|
|
227
|
-
}
|
|
228
|
-
catch {
|
|
229
|
-
return false;
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
/** Retire a canary skill back to draft (impact loop's rejection path) -
|
|
233
|
-
* nothing is ever deleted (append-only red line). */
|
|
234
|
-
export async function retireCanary(home, name) {
|
|
235
|
-
const safe = sanitize(name);
|
|
236
|
-
await mkdir(join(home, 'skills', 'draft'), { recursive: true });
|
|
237
|
-
try {
|
|
238
|
-
const dest = join(home, 'skills', 'draft', safe);
|
|
239
|
-
try {
|
|
240
|
-
await rm(dest, { recursive: true, force: true });
|
|
241
|
-
}
|
|
242
|
-
catch { /* not there */ }
|
|
243
|
-
await rename(join(home, 'skills', 'canary', safe), dest);
|
|
244
|
-
return true;
|
|
245
|
-
}
|
|
246
|
-
catch {
|
|
247
|
-
return false;
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
/** Restore the most recent archived snapshot of a skill back to active. */
|
|
251
|
-
export async function rollbackSkill(home, name) {
|
|
252
|
-
const safe = sanitize(name);
|
|
253
|
-
const archiveRoot = join(home, 'skills', 'archive');
|
|
254
|
-
let snapshots;
|
|
255
|
-
try {
|
|
256
|
-
snapshots = (await readdir(archiveRoot)).filter((d) => d.endsWith(`_${safe}`)).sort();
|
|
257
|
-
}
|
|
258
|
-
catch {
|
|
259
|
-
return false;
|
|
260
|
-
}
|
|
261
|
-
const latest = snapshots.pop();
|
|
262
|
-
if (!latest)
|
|
263
|
-
return false;
|
|
264
|
-
await mkdir(join(home, 'skills', 'active'), { recursive: true });
|
|
265
|
-
// Move the regressed incumbent out of the way first (rename onto an
|
|
266
|
-
// existing directory fails on Windows), keeping it as a rejected snapshot.
|
|
267
|
-
for (const cur of [join(home, 'skills', 'active', safe), join(home, 'skills', safe)]) {
|
|
268
|
-
try {
|
|
269
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
270
|
-
await rename(cur, join(archiveRoot, `rejected-${stamp}_${safe}`));
|
|
271
|
-
break;
|
|
272
|
-
}
|
|
273
|
-
catch {
|
|
274
|
-
/* nothing at this path */
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
await rename(join(archiveRoot, latest), join(home, 'skills', 'active', safe));
|
|
278
|
-
return true;
|
|
279
|
-
}
|
|
280
|
-
/** Demote an active skill back to draft without deleting anything. */
|
|
281
|
-
export async function unpromoteSkill(home, name) {
|
|
282
|
-
const safe = sanitize(name);
|
|
283
|
-
for (const from of [join(home, 'skills', 'active', safe), join(home, 'skills', safe)]) {
|
|
284
|
-
try {
|
|
285
|
-
await mkdir(join(home, 'skills', 'draft'), { recursive: true });
|
|
286
|
-
await rename(from, join(home, 'skills', 'draft', safe));
|
|
287
|
-
return true;
|
|
288
|
-
}
|
|
289
|
-
catch {
|
|
290
|
-
/* try next */
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
return false;
|
|
294
|
-
}
|
|
295
|
-
export async function deleteDraft(home, name) {
|
|
296
|
-
try {
|
|
297
|
-
await rm(join(home, 'skills', 'draft', sanitize(name)), { recursive: true, force: true });
|
|
298
|
-
return true;
|
|
299
|
-
}
|
|
300
|
-
catch {
|
|
301
|
-
return false;
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
function sanitize(name) {
|
|
305
|
-
return name.replace(/[^a-zA-Z0-9_-]/g, '-').slice(0, 60) || 'skill';
|
|
306
|
-
}
|
|
307
|
-
function parseDescription(text) {
|
|
308
|
-
// frontmatter "description:" line, or first non-heading non-empty line
|
|
309
|
-
const fm = text.match(/^---\n([\s\S]*?)\n---/);
|
|
310
|
-
if (fm) {
|
|
311
|
-
const m = fm[1].match(/^description:\s*(.+)$/m);
|
|
312
|
-
if (m)
|
|
313
|
-
return m[1].trim().slice(0, 120);
|
|
314
|
-
}
|
|
315
|
-
for (const line of text.split('\n')) {
|
|
316
|
-
const t = line.trim();
|
|
317
|
-
if (t && !t.startsWith('#') && !t.startsWith('---'))
|
|
318
|
-
return t.slice(0, 120);
|
|
319
|
-
}
|
|
320
|
-
return '';
|
|
321
|
-
}
|