@theglitchking/semantic-memory 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +90 -0
- package/LICENSES/Apache-2.0.txt +202 -0
- package/README.md +49 -29
- package/THIRD_PARTY_NOTICES.md +17 -0
- package/commands/jev-mode.md +18 -0
- package/commands/skill-loop.md +23 -0
- package/dist/{chunk-7PNF3AP2.js → chunk-34VN47IW.js} +2 -2
- package/dist/chunk-3FPUBAEW.js +113 -0
- package/dist/chunk-3FPUBAEW.js.map +1 -0
- package/dist/{chunk-WZEKIUYG.js → chunk-6TIKWOY4.js} +10 -8
- package/dist/{chunk-WZEKIUYG.js.map → chunk-6TIKWOY4.js.map} +1 -1
- package/dist/{chunk-NBY4TRP7.js → chunk-7M34JDTH.js} +3 -3
- package/dist/{chunk-FTRXH2WC.js → chunk-7UAVDOI6.js} +15 -12
- package/dist/chunk-7UAVDOI6.js.map +1 -0
- package/dist/{chunk-BBP3DG2U.js → chunk-AIXQFTSB.js} +25 -26
- package/dist/{chunk-BBP3DG2U.js.map → chunk-AIXQFTSB.js.map} +1 -1
- package/dist/chunk-BEACL25M.js +224 -0
- package/dist/chunk-BEACL25M.js.map +1 -0
- package/dist/chunk-CATPGINC.js +186 -0
- package/dist/chunk-CATPGINC.js.map +1 -0
- package/dist/{chunk-IGHCVAAL.js → chunk-CJUY55GG.js} +6 -3
- package/dist/{chunk-IGHCVAAL.js.map → chunk-CJUY55GG.js.map} +1 -1
- package/dist/chunk-DBFD3MTT.js +130 -0
- package/dist/chunk-DBFD3MTT.js.map +1 -0
- package/dist/{chunk-XAXK7NM4.js → chunk-EXAUVIGW.js} +9 -7
- package/dist/{chunk-XAXK7NM4.js.map → chunk-EXAUVIGW.js.map} +1 -1
- package/dist/{chunk-LGQPP5FQ.js → chunk-EYVFC745.js} +2 -2
- package/dist/chunk-GYR7U6KA.js +80 -0
- package/dist/chunk-GYR7U6KA.js.map +1 -0
- package/dist/{chunk-OEF5U5GV.js → chunk-H2BES6FU.js} +10 -2
- package/dist/{chunk-OEF5U5GV.js.map → chunk-H2BES6FU.js.map} +1 -1
- package/dist/chunk-IEZVFKUT.js +157 -0
- package/dist/chunk-IEZVFKUT.js.map +1 -0
- package/dist/chunk-J7LYDRML.js +9 -0
- package/dist/chunk-J7LYDRML.js.map +1 -0
- package/dist/chunk-MQS7JEBD.js +137 -0
- package/dist/chunk-MQS7JEBD.js.map +1 -0
- package/dist/chunk-NCGM36NX.js +118 -0
- package/dist/chunk-NCGM36NX.js.map +1 -0
- package/dist/chunk-NIZLBNH4.js +165 -0
- package/dist/chunk-NIZLBNH4.js.map +1 -0
- package/dist/chunk-OT3MV7XF.js +123 -0
- package/dist/chunk-OT3MV7XF.js.map +1 -0
- package/dist/chunk-P3GLY42T.js +291 -0
- package/dist/chunk-P3GLY42T.js.map +1 -0
- package/dist/chunk-PJXYCCI2.js +177 -0
- package/dist/chunk-PJXYCCI2.js.map +1 -0
- package/dist/chunk-R6GNP7DS.js +87 -0
- package/dist/chunk-R6GNP7DS.js.map +1 -0
- package/dist/{chunk-44BQDRXD.js → chunk-SHSOCFJH.js} +4 -1
- package/dist/chunk-SHSOCFJH.js.map +1 -0
- package/dist/chunk-TKHWVISV.js +71 -0
- package/dist/chunk-TKHWVISV.js.map +1 -0
- package/dist/{chunk-CAX46XLY.js → chunk-VBLCUWT4.js} +23 -12
- package/dist/chunk-VBLCUWT4.js.map +1 -0
- package/dist/chunk-VQCNWRTL.js +121 -0
- package/dist/chunk-VQCNWRTL.js.map +1 -0
- package/dist/{chunk-SX427M2R.js → chunk-XKGK643R.js} +2 -2
- package/dist/{chunk-FURCIJBO.js → chunk-Y7H76Q7F.js} +6 -3
- package/dist/chunk-Y7H76Q7F.js.map +1 -0
- package/dist/chunk-ZSWI3Y4Y.js +232 -0
- package/dist/chunk-ZSWI3Y4Y.js.map +1 -0
- package/dist/cli/index.js +389 -60
- package/dist/cli/index.js.map +1 -1
- package/dist/core/features.d.ts +38 -0
- package/dist/core/features.js +28 -0
- package/dist/core/index.d.ts +285 -2
- package/dist/core/index.js +170 -20
- package/dist/core/index.js.map +1 -1
- package/dist/core/jev.d.ts +35 -0
- package/dist/core/jev.js +20 -0
- package/dist/core/redact.js +1 -1
- package/dist/core/skill-curator.d.ts +1 -0
- package/dist/core/skill-curator.js +41 -0
- package/dist/core/skill-evidence.d.ts +25 -0
- package/dist/core/skill-evidence.js +23 -0
- package/dist/core/skill-queue.d.ts +40 -0
- package/dist/core/skill-queue.js +38 -0
- package/dist/core/skill-review.d.ts +3 -0
- package/dist/core/skill-review.js +29 -0
- package/dist/core/skill-worker.d.ts +21 -0
- package/dist/core/skill-worker.js +70 -0
- package/dist/core/skill-worker.js.map +1 -0
- package/dist/{dossier-RWTIMGZN.js → dossier-RXBLQOXC.js} +5 -4
- package/dist/failure-eval-IUJO47UC.js +18 -0
- package/dist/healthcheck-S7D6YNTA.js +21 -0
- package/dist/indexer-RZPH2PVL.js +9 -0
- package/dist/{lexicon-IVNJTMWE.js → lexicon-SEJSLBHO.js} +6 -5
- package/dist/lexicon-SEJSLBHO.js.map +1 -0
- package/dist/lint-3JNLRM5U.js +18 -0
- package/dist/lint-3JNLRM5U.js.map +1 -0
- package/dist/log-RM2NDNPH.js +11 -0
- package/dist/log-RM2NDNPH.js.map +1 -0
- package/dist/matter-SNJ2WDL6.js +8 -0
- package/dist/matter-SNJ2WDL6.js.map +1 -0
- package/dist/mcp/server.js +43 -38
- package/dist/mcp/server.js.map +1 -1
- package/dist/{profile-UE4YVTDC.js → profile-SNULTJ27.js} +6 -5
- package/dist/profile-SNULTJ27.js.map +1 -0
- package/dist/{schema-LY6533NJ.js → schema-RDYUK6CC.js} +3 -2
- package/dist/schema-RDYUK6CC.js.map +1 -0
- package/dist/{search-MZC4DC7H.js → search-325ULY57.js} +4 -3
- package/dist/{search-MZC4DC7H.js.map → search-325ULY57.js.map} +1 -1
- package/dist/skill-curator-D7HHZaUI.d.ts +71 -0
- package/dist/skill-land-XU2APPBJ.js +31 -0
- package/dist/skill-land-XU2APPBJ.js.map +1 -0
- package/dist/skill-ledger-AYXVNFX7.js +18 -0
- package/dist/skill-ledger-AYXVNFX7.js.map +1 -0
- package/dist/skill-merge-GXBQLBZA.js +24 -0
- package/dist/skill-merge-GXBQLBZA.js.map +1 -0
- package/dist/skill-review-BaW5n0Zg.d.ts +179 -0
- package/dist/skill-schema-RVB3EHEF.js +30 -0
- package/dist/skill-schema-RVB3EHEF.js.map +1 -0
- package/dist/{telemetry-B4KP3UVE.js → telemetry-YAG3VA4Y.js} +4 -4
- package/dist/telemetry-YAG3VA4Y.js.map +1 -0
- package/docs/regression-snapshot.md +5 -5
- package/hooks/hooks.json +26 -1
- package/hooks/jev-bridge.py +109 -0
- package/hooks/session-start.js +5 -0
- package/hooks/skill-loop.js +65 -0
- package/hooks/vault-context.js +104 -6
- package/package.json +4 -2
- package/dist/chunk-44BQDRXD.js.map +0 -1
- package/dist/chunk-5F5ZIAW2.js +0 -79
- package/dist/chunk-5F5ZIAW2.js.map +0 -1
- package/dist/chunk-CAX46XLY.js.map +0 -1
- package/dist/chunk-FTRXH2WC.js.map +0 -1
- package/dist/chunk-FURCIJBO.js.map +0 -1
- package/dist/failure-eval-37JRNTIW.js +0 -10
- package/dist/healthcheck-3FSNL2U7.js +0 -20
- package/dist/indexer-RAKJ7JR3.js +0 -8
- package/dist/lint-36WTW762.js +0 -17
- package/dist/log-YIXF6CYM.js +0 -11
- /package/dist/{chunk-7PNF3AP2.js.map → chunk-34VN47IW.js.map} +0 -0
- /package/dist/{chunk-NBY4TRP7.js.map → chunk-7M34JDTH.js.map} +0 -0
- /package/dist/{chunk-LGQPP5FQ.js.map → chunk-EYVFC745.js.map} +0 -0
- /package/dist/{chunk-SX427M2R.js.map → chunk-XKGK643R.js.map} +0 -0
- /package/dist/{dossier-RWTIMGZN.js.map → core/features.js.map} +0 -0
- /package/dist/{failure-eval-37JRNTIW.js.map → core/jev.js.map} +0 -0
- /package/dist/{healthcheck-3FSNL2U7.js.map → core/skill-curator.js.map} +0 -0
- /package/dist/{indexer-RAKJ7JR3.js.map → core/skill-evidence.js.map} +0 -0
- /package/dist/{lexicon-IVNJTMWE.js.map → core/skill-queue.js.map} +0 -0
- /package/dist/{lint-36WTW762.js.map → core/skill-review.js.map} +0 -0
- /package/dist/{log-YIXF6CYM.js.map → dossier-RXBLQOXC.js.map} +0 -0
- /package/dist/{profile-UE4YVTDC.js.map → failure-eval-IUJO47UC.js.map} +0 -0
- /package/dist/{schema-LY6533NJ.js.map → healthcheck-S7D6YNTA.js.map} +0 -0
- /package/dist/{telemetry-B4KP3UVE.js.map → indexer-RZPH2PVL.js.map} +0 -0
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import {
|
|
2
|
+
runClaude,
|
|
3
|
+
skillsCalled
|
|
4
|
+
} from "./chunk-IEZVFKUT.js";
|
|
5
|
+
import {
|
|
6
|
+
parseSkill
|
|
7
|
+
} from "./chunk-ZSWI3Y4Y.js";
|
|
8
|
+
|
|
9
|
+
// src/core/skill-eval/sets.ts
|
|
10
|
+
import { createHash } from "crypto";
|
|
11
|
+
var STOP = /* @__PURE__ */ new Set(["the", "and", "for", "with", "when", "this", "that", "use", "run", "you", "are", "not", "can", "how", "what", "from", "into", "your"]);
|
|
12
|
+
var terms = (s) => new Set(s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length >= 3 && !STOP.has(t)));
|
|
13
|
+
var splitOf = (text) => createHash("sha256").update(text).digest().readUInt32BE(0) % 100 < 60 ? "train" : "heldout";
|
|
14
|
+
function buildEvalSet(skill, candidates, opts = {}) {
|
|
15
|
+
const sessions = new Set(skill.meta?.provenance.sessions ?? []);
|
|
16
|
+
const own = terms([skill.name, skill.description, ...skill.meta?.triggers ?? []].join(" "));
|
|
17
|
+
const seen = /* @__PURE__ */ new Set();
|
|
18
|
+
const positives = [];
|
|
19
|
+
const near = [];
|
|
20
|
+
for (const c of candidates) {
|
|
21
|
+
const text = c.text.trim();
|
|
22
|
+
if (!text || seen.has(text)) continue;
|
|
23
|
+
seen.add(text);
|
|
24
|
+
if (sessions.has(c.session)) positives.push(text);
|
|
25
|
+
else {
|
|
26
|
+
const overlap = [...terms(text)].filter((t) => own.has(t)).length;
|
|
27
|
+
if (overlap > 0) near.push({ text, overlap });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
near.sort((a, b) => b.overlap - a.overlap || a.text.localeCompare(b.text));
|
|
31
|
+
const negatives = near.slice(0, opts.maxNegatives ?? Math.max(positives.length, 4)).map((n) => n.text);
|
|
32
|
+
return [...positives.map((text) => ({ text, should_trigger: true })), ...negatives.map((text) => ({ text, should_trigger: false }))].map((p) => ({ ...p, split: splitOf(p.text) }));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// src/core/skill-eval/trigger.ts
|
|
36
|
+
import { createHash as createHash2 } from "crypto";
|
|
37
|
+
import { appendFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "fs";
|
|
38
|
+
import { homedir, tmpdir } from "os";
|
|
39
|
+
import { dirname, join } from "path";
|
|
40
|
+
var MAX_TRIGGER_RUNS = 3;
|
|
41
|
+
var evalCachePath = (env = process.env) => env.SEMANTIC_MEMORY_SKILL_EVAL_CACHE || join(env.HOME || homedir(), ".semantic-memory", "skill-eval-cache.jsonl");
|
|
42
|
+
var sha256 = (s) => createHash2("sha256").update(s).digest("hex");
|
|
43
|
+
var rate = (xs) => xs.length ? xs.filter(Boolean).length / xs.length : null;
|
|
44
|
+
function ratesOf(rs) {
|
|
45
|
+
return {
|
|
46
|
+
n: rs.length,
|
|
47
|
+
positiveRate: rate(rs.filter((r) => r.should_trigger).map((r) => r.triggered)),
|
|
48
|
+
falseTriggerRate: rate(rs.filter((r) => !r.should_trigger).map((r) => r.triggered)),
|
|
49
|
+
accuracy: rate(rs.map((r) => r.triggered === r.should_trigger))
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
function makeScratch(repoRoot, name, skillRaw) {
|
|
53
|
+
const dir = mkdtempSync(join(tmpdir(), "sm-skill-eval-"));
|
|
54
|
+
const skills = join(dir, ".claude", "skills");
|
|
55
|
+
const src = join(repoRoot, ".claude", "skills");
|
|
56
|
+
if (existsSync(src)) {
|
|
57
|
+
for (const e of readdirSync(src)) if (e !== name) cpSync(join(src, e), join(skills, e), { recursive: true });
|
|
58
|
+
}
|
|
59
|
+
mkdirSync(join(skills, name), { recursive: true });
|
|
60
|
+
writeFileSync(join(skills, name, "SKILL.md"), skillRaw);
|
|
61
|
+
return dir;
|
|
62
|
+
}
|
|
63
|
+
var cacheKey = (hash, prompt) => `${hash}
|
|
64
|
+
${sha256(prompt)}`;
|
|
65
|
+
function loadCache(path) {
|
|
66
|
+
const m = /* @__PURE__ */ new Map();
|
|
67
|
+
let raw = "";
|
|
68
|
+
try {
|
|
69
|
+
raw = readFileSync(path, "utf8");
|
|
70
|
+
} catch {
|
|
71
|
+
}
|
|
72
|
+
for (const line of raw.split("\n")) {
|
|
73
|
+
try {
|
|
74
|
+
const r = JSON.parse(line);
|
|
75
|
+
m.set(`${r.skill}
|
|
76
|
+
${r.prompt}`, { triggers: r.triggers, runs: r.runs });
|
|
77
|
+
} catch {
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return m;
|
|
81
|
+
}
|
|
82
|
+
async function runTrigger(prompts, o) {
|
|
83
|
+
const env = o.env ?? process.env;
|
|
84
|
+
const skill = parseSkill(o.skillRaw);
|
|
85
|
+
const hash = sha256(o.skillRaw);
|
|
86
|
+
const cachePath = evalCachePath(env);
|
|
87
|
+
const cache = loadCache(cachePath);
|
|
88
|
+
const scratch = makeScratch(o.repoRoot, skill.name, o.skillRaw);
|
|
89
|
+
let started = 0;
|
|
90
|
+
const callOnce = async (text) => {
|
|
91
|
+
started++;
|
|
92
|
+
const r = await runClaude({ prompt: text, cwd: scratch, tools: ["Skill", "Read", "Glob", "Grep"], maxTurns: 2, output: "stream-json", env, timeoutMs: o.timeoutMs });
|
|
93
|
+
return r.ok ? skillsCalled(r.events).includes(skill.name) : r.error ?? "claude failed";
|
|
94
|
+
};
|
|
95
|
+
try {
|
|
96
|
+
const results = await Promise.all(
|
|
97
|
+
prompts.map(async (p) => {
|
|
98
|
+
const hit = cache.get(cacheKey(hash, p.text));
|
|
99
|
+
if (hit) return { ...p, ...hit, triggered: hit.triggers * 2 > hit.runs, cached: true };
|
|
100
|
+
const votes = [];
|
|
101
|
+
let error;
|
|
102
|
+
for (let attempt = 0; attempt < MAX_TRIGGER_RUNS + 1 && votes.length < MAX_TRIGGER_RUNS; attempt++) {
|
|
103
|
+
if (votes.length === 2 && votes[0] === votes[1]) break;
|
|
104
|
+
const v = await callOnce(p.text);
|
|
105
|
+
if (typeof v === "string") error = v;
|
|
106
|
+
else votes.push(v);
|
|
107
|
+
}
|
|
108
|
+
const triggers = votes.filter(Boolean).length;
|
|
109
|
+
if (votes.length) {
|
|
110
|
+
mkdirSync(dirname(cachePath), { recursive: true });
|
|
111
|
+
appendFileSync(cachePath, JSON.stringify({ skill: hash, prompt: sha256(p.text), triggers, runs: votes.length }) + "\n");
|
|
112
|
+
}
|
|
113
|
+
return { ...p, triggers, runs: votes.length, triggered: triggers * 2 > votes.length, cached: false, ...votes.length ? {} : { error } };
|
|
114
|
+
})
|
|
115
|
+
);
|
|
116
|
+
return {
|
|
117
|
+
skill: skill.name,
|
|
118
|
+
results,
|
|
119
|
+
train: ratesOf(results.filter((r) => r.split === "train")),
|
|
120
|
+
heldout: ratesOf(results.filter((r) => r.split === "heldout")),
|
|
121
|
+
runs: started
|
|
122
|
+
};
|
|
123
|
+
} finally {
|
|
124
|
+
rmSync(scratch, { recursive: true, force: true });
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export {
|
|
129
|
+
splitOf,
|
|
130
|
+
buildEvalSet,
|
|
131
|
+
MAX_TRIGGER_RUNS,
|
|
132
|
+
evalCachePath,
|
|
133
|
+
ratesOf,
|
|
134
|
+
makeScratch,
|
|
135
|
+
runTrigger
|
|
136
|
+
};
|
|
137
|
+
//# sourceMappingURL=chunk-MQS7JEBD.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/core/skill-eval/sets.ts","../src/core/skill-eval/trigger.ts"],"sourcesContent":["/**\n * Trigger eval sets (v1.7): positives are prompts from the skill's own provenance sessions, near-miss\n * negatives are prompts from other sessions that share terms with it. The caller supplies the prompts\n * (this module reads no transcripts). Split ~60/40 into train / held-out by hashing the prompt text.\n */\nimport { createHash } from \"node:crypto\";\nimport type { ParsedSkill } from \"../skill-schema.js\";\n\nexport interface CandidatePrompt {\n session: string;\n text: string;\n}\n\nexport interface EvalPrompt {\n text: string;\n should_trigger: boolean;\n split: \"train\" | \"heldout\";\n}\n\nconst STOP = new Set([\"the\", \"and\", \"for\", \"with\", \"when\", \"this\", \"that\", \"use\", \"run\", \"you\", \"are\", \"not\", \"can\", \"how\", \"what\", \"from\", \"into\", \"your\"]);\nconst terms = (s: string) => new Set(s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length >= 3 && !STOP.has(t)));\n\n/** Same prompt, same side of the split, on every machine. */\nexport const splitOf = (text: string): EvalPrompt[\"split\"] => (createHash(\"sha256\").update(text).digest().readUInt32BE(0) % 100 < 60 ? \"train\" : \"heldout\");\n\nexport function buildEvalSet(skill: ParsedSkill, candidates: CandidatePrompt[], opts: { maxNegatives?: number } = {}): EvalPrompt[] {\n const sessions = new Set(skill.meta?.provenance.sessions ?? []);\n const own = terms([skill.name, skill.description, ...(skill.meta?.triggers ?? [])].join(\" \"));\n const seen = new Set<string>();\n const positives: string[] = [];\n const near: { text: string; overlap: number }[] = [];\n for (const c of candidates) {\n const text = c.text.trim();\n if (!text || seen.has(text)) continue;\n seen.add(text);\n if (sessions.has(c.session)) positives.push(text);\n else {\n const overlap = [...terms(text)].filter((t) => own.has(t)).length;\n if (overlap > 0) near.push({ text, overlap });\n }\n }\n near.sort((a, b) => b.overlap - a.overlap || a.text.localeCompare(b.text));\n const negatives = near.slice(0, opts.maxNegatives ?? Math.max(positives.length, 4)).map((n) => n.text);\n return [...positives.map((text) => ({ text, should_trigger: true })), ...negatives.map((text) => ({ text, should_trigger: false }))].map((p) => ({ ...p, split: splitOf(p.text) }));\n}\n","/**\n * Trigger eval (v1.7): does Claude reach for the skill when it should, and stay away when it should not?\n * Each prompt runs in a scratch directory (outside the repo) holding the skill under test next to copies of\n * the repo's other skills, so it competes like it would for real. Up to 3 runs per prompt, stopping once the\n * first two agree. Results are cached by (sha256 of the skill file, prompt): an unchanged skill costs no runs.\n */\nimport { createHash } from \"node:crypto\";\nimport { appendFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from \"node:fs\";\nimport { homedir, tmpdir } from \"node:os\";\nimport { dirname, join } from \"node:path\";\nimport { runClaude, skillsCalled } from \"../claude-run.js\";\nimport { parseSkill } from \"../skill-schema.js\";\nimport type { EvalPrompt } from \"./sets.js\";\n\nexport const MAX_TRIGGER_RUNS = 3;\n\nexport const evalCachePath = (env = process.env) =>\n env.SEMANTIC_MEMORY_SKILL_EVAL_CACHE || join(env.HOME || homedir(), \".semantic-memory\", \"skill-eval-cache.jsonl\");\n\nexport const sha256 = (s: string) => createHash(\"sha256\").update(s).digest(\"hex\");\n\nexport interface PromptTrigger extends EvalPrompt {\n triggers: number;\n runs: number;\n /** a majority of the runs called the skill */\n triggered: boolean;\n cached: boolean;\n error?: string;\n}\n\nexport interface Rates {\n n: number;\n /** share of should-trigger prompts that triggered; null when there are none */\n positiveRate: number | null;\n /** share of near-miss prompts that triggered; null when there are none */\n falseTriggerRate: number | null;\n /** prompts that behaved as expected / n */\n accuracy: number | null;\n}\n\nexport interface TriggerReport {\n skill: string;\n results: PromptTrigger[];\n train: Rates;\n heldout: Rates;\n /** claude runs started (0 when everything was cached) */\n runs: number;\n}\n\nconst rate = (xs: boolean[]) => (xs.length ? xs.filter(Boolean).length / xs.length : null);\n\nexport function ratesOf(rs: PromptTrigger[]): Rates {\n return {\n n: rs.length,\n positiveRate: rate(rs.filter((r) => r.should_trigger).map((r) => r.triggered)),\n falseTriggerRate: rate(rs.filter((r) => !r.should_trigger).map((r) => r.triggered)),\n accuracy: rate(rs.map((r) => r.triggered === r.should_trigger)),\n };\n}\n\n/** A scratch dir with the skill under test and copies of the repo's other project skills. The caller removes it. */\nexport function makeScratch(repoRoot: string, name: string, skillRaw: string): string {\n const dir = mkdtempSync(join(tmpdir(), \"sm-skill-eval-\"));\n const skills = join(dir, \".claude\", \"skills\");\n const src = join(repoRoot, \".claude\", \"skills\");\n if (existsSync(src)) {\n for (const e of readdirSync(src)) if (e !== name) cpSync(join(src, e), join(skills, e), { recursive: true });\n }\n mkdirSync(join(skills, name), { recursive: true });\n writeFileSync(join(skills, name, \"SKILL.md\"), skillRaw);\n return dir;\n}\n\ntype Cache = Map<string, { triggers: number; runs: number }>;\n// prompts come from real sessions: the cache (in $HOME, shared by every repo) keeps only their hash\nconst cacheKey = (hash: string, prompt: string) => `${hash}\\n${sha256(prompt)}`;\n\nfunction loadCache(path: string): Cache {\n const m: Cache = new Map();\n let raw = \"\";\n try {\n raw = readFileSync(path, \"utf8\");\n } catch {}\n for (const line of raw.split(\"\\n\")) {\n try {\n const r = JSON.parse(line);\n m.set(`${r.skill}\\n${r.prompt}`, { triggers: r.triggers, runs: r.runs });\n } catch {}\n }\n return m;\n}\n\nexport interface TriggerOptions {\n repoRoot: string;\n /** the skill file as it is on disk (or a candidate version of it) */\n skillRaw: string;\n env?: NodeJS.ProcessEnv;\n timeoutMs?: number;\n}\n\nexport async function runTrigger(prompts: EvalPrompt[], o: TriggerOptions): Promise<TriggerReport> {\n const env = o.env ?? process.env;\n const skill = parseSkill(o.skillRaw);\n const hash = sha256(o.skillRaw);\n const cachePath = evalCachePath(env);\n const cache = loadCache(cachePath);\n const scratch = makeScratch(o.repoRoot, skill.name, o.skillRaw);\n let started = 0;\n const callOnce = async (text: string): Promise<boolean | string> => {\n started++;\n const r = await runClaude({ prompt: text, cwd: scratch, tools: [\"Skill\", \"Read\", \"Glob\", \"Grep\"], maxTurns: 2, output: \"stream-json\", env, timeoutMs: o.timeoutMs });\n return r.ok ? skillsCalled(r.events).includes(skill.name) : (r.error ?? \"claude failed\");\n };\n try {\n const results = await Promise.all(\n prompts.map(async (p): Promise<PromptTrigger> => {\n const hit = cache.get(cacheKey(hash, p.text));\n if (hit) return { ...p, ...hit, triggered: hit.triggers * 2 > hit.runs, cached: true };\n const votes: boolean[] = [];\n let error: string | undefined;\n // errored runs are retried inside the same budget of attempts, never counted as a vote\n for (let attempt = 0; attempt < MAX_TRIGGER_RUNS + 1 && votes.length < MAX_TRIGGER_RUNS; attempt++) {\n if (votes.length === 2 && votes[0] === votes[1]) break;\n const v = await callOnce(p.text);\n if (typeof v === \"string\") error = v;\n else votes.push(v);\n }\n const triggers = votes.filter(Boolean).length;\n if (votes.length) {\n mkdirSync(dirname(cachePath), { recursive: true });\n appendFileSync(cachePath, JSON.stringify({ skill: hash, prompt: sha256(p.text), triggers, runs: votes.length }) + \"\\n\");\n }\n return { ...p, triggers, runs: votes.length, triggered: triggers * 2 > votes.length, cached: false, ...(votes.length ? {} : { error }) };\n }),\n );\n return {\n skill: skill.name,\n results,\n train: ratesOf(results.filter((r) => r.split === \"train\")),\n heldout: ratesOf(results.filter((r) => r.split === \"heldout\")),\n runs: started,\n };\n } finally {\n rmSync(scratch, { recursive: true, force: true });\n }\n}\n"],"mappings":";;;;;;;;;AAKA,SAAS,kBAAkB;AAc3B,IAAM,OAAO,oBAAI,IAAI,CAAC,OAAO,OAAO,OAAO,QAAQ,QAAQ,QAAQ,QAAQ,OAAO,OAAO,OAAO,OAAO,OAAO,OAAO,OAAO,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AAC3J,IAAM,QAAQ,CAAC,MAAc,IAAI,IAAI,EAAE,YAAY,EAAE,MAAM,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AAG9G,IAAM,UAAU,CAAC,SAAuC,WAAW,QAAQ,EAAE,OAAO,IAAI,EAAE,OAAO,EAAE,aAAa,CAAC,IAAI,MAAM,KAAK,UAAU;AAE1I,SAAS,aAAa,OAAoB,YAA+B,OAAkC,CAAC,GAAiB;AAClI,QAAM,WAAW,IAAI,IAAI,MAAM,MAAM,WAAW,YAAY,CAAC,CAAC;AAC9D,QAAM,MAAM,MAAM,CAAC,MAAM,MAAM,MAAM,aAAa,GAAI,MAAM,MAAM,YAAY,CAAC,CAAE,EAAE,KAAK,GAAG,CAAC;AAC5F,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,YAAsB,CAAC;AAC7B,QAAM,OAA4C,CAAC;AACnD,aAAW,KAAK,YAAY;AAC1B,UAAM,OAAO,EAAE,KAAK,KAAK;AACzB,QAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,EAAG;AAC7B,SAAK,IAAI,IAAI;AACb,QAAI,SAAS,IAAI,EAAE,OAAO,EAAG,WAAU,KAAK,IAAI;AAAA,SAC3C;AACH,YAAM,UAAU,CAAC,GAAG,MAAM,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE;AAC3D,UAAI,UAAU,EAAG,MAAK,KAAK,EAAE,MAAM,QAAQ,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,OAAK,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,KAAK,cAAc,EAAE,IAAI,CAAC;AACzE,QAAM,YAAY,KAAK,MAAM,GAAG,KAAK,gBAAgB,KAAK,IAAI,UAAU,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AACrG,SAAO,CAAC,GAAG,UAAU,IAAI,CAAC,UAAU,EAAE,MAAM,gBAAgB,KAAK,EAAE,GAAG,GAAG,UAAU,IAAI,CAAC,UAAU,EAAE,MAAM,gBAAgB,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,OAAO,QAAQ,EAAE,IAAI,EAAE,EAAE;AACpL;;;ACtCA,SAAS,cAAAA,mBAAkB;AAC3B,SAAS,gBAAgB,QAAQ,YAAY,WAAW,aAAa,cAAc,aAAa,QAAQ,qBAAqB;AAC7H,SAAS,SAAS,cAAc;AAChC,SAAS,SAAS,YAAY;AAKvB,IAAM,mBAAmB;AAEzB,IAAM,gBAAgB,CAAC,MAAM,QAAQ,QAC1C,IAAI,oCAAoC,KAAK,IAAI,QAAQ,QAAQ,GAAG,oBAAoB,wBAAwB;AAE3G,IAAM,SAAS,CAAC,MAAcC,YAAW,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,KAAK;AA8BhF,IAAM,OAAO,CAAC,OAAmB,GAAG,SAAS,GAAG,OAAO,OAAO,EAAE,SAAS,GAAG,SAAS;AAE9E,SAAS,QAAQ,IAA4B;AAClD,SAAO;AAAA,IACL,GAAG,GAAG;AAAA,IACN,cAAc,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC;AAAA,IAC7E,kBAAkB,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC;AAAA,IAClF,UAAU,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,cAAc,CAAC;AAAA,EAChE;AACF;AAGO,SAAS,YAAY,UAAkB,MAAc,UAA0B;AACpF,QAAM,MAAM,YAAY,KAAK,OAAO,GAAG,gBAAgB,CAAC;AACxD,QAAM,SAAS,KAAK,KAAK,WAAW,QAAQ;AAC5C,QAAM,MAAM,KAAK,UAAU,WAAW,QAAQ;AAC9C,MAAI,WAAW,GAAG,GAAG;AACnB,eAAW,KAAK,YAAY,GAAG,EAAG,KAAI,MAAM,KAAM,QAAO,KAAK,KAAK,CAAC,GAAG,KAAK,QAAQ,CAAC,GAAG,EAAE,WAAW,KAAK,CAAC;AAAA,EAC7G;AACA,YAAU,KAAK,QAAQ,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,gBAAc,KAAK,QAAQ,MAAM,UAAU,GAAG,QAAQ;AACtD,SAAO;AACT;AAIA,IAAM,WAAW,CAAC,MAAc,WAAmB,GAAG,IAAI;AAAA,EAAK,OAAO,MAAM,CAAC;AAE7E,SAAS,UAAU,MAAqB;AACtC,QAAM,IAAW,oBAAI,IAAI;AACzB,MAAI,MAAM;AACV,MAAI;AACF,UAAM,aAAa,MAAM,MAAM;AAAA,EACjC,QAAQ;AAAA,EAAC;AACT,aAAW,QAAQ,IAAI,MAAM,IAAI,GAAG;AAClC,QAAI;AACF,YAAM,IAAI,KAAK,MAAM,IAAI;AACzB,QAAE,IAAI,GAAG,EAAE,KAAK;AAAA,EAAK,EAAE,MAAM,IAAI,EAAE,UAAU,EAAE,UAAU,MAAM,EAAE,KAAK,CAAC;AAAA,IACzE,QAAQ;AAAA,IAAC;AAAA,EACX;AACA,SAAO;AACT;AAUA,eAAsB,WAAW,SAAuB,GAA2C;AACjG,QAAM,MAAM,EAAE,OAAO,QAAQ;AAC7B,QAAM,QAAQ,WAAW,EAAE,QAAQ;AACnC,QAAM,OAAO,OAAO,EAAE,QAAQ;AAC9B,QAAM,YAAY,cAAc,GAAG;AACnC,QAAM,QAAQ,UAAU,SAAS;AACjC,QAAM,UAAU,YAAY,EAAE,UAAU,MAAM,MAAM,EAAE,QAAQ;AAC9D,MAAI,UAAU;AACd,QAAM,WAAW,OAAO,SAA4C;AAClE;AACA,UAAM,IAAI,MAAM,UAAU,EAAE,QAAQ,MAAM,KAAK,SAAS,OAAO,CAAC,SAAS,QAAQ,QAAQ,MAAM,GAAG,UAAU,GAAG,QAAQ,eAAe,KAAK,WAAW,EAAE,UAAU,CAAC;AACnK,WAAO,EAAE,KAAK,aAAa,EAAE,MAAM,EAAE,SAAS,MAAM,IAAI,IAAK,EAAE,SAAS;AAAA,EAC1E;AACA,MAAI;AACF,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,QAAQ,IAAI,OAAO,MAA8B;AAC/C,cAAM,MAAM,MAAM,IAAI,SAAS,MAAM,EAAE,IAAI,CAAC;AAC5C,YAAI,IAAK,QAAO,EAAE,GAAG,GAAG,GAAG,KAAK,WAAW,IAAI,WAAW,IAAI,IAAI,MAAM,QAAQ,KAAK;AACrF,cAAM,QAAmB,CAAC;AAC1B,YAAI;AAEJ,iBAAS,UAAU,GAAG,UAAU,mBAAmB,KAAK,MAAM,SAAS,kBAAkB,WAAW;AAClG,cAAI,MAAM,WAAW,KAAK,MAAM,CAAC,MAAM,MAAM,CAAC,EAAG;AACjD,gBAAM,IAAI,MAAM,SAAS,EAAE,IAAI;AAC/B,cAAI,OAAO,MAAM,SAAU,SAAQ;AAAA,cAC9B,OAAM,KAAK,CAAC;AAAA,QACnB;AACA,cAAM,WAAW,MAAM,OAAO,OAAO,EAAE;AACvC,YAAI,MAAM,QAAQ;AAChB,oBAAU,QAAQ,SAAS,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,yBAAe,WAAW,KAAK,UAAU,EAAE,OAAO,MAAM,QAAQ,OAAO,EAAE,IAAI,GAAG,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,IAAI;AAAA,QACxH;AACA,eAAO,EAAE,GAAG,GAAG,UAAU,MAAM,MAAM,QAAQ,WAAW,WAAW,IAAI,MAAM,QAAQ,QAAQ,OAAO,GAAI,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,EAAG;AAAA,MACzI,CAAC;AAAA,IACH;AACA,WAAO;AAAA,MACL,OAAO,MAAM;AAAA,MACb;AAAA,MACA,OAAO,QAAQ,QAAQ,OAAO,CAAC,MAAM,EAAE,UAAU,OAAO,CAAC;AAAA,MACzD,SAAS,QAAQ,QAAQ,OAAO,CAAC,MAAM,EAAE,UAAU,SAAS,CAAC;AAAA,MAC7D,MAAM;AAAA,IACR;AAAA,EACF,UAAE;AACA,WAAO,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,EAClD;AACF;","names":["createHash","createHash"]}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import {
|
|
2
|
+
materializeDraft,
|
|
3
|
+
parseReviewerJson
|
|
4
|
+
} from "./chunk-P3GLY42T.js";
|
|
5
|
+
import {
|
|
6
|
+
runClaude
|
|
7
|
+
} from "./chunk-IEZVFKUT.js";
|
|
8
|
+
import {
|
|
9
|
+
findSkills
|
|
10
|
+
} from "./chunk-ZSWI3Y4Y.js";
|
|
11
|
+
import {
|
|
12
|
+
featureOn,
|
|
13
|
+
readRepoConfig
|
|
14
|
+
} from "./chunk-VQCNWRTL.js";
|
|
15
|
+
import {
|
|
16
|
+
askJev,
|
|
17
|
+
jevVerdict
|
|
18
|
+
} from "./chunk-GYR7U6KA.js";
|
|
19
|
+
|
|
20
|
+
// src/core/skill-merge.ts
|
|
21
|
+
var MERGE_MIN_OVERLAP = 0.5;
|
|
22
|
+
var words = (s) => new Set(
|
|
23
|
+
[s.skill.name, s.skill.description, ...s.skill.meta?.triggers ?? [], ...s.skill.meta?.applies_to.paths ?? [], ...s.skill.meta?.applies_to.commands ?? []].join(" ").toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 2)
|
|
24
|
+
);
|
|
25
|
+
function mergeCandidates(repo, env = process.env) {
|
|
26
|
+
const learned = findSkills(repo, { home: env.HOME, global: false }).filter((s) => s.learned && s.skill.meta && !s.skill.meta.pinned);
|
|
27
|
+
const out = [];
|
|
28
|
+
for (let i = 0; i < learned.length; i++) {
|
|
29
|
+
for (let j = i + 1; j < learned.length; j++) {
|
|
30
|
+
const x = words(learned[i]), y = words(learned[j]);
|
|
31
|
+
const shared = [...x].filter((w) => y.has(w)).length;
|
|
32
|
+
const overlap = shared / (x.size + y.size - shared || 1);
|
|
33
|
+
if (overlap >= MERGE_MIN_OVERLAP) out.push({ a: learned[i], b: learned[j], overlap });
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return out.sort((p, q) => q.overlap - p.overlap);
|
|
37
|
+
}
|
|
38
|
+
var consolidateOn = (repo) => readRepoConfig(repo).skill_loop_consolidate === true;
|
|
39
|
+
var show = (s) => `### ${s.skill.name}
|
|
40
|
+
${s.skill.description}
|
|
41
|
+
|
|
42
|
+
${s.skill.body}`;
|
|
43
|
+
var MERGE_PROMPT = `Two learned skills of this repository may describe the same procedure. If they do (or one contains the
|
|
44
|
+
other), write ONE skill that keeps every step and pitfall that is still true, in the same format as the inputs:
|
|
45
|
+
sections "## When to use", "## Steps", "## Pitfalls", "## Verify". Never invent steps. If they are different
|
|
46
|
+
procedures, say so.
|
|
47
|
+
|
|
48
|
+
Answer with one JSON object and nothing else:
|
|
49
|
+
{"drafts": [{"kind": "new", "name": "<kebab-case>", "description": "<what + Use when ...>", "scope": "project",
|
|
50
|
+
"applies_to": {"paths": [], "commands": []}, "triggers": [], "provenance": {"sessions": [], "failures": [], "corrections": []},
|
|
51
|
+
"cites": [], "body": "..."}]} \u2014 or {"drafts": [], "none_reason": "<why they are different>"}`;
|
|
52
|
+
async function proposeMerges(repo, env = process.env) {
|
|
53
|
+
const drafted = [];
|
|
54
|
+
const skipped = [];
|
|
55
|
+
const jev = featureOn("jev", repo, env);
|
|
56
|
+
const used = /* @__PURE__ */ new Set();
|
|
57
|
+
for (const { a, b } of mergeCandidates(repo, env)) {
|
|
58
|
+
if (used.has(a.path) || used.has(b.path)) continue;
|
|
59
|
+
const pair = `${a.skill.name} + ${b.skill.name}`;
|
|
60
|
+
if (jev) {
|
|
61
|
+
const r2 = await askJev(
|
|
62
|
+
{
|
|
63
|
+
cwd: repo,
|
|
64
|
+
caller: "semantic-memory-skill-loop",
|
|
65
|
+
state: `${show(a)}
|
|
66
|
+
|
|
67
|
+
${show(b)}`.slice(0, 2e3),
|
|
68
|
+
questions: [
|
|
69
|
+
{ key: "same", instructions: "Do these two skills describe the same procedure, so that one could replace the other?" },
|
|
70
|
+
{ key: "subset", instructions: "Is every step of one of these skills already contained in the other skill?" }
|
|
71
|
+
]
|
|
72
|
+
},
|
|
73
|
+
{ env, timeoutMs: 3e4 }
|
|
74
|
+
);
|
|
75
|
+
if (jevVerdict(r2.scores.same) !== true && jevVerdict(r2.scores.subset) !== true) {
|
|
76
|
+
skipped.push(`${pair}: Jev did not say duplicate (${r2.error ?? JSON.stringify(r2.scores)})`);
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const run = await runClaude({ prompt: `${MERGE_PROMPT}
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
${show(a)}
|
|
85
|
+
|
|
86
|
+
${show(b)}`, cwd: repo, tools: [], maxTurns: 1, env });
|
|
87
|
+
const answer = run.ok ? parseReviewerJson(run.result) : null;
|
|
88
|
+
const d = answer?.drafts[0];
|
|
89
|
+
if (!d) {
|
|
90
|
+
skipped.push(`${pair}: ${answer?.none_reason ?? run.error ?? "no merge proposed"}`);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const prov = (k) => [.../* @__PURE__ */ new Set([...a.skill.meta.provenance[k] ?? [], ...b.skill.meta.provenance[k] ?? []])];
|
|
94
|
+
const job = { session_id: prov("sessions")[0] ?? "merge", repo, transcript_path: null, reason: `merge of ${pair}`, ts: (/* @__PURE__ */ new Date()).toISOString(), evidence: { failures: prov("failures"), corrections: prov("corrections").length, skills_used: [], tool_calls: 0 } };
|
|
95
|
+
const r = await materializeDraft(
|
|
96
|
+
repo,
|
|
97
|
+
job,
|
|
98
|
+
{ ...d, kind: "new", provenance: { sessions: prov("sessions"), failures: prov("failures"), corrections: prov("corrections") } },
|
|
99
|
+
findSkills(repo, { home: env.HOME }).filter((s) => s.path !== a.path && s.path !== b.path),
|
|
100
|
+
async () => ({ scope: "project", globalCandidate: false, why: `merge of ${pair}` }),
|
|
101
|
+
env,
|
|
102
|
+
{ supersedes: [a.path, b.path] }
|
|
103
|
+
);
|
|
104
|
+
if (r.id) {
|
|
105
|
+
drafted.push(r.id);
|
|
106
|
+
used.add(a.path).add(b.path);
|
|
107
|
+
} else skipped.push(`${pair}: ${r.rejected}`);
|
|
108
|
+
}
|
|
109
|
+
return { drafted, skipped };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export {
|
|
113
|
+
MERGE_MIN_OVERLAP,
|
|
114
|
+
mergeCandidates,
|
|
115
|
+
consolidateOn,
|
|
116
|
+
proposeMerges
|
|
117
|
+
};
|
|
118
|
+
//# sourceMappingURL=chunk-NCGM36NX.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/core/skill-merge.ts"],"sourcesContent":["/**\n * Merging learned skills (v1.7) — opt-in only: `skill_loop_consolidate: true` in the repo config, or\n * `skill-loop curate --merge`. Candidate pairs come from word overlap of name, description, triggers and\n * applies_to. Jev mode: Jev judges each pair (same procedure / one contains the other, >= 0.95) and only a\n * yes goes on to Claude for the merged text; without jev mode Claude judges and drafts in one call. Either\n * way the result is a review draft that `supersedes` both — nothing merges without a person.\n */\nimport { featureOn, readRepoConfig } from \"./features.js\";\nimport { askJev, jevVerdict } from \"./jev.js\";\nimport { runClaude } from \"./claude-run.js\";\nimport { findSkills, type SkillFile } from \"./skill-schema.js\";\nimport { materializeDraft, parseReviewerJson } from \"./skill-review.js\";\n\nexport const MERGE_MIN_OVERLAP = 0.5;\n\nconst words = (s: SkillFile) =>\n new Set(\n [s.skill.name, s.skill.description, ...(s.skill.meta?.triggers ?? []), ...(s.skill.meta?.applies_to.paths ?? []), ...(s.skill.meta?.applies_to.commands ?? [])]\n .join(\" \")\n .toLowerCase()\n .split(/[^a-z0-9]+/)\n .filter((w) => w.length > 2),\n );\n\nexport function mergeCandidates(repo: string, env = process.env): { a: SkillFile; b: SkillFile; overlap: number }[] {\n const learned = findSkills(repo, { home: env.HOME, global: false }).filter((s) => s.learned && s.skill.meta && !s.skill.meta.pinned);\n const out: { a: SkillFile; b: SkillFile; overlap: number }[] = [];\n for (let i = 0; i < learned.length; i++) {\n for (let j = i + 1; j < learned.length; j++) {\n const x = words(learned[i]), y = words(learned[j]);\n const shared = [...x].filter((w) => y.has(w)).length;\n const overlap = shared / (x.size + y.size - shared || 1);\n if (overlap >= MERGE_MIN_OVERLAP) out.push({ a: learned[i], b: learned[j], overlap });\n }\n }\n return out.sort((p, q) => q.overlap - p.overlap);\n}\n\nexport const consolidateOn = (repo: string) => readRepoConfig(repo).skill_loop_consolidate === true;\n\nconst show = (s: SkillFile) => `### ${s.skill.name}\\n${s.skill.description}\\n\\n${s.skill.body}`;\n\nconst MERGE_PROMPT = `Two learned skills of this repository may describe the same procedure. If they do (or one contains the\nother), write ONE skill that keeps every step and pitfall that is still true, in the same format as the inputs:\nsections \"## When to use\", \"## Steps\", \"## Pitfalls\", \"## Verify\". Never invent steps. If they are different\nprocedures, say so.\n\nAnswer with one JSON object and nothing else:\n{\"drafts\": [{\"kind\": \"new\", \"name\": \"<kebab-case>\", \"description\": \"<what + Use when ...>\", \"scope\": \"project\",\n \"applies_to\": {\"paths\": [], \"commands\": []}, \"triggers\": [], \"provenance\": {\"sessions\": [], \"failures\": [], \"corrections\": []},\n \"cites\": [], \"body\": \"...\"}]} — or {\"drafts\": [], \"none_reason\": \"<why they are different>\"}`;\n\nexport async function proposeMerges(repo: string, env = process.env): Promise<{ drafted: string[]; skipped: string[] }> {\n const drafted: string[] = [];\n const skipped: string[] = [];\n const jev = featureOn(\"jev\", repo, env);\n const used = new Set<string>();\n for (const { a, b } of mergeCandidates(repo, env)) {\n if (used.has(a.path) || used.has(b.path)) continue;\n const pair = `${a.skill.name} + ${b.skill.name}`;\n if (jev) {\n const r = await askJev(\n {\n cwd: repo,\n caller: \"semantic-memory-skill-loop\",\n state: `${show(a)}\\n\\n${show(b)}`.slice(0, 2000),\n questions: [\n { key: \"same\", instructions: \"Do these two skills describe the same procedure, so that one could replace the other?\" },\n { key: \"subset\", instructions: \"Is every step of one of these skills already contained in the other skill?\" },\n ],\n },\n { env, timeoutMs: 30_000 },\n );\n if (jevVerdict(r.scores.same) !== true && jevVerdict(r.scores.subset) !== true) {\n skipped.push(`${pair}: Jev did not say duplicate (${r.error ?? JSON.stringify(r.scores)})`);\n continue;\n }\n }\n const run = await runClaude({ prompt: `${MERGE_PROMPT}\\n\\n---\\n\\n${show(a)}\\n\\n${show(b)}`, cwd: repo, tools: [], maxTurns: 1, env });\n const answer = run.ok ? parseReviewerJson(run.result) : null;\n const d = answer?.drafts[0];\n if (!d) {\n skipped.push(`${pair}: ${answer?.none_reason ?? run.error ?? \"no merge proposed\"}`);\n continue;\n }\n const prov = (k: \"sessions\" | \"failures\" | \"corrections\") => [...new Set([...(a.skill.meta!.provenance[k] ?? []), ...(b.skill.meta!.provenance[k] ?? [])])];\n const job = { session_id: prov(\"sessions\")[0] ?? \"merge\", repo, transcript_path: null, reason: `merge of ${pair}`, ts: new Date().toISOString(), evidence: { failures: prov(\"failures\"), corrections: prov(\"corrections\").length, skills_used: [], tool_calls: 0 } };\n const r = await materializeDraft(\n repo,\n job,\n { ...d, kind: \"new\", provenance: { sessions: prov(\"sessions\"), failures: prov(\"failures\"), corrections: prov(\"corrections\") } },\n findSkills(repo, { home: env.HOME }).filter((s) => s.path !== a.path && s.path !== b.path),\n async () => ({ scope: \"project\", globalCandidate: false, why: `merge of ${pair}` }),\n env,\n { supersedes: [a.path, b.path] },\n );\n if (r.id) {\n drafted.push(r.id);\n used.add(a.path).add(b.path);\n } else skipped.push(`${pair}: ${r.rejected}`);\n }\n return { drafted, skipped };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAaO,IAAM,oBAAoB;AAEjC,IAAM,QAAQ,CAAC,MACb,IAAI;AAAA,EACF,CAAC,EAAE,MAAM,MAAM,EAAE,MAAM,aAAa,GAAI,EAAE,MAAM,MAAM,YAAY,CAAC,GAAI,GAAI,EAAE,MAAM,MAAM,WAAW,SAAS,CAAC,GAAI,GAAI,EAAE,MAAM,MAAM,WAAW,YAAY,CAAC,CAAE,EAC3J,KAAK,GAAG,EACR,YAAY,EACZ,MAAM,YAAY,EAClB,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AAC/B;AAEK,SAAS,gBAAgB,MAAc,MAAM,QAAQ,KAAwD;AAClH,QAAM,UAAU,WAAW,MAAM,EAAE,MAAM,IAAI,MAAM,QAAQ,MAAM,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM;AACnI,QAAM,MAAyD,CAAC;AAChE,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,aAAS,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AAC3C,YAAM,IAAI,MAAM,QAAQ,CAAC,CAAC,GAAG,IAAI,MAAM,QAAQ,CAAC,CAAC;AACjD,YAAM,SAAS,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE;AAC9C,YAAM,UAAU,UAAU,EAAE,OAAO,EAAE,OAAO,UAAU;AACtD,UAAI,WAAW,kBAAmB,KAAI,KAAK,EAAE,GAAG,QAAQ,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC;AAAA,IACtF;AAAA,EACF;AACA,SAAO,IAAI,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO;AACjD;AAEO,IAAM,gBAAgB,CAAC,SAAiB,eAAe,IAAI,EAAE,2BAA2B;AAE/F,IAAM,OAAO,CAAC,MAAiB,OAAO,EAAE,MAAM,IAAI;AAAA,EAAK,EAAE,MAAM,WAAW;AAAA;AAAA,EAAO,EAAE,MAAM,IAAI;AAE7F,IAAM,eAAe;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAUrB,eAAsB,cAAc,MAAc,MAAM,QAAQ,KAAwD;AACtH,QAAM,UAAoB,CAAC;AAC3B,QAAM,UAAoB,CAAC;AAC3B,QAAM,MAAM,UAAU,OAAO,MAAM,GAAG;AACtC,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,EAAE,GAAG,EAAE,KAAK,gBAAgB,MAAM,GAAG,GAAG;AACjD,QAAI,KAAK,IAAI,EAAE,IAAI,KAAK,KAAK,IAAI,EAAE,IAAI,EAAG;AAC1C,UAAM,OAAO,GAAG,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,IAAI;AAC9C,QAAI,KAAK;AACP,YAAMA,KAAI,MAAM;AAAA,QACd;AAAA,UACE,KAAK;AAAA,UACL,QAAQ;AAAA,UACR,OAAO,GAAG,KAAK,CAAC,CAAC;AAAA;AAAA,EAAO,KAAK,CAAC,CAAC,GAAG,MAAM,GAAG,GAAI;AAAA,UAC/C,WAAW;AAAA,YACT,EAAE,KAAK,QAAQ,cAAc,wFAAwF;AAAA,YACrH,EAAE,KAAK,UAAU,cAAc,6EAA6E;AAAA,UAC9G;AAAA,QACF;AAAA,QACA,EAAE,KAAK,WAAW,IAAO;AAAA,MAC3B;AACA,UAAI,WAAWA,GAAE,OAAO,IAAI,MAAM,QAAQ,WAAWA,GAAE,OAAO,MAAM,MAAM,MAAM;AAC9E,gBAAQ,KAAK,GAAG,IAAI,gCAAgCA,GAAE,SAAS,KAAK,UAAUA,GAAE,MAAM,CAAC,GAAG;AAC1F;AAAA,MACF;AAAA,IACF;AACA,UAAM,MAAM,MAAM,UAAU,EAAE,QAAQ,GAAG,YAAY;AAAA;AAAA;AAAA;AAAA,EAAc,KAAK,CAAC,CAAC;AAAA;AAAA,EAAO,KAAK,CAAC,CAAC,IAAI,KAAK,MAAM,OAAO,CAAC,GAAG,UAAU,GAAG,IAAI,CAAC;AACpI,UAAM,SAAS,IAAI,KAAK,kBAAkB,IAAI,MAAM,IAAI;AACxD,UAAM,IAAI,QAAQ,OAAO,CAAC;AAC1B,QAAI,CAAC,GAAG;AACN,cAAQ,KAAK,GAAG,IAAI,KAAK,QAAQ,eAAe,IAAI,SAAS,mBAAmB,EAAE;AAClF;AAAA,IACF;AACA,UAAM,OAAO,CAAC,MAA+C,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAI,EAAE,MAAM,KAAM,WAAW,CAAC,KAAK,CAAC,GAAI,GAAI,EAAE,MAAM,KAAM,WAAW,CAAC,KAAK,CAAC,CAAE,CAAC,CAAC;AAC1J,UAAM,MAAM,EAAE,YAAY,KAAK,UAAU,EAAE,CAAC,KAAK,SAAS,MAAM,iBAAiB,MAAM,QAAQ,YAAY,IAAI,IAAI,KAAI,oBAAI,KAAK,GAAE,YAAY,GAAG,UAAU,EAAE,UAAU,KAAK,UAAU,GAAG,aAAa,KAAK,aAAa,EAAE,QAAQ,aAAa,CAAC,GAAG,YAAY,EAAE,EAAE;AACnQ,UAAM,IAAI,MAAM;AAAA,MACd;AAAA,MACA;AAAA,MACA,EAAE,GAAG,GAAG,MAAM,OAAO,YAAY,EAAE,UAAU,KAAK,UAAU,GAAG,UAAU,KAAK,UAAU,GAAG,aAAa,KAAK,aAAa,EAAE,EAAE;AAAA,MAC9H,WAAW,MAAM,EAAE,MAAM,IAAI,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI;AAAA,MACzF,aAAa,EAAE,OAAO,WAAW,iBAAiB,OAAO,KAAK,YAAY,IAAI,GAAG;AAAA,MACjF;AAAA,MACA,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;AAAA,IACjC;AACA,QAAI,EAAE,IAAI;AACR,cAAQ,KAAK,EAAE,EAAE;AACjB,WAAK,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;AAAA,IAC7B,MAAO,SAAQ,KAAK,GAAG,IAAI,KAAK,EAAE,QAAQ,EAAE;AAAA,EAC9C;AACA,SAAO,EAAE,SAAS,QAAQ;AAC5B;","names":["r"]}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import {
|
|
2
|
+
readEvidence,
|
|
3
|
+
stateDir
|
|
4
|
+
} from "./chunk-R6GNP7DS.js";
|
|
5
|
+
|
|
6
|
+
// src/core/skill-queue.ts
|
|
7
|
+
import { appendFileSync, existsSync, mkdirSync, openSync, writeSync, closeSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync, statSync } from "fs";
|
|
8
|
+
import { homedir } from "os";
|
|
9
|
+
import { join } from "path";
|
|
10
|
+
var smHome = (env = process.env) => env.SEMANTIC_MEMORY_STATE_HOME || join(env.HOME || homedir(), ".semantic-memory");
|
|
11
|
+
var queueDir = (env = process.env) => join(smHome(env), "review-queue");
|
|
12
|
+
var runsPath = (env = process.env) => join(smHome(env), "skill-loop-runs.jsonl");
|
|
13
|
+
var lockPath = (repo) => join(stateDir(repo), "skill-loop.lock");
|
|
14
|
+
var logPath = (repo) => join(stateDir(repo), "skill-loop.log");
|
|
15
|
+
var dailyRunCap = (env = process.env) => Math.max(0, Number(env.SEMANTIC_MEMORY_SKILL_LOOP_DAILY_RUNS ?? 10) || 0);
|
|
16
|
+
var minToolCalls = (env = process.env) => Number(env.SEMANTIC_MEMORY_SKILL_LOOP_MIN_TOOL_CALLS) || 20;
|
|
17
|
+
var TRANSCRIPT_SCAN_MAX = 50 * 1024 * 1024;
|
|
18
|
+
var safeName = (s) => s.replace(/[^A-Za-z0-9_.-]/g, "_").slice(0, 120);
|
|
19
|
+
function countToolCalls(transcript) {
|
|
20
|
+
if (!transcript) return 0;
|
|
21
|
+
try {
|
|
22
|
+
if (statSync(transcript).size > TRANSCRIPT_SCAN_MAX) return 0;
|
|
23
|
+
return readFileSync(transcript, "utf8").split('"type":"tool_use"').length - 1;
|
|
24
|
+
} catch {
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function sessionJob(repo, input, env = process.env) {
|
|
29
|
+
const session_id = typeof input.session_id === "string" ? input.session_id : "";
|
|
30
|
+
if (!session_id) return null;
|
|
31
|
+
const failures = readEvidence(repo, "failures").filter((f) => f.session_id === session_id);
|
|
32
|
+
const corrections = readEvidence(repo, "corrections").filter((c) => c.session_id === session_id).length;
|
|
33
|
+
const skills_used = [...new Set(readEvidence(repo, "usage").filter((u) => u.session_id === session_id).map((u) => String(u.skill)))];
|
|
34
|
+
const transcript_path = typeof input.transcript_path === "string" ? input.transcript_path : null;
|
|
35
|
+
const tool_calls = failures.length || corrections ? 0 : countToolCalls(transcript_path);
|
|
36
|
+
const why = failures.length ? `${failures.length} failure(s)` : corrections ? `${corrections} correction(s)` : tool_calls >= minToolCalls(env) ? `${tool_calls} tool calls` : "";
|
|
37
|
+
if (!why) return null;
|
|
38
|
+
return {
|
|
39
|
+
session_id,
|
|
40
|
+
repo,
|
|
41
|
+
transcript_path,
|
|
42
|
+
reason: `${why}; ended: ${input.reason ?? "unknown"}`,
|
|
43
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
44
|
+
evidence: { failures: failures.map((f) => String(f.id ?? "")).filter(Boolean), corrections, skills_used, tool_calls }
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function enqueue(job, env = process.env) {
|
|
48
|
+
mkdirSync(queueDir(env), { recursive: true });
|
|
49
|
+
const path = join(queueDir(env), `${safeName(job.session_id)}.json`);
|
|
50
|
+
if (["done", "failed"].some((d) => existsSync(join(queueDir(env), d, `${safeName(job.session_id)}.json`)))) return path;
|
|
51
|
+
writeFileSync(path, JSON.stringify(job, null, 1));
|
|
52
|
+
return path;
|
|
53
|
+
}
|
|
54
|
+
function pendingJobs(repo, env = process.env) {
|
|
55
|
+
let files = [];
|
|
56
|
+
try {
|
|
57
|
+
files = readdirSync(queueDir(env)).filter((f) => f.endsWith(".json"));
|
|
58
|
+
} catch {
|
|
59
|
+
return [];
|
|
60
|
+
}
|
|
61
|
+
const jobs = [];
|
|
62
|
+
for (const f of files) {
|
|
63
|
+
try {
|
|
64
|
+
const path = join(queueDir(env), f);
|
|
65
|
+
const job = JSON.parse(readFileSync(path, "utf8"));
|
|
66
|
+
if (job.repo === repo) jobs.push({ path, job });
|
|
67
|
+
} catch {
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return jobs.sort((a, b) => a.job.ts.localeCompare(b.job.ts));
|
|
71
|
+
}
|
|
72
|
+
function finishJob(path, outcome, result, env = process.env) {
|
|
73
|
+
const dir = join(queueDir(env), outcome);
|
|
74
|
+
mkdirSync(dir, { recursive: true });
|
|
75
|
+
const job = JSON.parse(readFileSync(path, "utf8"));
|
|
76
|
+
writeFileSync(path, JSON.stringify({ ...job, finished: (/* @__PURE__ */ new Date()).toISOString(), result }, null, 1));
|
|
77
|
+
renameSync(path, join(dir, path.split("/").pop()));
|
|
78
|
+
}
|
|
79
|
+
function bumpAttempts(path) {
|
|
80
|
+
const job = JSON.parse(readFileSync(path, "utf8"));
|
|
81
|
+
job.attempts = (job.attempts ?? 0) + 1;
|
|
82
|
+
writeFileSync(path, JSON.stringify(job, null, 1));
|
|
83
|
+
return job.attempts;
|
|
84
|
+
}
|
|
85
|
+
var localDay = (d = /* @__PURE__ */ new Date()) => `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
|
|
86
|
+
function runsToday(env = process.env, now = /* @__PURE__ */ new Date()) {
|
|
87
|
+
try {
|
|
88
|
+
const day = localDay(now);
|
|
89
|
+
return readFileSync(runsPath(env), "utf8").split("\n").filter((l) => l.includes(`"day":"${day}"`)).length;
|
|
90
|
+
} catch {
|
|
91
|
+
return 0;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
function recordRun(repo, session_id, env = process.env, now = /* @__PURE__ */ new Date()) {
|
|
95
|
+
mkdirSync(smHome(env), { recursive: true });
|
|
96
|
+
appendFileSync(runsPath(env), JSON.stringify({ day: localDay(now), ts: now.toISOString(), repo, session_id }) + "\n");
|
|
97
|
+
}
|
|
98
|
+
var alive = (pid) => {
|
|
99
|
+
try {
|
|
100
|
+
process.kill(pid, 0);
|
|
101
|
+
return true;
|
|
102
|
+
} catch (e) {
|
|
103
|
+
return e.code === "EPERM";
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
function lockHolder(repo) {
|
|
107
|
+
try {
|
|
108
|
+
const pid = Number(readFileSync(lockPath(repo), "utf8").trim());
|
|
109
|
+
return pid && alive(pid) ? pid : null;
|
|
110
|
+
} catch {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
function takeLock(repo) {
|
|
115
|
+
mkdirSync(stateDir(repo), { recursive: true });
|
|
116
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
117
|
+
try {
|
|
118
|
+
const fd = openSync(lockPath(repo), "wx");
|
|
119
|
+
writeSync(fd, `${process.pid}
|
|
120
|
+
`);
|
|
121
|
+
closeSync(fd);
|
|
122
|
+
return () => {
|
|
123
|
+
try {
|
|
124
|
+
if (Number(readFileSync(lockPath(repo), "utf8").trim()) === process.pid) unlinkSync(lockPath(repo));
|
|
125
|
+
} catch {
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
} catch (e) {
|
|
129
|
+
if (e.code !== "EEXIST") throw e;
|
|
130
|
+
if (lockHolder(repo) !== null) return null;
|
|
131
|
+
try {
|
|
132
|
+
unlinkSync(lockPath(repo));
|
|
133
|
+
} catch {
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
function log(repo, line) {
|
|
140
|
+
try {
|
|
141
|
+
mkdirSync(stateDir(repo), { recursive: true });
|
|
142
|
+
appendFileSync(logPath(repo), `${(/* @__PURE__ */ new Date()).toISOString()} ${line}
|
|
143
|
+
`);
|
|
144
|
+
} catch {
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export {
|
|
149
|
+
smHome,
|
|
150
|
+
queueDir,
|
|
151
|
+
lockPath,
|
|
152
|
+
logPath,
|
|
153
|
+
dailyRunCap,
|
|
154
|
+
sessionJob,
|
|
155
|
+
enqueue,
|
|
156
|
+
pendingJobs,
|
|
157
|
+
finishJob,
|
|
158
|
+
bumpAttempts,
|
|
159
|
+
runsToday,
|
|
160
|
+
recordRun,
|
|
161
|
+
lockHolder,
|
|
162
|
+
takeLock,
|
|
163
|
+
log
|
|
164
|
+
};
|
|
165
|
+
//# sourceMappingURL=chunk-NIZLBNH4.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/core/skill-queue.ts"],"sourcesContent":["/**\n * The skill loop's review queue (v1.7). SessionEnd writes one job per session that has signal and starts\n * the worker detached; the worker drains this repo's jobs oldest first, inside a global daily cap. The\n * session never waits: SessionEnd hooks get 1.5 s by default, so the hook only writes and spawns.\n *\n * ~/.semantic-memory/review-queue/<session>.json pending jobs (all repos)\n * ~/.semantic-memory/review-queue/{done,failed}/ finished jobs, never re-reviewed\n * ~/.semantic-memory/skill-loop-runs.jsonl one line per review run (the daily cap)\n * <repo>/.claude/.semantic-memory/skill-loop.lock the repo's worker (pid)\n * <repo>/.claude/.semantic-memory/skill-loop.log what the worker did\n *\n * Standalone for hooks (no embedder).\n */\nimport { appendFileSync, existsSync, mkdirSync, openSync, writeSync, closeSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport { readEvidence, stateDir } from \"./skill-evidence.js\";\n\nexport interface ReviewJob {\n session_id: string;\n repo: string;\n transcript_path: string | null;\n reason: string;\n ts: string;\n evidence: { failures: string[]; corrections: number; skills_used: string[]; tool_calls: number };\n attempts?: number;\n}\n\nexport const smHome = (env = process.env) => env.SEMANTIC_MEMORY_STATE_HOME || join(env.HOME || homedir(), \".semantic-memory\");\nexport const queueDir = (env = process.env) => join(smHome(env), \"review-queue\");\nconst runsPath = (env = process.env) => join(smHome(env), \"skill-loop-runs.jsonl\");\nexport const lockPath = (repo: string) => join(stateDir(repo), \"skill-loop.lock\");\nexport const logPath = (repo: string) => join(stateDir(repo), \"skill-loop.log\");\n\nexport const dailyRunCap = (env = process.env) => Math.max(0, Number(env.SEMANTIC_MEMORY_SKILL_LOOP_DAILY_RUNS ?? 10) || 0);\nconst minToolCalls = (env = process.env) => Number(env.SEMANTIC_MEMORY_SKILL_LOOP_MIN_TOOL_CALLS) || 20;\n// shortcut: counts tool calls by scanning the transcript text; above this size the count is skipped (signal then\n// needs a failure or a correction). Upgrade if long tool-only sessions should qualify.\nconst TRANSCRIPT_SCAN_MAX = 50 * 1024 * 1024;\n\nconst safeName = (s: string) => s.replace(/[^A-Za-z0-9_.-]/g, \"_\").slice(0, 120);\n\nfunction countToolCalls(transcript: string | null): number {\n if (!transcript) return 0;\n try {\n if (statSync(transcript).size > TRANSCRIPT_SCAN_MAX) return 0;\n return readFileSync(transcript, \"utf8\").split('\"type\":\"tool_use\"').length - 1;\n } catch {\n return 0;\n }\n}\n\n/** The job SessionEnd would write for this session, or null when the session has nothing to learn from. */\nexport function sessionJob(repo: string, input: { session_id?: string; transcript_path?: string; reason?: string }, env = process.env): ReviewJob | null {\n const session_id = typeof input.session_id === \"string\" ? input.session_id : \"\";\n if (!session_id) return null;\n const failures = readEvidence(repo, \"failures\").filter((f) => f.session_id === session_id);\n const corrections = readEvidence(repo, \"corrections\").filter((c) => c.session_id === session_id).length;\n const skills_used = [...new Set(readEvidence(repo, \"usage\").filter((u) => u.session_id === session_id).map((u) => String(u.skill)))];\n const transcript_path = typeof input.transcript_path === \"string\" ? input.transcript_path : null;\n const tool_calls = failures.length || corrections ? 0 : countToolCalls(transcript_path);\n const why = failures.length ? `${failures.length} failure(s)` : corrections ? `${corrections} correction(s)` : tool_calls >= minToolCalls(env) ? `${tool_calls} tool calls` : \"\";\n if (!why) return null;\n return {\n session_id,\n repo,\n transcript_path,\n reason: `${why}; ended: ${input.reason ?? \"unknown\"}`,\n ts: new Date().toISOString(),\n evidence: { failures: failures.map((f) => String(f.id ?? \"\")).filter(Boolean), corrections, skills_used, tool_calls },\n };\n}\n\nexport function enqueue(job: ReviewJob, env = process.env): string {\n mkdirSync(queueDir(env), { recursive: true });\n const path = join(queueDir(env), `${safeName(job.session_id)}.json`);\n // a session that ends twice (e.g. /clear then exit) is one job, and one already reviewed stays done\n if ([\"done\", \"failed\"].some((d) => existsSync(join(queueDir(env), d, `${safeName(job.session_id)}.json`)))) return path;\n writeFileSync(path, JSON.stringify(job, null, 1));\n return path;\n}\n\nexport function pendingJobs(repo: string, env = process.env): { path: string; job: ReviewJob }[] {\n let files: string[] = [];\n try {\n files = readdirSync(queueDir(env)).filter((f) => f.endsWith(\".json\"));\n } catch {\n return [];\n }\n const jobs: { path: string; job: ReviewJob }[] = [];\n for (const f of files) {\n try {\n const path = join(queueDir(env), f);\n const job = JSON.parse(readFileSync(path, \"utf8\")) as ReviewJob;\n if (job.repo === repo) jobs.push({ path, job });\n } catch {}\n }\n return jobs.sort((a, b) => a.job.ts.localeCompare(b.job.ts));\n}\n\nexport function finishJob(path: string, outcome: \"done\" | \"failed\", result: Record<string, unknown>, env = process.env): void {\n const dir = join(queueDir(env), outcome);\n mkdirSync(dir, { recursive: true });\n const job = JSON.parse(readFileSync(path, \"utf8\"));\n writeFileSync(path, JSON.stringify({ ...job, finished: new Date().toISOString(), result }, null, 1));\n renameSync(path, join(dir, path.split(\"/\").pop()!));\n}\n\nexport function bumpAttempts(path: string): number {\n const job = JSON.parse(readFileSync(path, \"utf8\")) as ReviewJob;\n job.attempts = (job.attempts ?? 0) + 1;\n writeFileSync(path, JSON.stringify(job, null, 1));\n return job.attempts;\n}\n\nconst localDay = (d = new Date()) => `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, \"0\")}-${String(d.getDate()).padStart(2, \"0\")}`;\n\nexport function runsToday(env = process.env, now = new Date()): number {\n try {\n const day = localDay(now);\n return readFileSync(runsPath(env), \"utf8\").split(\"\\n\").filter((l) => l.includes(`\"day\":\"${day}\"`)).length;\n } catch {\n return 0;\n }\n}\n\nexport function recordRun(repo: string, session_id: string, env = process.env, now = new Date()): void {\n mkdirSync(smHome(env), { recursive: true });\n appendFileSync(runsPath(env), JSON.stringify({ day: localDay(now), ts: now.toISOString(), repo, session_id }) + \"\\n\");\n}\n\nconst alive = (pid: number) => {\n try {\n process.kill(pid, 0);\n return true;\n } catch (e) {\n return (e as NodeJS.ErrnoException).code === \"EPERM\";\n }\n};\n\nexport function lockHolder(repo: string): number | null {\n try {\n const pid = Number(readFileSync(lockPath(repo), \"utf8\").trim());\n return pid && alive(pid) ? pid : null;\n } catch {\n return null;\n }\n}\n\n/** Take the repo's worker lock (reclaiming a dead holder's). Returns a release function, or null if held. */\nexport function takeLock(repo: string): (() => void) | null {\n mkdirSync(stateDir(repo), { recursive: true });\n for (let attempt = 0; attempt < 2; attempt++) {\n try {\n const fd = openSync(lockPath(repo), \"wx\");\n writeSync(fd, `${process.pid}\\n`);\n closeSync(fd);\n return () => {\n try {\n if (Number(readFileSync(lockPath(repo), \"utf8\").trim()) === process.pid) unlinkSync(lockPath(repo));\n } catch {}\n };\n } catch (e) {\n if ((e as NodeJS.ErrnoException).code !== \"EEXIST\") throw e;\n if (lockHolder(repo) !== null) return null;\n try {\n unlinkSync(lockPath(repo));\n } catch {}\n }\n }\n return null;\n}\n\nexport function log(repo: string, line: string): void {\n try {\n mkdirSync(stateDir(repo), { recursive: true });\n appendFileSync(logPath(repo), `${new Date().toISOString()} ${line}\\n`);\n } catch {}\n}\n"],"mappings":";;;;;;AAaA,SAAS,gBAAgB,YAAY,WAAW,UAAU,WAAW,WAAW,cAAc,aAAa,YAAY,YAAY,eAAe,gBAAgB;AAClK,SAAS,eAAe;AACxB,SAAS,YAAY;AAad,IAAM,SAAS,CAAC,MAAM,QAAQ,QAAQ,IAAI,8BAA8B,KAAK,IAAI,QAAQ,QAAQ,GAAG,kBAAkB;AACtH,IAAM,WAAW,CAAC,MAAM,QAAQ,QAAQ,KAAK,OAAO,GAAG,GAAG,cAAc;AAC/E,IAAM,WAAW,CAAC,MAAM,QAAQ,QAAQ,KAAK,OAAO,GAAG,GAAG,uBAAuB;AAC1E,IAAM,WAAW,CAAC,SAAiB,KAAK,SAAS,IAAI,GAAG,iBAAiB;AACzE,IAAM,UAAU,CAAC,SAAiB,KAAK,SAAS,IAAI,GAAG,gBAAgB;AAEvE,IAAM,cAAc,CAAC,MAAM,QAAQ,QAAQ,KAAK,IAAI,GAAG,OAAO,IAAI,yCAAyC,EAAE,KAAK,CAAC;AAC1H,IAAM,eAAe,CAAC,MAAM,QAAQ,QAAQ,OAAO,IAAI,yCAAyC,KAAK;AAGrG,IAAM,sBAAsB,KAAK,OAAO;AAExC,IAAM,WAAW,CAAC,MAAc,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,GAAG;AAE/E,SAAS,eAAe,YAAmC;AACzD,MAAI,CAAC,WAAY,QAAO;AACxB,MAAI;AACF,QAAI,SAAS,UAAU,EAAE,OAAO,oBAAqB,QAAO;AAC5D,WAAO,aAAa,YAAY,MAAM,EAAE,MAAM,mBAAmB,EAAE,SAAS;AAAA,EAC9E,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,WAAW,MAAc,OAA2E,MAAM,QAAQ,KAAuB;AACvJ,QAAM,aAAa,OAAO,MAAM,eAAe,WAAW,MAAM,aAAa;AAC7E,MAAI,CAAC,WAAY,QAAO;AACxB,QAAM,WAAW,aAAa,MAAM,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,eAAe,UAAU;AACzF,QAAM,cAAc,aAAa,MAAM,aAAa,EAAE,OAAO,CAAC,MAAM,EAAE,eAAe,UAAU,EAAE;AACjG,QAAM,cAAc,CAAC,GAAG,IAAI,IAAI,aAAa,MAAM,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,eAAe,UAAU,EAAE,IAAI,CAAC,MAAM,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;AACnI,QAAM,kBAAkB,OAAO,MAAM,oBAAoB,WAAW,MAAM,kBAAkB;AAC5F,QAAM,aAAa,SAAS,UAAU,cAAc,IAAI,eAAe,eAAe;AACtF,QAAM,MAAM,SAAS,SAAS,GAAG,SAAS,MAAM,gBAAgB,cAAc,GAAG,WAAW,mBAAmB,cAAc,aAAa,GAAG,IAAI,GAAG,UAAU,gBAAgB;AAC9K,MAAI,CAAC,IAAK,QAAO;AACjB,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,GAAG,GAAG,YAAY,MAAM,UAAU,SAAS;AAAA,IACnD,KAAI,oBAAI,KAAK,GAAE,YAAY;AAAA,IAC3B,UAAU,EAAE,UAAU,SAAS,IAAI,CAAC,MAAM,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,OAAO,GAAG,aAAa,aAAa,WAAW;AAAA,EACtH;AACF;AAEO,SAAS,QAAQ,KAAgB,MAAM,QAAQ,KAAa;AACjE,YAAU,SAAS,GAAG,GAAG,EAAE,WAAW,KAAK,CAAC;AAC5C,QAAM,OAAO,KAAK,SAAS,GAAG,GAAG,GAAG,SAAS,IAAI,UAAU,CAAC,OAAO;AAEnE,MAAI,CAAC,QAAQ,QAAQ,EAAE,KAAK,CAAC,MAAM,WAAW,KAAK,SAAS,GAAG,GAAG,GAAG,GAAG,SAAS,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,EAAG,QAAO;AACnH,gBAAc,MAAM,KAAK,UAAU,KAAK,MAAM,CAAC,CAAC;AAChD,SAAO;AACT;AAEO,SAAS,YAAY,MAAc,MAAM,QAAQ,KAAyC;AAC/F,MAAI,QAAkB,CAAC;AACvB,MAAI;AACF,YAAQ,YAAY,SAAS,GAAG,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,CAAC;AAAA,EACtE,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,QAAM,OAA2C,CAAC;AAClD,aAAW,KAAK,OAAO;AACrB,QAAI;AACF,YAAM,OAAO,KAAK,SAAS,GAAG,GAAG,CAAC;AAClC,YAAM,MAAM,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;AACjD,UAAI,IAAI,SAAS,KAAM,MAAK,KAAK,EAAE,MAAM,IAAI,CAAC;AAAA,IAChD,QAAQ;AAAA,IAAC;AAAA,EACX;AACA,SAAO,KAAK,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,GAAG,cAAc,EAAE,IAAI,EAAE,CAAC;AAC7D;AAEO,SAAS,UAAU,MAAc,SAA4B,QAAiC,MAAM,QAAQ,KAAW;AAC5H,QAAM,MAAM,KAAK,SAAS,GAAG,GAAG,OAAO;AACvC,YAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAClC,QAAM,MAAM,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;AACjD,gBAAc,MAAM,KAAK,UAAU,EAAE,GAAG,KAAK,WAAU,oBAAI,KAAK,GAAE,YAAY,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;AACnG,aAAW,MAAM,KAAK,KAAK,KAAK,MAAM,GAAG,EAAE,IAAI,CAAE,CAAC;AACpD;AAEO,SAAS,aAAa,MAAsB;AACjD,QAAM,MAAM,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;AACjD,MAAI,YAAY,IAAI,YAAY,KAAK;AACrC,gBAAc,MAAM,KAAK,UAAU,KAAK,MAAM,CAAC,CAAC;AAChD,SAAO,IAAI;AACb;AAEA,IAAM,WAAW,CAAC,IAAI,oBAAI,KAAK,MAAM,GAAG,EAAE,YAAY,CAAC,IAAI,OAAO,EAAE,SAAS,IAAI,CAAC,EAAE,SAAS,GAAG,GAAG,CAAC,IAAI,OAAO,EAAE,QAAQ,CAAC,EAAE,SAAS,GAAG,GAAG,CAAC;AAErI,SAAS,UAAU,MAAM,QAAQ,KAAK,MAAM,oBAAI,KAAK,GAAW;AACrE,MAAI;AACF,UAAM,MAAM,SAAS,GAAG;AACxB,WAAO,aAAa,SAAS,GAAG,GAAG,MAAM,EAAE,MAAM,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,GAAG,GAAG,CAAC,EAAE;AAAA,EACrG,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,UAAU,MAAc,YAAoB,MAAM,QAAQ,KAAK,MAAM,oBAAI,KAAK,GAAS;AACrG,YAAU,OAAO,GAAG,GAAG,EAAE,WAAW,KAAK,CAAC;AAC1C,iBAAe,SAAS,GAAG,GAAG,KAAK,UAAU,EAAE,KAAK,SAAS,GAAG,GAAG,IAAI,IAAI,YAAY,GAAG,MAAM,WAAW,CAAC,IAAI,IAAI;AACtH;AAEA,IAAM,QAAQ,CAAC,QAAgB;AAC7B,MAAI;AACF,YAAQ,KAAK,KAAK,CAAC;AACnB,WAAO;AAAA,EACT,SAAS,GAAG;AACV,WAAQ,EAA4B,SAAS;AAAA,EAC/C;AACF;AAEO,SAAS,WAAW,MAA6B;AACtD,MAAI;AACF,UAAM,MAAM,OAAO,aAAa,SAAS,IAAI,GAAG,MAAM,EAAE,KAAK,CAAC;AAC9D,WAAO,OAAO,MAAM,GAAG,IAAI,MAAM;AAAA,EACnC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,SAAS,MAAmC;AAC1D,YAAU,SAAS,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AAC7C,WAAS,UAAU,GAAG,UAAU,GAAG,WAAW;AAC5C,QAAI;AACF,YAAM,KAAK,SAAS,SAAS,IAAI,GAAG,IAAI;AACxC,gBAAU,IAAI,GAAG,QAAQ,GAAG;AAAA,CAAI;AAChC,gBAAU,EAAE;AACZ,aAAO,MAAM;AACX,YAAI;AACF,cAAI,OAAO,aAAa,SAAS,IAAI,GAAG,MAAM,EAAE,KAAK,CAAC,MAAM,QAAQ,IAAK,YAAW,SAAS,IAAI,CAAC;AAAA,QACpG,QAAQ;AAAA,QAAC;AAAA,MACX;AAAA,IACF,SAAS,GAAG;AACV,UAAK,EAA4B,SAAS,SAAU,OAAM;AAC1D,UAAI,WAAW,IAAI,MAAM,KAAM,QAAO;AACtC,UAAI;AACF,mBAAW,SAAS,IAAI,CAAC;AAAA,MAC3B,QAAQ;AAAA,MAAC;AAAA,IACX;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,IAAI,MAAc,MAAoB;AACpD,MAAI;AACF,cAAU,SAAS,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AAC7C,mBAAe,QAAQ,IAAI,GAAG,IAAG,oBAAI,KAAK,GAAE,YAAY,CAAC,IAAI,IAAI;AAAA,CAAI;AAAA,EACvE,QAAQ;AAAA,EAAC;AACX;","names":[]}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import {
|
|
2
|
+
runTrigger
|
|
3
|
+
} from "./chunk-MQS7JEBD.js";
|
|
4
|
+
import {
|
|
5
|
+
runClaude
|
|
6
|
+
} from "./chunk-IEZVFKUT.js";
|
|
7
|
+
import {
|
|
8
|
+
parseSkill
|
|
9
|
+
} from "./chunk-ZSWI3Y4Y.js";
|
|
10
|
+
import {
|
|
11
|
+
matter_default
|
|
12
|
+
} from "./chunk-J7LYDRML.js";
|
|
13
|
+
|
|
14
|
+
// src/core/skill-eval/prompts/improve-description.ts
|
|
15
|
+
var DESCRIPTION_LIMIT = 1024;
|
|
16
|
+
var list = (rs) => rs.map((r) => ` - "${r.text}" (triggered ${r.triggers}/${r.runs} times)
|
|
17
|
+
`).join("");
|
|
18
|
+
function improveDescriptionPrompt(a) {
|
|
19
|
+
let p = `You are optimizing a skill description for a Claude Code skill called "${a.name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details.
|
|
20
|
+
|
|
21
|
+
The description appears in Claude's "available_skills" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones.
|
|
22
|
+
|
|
23
|
+
Here's the current description:
|
|
24
|
+
<current_description>
|
|
25
|
+
"${a.description}"
|
|
26
|
+
</current_description>
|
|
27
|
+
|
|
28
|
+
Current scores (train ${a.score}):
|
|
29
|
+
<scores_summary>
|
|
30
|
+
`;
|
|
31
|
+
if (a.missed.length) p += `FAILED TO TRIGGER (should have triggered but didn't):
|
|
32
|
+
${list(a.missed)}
|
|
33
|
+
`;
|
|
34
|
+
if (a.falseTriggers.length) p += `FALSE TRIGGERS (triggered but shouldn't have):
|
|
35
|
+
${list(a.falseTriggers)}
|
|
36
|
+
`;
|
|
37
|
+
if (a.history.length) {
|
|
38
|
+
p += "PREVIOUS ATTEMPTS (do NOT repeat these -- try something structurally different):\n\n";
|
|
39
|
+
for (const h of a.history) p += `<attempt train=${h.score}>
|
|
40
|
+
Description: "${h.description}"
|
|
41
|
+
</attempt>
|
|
42
|
+
|
|
43
|
+
`;
|
|
44
|
+
}
|
|
45
|
+
return p + `</scores_summary>
|
|
46
|
+
|
|
47
|
+
Skill content (for context on what the skill does):
|
|
48
|
+
<skill_content>
|
|
49
|
+
${a.skillContent}
|
|
50
|
+
</skill_content>
|
|
51
|
+
|
|
52
|
+
Based on the failures, write a new and improved description that is more likely to trigger correctly. Do not overfit: don't produce an ever-expanding list of specific queries. Generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The description is injected into ALL queries, so keep it to about 100-200 words. There is a hard limit of ${DESCRIPTION_LIMIT} characters -- stay comfortably under it.
|
|
53
|
+
|
|
54
|
+
Tips:
|
|
55
|
+
- Phrase it in the imperative -- "Use this skill for" rather than "this skill does".
|
|
56
|
+
- Focus on the user's intent, what they are trying to achieve, vs. the implementation details.
|
|
57
|
+
- The description competes with other skills for Claude's attention -- make it distinctive and immediately recognizable.
|
|
58
|
+
- If you're getting lots of failures after repeated attempts, change things up: different sentence structures or wordings.
|
|
59
|
+
|
|
60
|
+
Please respond with only the new description text in <new_description> tags, nothing else.`;
|
|
61
|
+
}
|
|
62
|
+
var shortenPrompt = (base, tooLong) => `${base}
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
A previous attempt produced this description, which at ${tooLong.length} characters is over the ${DESCRIPTION_LIMIT}-character hard limit:
|
|
67
|
+
|
|
68
|
+
"${tooLong}"
|
|
69
|
+
|
|
70
|
+
Rewrite it to be under ${DESCRIPTION_LIMIT} characters while keeping the most important trigger words and intent coverage. Respond with only the new description in <new_description> tags.`;
|
|
71
|
+
var parseNewDescription = (text) => (text.match(/<new_description>([\s\S]*?)<\/new_description>/)?.[1] ?? text).trim().replace(/^"|"$/g, "").trim();
|
|
72
|
+
|
|
73
|
+
// src/core/skill-eval/tune.ts
|
|
74
|
+
var MAX_TUNE_ITERATIONS = 3;
|
|
75
|
+
function withDescription(raw, description) {
|
|
76
|
+
const p = matter_default(raw);
|
|
77
|
+
return matter_default.stringify(p.content, { ...p.data, description });
|
|
78
|
+
}
|
|
79
|
+
var score = (r) => r.train.accuracy;
|
|
80
|
+
var frac = (r) => `${r.results.filter((x) => x.split === "train" && x.triggered === x.should_trigger).length}/${r.train.n}`;
|
|
81
|
+
async function tuneDescription(o) {
|
|
82
|
+
const skill = parseSkill(o.skillRaw);
|
|
83
|
+
const tr = (raw) => runTrigger(o.prompts, { repoRoot: o.repoRoot, skillRaw: raw, env: o.env });
|
|
84
|
+
const cand = (description, r) => ({ description, train: score(r), heldout: r.heldout.accuracy });
|
|
85
|
+
let last = await tr(o.skillRaw);
|
|
86
|
+
const original = cand(skill.description, last);
|
|
87
|
+
const candidates = [original];
|
|
88
|
+
const history = [];
|
|
89
|
+
for (let i = 0; i < Math.min(o.maxIterations ?? MAX_TUNE_ITERATIONS, MAX_TUNE_ITERATIONS); i++) {
|
|
90
|
+
const prev = candidates.at(-1);
|
|
91
|
+
history.push({ description: prev.description, score: frac(last) });
|
|
92
|
+
const train = last.results.filter((r2) => r2.split === "train");
|
|
93
|
+
const prompt = improveDescriptionPrompt({
|
|
94
|
+
name: skill.name,
|
|
95
|
+
skillContent: skill.body,
|
|
96
|
+
description: prev.description,
|
|
97
|
+
score: frac(last),
|
|
98
|
+
missed: train.filter((r2) => r2.should_trigger && !r2.triggered),
|
|
99
|
+
falseTriggers: train.filter((r2) => !r2.should_trigger && r2.triggered),
|
|
100
|
+
history: history.slice(0, -1)
|
|
101
|
+
});
|
|
102
|
+
const env = o.env;
|
|
103
|
+
let r = await runClaude({ prompt, cwd: o.repoRoot, tools: [], maxTurns: 1, env });
|
|
104
|
+
if (!r.ok) break;
|
|
105
|
+
let description = parseNewDescription(r.result);
|
|
106
|
+
if (description.length > DESCRIPTION_LIMIT) {
|
|
107
|
+
r = await runClaude({ prompt: shortenPrompt(prompt, description), cwd: o.repoRoot, tools: [], maxTurns: 1, env });
|
|
108
|
+
if (r.ok) description = parseNewDescription(r.result);
|
|
109
|
+
}
|
|
110
|
+
if (!description || description.length > DESCRIPTION_LIMIT || candidates.some((c) => c.description === description)) continue;
|
|
111
|
+
last = await tr(withDescription(o.skillRaw, description));
|
|
112
|
+
candidates.push(cand(description, last));
|
|
113
|
+
}
|
|
114
|
+
const best = candidates.reduce((b, c) => (c.heldout ?? -1) > (b.heldout ?? -1) || (c.heldout ?? -1) === (b.heldout ?? -1) && (c.train ?? -1) > (b.train ?? -1) ? c : b);
|
|
115
|
+
return { original, best, improved: best !== original, candidates };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export {
|
|
119
|
+
MAX_TUNE_ITERATIONS,
|
|
120
|
+
withDescription,
|
|
121
|
+
tuneDescription
|
|
122
|
+
};
|
|
123
|
+
//# sourceMappingURL=chunk-OT3MV7XF.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/core/skill-eval/prompts/improve-description.ts","../src/core/skill-eval/tune.ts"],"sourcesContent":["/**\n * Adapted from Anthropic's skill-creator (https://github.com/anthropics/claude-plugins-official, Apache License 2.0); modified for semantic-memory.\n */\n\nexport interface DescriptionAttempt {\n description: string;\n /** \"passed/total\" on the train prompts */\n score: string;\n}\n\nexport interface PromptResult {\n text: string;\n triggers: number;\n runs: number;\n}\n\nexport const DESCRIPTION_LIMIT = 1024;\n\nconst list = (rs: PromptResult[]) => rs.map((r) => ` - \"${r.text}\" (triggered ${r.triggers}/${r.runs} times)\\n`).join(\"\");\n\n/** Ask for a better description from the train failures only (held-out results never reach the proposer). */\nexport function improveDescriptionPrompt(a: {\n name: string;\n skillContent: string;\n description: string;\n score: string;\n missed: PromptResult[];\n falseTriggers: PromptResult[];\n history: DescriptionAttempt[];\n}): string {\n let p = `You are optimizing a skill description for a Claude Code skill called \"${a.name}\". A \"skill\" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details.\n\nThe description appears in Claude's \"available_skills\" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones.\n\nHere's the current description:\n<current_description>\n\"${a.description}\"\n</current_description>\n\nCurrent scores (train ${a.score}):\n<scores_summary>\n`;\n if (a.missed.length) p += `FAILED TO TRIGGER (should have triggered but didn't):\\n${list(a.missed)}\\n`;\n if (a.falseTriggers.length) p += `FALSE TRIGGERS (triggered but shouldn't have):\\n${list(a.falseTriggers)}\\n`;\n if (a.history.length) {\n p += \"PREVIOUS ATTEMPTS (do NOT repeat these -- try something structurally different):\\n\\n\";\n for (const h of a.history) p += `<attempt train=${h.score}>\\nDescription: \"${h.description}\"\\n</attempt>\\n\\n`;\n }\n return (\n p +\n `</scores_summary>\n\nSkill content (for context on what the skill does):\n<skill_content>\n${a.skillContent}\n</skill_content>\n\nBased on the failures, write a new and improved description that is more likely to trigger correctly. Do not overfit: don't produce an ever-expanding list of specific queries. Generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The description is injected into ALL queries, so keep it to about 100-200 words. There is a hard limit of ${DESCRIPTION_LIMIT} characters -- stay comfortably under it.\n\nTips:\n- Phrase it in the imperative -- \"Use this skill for\" rather than \"this skill does\".\n- Focus on the user's intent, what they are trying to achieve, vs. the implementation details.\n- The description competes with other skills for Claude's attention -- make it distinctive and immediately recognizable.\n- If you're getting lots of failures after repeated attempts, change things up: different sentence structures or wordings.\n\nPlease respond with only the new description text in <new_description> tags, nothing else.`\n );\n}\n\n/** The follow-up for a description over the limit. */\nexport const shortenPrompt = (base: string, tooLong: string) =>\n `${base}\\n\\n---\\n\\nA previous attempt produced this description, which at ${tooLong.length} characters is over the ${DESCRIPTION_LIMIT}-character hard limit:\\n\\n\"${tooLong}\"\\n\\n` +\n `Rewrite it to be under ${DESCRIPTION_LIMIT} characters while keeping the most important trigger words and intent coverage. Respond with only the new description in <new_description> tags.`;\n\n/** The description inside <new_description> tags, or the whole reply when untagged. */\nexport const parseNewDescription = (text: string): string =>\n (text.match(/<new_description>([\\s\\S]*?)<\\/new_description>/)?.[1] ?? text).trim().replace(/^\"|\"$/g, \"\").trim();\n","/**\n * Description tuning (v1.7): propose better descriptions from the train failures, score every candidate\n * with the trigger eval, keep the best by held-out score. At most 3 iterations. Returns the proposal and the\n * scores — it never writes the skill file.\n */\nimport { runClaude } from \"../claude-run.js\";\nimport matter from \"../matter.js\";\nimport { parseSkill } from \"../skill-schema.js\";\nimport { DESCRIPTION_LIMIT, improveDescriptionPrompt, parseNewDescription, shortenPrompt, type DescriptionAttempt } from \"./prompts/improve-description.js\";\nimport type { EvalPrompt } from \"./sets.js\";\nimport { runTrigger, type TriggerReport } from \"./trigger.js\";\n\nexport const MAX_TUNE_ITERATIONS = 3;\n\nexport interface TuneCandidate {\n description: string;\n train: number | null;\n heldout: number | null;\n}\n\nexport interface TuneReport {\n original: TuneCandidate;\n best: TuneCandidate;\n /** the best candidate is not the original */\n improved: boolean;\n candidates: TuneCandidate[];\n}\n\n/** The skill file with its frontmatter description replaced and nothing else changed. */\nexport function withDescription(raw: string, description: string): string {\n const p = matter(raw);\n return matter.stringify(p.content, { ...p.data, description });\n}\n\nconst score = (r: TriggerReport): TuneCandidate[\"train\"] => r.train.accuracy;\nconst frac = (r: TriggerReport) => `${r.results.filter((x) => x.split === \"train\" && x.triggered === x.should_trigger).length}/${r.train.n}`;\n\nexport async function tuneDescription(o: { repoRoot: string; skillRaw: string; prompts: EvalPrompt[]; env?: NodeJS.ProcessEnv; maxIterations?: number }): Promise<TuneReport> {\n const skill = parseSkill(o.skillRaw);\n const tr = (raw: string) => runTrigger(o.prompts, { repoRoot: o.repoRoot, skillRaw: raw, env: o.env });\n const cand = (description: string, r: TriggerReport): TuneCandidate => ({ description, train: score(r), heldout: r.heldout.accuracy });\n let last = await tr(o.skillRaw);\n const original = cand(skill.description, last);\n const candidates = [original];\n const history: DescriptionAttempt[] = [];\n for (let i = 0; i < Math.min(o.maxIterations ?? MAX_TUNE_ITERATIONS, MAX_TUNE_ITERATIONS); i++) {\n const prev = candidates.at(-1)!;\n history.push({ description: prev.description, score: frac(last) });\n // only train results go to the proposer; held-out stays unseen so it can judge\n const train = last.results.filter((r) => r.split === \"train\");\n const prompt = improveDescriptionPrompt({\n name: skill.name,\n skillContent: skill.body,\n description: prev.description,\n score: frac(last),\n missed: train.filter((r) => r.should_trigger && !r.triggered),\n falseTriggers: train.filter((r) => !r.should_trigger && r.triggered),\n history: history.slice(0, -1),\n });\n const env = o.env;\n let r = await runClaude({ prompt, cwd: o.repoRoot, tools: [], maxTurns: 1, env });\n if (!r.ok) break;\n let description = parseNewDescription(r.result);\n if (description.length > DESCRIPTION_LIMIT) {\n r = await runClaude({ prompt: shortenPrompt(prompt, description), cwd: o.repoRoot, tools: [], maxTurns: 1, env });\n if (r.ok) description = parseNewDescription(r.result);\n }\n if (!description || description.length > DESCRIPTION_LIMIT || candidates.some((c) => c.description === description)) continue;\n last = await tr(withDescription(o.skillRaw, description));\n candidates.push(cand(description, last));\n }\n // best by held-out, ties by train, then the earlier candidate (the original wins a full tie)\n const best = candidates.reduce((b, c) => ((c.heldout ?? -1) > (b.heldout ?? -1) || ((c.heldout ?? -1) === (b.heldout ?? -1) && (c.train ?? -1) > (b.train ?? -1)) ? c : b));\n return { original, best, improved: best !== original, candidates };\n}\n\n"],"mappings":";;;;;;;;;;;;;;AAgBO,IAAM,oBAAoB;AAEjC,IAAM,OAAO,CAAC,OAAuB,GAAG,IAAI,CAAC,MAAM,QAAQ,EAAE,IAAI,gBAAgB,EAAE,QAAQ,IAAI,EAAE,IAAI;AAAA,CAAW,EAAE,KAAK,EAAE;AAGlH,SAAS,yBAAyB,GAQ9B;AACT,MAAI,IAAI,0EAA0E,EAAE,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAMvF,EAAE,WAAW;AAAA;AAAA;AAAA,wBAGQ,EAAE,KAAK;AAAA;AAAA;AAG7B,MAAI,EAAE,OAAO,OAAQ,MAAK;AAAA,EAA0D,KAAK,EAAE,MAAM,CAAC;AAAA;AAClG,MAAI,EAAE,cAAc,OAAQ,MAAK;AAAA,EAAmD,KAAK,EAAE,aAAa,CAAC;AAAA;AACzG,MAAI,EAAE,QAAQ,QAAQ;AACpB,SAAK;AACL,eAAW,KAAK,EAAE,QAAS,MAAK,kBAAkB,EAAE,KAAK;AAAA,gBAAoB,EAAE,WAAW;AAAA;AAAA;AAAA;AAAA,EAC5F;AACA,SACE,IACA;AAAA;AAAA;AAAA;AAAA,EAIF,EAAE,YAAY;AAAA;AAAA;AAAA,6ZAG6Y,iBAAiB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAU9a;AAGO,IAAM,gBAAgB,CAAC,MAAc,YAC1C,GAAG,IAAI;AAAA;AAAA;AAAA;AAAA,yDAAqE,QAAQ,MAAM,2BAA2B,iBAAiB;AAAA;AAAA,GAA8B,OAAO;AAAA;AAAA,yBACjJ,iBAAiB;AAGtC,IAAM,sBAAsB,CAAC,UACjC,KAAK,MAAM,gDAAgD,IAAI,CAAC,KAAK,MAAM,KAAK,EAAE,QAAQ,UAAU,EAAE,EAAE,KAAK;;;AChEzG,IAAM,sBAAsB;AAiB5B,SAAS,gBAAgB,KAAa,aAA6B;AACxE,QAAM,IAAI,eAAO,GAAG;AACpB,SAAO,eAAO,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAC/D;AAEA,IAAM,QAAQ,CAAC,MAA6C,EAAE,MAAM;AACpE,IAAM,OAAO,CAAC,MAAqB,GAAG,EAAE,QAAQ,OAAO,CAAC,MAAM,EAAE,UAAU,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,IAAI,EAAE,MAAM,CAAC;AAE1I,eAAsB,gBAAgB,GAAwI;AAC5K,QAAM,QAAQ,WAAW,EAAE,QAAQ;AACnC,QAAM,KAAK,CAAC,QAAgB,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,UAAU,KAAK,KAAK,EAAE,IAAI,CAAC;AACrG,QAAM,OAAO,CAAC,aAAqB,OAAqC,EAAE,aAAa,OAAO,MAAM,CAAC,GAAG,SAAS,EAAE,QAAQ,SAAS;AACpI,MAAI,OAAO,MAAM,GAAG,EAAE,QAAQ;AAC9B,QAAM,WAAW,KAAK,MAAM,aAAa,IAAI;AAC7C,QAAM,aAAa,CAAC,QAAQ;AAC5B,QAAM,UAAgC,CAAC;AACvC,WAAS,IAAI,GAAG,IAAI,KAAK,IAAI,EAAE,iBAAiB,qBAAqB,mBAAmB,GAAG,KAAK;AAC9F,UAAM,OAAO,WAAW,GAAG,EAAE;AAC7B,YAAQ,KAAK,EAAE,aAAa,KAAK,aAAa,OAAO,KAAK,IAAI,EAAE,CAAC;AAEjE,UAAM,QAAQ,KAAK,QAAQ,OAAO,CAACA,OAAMA,GAAE,UAAU,OAAO;AAC5D,UAAM,SAAS,yBAAyB;AAAA,MACtC,MAAM,MAAM;AAAA,MACZ,cAAc,MAAM;AAAA,MACpB,aAAa,KAAK;AAAA,MAClB,OAAO,KAAK,IAAI;AAAA,MAChB,QAAQ,MAAM,OAAO,CAACA,OAAMA,GAAE,kBAAkB,CAACA,GAAE,SAAS;AAAA,MAC5D,eAAe,MAAM,OAAO,CAACA,OAAM,CAACA,GAAE,kBAAkBA,GAAE,SAAS;AAAA,MACnE,SAAS,QAAQ,MAAM,GAAG,EAAE;AAAA,IAC9B,CAAC;AACD,UAAM,MAAM,EAAE;AACd,QAAI,IAAI,MAAM,UAAU,EAAE,QAAQ,KAAK,EAAE,UAAU,OAAO,CAAC,GAAG,UAAU,GAAG,IAAI,CAAC;AAChF,QAAI,CAAC,EAAE,GAAI;AACX,QAAI,cAAc,oBAAoB,EAAE,MAAM;AAC9C,QAAI,YAAY,SAAS,mBAAmB;AAC1C,UAAI,MAAM,UAAU,EAAE,QAAQ,cAAc,QAAQ,WAAW,GAAG,KAAK,EAAE,UAAU,OAAO,CAAC,GAAG,UAAU,GAAG,IAAI,CAAC;AAChH,UAAI,EAAE,GAAI,eAAc,oBAAoB,EAAE,MAAM;AAAA,IACtD;AACA,QAAI,CAAC,eAAe,YAAY,SAAS,qBAAqB,WAAW,KAAK,CAAC,MAAM,EAAE,gBAAgB,WAAW,EAAG;AACrH,WAAO,MAAM,GAAG,gBAAgB,EAAE,UAAU,WAAW,CAAC;AACxD,eAAW,KAAK,KAAK,aAAa,IAAI,CAAC;AAAA,EACzC;AAEA,QAAM,OAAO,WAAW,OAAO,CAAC,GAAG,OAAQ,EAAE,WAAW,OAAO,EAAE,WAAW,QAAS,EAAE,WAAW,SAAS,EAAE,WAAW,QAAQ,EAAE,SAAS,OAAO,EAAE,SAAS,MAAO,IAAI,CAAE;AAC1K,SAAO,EAAE,UAAU,MAAM,UAAU,SAAS,UAAU,WAAW;AACnE;","names":["r"]}
|