@ericrisco/rsc 1.0.9 → 1.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -13
- package/manifest.json +133 -2
- package/package.json +6 -4
- package/scripts/consult.js +75 -42
- package/scripts/drift-check.js +316 -0
- package/scripts/lib/default-skill-floor.js +11 -0
- package/scripts/lib/domains.js +3 -3
- package/scripts/lib/recommend.js +1 -0
- package/scripts/lib/text-rank.js +112 -0
- package/scripts/lib/ui.js +3 -3
- package/scripts/routing-audit.js +163 -0
- package/scripts/rsc.js +11 -10
- package/skills/author-skill/references/rsc-conventions.md +1 -1
- package/skills/bro/SKILL.md +68 -0
- package/skills/bro/evals/README.md +22 -0
- package/skills/bro/evals/cases.yaml +58 -0
- package/skills/data-cleaning/references/normalization-recipes.md +1 -1
- package/skills/decision-challenge/SKILL.md +113 -0
- package/skills/decision-challenge/evals/README.md +3 -0
- package/skills/decision-challenge/evals/cases.yaml +42 -0
- package/skills/deprecation/SKILL.md +123 -0
- package/skills/deprecation/evals/README.md +3 -0
- package/skills/deprecation/evals/cases.yaml +42 -0
- package/skills/harness/references/wiki-gaps-template.md +4 -0
- package/skills/idea-refinement/SKILL.md +121 -0
- package/skills/idea-refinement/evals/README.md +3 -0
- package/skills/idea-refinement/evals/cases.yaml +42 -0
- package/skills/observability/SKILL.md +6 -0
- package/skills/observability/evals/cases.yaml +2 -0
- package/skills/performance/SKILL.md +13 -0
- package/skills/performance/evals/cases.yaml +2 -0
- package/skills/simplify-code/SKILL.md +101 -0
- package/skills/simplify-code/evals/README.md +3 -0
- package/skills/simplify-code/evals/cases.yaml +42 -0
- package/skills/solid-js/references/router-and-start.md +1 -1
- package/skills/source-grounded-development/SKILL.md +108 -0
- package/skills/source-grounded-development/evals/README.md +3 -0
- package/skills/source-grounded-development/evals/cases.yaml +42 -0
- package/skills/suggest/evals/cases.yaml +1 -1
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// drift-check.js — do the paths the harness claims exist actually exist?
|
|
3
|
+
//
|
|
4
|
+
// The harness writes knowledge about itself in three places and every one of them can name a
|
|
5
|
+
// file that has since moved. Measured before this existed: 67 of 688 markdown links in the
|
|
6
|
+
// published catalog did not resolve, and two were real wrong-depth bugs shipping to users
|
|
7
|
+
// (`data-cleaning` → duckdb, `solid-js` → vercel, both missing a `../`). The private wiki had a
|
|
8
|
+
// plan marked "implementada" naming three files that never existed. Nothing checked any of it,
|
|
9
|
+
// while the memory instructions already ordered "verify it still exists before recommending it"
|
|
10
|
+
// — a binding rule resting on the model remembering. That is the pattern P2 exists to kill.
|
|
11
|
+
//
|
|
12
|
+
// THE ONE IDEA THAT MAKES THIS VIABLE: **code shows, prose points.** A link inside a fenced
|
|
13
|
+
// block or an inline-code span is being *displayed* as syntax; a link in prose is *offered to
|
|
14
|
+
// follow*. That distinction is already in the markup, so it needs no markers to author and no
|
|
15
|
+
// allowlist to maintain (P3 — the content is the ledger). Applied to the catalog it takes 67
|
|
16
|
+
// unresolved links down to 4 real ones, which is the difference between a gate people act on
|
|
17
|
+
// and a gate they learn to ignore.
|
|
18
|
+
//
|
|
19
|
+
// Two modes, because the precision a check needs scales with what it costs to be wrong (P7):
|
|
20
|
+
// catalog blocking. Prose links only, resolved exactly as a markdown reader would (relative
|
|
21
|
+
// to the document). Precision first: a false positive here fails a build.
|
|
22
|
+
// knowledge report-only. Prose links PLUS paths in inline code, resolved against several
|
|
23
|
+
// plausible bases. Recall first: a human triages, nothing blocks, and the wiki's
|
|
24
|
+
// worst finding lives in backticks — precision-only extraction would miss it.
|
|
25
|
+
//
|
|
26
|
+
// It never writes. The output is a report; adopting it is a human's job.
|
|
27
|
+
import { readdirSync, readFileSync, existsSync, realpathSync } from 'node:fs';
|
|
28
|
+
import { join, dirname, resolve, relative, basename } from 'node:path';
|
|
29
|
+
import { fileURLToPath } from 'node:url';
|
|
30
|
+
import { homedir } from 'node:os';
|
|
31
|
+
|
|
32
|
+
const REPO = join(dirname(fileURLToPath(import.meta.url)), '..');
|
|
33
|
+
|
|
34
|
+
// ── extraction ────────────────────────────────────────────────────────────────────────────
|
|
35
|
+
// Both strippers keep the line count (and stripInlineCode the column offsets) intact, so a
|
|
36
|
+
// finding can name the line a human has to open.
|
|
37
|
+
|
|
38
|
+
// Fenced blocks, line-scanned rather than regexed: an UNTERMINATED fence must swallow the rest
|
|
39
|
+
// of the file, which a paired regex silently declines to do — leaving example links in a
|
|
40
|
+
// half-open block looking like prose.
|
|
41
|
+
export function stripFences(text) {
|
|
42
|
+
const out = [];
|
|
43
|
+
let fence = null;
|
|
44
|
+
for (const line of String(text).split('\n')) {
|
|
45
|
+
const m = line.match(/^([ \t]*)(`{3,}|~{3,})(.*)$/);
|
|
46
|
+
if (fence) {
|
|
47
|
+
if (m && m[2][0] === fence.char && m[2].length >= fence.len && m[3].trim() === '') fence = null;
|
|
48
|
+
out.push('');
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (m) {
|
|
52
|
+
fence = { char: m[2][0], len: m[2].length };
|
|
53
|
+
out.push('');
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
out.push(line);
|
|
57
|
+
}
|
|
58
|
+
return out.join('\n');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Inline code spans, per line so a span can never eat across a newline. An UNBALANCED backtick
|
|
62
|
+
// run is left as prose on purpose: swallowing it would hide a real link, and a false negative
|
|
63
|
+
// is the failure mode this whole check exists to prevent.
|
|
64
|
+
export function stripInlineCode(text) {
|
|
65
|
+
return String(text)
|
|
66
|
+
.split('\n')
|
|
67
|
+
.map((line) => line.replace(/(`+)(?:(?!\1)[^`])*\1/g, (m) => ' '.repeat(m.length)))
|
|
68
|
+
.join('\n');
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const FILE_EXT = /\.(md|markdown|ya?ml|json|jsonc|sh|bash|mjs|cjs|js|ts|txt|toml)$/i;
|
|
72
|
+
const NOT_A_PATH = /^(https?:|mailto:|tel:|#|\/\/)/i;
|
|
73
|
+
const isPathLike = (t) => !NOT_A_PATH.test(t) && (FILE_EXT.test(t) || t.endsWith('/'));
|
|
74
|
+
|
|
75
|
+
// Inline links only — `[text](target)`. Reference-style links and autolinks are deliberately
|
|
76
|
+
// out: `<path>` carries angle brackets and is classified as a placeholder anyway.
|
|
77
|
+
const LINK_RE = /\[[^\]]*\]\(\s*([^)\s]+?)\s*\)/g;
|
|
78
|
+
|
|
79
|
+
export function extractProseLinks(text) {
|
|
80
|
+
const prose = stripInlineCode(stripFences(text));
|
|
81
|
+
const out = [];
|
|
82
|
+
prose.split('\n').forEach((line, i) => {
|
|
83
|
+
for (const m of line.matchAll(LINK_RE)) {
|
|
84
|
+
const target = m[1].replace(/^<|>$/g, '');
|
|
85
|
+
if (isPathLike(target)) out.push({ target, line: i + 1, kind: 'link' });
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Report mode only: a path asserted inside backticks. Noisier by nature (this is where syntax
|
|
92
|
+
// examples live), which is exactly why it never gates a build.
|
|
93
|
+
const TOP_DIRS = /^(scripts|targets|skills|tests|bin|site|schema|02-DOCS|\.github|\.rsc)(\/|$)/;
|
|
94
|
+
export function extractInlinePaths(text) {
|
|
95
|
+
const noFences = stripFences(text);
|
|
96
|
+
const out = [];
|
|
97
|
+
noFences.split('\n').forEach((line, i) => {
|
|
98
|
+
for (const m of line.matchAll(/(`+)((?:(?!\1)[^`])*)\1/g)) {
|
|
99
|
+
const t = m[2].trim();
|
|
100
|
+
if (!t || /\s/.test(t) || NOT_A_PATH.test(t)) continue;
|
|
101
|
+
if (!/\//.test(t)) continue; // a bare filename is a mention, not a location
|
|
102
|
+
if (TOP_DIRS.test(t) || FILE_EXT.test(t)) out.push({ target: t, line: i + 1, kind: 'inline' });
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
return out;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// ── classification ────────────────────────────────────────────────────────────────────────
|
|
109
|
+
// Every rule is syntactic and narrow. Nothing here guesses from file names or wording: a
|
|
110
|
+
// classifier that reasons about intent is a classifier that learns to stay quiet, and a quiet
|
|
111
|
+
// detector is indistinguishable from no detector.
|
|
112
|
+
export const CLASSES = {
|
|
113
|
+
// A shape, not a path: `skills/<ID>/`, `references/*.md`, `{slug}.md`.
|
|
114
|
+
placeholder: (t) => /[<>*{}]/.test(t),
|
|
115
|
+
// The file exists; the line number moves on its own. Anchors are recognised, not chased.
|
|
116
|
+
anchored: (t) => /:\d+$/.test(t) || t.includes('#'),
|
|
117
|
+
// Born at runtime, never in a repo. Note the trailing slash: `.rsc-notes.md` is NOT this.
|
|
118
|
+
runtime: (t) => t.startsWith('~') || /(^|\/)\.rsc\//.test(t),
|
|
119
|
+
// A site route, not a file: `/llms.txt`, `/skills/`. Found by running the report on the real
|
|
120
|
+
// wiki, where six route mentions arrived as "missing files". A genuinely absolute filesystem
|
|
121
|
+
// path in these documents would start with a home or volume directory, and the P9 test in
|
|
122
|
+
// tests/drift-check.test.js already forbids shipping one.
|
|
123
|
+
'web-route': (t) => t.startsWith('/'),
|
|
124
|
+
// The INSTALLED user's project, which the catalog legitimately talks about and does not
|
|
125
|
+
// contain. Segment-exact, so `rawdata/` and `02-DOCSX/` stay real claims.
|
|
126
|
+
'user-project': (t) => /(^|\/)(02-DOCS|raw)\//.test(t),
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
export function classify(target, { mode = 'catalog' } = {}) {
|
|
130
|
+
for (const name of ['placeholder', 'anchored', 'runtime', 'web-route']) {
|
|
131
|
+
if (CLASSES[name](target)) return name;
|
|
132
|
+
}
|
|
133
|
+
// In knowledge mode this repo IS the project, so `02-DOCS/…` is a real claim about real files.
|
|
134
|
+
if (mode === 'catalog' && CLASSES['user-project'](target)) return 'user-project';
|
|
135
|
+
return 'real';
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// ── resolution ────────────────────────────────────────────────────────────────────────────
|
|
139
|
+
// Catalog: the document's own directory, and nothing else — that is what a reader following the
|
|
140
|
+
// link does, and being generous here would hide the exact wrong-depth bug this must catch.
|
|
141
|
+
// Knowledge: several plausible bases, because the wiki writes shorthand
|
|
142
|
+
// (`orient/references/x.md` meaning `skills/orient/references/x.md`) and 20 such references
|
|
143
|
+
// would otherwise arrive as false positives.
|
|
144
|
+
export function basesFor(docPath, { mode, root }) {
|
|
145
|
+
if (mode === 'catalog') return [dirname(docPath)];
|
|
146
|
+
return [dirname(docPath), root, join(root, '02-DOCS'), join(root, '02-DOCS', 'wiki'), join(root, 'skills')];
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const resolves = (target, bases) => bases.some((b) => existsSync(resolve(b, target)));
|
|
150
|
+
|
|
151
|
+
// Knowledge mode only. The authoring standards write per-skill paths generically — "every skill
|
|
152
|
+
// carries `evals/cases.yaml`" — which is a claim about a shape shared by 258 directories, not a
|
|
153
|
+
// location. Resolving it against any one skill is the honest reading; treating it as drift
|
|
154
|
+
// produced five findings that were never wrong.
|
|
155
|
+
function resolvesInAnySkill(target, root) {
|
|
156
|
+
if (target.startsWith('.') || target.startsWith('/')) return false;
|
|
157
|
+
const skillsDir = join(root, 'skills');
|
|
158
|
+
if (!existsSync(skillsDir)) return false;
|
|
159
|
+
for (const e of readdirSync(skillsDir, { withFileTypes: true })) {
|
|
160
|
+
if (e.isDirectory() && existsSync(join(skillsDir, e.name, target))) return true;
|
|
161
|
+
}
|
|
162
|
+
return false;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// ── walking ───────────────────────────────────────────────────────────────────────────────
|
|
166
|
+
function markdownFiles(dir, out = []) {
|
|
167
|
+
for (const e of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
168
|
+
if (e.name === 'node_modules' || e.name.startsWith('.git')) continue;
|
|
169
|
+
const p = join(dir, e.name);
|
|
170
|
+
if (e.isDirectory()) markdownFiles(p, out);
|
|
171
|
+
else if (/\.(md|markdown)$/i.test(e.name)) out.push(p);
|
|
172
|
+
}
|
|
173
|
+
return out;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Claude Code's auto-memory for a project lives under a slug of its absolute path. Derived from
|
|
177
|
+
// HOME and the repo path — never a hardcoded user, because this file ships in a public package.
|
|
178
|
+
export function memoryDir(root, home = homedir()) {
|
|
179
|
+
return join(home, '.claude', 'projects', String(root).replace(/\//g, '-'), 'memory');
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Scan one tree. Returns findings plus the number of claims actually examined — a check that
|
|
183
|
+
// silently stops looking would otherwise report "all clear" forever (see the count assertion in
|
|
184
|
+
// tests/drift-check.test.js).
|
|
185
|
+
export function scanTree(dir, { mode, root = REPO, label = null } = {}) {
|
|
186
|
+
const findings = [];
|
|
187
|
+
let claims = 0;
|
|
188
|
+
let skipped = 0;
|
|
189
|
+
for (const doc of markdownFiles(dir)) {
|
|
190
|
+
const text = readFileSync(doc, 'utf8');
|
|
191
|
+
const raw = mode === 'catalog'
|
|
192
|
+
? extractProseLinks(text)
|
|
193
|
+
: [...extractProseLinks(text), ...extractInlinePaths(text)];
|
|
194
|
+
const bases = basesFor(doc, { mode, root });
|
|
195
|
+
for (const c of raw) {
|
|
196
|
+
const cls = classify(c.target, { mode });
|
|
197
|
+
if (cls !== 'real') {
|
|
198
|
+
skipped++;
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
claims++;
|
|
202
|
+
if (mode === 'knowledge' && resolvesInAnySkill(c.target, root)) continue;
|
|
203
|
+
if (!resolves(c.target, bases)) {
|
|
204
|
+
findings.push({
|
|
205
|
+
doc: relative(root, doc),
|
|
206
|
+
line: c.line,
|
|
207
|
+
target: c.target,
|
|
208
|
+
kind: c.kind,
|
|
209
|
+
tree: label || basename(dir),
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
return { findings, claims, skipped };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// ── the two modes ─────────────────────────────────────────────────────────────────────────
|
|
218
|
+
export class MissingRoot extends Error {}
|
|
219
|
+
|
|
220
|
+
// Blocking. A broken link in the catalog is broken product: the user follows a pointer to
|
|
221
|
+
// nowhere and cannot tell that from having misunderstood the skill.
|
|
222
|
+
export function checkCatalog({ root = REPO } = {}) {
|
|
223
|
+
const dir = join(root, 'skills');
|
|
224
|
+
if (!existsSync(dir)) throw new MissingRoot(`no catalog to check: ${relative(root, dir) || dir} does not exist`);
|
|
225
|
+
return scanTree(dir, { mode: 'catalog', root, label: 'skills' });
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// Report-only. Inputs are the harness's own artifacts — written by a trusted agent, and for the
|
|
229
|
+
// wiki already reviewed. Session transcripts are NOT an input here and must not become one: they
|
|
230
|
+
// carry web pages, third-party READMEs and tool output, and anything derived from them would be
|
|
231
|
+
// loaded into every later session.
|
|
232
|
+
export function checkKnowledge({ root = REPO, home = homedir() } = {}) {
|
|
233
|
+
const trees = [
|
|
234
|
+
{ dir: join(root, '02-DOCS', 'wiki'), label: 'wiki' },
|
|
235
|
+
{ dir: memoryDir(root, home), label: 'memory' },
|
|
236
|
+
].filter((t) => existsSync(t.dir));
|
|
237
|
+
if (!trees.length) throw new MissingRoot('no knowledge tree found (02-DOCS/wiki and auto-memory are both absent)');
|
|
238
|
+
const all = { findings: [], claims: 0, skipped: 0, trees: trees.map((t) => t.label) };
|
|
239
|
+
for (const t of trees) {
|
|
240
|
+
const r = scanTree(t.dir, { mode: 'knowledge', root, label: t.label });
|
|
241
|
+
all.findings.push(...r.findings);
|
|
242
|
+
all.claims += r.claims;
|
|
243
|
+
all.skipped += r.skipped;
|
|
244
|
+
}
|
|
245
|
+
return all;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// ── CLI ───────────────────────────────────────────────────────────────────────────────────
|
|
249
|
+
// Every denial carries its own exit (P6). The second half of the recovery matters as much as the
|
|
250
|
+
// first: an example link is fixed by putting it in code, and that is not a loophole — it is the
|
|
251
|
+
// rule this check is built on, stated where someone hitting the wall will read it.
|
|
252
|
+
const RECOVER = [
|
|
253
|
+
'Recover: fix the link so it resolves from the document that contains it,',
|
|
254
|
+
' or — if it is showing what a link looks like rather than pointing at a file —',
|
|
255
|
+
' wrap it in backticks or a fenced block. Code shows; prose points.',
|
|
256
|
+
].join('\n');
|
|
257
|
+
|
|
258
|
+
const report = (f) => ` ${f.doc}:${f.line} → ${f.target}${f.kind === 'inline' ? ' [inline]' : ''}`;
|
|
259
|
+
|
|
260
|
+
function main(argv) {
|
|
261
|
+
const wantKnowledge = argv.includes('--knowledge') || argv.includes('--all');
|
|
262
|
+
const wantCatalog = !argv.includes('--knowledge') || argv.includes('--all');
|
|
263
|
+
let failed = false;
|
|
264
|
+
|
|
265
|
+
if (wantCatalog) {
|
|
266
|
+
let r;
|
|
267
|
+
try {
|
|
268
|
+
r = checkCatalog({});
|
|
269
|
+
} catch (e) {
|
|
270
|
+
// Distinct exit and message: "could not run" must never be reported as "nothing wrong".
|
|
271
|
+
console.error(`drift-check: ${e.message}`);
|
|
272
|
+
console.error('Recover: run this from the repo root, where skills/ lives.');
|
|
273
|
+
process.exit(2);
|
|
274
|
+
}
|
|
275
|
+
console.log(`drift-check catalog: ${r.claims} prose link claim(s) checked, ${r.skipped} classified as non-claims`);
|
|
276
|
+
if (r.findings.length) {
|
|
277
|
+
console.log(`\nFAIL: ${r.findings.length} link(s) do not resolve:\n${r.findings.map(report).join('\n')}\n`);
|
|
278
|
+
console.log(RECOVER);
|
|
279
|
+
failed = true;
|
|
280
|
+
} else {
|
|
281
|
+
console.log('RESULT: PASS — every prose link in the catalog resolves');
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
if (wantKnowledge) {
|
|
286
|
+
try {
|
|
287
|
+
const k = checkKnowledge({});
|
|
288
|
+
console.log(`\ndrift-check knowledge (${k.trees.join(', ')}): ${k.claims} claim(s) checked, ${k.skipped} classified as non-claims`);
|
|
289
|
+
if (k.findings.length) {
|
|
290
|
+
console.log(`${k.findings.length} claim(s) do not resolve — advisory, nothing is blocked:\n${k.findings.map(report).join('\n')}`);
|
|
291
|
+
console.log('\nEach is either drift (fix the document) or a deliberate mention of something absent (leave it).');
|
|
292
|
+
} else {
|
|
293
|
+
console.log('no drift found');
|
|
294
|
+
}
|
|
295
|
+
} catch (e) {
|
|
296
|
+
// Advisory tree missing is normal (02-DOCS is gitignored; a fresh install has no wiki).
|
|
297
|
+
console.log(`\ndrift-check knowledge: skipped — ${e.message}`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
process.exit(failed ? 1 : 0);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// Compared through realpath, not by string: on macOS the temp dir is a symlink, so argv[1] and
|
|
305
|
+
// import.meta.url disagree for any script under it and the common
|
|
306
|
+
// `import.meta.url === \`file://${process.argv[1]}\`` guard silently declines to run —
|
|
307
|
+
// which looks exactly like a clean pass.
|
|
308
|
+
function invokedDirectly() {
|
|
309
|
+
try {
|
|
310
|
+
return !!process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
|
|
311
|
+
} catch {
|
|
312
|
+
return false;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
if (invokedDirectly()) main(process.argv.slice(2));
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// Skills every installation path equips, even when the user picks a profile,
|
|
2
|
+
// adds one skill directly, or browses the catalog by hand.
|
|
3
|
+
//
|
|
4
|
+
// `orient` and `suggest` are runtime always-on. `bro` is installed by default
|
|
5
|
+
// but remains conditionally loaded: its description routes human-language asks
|
|
6
|
+
// without paying for its body on unrelated turns.
|
|
7
|
+
export const DEFAULT_SKILL_FLOOR = Object.freeze(['orient', 'suggest', 'bro']);
|
|
8
|
+
|
|
9
|
+
export function withDefaultSkillFloor(skillIds = []) {
|
|
10
|
+
return [...new Set([...DEFAULT_SKILL_FLOOR, ...skillIds])];
|
|
11
|
+
}
|
package/scripts/lib/domains.js
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
// catalog grows deliberately, never silently.
|
|
5
5
|
|
|
6
6
|
export const DOMAINS = [
|
|
7
|
-
{ title: 'Core & control plane', ids: ['init', 'harness', 'orient', 'suggest', 'author-skill', 'sdd-init'] },
|
|
8
|
-
{ title: 'Spec-Driven Development', ids: ['sdd', 'constitution', 'specify', 'clarify', 'plan', 'tasks', 'analyze', 'implement', 'verify', 'review', 'ship', 'debug', 'worktrees', 'parallel'] },
|
|
7
|
+
{ title: 'Core & control plane', ids: ['init', 'harness', 'orient', 'suggest', 'bro', 'author-skill', 'sdd-init'] },
|
|
8
|
+
{ title: 'Spec-Driven Development', ids: ['sdd', 'constitution', 'idea-refinement', 'specify', 'clarify', 'plan', 'tasks', 'analyze', 'decision-challenge', 'implement', 'source-grounded-development', 'verify', 'review', 'simplify-code', 'ship', 'debug', 'worktrees', 'parallel'] },
|
|
9
9
|
{ title: 'Run a business', ids: ['finance-ops', 'invoicing', 'bookkeeping', 'pricing', 'sales-pipeline', 'lead-gen', 'cold-outreach', 'proposals', 'contracts', 'customer-support', 'client-onboarding', 'retention', 'hiring', 'people-ops', 'inventory', 'logistics-ops', 'procurement', 'meeting-notes', 'sop-builder', 'project-ops'] },
|
|
10
10
|
{ title: 'Raise & model money', ids: ['pitch-deck', 'investor-materials', 'financial-model', 'fundraising', 'unit-economics', 'grants'] },
|
|
11
11
|
{ title: 'Legal, privacy & compliance', ids: ['gdpr-privacy', 'terms-conditions', 'compliance', 'data-policy', 'ip-trademark'] },
|
|
@@ -22,7 +22,7 @@ export const DOMAINS = [
|
|
|
22
22
|
{ title: 'Game development', ids: ['godot', 'unity', 'unreal', 'game-design', 'game-storytelling', 'level-design', 'gamedev-shaders', 'gamedev-multiplayer', 'gamedev-physics', 'gamedev-pathing', 'gamedev-shipping'] },
|
|
23
23
|
{ title: 'Databases & data layer', ids: ['postgresdb', 'mysql', 'mongodb', 'redis', 'supabase', 'neon', 'planetscale', 'sqlite-turso', 'prisma-orm', 'drizzle-orm', 'firebase', 'dynamodb', 'vector-db', 'clickhouse-analytics', 'duckdb', 'db-migrations', 'backups'] },
|
|
24
24
|
{ title: 'Ship & operate — platforms', ids: ['vercel', 'netlify', 'cloudflare', 'railway', 'render', 'fly-io', 'coolify', 'hetzner', 'digitalocean', 'aws-essentials', 'gcp-essentials'] },
|
|
25
|
-
{ title: 'Ship & operate — devops', ids: ['docker', 'github-actions', 'git-workflow', 'domains-dns', 'monitoring', 'email-deliverability', 'scaling', 'deployment'] },
|
|
25
|
+
{ title: 'Ship & operate — devops', ids: ['docker', 'github-actions', 'git-workflow', 'domains-dns', 'monitoring', 'email-deliverability', 'scaling', 'deployment', 'deprecation'] },
|
|
26
26
|
{ title: 'Ship & operate — quality & security', ids: ['code-review', 'security-scan', 'secure-coding', 'testing-py', 'testing-web', 'testing-go', 'e2e-testing', 'accessibility', 'performance', 'error-handling', 'observability'] },
|
|
27
27
|
{ title: 'Design & content craft', ids: ['design', 'presentations', 'course-storytelling', 'course-builder', 'technical-writing', 'translation-l10n'] },
|
|
28
28
|
{ title: 'Knowledge & meta', ids: ['knowledge-ops', 'codebase-onboarding', 'research-ops', 'decision-records', 'continuous-learning', 'skill-scout', 'context-budget', 'roast-me', 'fable-operator'] },
|
package/scripts/lib/recommend.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { skillById } from './manifest.js';
|
|
2
2
|
|
|
3
3
|
const OUTCOMES = {
|
|
4
|
+
bro: 'Writing that sounds natural and human',
|
|
4
5
|
suggest: 'An assistant that proposes what you need as you go',
|
|
5
6
|
harness: 'A tidy workspace: connect and document your company',
|
|
6
7
|
init: 'Guided project bootstrap',
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
const STOP = new Set([
|
|
2
|
+
'a', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'before', 'by', 'for', 'from',
|
|
3
|
+
'in', 'into', 'is', 'it', 'its', 'my', 'need', 'needs', 'of', 'on', 'or', 'our',
|
|
4
|
+
'so', 'that', 'the', 'them', 'this', 'to', 'use', 'want', 'we', 'when', 'with',
|
|
5
|
+
'you', 'your', 'help', 'me', 'i',
|
|
6
|
+
'un', 'una', 'unos', 'unas', 'con', 'de', 'del', 'la', 'el', 'los', 'las',
|
|
7
|
+
'para', 'que', 'porque', 'en', 'mi', 'mis', 'quiero', 'hacer', 'como',
|
|
8
|
+
'amb', 'aquest', 'aquesta', 'els', 'les', 'per', 'vull', 'vols',
|
|
9
|
+
]);
|
|
10
|
+
|
|
11
|
+
function fold(text) {
|
|
12
|
+
return text.normalize('NFD').replace(/\p{M}/gu, '');
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function stem(token) {
|
|
16
|
+
// Deliberately light: enough to join "simplify/simplified" and
|
|
17
|
+
// "conflicts/conflict" without pretending to be a language model.
|
|
18
|
+
for (const suffix of ['ally', 'ing', 'ed', 'es', 'al']) {
|
|
19
|
+
if (token.length > suffix.length + 3 && token.endsWith(suffix)) {
|
|
20
|
+
token = token.slice(0, -suffix.length);
|
|
21
|
+
break;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
if (token.length > 3 && token.endsWith('s') && !token.endsWith('ss')) token = token.slice(0, -1);
|
|
25
|
+
if (token.length > 4 && token.endsWith('e')) token = token.slice(0, -1);
|
|
26
|
+
if (
|
|
27
|
+
token.length > 4
|
|
28
|
+
&& token[token.length - 1] === token[token.length - 2]
|
|
29
|
+
&& !'aeiou'.includes(token[token.length - 1])
|
|
30
|
+
) token = token.slice(0, -1);
|
|
31
|
+
if (token.length > 3 && token.endsWith('y')) token = `${token.slice(0, -1)}i`;
|
|
32
|
+
return token;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function tokenize(text) {
|
|
36
|
+
return fold(text.toLowerCase())
|
|
37
|
+
.replace(/[^\p{L}\p{N}\s-]/gu, ' ')
|
|
38
|
+
.split(/[\s-]+/)
|
|
39
|
+
.filter((token) => token.length > 2 && !STOP.has(token))
|
|
40
|
+
.map(stem);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function frequencies(tokens) {
|
|
44
|
+
const result = new Map();
|
|
45
|
+
for (const token of tokens) result.set(token, (result.get(token) || 0) + 1);
|
|
46
|
+
return result;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function vector(frequency, idf) {
|
|
50
|
+
const result = new Map();
|
|
51
|
+
for (const [term, count] of frequency) result.set(term, count * idf(term));
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function cosine(left, right) {
|
|
56
|
+
let dot = 0;
|
|
57
|
+
let leftNorm = 0;
|
|
58
|
+
let rightNorm = 0;
|
|
59
|
+
for (const [term, weight] of left) {
|
|
60
|
+
leftNorm += weight * weight;
|
|
61
|
+
const other = right.get(term);
|
|
62
|
+
if (other) dot += weight * other;
|
|
63
|
+
}
|
|
64
|
+
for (const weight of right.values()) rightNorm += weight * weight;
|
|
65
|
+
if (!leftNorm || !rightNorm) return 0;
|
|
66
|
+
return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function buildTextCorpus(skills) {
|
|
70
|
+
const frequenciesById = new Map();
|
|
71
|
+
for (const skill of skills) {
|
|
72
|
+
const idTokens = tokenize(skill.id.replace(/-/g, ' '));
|
|
73
|
+
const tagTokens = tokenize((skill.tags || []).join(' '));
|
|
74
|
+
frequenciesById.set(skill.id, frequencies([
|
|
75
|
+
...idTokens, ...idTokens, ...idTokens,
|
|
76
|
+
...tagTokens, ...tagTokens,
|
|
77
|
+
...tokenize(skill.description),
|
|
78
|
+
]));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const documentFrequency = new Map();
|
|
82
|
+
for (const terms of frequenciesById.values()) {
|
|
83
|
+
for (const term of terms.keys()) {
|
|
84
|
+
documentFrequency.set(term, (documentFrequency.get(term) || 0) + 1);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
const size = frequenciesById.size;
|
|
88
|
+
const idf = (term) => Math.log(1 + size / (1 + (documentFrequency.get(term) || 0)));
|
|
89
|
+
const vectors = new Map();
|
|
90
|
+
for (const [id, terms] of frequenciesById) vectors.set(id, vector(terms, idf));
|
|
91
|
+
return { idf, vectors };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function rankText(corpus, prompt) {
|
|
95
|
+
const query = vector(frequencies(tokenize(prompt)), corpus.idf);
|
|
96
|
+
return [...corpus.vectors]
|
|
97
|
+
.map(([id, skillVector]) => ({ id, score: cosine(query, skillVector) }))
|
|
98
|
+
.filter((row) => row.score > 0)
|
|
99
|
+
.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function descriptionCollisions(corpus, minimum = 0.5) {
|
|
103
|
+
const rows = [...corpus.vectors];
|
|
104
|
+
const collisions = [];
|
|
105
|
+
for (let left = 0; left < rows.length; left += 1) {
|
|
106
|
+
for (let right = left + 1; right < rows.length; right += 1) {
|
|
107
|
+
const score = cosine(rows[left][1], rows[right][1]);
|
|
108
|
+
if (score >= minimum) collisions.push({ left: rows[left][0], right: rows[right][0], score });
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return collisions.sort((a, b) => b.score - a.score || a.left.localeCompare(b.left));
|
|
112
|
+
}
|
package/scripts/lib/ui.js
CHANGED
|
@@ -92,11 +92,11 @@ function frame(phase, cols, sparks, white) {
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
// Animated ASCII wordmark — the exaggerated "WOW". Static plain text when not a TTY.
|
|
95
|
-
export async function banner() {
|
|
95
|
+
export async function banner(skillCount = 'all') {
|
|
96
96
|
if (!stdout.isTTY) {
|
|
97
97
|
say('');
|
|
98
98
|
for (const l of ART) say(l);
|
|
99
|
-
say(
|
|
99
|
+
say(` ${skillCount} skills · one CLI · zero bloat`);
|
|
100
100
|
return;
|
|
101
101
|
}
|
|
102
102
|
const rows = ART.length;
|
|
@@ -133,7 +133,7 @@ export async function banner() {
|
|
|
133
133
|
// settle on a final rainbow snapshot
|
|
134
134
|
stdout.write(`\x1b[${rows}A`);
|
|
135
135
|
stdout.write(frame(15, W));
|
|
136
|
-
say(C.dim(
|
|
136
|
+
say(C.dim(` ${skillCount} skills · one CLI · zero bloat`));
|
|
137
137
|
stdout.write('\x1b[?25h'); // show cursor
|
|
138
138
|
}
|
|
139
139
|
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
|
4
|
+
import { dirname, join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { createRanker } from './consult.js';
|
|
7
|
+
import { loadManifest } from './lib/manifest.js';
|
|
8
|
+
import { buildTextCorpus, descriptionCollisions } from './lib/text-rank.js';
|
|
9
|
+
|
|
10
|
+
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
|
11
|
+
const FLOORS = {
|
|
12
|
+
rank1: 44,
|
|
13
|
+
top5: 70,
|
|
14
|
+
routedTop5: 85,
|
|
15
|
+
maxNegativeSelfRank1: 6,
|
|
16
|
+
maxCollision: 0.80,
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
function unquote(value) {
|
|
20
|
+
const trimmed = value.trim();
|
|
21
|
+
if (trimmed.startsWith('"')) return JSON.parse(trimmed);
|
|
22
|
+
return trimmed.replace(/^['"]|['"]$/g, '');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function loadRoutingCases(root = ROOT) {
|
|
26
|
+
const cases = [];
|
|
27
|
+
const skillsRoot = join(root, 'skills');
|
|
28
|
+
for (const skill of readdirSync(skillsRoot).sort()) {
|
|
29
|
+
const file = join(skillsRoot, skill, 'evals', 'cases.yaml');
|
|
30
|
+
if (!existsSync(file)) continue;
|
|
31
|
+
const lines = readFileSync(file, 'utf8').split(/\r?\n/);
|
|
32
|
+
let section = '';
|
|
33
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
34
|
+
const line = lines[index];
|
|
35
|
+
if (/^should_trigger:/.test(line)) section = 'positive';
|
|
36
|
+
else if (/^should_not_trigger:/.test(line)) section = 'negative';
|
|
37
|
+
else if (/^capability:/.test(line)) section = '';
|
|
38
|
+
const prompt = line.match(/^ - prompt:\s*(".*")$/);
|
|
39
|
+
if (!prompt || !section) continue;
|
|
40
|
+
|
|
41
|
+
let routeTo = null;
|
|
42
|
+
if (section === 'negative') {
|
|
43
|
+
for (let next = index + 1; next < Math.min(lines.length, index + 8); next += 1) {
|
|
44
|
+
if (/^ - prompt:|^[A-Za-z_]+:/.test(lines[next])) break;
|
|
45
|
+
const route = lines[next].match(/^\s+route_to:\s*(.+?)\s*$/);
|
|
46
|
+
if (route) {
|
|
47
|
+
routeTo = unquote(route[1]);
|
|
48
|
+
break;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
cases.push({ skill, section, prompt: unquote(prompt[1]), routeTo });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return cases;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function percentage(value, total) {
|
|
59
|
+
return total ? (100 * value) / total : 0;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function singleCatalogRoute(routeTo, ids) {
|
|
63
|
+
if (!routeTo || routeTo === 'none' || routeTo.startsWith('external:')) return null;
|
|
64
|
+
if (/[|,]/.test(routeTo) || routeTo.includes(' or ')) return null;
|
|
65
|
+
return ids.has(routeTo) ? routeTo : null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export async function auditRouting(manifest, cases) {
|
|
69
|
+
const ids = new Set(manifest.skills.map(({ id }) => id));
|
|
70
|
+
const ranker = await createRanker(manifest);
|
|
71
|
+
const result = {
|
|
72
|
+
positives: 0,
|
|
73
|
+
rank1: 0,
|
|
74
|
+
top3: 0,
|
|
75
|
+
top5: 0,
|
|
76
|
+
negatives: 0,
|
|
77
|
+
negativeSelfRank1: 0,
|
|
78
|
+
routed: 0,
|
|
79
|
+
routedTop5: 0,
|
|
80
|
+
routeBeatsOwner: 0,
|
|
81
|
+
missedBySkill: new Map(),
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
try {
|
|
85
|
+
for (const row of cases) {
|
|
86
|
+
const ranked = ranker.rank(row.prompt).map(({ id }) => id);
|
|
87
|
+
if (row.section === 'positive') {
|
|
88
|
+
result.positives += 1;
|
|
89
|
+
const position = ranked.indexOf(row.skill);
|
|
90
|
+
if (position === 0) result.rank1 += 1;
|
|
91
|
+
if (position >= 0 && position < 3) result.top3 += 1;
|
|
92
|
+
if (position >= 0 && position < 5) result.top5 += 1;
|
|
93
|
+
else result.missedBySkill.set(row.skill, (result.missedBySkill.get(row.skill) || 0) + 1);
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
result.negatives += 1;
|
|
98
|
+
if (ranked[0] === row.skill) result.negativeSelfRank1 += 1;
|
|
99
|
+
const target = singleCatalogRoute(row.routeTo, ids);
|
|
100
|
+
if (!target) continue;
|
|
101
|
+
result.routed += 1;
|
|
102
|
+
const targetPosition = ranked.indexOf(target);
|
|
103
|
+
const ownerPosition = ranked.indexOf(row.skill);
|
|
104
|
+
if (targetPosition >= 0 && targetPosition < 5) result.routedTop5 += 1;
|
|
105
|
+
if (targetPosition >= 0 && (ownerPosition < 0 || targetPosition < ownerPosition)) {
|
|
106
|
+
result.routeBeatsOwner += 1;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
} finally {
|
|
110
|
+
ranker.close();
|
|
111
|
+
}
|
|
112
|
+
return result;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
async function main() {
|
|
116
|
+
const manifest = loadManifest();
|
|
117
|
+
const cases = loadRoutingCases();
|
|
118
|
+
const result = await auditRouting(manifest, cases);
|
|
119
|
+
const collisions = descriptionCollisions(buildTextCorpus(manifest.skills), 0.45);
|
|
120
|
+
const rates = {
|
|
121
|
+
rank1: percentage(result.rank1, result.positives),
|
|
122
|
+
top3: percentage(result.top3, result.positives),
|
|
123
|
+
top5: percentage(result.top5, result.positives),
|
|
124
|
+
negativeSelfRank1: percentage(result.negativeSelfRank1, result.negatives),
|
|
125
|
+
routedTop5: percentage(result.routedTop5, result.routed),
|
|
126
|
+
routeBeatsOwner: percentage(result.routeBeatsOwner, result.routed),
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
console.log(`routing-audit: ${manifest.skills.length} skills · ${cases.length} prompts`);
|
|
130
|
+
console.log(
|
|
131
|
+
`positive recall: rank-1 ${rates.rank1.toFixed(1)}% · top-3 ${rates.top3.toFixed(1)}% · top-5 ${rates.top5.toFixed(1)}%`,
|
|
132
|
+
);
|
|
133
|
+
console.log(
|
|
134
|
+
`negative owner-at-1: ${rates.negativeSelfRank1.toFixed(1)}% · declared route top-5 ${rates.routedTop5.toFixed(1)}% · route beats owner ${rates.routeBeatsOwner.toFixed(1)}%`,
|
|
135
|
+
);
|
|
136
|
+
console.log('highest description overlaps:');
|
|
137
|
+
for (const row of collisions.slice(0, 10)) {
|
|
138
|
+
console.log(` ${row.score.toFixed(2)} ${row.left} ↔ ${row.right}`);
|
|
139
|
+
}
|
|
140
|
+
const misses = [...result.missedBySkill].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
|
141
|
+
console.log(`largest top-5 gaps: ${misses.slice(0, 10).map(([id, count]) => `${id} (${count})`).join(', ')}`);
|
|
142
|
+
|
|
143
|
+
const failures = [];
|
|
144
|
+
if (rates.rank1 < FLOORS.rank1) failures.push(`rank-1 ${rates.rank1.toFixed(1)}% < ${FLOORS.rank1}%`);
|
|
145
|
+
if (rates.top5 < FLOORS.top5) failures.push(`top-5 ${rates.top5.toFixed(1)}% < ${FLOORS.top5}%`);
|
|
146
|
+
if (rates.routedTop5 < FLOORS.routedTop5) {
|
|
147
|
+
failures.push(`declared-route top-5 ${rates.routedTop5.toFixed(1)}% < ${FLOORS.routedTop5}%`);
|
|
148
|
+
}
|
|
149
|
+
if (rates.negativeSelfRank1 > FLOORS.maxNegativeSelfRank1) {
|
|
150
|
+
failures.push(`negative owner-at-1 ${rates.negativeSelfRank1.toFixed(1)}% > ${FLOORS.maxNegativeSelfRank1}%`);
|
|
151
|
+
}
|
|
152
|
+
if (collisions[0]?.score >= FLOORS.maxCollision) {
|
|
153
|
+
failures.push(`description collision ${collisions[0].score.toFixed(2)} >= ${FLOORS.maxCollision}`);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
if (failures.length) {
|
|
157
|
+
console.error(`FAIL: ${failures.join('; ')}`);
|
|
158
|
+
process.exit(1);
|
|
159
|
+
}
|
|
160
|
+
console.log('PASS');
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if (import.meta.url === `file://${process.argv[1]}`) main();
|