@chimix/agentscan 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +475 -0
- package/package.json +56 -0
- package/src/analyze.ts +82 -0
- package/src/checks/budgets.ts +189 -0
- package/src/checks/index.ts +738 -0
- package/src/checks/make.ts +37 -0
- package/src/cli.ts +230 -0
- package/src/commands/check.ts +64 -0
- package/src/commands/explain.ts +64 -0
- package/src/commands/init.ts +42 -0
- package/src/commands/rules.ts +36 -0
- package/src/config/load.ts +26 -0
- package/src/config/schema.ts +63 -0
- package/src/discover/index.ts +888 -0
- package/src/facts/extract.ts +273 -0
- package/src/facts/semver.ts +41 -0
- package/src/facts/types.ts +113 -0
- package/src/report/clipboard.ts +90 -0
- package/src/report/exit-code.ts +36 -0
- package/src/report/json.ts +36 -0
- package/src/report/prompt.ts +102 -0
- package/src/report/safe.ts +0 -0
- package/src/report/score.ts +47 -0
- package/src/report/sort.ts +26 -0
- package/src/report/text.ts +142 -0
- package/src/version.ts +1 -0
|
@@ -0,0 +1,738 @@
|
|
|
1
|
+
import { basename } from "node:path";
|
|
2
|
+
import type { Facts, Finding, SkillFact } from "../facts/types";
|
|
3
|
+
import { runBudgets } from "./budgets";
|
|
4
|
+
import { make } from "./make";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Structural config checks.
|
|
8
|
+
*
|
|
9
|
+
* Every assumption these encode about how agent configuration works is recorded
|
|
10
|
+
* in docs/spec/, with the URL it came from and the date. Two checks written from
|
|
11
|
+
* observation instead — a nine-name hook-event list where the spec has 31, and a
|
|
12
|
+
* "frontmatter name must equal the directory" rule the spec contradicts —
|
|
13
|
+
* produced 25 of the 37 findings this tool reported across 17 projects. Before
|
|
14
|
+
* adding a check, find the spec line and record it there.
|
|
15
|
+
*
|
|
16
|
+
* Each check here validates one discovered item against its own file on disk.
|
|
17
|
+
* The aggregate size judgements — how long AGENTS.md is, how many MCP servers
|
|
18
|
+
* there are — live in budgets.ts, which runs from the same entry point and is
|
|
19
|
+
* declared in the same list.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export type CheckOptions = {
|
|
23
|
+
/** Flag a project that has skills but no skills-lock.json. Off by default. */
|
|
24
|
+
requireLock?: boolean;
|
|
25
|
+
/** Byte ceiling for all skill names + descriptions loaded at startup. */
|
|
26
|
+
skillDescriptionBytes?: number;
|
|
27
|
+
/** Budget ceilings. Absent means "do not run the budget checks at all". */
|
|
28
|
+
budgets?: {
|
|
29
|
+
agentsMdLines: number;
|
|
30
|
+
claudeMdLines: number;
|
|
31
|
+
agents: number;
|
|
32
|
+
mcp: number;
|
|
33
|
+
};
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Every id `runChecks` can emit — the whole of what this tool can report, and
|
|
38
|
+
* what `agentscan rules` prints. Keep in sync with the functions below and in
|
|
39
|
+
* budgets.ts; the `declared ids and emitted ids are the same set` test fails in
|
|
40
|
+
* both directions if they drift.
|
|
41
|
+
*/
|
|
42
|
+
export const STRUCTURAL_CHECKS: { id: string; description: string }[] = [
|
|
43
|
+
{
|
|
44
|
+
id: "config.unreadable",
|
|
45
|
+
description: "Config file is not valid JSON or cannot be read",
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
id: "hook.unknown-event",
|
|
49
|
+
description: "Hook registered under an event name that is never dispatched",
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
id: "hook.missing-script",
|
|
53
|
+
description: "Hook points at a script that does not exist, so it never runs",
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
id: "skill.missing-skill-md",
|
|
57
|
+
description: "Skill directory has no SKILL.md",
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
id: "skill.missing-frontmatter",
|
|
61
|
+
description: "SKILL.md has no YAML frontmatter block",
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
id: "skill.missing-description",
|
|
65
|
+
description: "SKILL.md frontmatter has no description (recommended field)",
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
id: "skill.not-in-lock",
|
|
69
|
+
description: "Skill on disk is not tracked by skills-lock.json",
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
id: "skill.locked-not-installed",
|
|
73
|
+
description: "skills-lock.json pins a skill that is not installed",
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
id: "skill.broken-reference",
|
|
77
|
+
description: "SKILL.md points at a bundled file that does not exist",
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
id: "skill.duplicate-description",
|
|
81
|
+
description: "Two or more skills share an identical description",
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
id: "skill.description-budget",
|
|
85
|
+
description: "Skill descriptions exceed the startup character budget",
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
id: "skill.no-lockfile",
|
|
89
|
+
description: "Skills present with no skills-lock.json (requireLock only)",
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
id: "agent.missing-frontmatter",
|
|
93
|
+
description: "Agent definition has no YAML frontmatter block",
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
id: "agent.missing-description",
|
|
97
|
+
description: "Agent frontmatter has no description",
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
id: "mcp.no-launch",
|
|
101
|
+
description: "MCP server declares neither command nor url",
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
id: "mcp.url-without-type",
|
|
105
|
+
description: "Remote MCP server has a url but no transport type",
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
id: "mcp.hardcoded-secret",
|
|
109
|
+
description: "MCP config contains a token-shaped literal",
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
id: "mcp.literal-env",
|
|
113
|
+
description: "MCP env value is a long literal instead of ${VAR}",
|
|
114
|
+
},
|
|
115
|
+
// Budgets — see budgets.ts. Size judgements, all info, never build-failing.
|
|
116
|
+
{
|
|
117
|
+
id: "budget.agents-md",
|
|
118
|
+
description:
|
|
119
|
+
"AGENTS.md longer than the point where added lines stop helping (>150 lines)",
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
id: "budget.claude-md",
|
|
123
|
+
description:
|
|
124
|
+
"CLAUDE.md longer than the instruction budget a model reliably follows (>200 lines)",
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
id: "budget.agents",
|
|
128
|
+
description:
|
|
129
|
+
"More agent definitions than a focused set (>8 files in .claude/agents)",
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
id: "budget.mcp",
|
|
133
|
+
description:
|
|
134
|
+
"Project MCP server count above the point where tool selection degrades",
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
id: "policy.package-manager-drift",
|
|
138
|
+
description: "Policy mentions npm install while packageManager is bun",
|
|
139
|
+
},
|
|
140
|
+
];
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Token shapes worth failing a build over. Deliberately narrow — no entropy
|
|
144
|
+
* guessing.
|
|
145
|
+
*
|
|
146
|
+
* ORDER IS SIGNIFICANT: the caller uses `.find()`, so a more specific prefix
|
|
147
|
+
* must come before any pattern that also matches it. `sk-ant-` is a subset of
|
|
148
|
+
* `sk-`, and reporting the wrong provider on a "rotate this key" finding sends
|
|
149
|
+
* the user to the wrong dashboard.
|
|
150
|
+
*/
|
|
151
|
+
const SECRET_PATTERNS: { label: string; re: RegExp }[] = [
|
|
152
|
+
{ label: "Anthropic key", re: /\bsk-ant-[A-Za-z0-9_-]{16,}/ },
|
|
153
|
+
// No embedded hyphens after the prefix. `sk-mcp-server-toolkit` is a package
|
|
154
|
+
// name, and calling it a leaked credential at severity error is unfalsifiable
|
|
155
|
+
// from the report, since the matched value is correctly never echoed.
|
|
156
|
+
{ label: "OpenAI-style key", re: /\bsk-[A-Za-z0-9_]{20,}\b/ },
|
|
157
|
+
{ label: "GitHub token", re: /\bgh[pousr]_[A-Za-z0-9]{20,}/ },
|
|
158
|
+
{ label: "Google API key", re: /\bAIza[A-Za-z0-9_-]{20,}/ },
|
|
159
|
+
{ label: "Slack token", re: /\bxox[baprs]-[A-Za-z0-9-]{10,}/ },
|
|
160
|
+
{ label: "AWS access key id", re: /\bAKIA[0-9A-Z]{16}\b/ },
|
|
161
|
+
];
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Strip source text out of a parser's message.
|
|
165
|
+
*
|
|
166
|
+
* Parsers quote the offending fragment back — `Unexpected identifier "ghp_…"`,
|
|
167
|
+
* `Unresolved alias …: ghp_…`. When the unparseable file is an MCP config or a
|
|
168
|
+
* SKILL.md that fragment can be a credential, and it reached both the text
|
|
169
|
+
* report and `--json`, which the README tells people to pipe into CI logs. A
|
|
170
|
+
* tool whose flagship check reports hardcoded secrets must not print one.
|
|
171
|
+
*
|
|
172
|
+
* Only the position survives, because only the position is actionable. Redacting
|
|
173
|
+
* by token shape was the alternative and is strictly worse: it catches six known
|
|
174
|
+
* shapes, and a parser can quote any source at all.
|
|
175
|
+
*/
|
|
176
|
+
function safeDetail(detail: string): string {
|
|
177
|
+
const position = detail.match(/\bline \d+(?:, column \d+)?/i)?.[0];
|
|
178
|
+
const kind = detail.split(/[:(]/)[0]?.trim().slice(0, 60);
|
|
179
|
+
const head = kind !== undefined && kind.length > 0 ? kind : "parse error";
|
|
180
|
+
return position === undefined ? head : `${head} at ${position}`;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function checkConfigErrors(facts: Facts): Finding[] {
|
|
184
|
+
return facts.configErrors.map((err) => {
|
|
185
|
+
const what =
|
|
186
|
+
err.kind === "invalid-json"
|
|
187
|
+
? "is not valid JSON"
|
|
188
|
+
: err.kind === "unreadable"
|
|
189
|
+
? "could not be read"
|
|
190
|
+
: "has an unexpected shape";
|
|
191
|
+
// One file can fail several ways at once — three invalid package.json
|
|
192
|
+
// fields gave three findings sharing one id, and `explain` reached only the
|
|
193
|
+
// first. The kind and detail disambiguate without changing the common case.
|
|
194
|
+
return make("config.unreadable", `config:${err.path}#${err.kind}:${safeDetail(err.detail)}`, {
|
|
195
|
+
action: "warn",
|
|
196
|
+
severity: "error",
|
|
197
|
+
message: `${basename(err.path)} ${what} — its contents are invisible to the scan`,
|
|
198
|
+
reason:
|
|
199
|
+
"An unparseable config is silently ignored by the tools that read it, so whatever it configured is simply not in effect.",
|
|
200
|
+
evidence: [
|
|
201
|
+
{ kind: "config", value: err.path },
|
|
202
|
+
{ kind: "detail", value: safeDetail(err.detail) },
|
|
203
|
+
],
|
|
204
|
+
suggest: `Fix the syntax in ${err.path}`,
|
|
205
|
+
});
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Every hook event Claude Code dispatches, from the official hooks reference.
|
|
211
|
+
*
|
|
212
|
+
* This list started at nine names guessed from what appeared in real projects,
|
|
213
|
+
* which made `hook.unknown-event` report `PostToolBatch` — a real event — as a
|
|
214
|
+
* dead hook, at severity error. Twenty-two valid names were missing. If this
|
|
215
|
+
* lags upstream again the escape hatch is
|
|
216
|
+
* `ignoreRules: ["hook.unknown-event"]`, but the right fix is to update it.
|
|
217
|
+
*
|
|
218
|
+
* Source: docs/spec/hook-events.md (read 2026-08-09)
|
|
219
|
+
*/
|
|
220
|
+
export const KNOWN_HOOK_EVENTS = new Set([
|
|
221
|
+
"SessionStart",
|
|
222
|
+
"Setup",
|
|
223
|
+
"UserPromptSubmit",
|
|
224
|
+
"UserPromptExpansion",
|
|
225
|
+
"PreToolUse",
|
|
226
|
+
"PermissionRequest",
|
|
227
|
+
"PermissionDenied",
|
|
228
|
+
"PostToolUse",
|
|
229
|
+
"PostToolUseFailure",
|
|
230
|
+
"PostToolBatch",
|
|
231
|
+
"Notification",
|
|
232
|
+
"MessageDisplay",
|
|
233
|
+
"SubagentStart",
|
|
234
|
+
"SubagentStop",
|
|
235
|
+
"TaskCreated",
|
|
236
|
+
"TaskCompleted",
|
|
237
|
+
"Stop",
|
|
238
|
+
"StopFailure",
|
|
239
|
+
"TeammateIdle",
|
|
240
|
+
"InstructionsLoaded",
|
|
241
|
+
"ConfigChange",
|
|
242
|
+
"CwdChanged",
|
|
243
|
+
"DirectoryAdded",
|
|
244
|
+
"FileChanged",
|
|
245
|
+
"WorktreeCreate",
|
|
246
|
+
"WorktreeRemove",
|
|
247
|
+
"PreCompact",
|
|
248
|
+
"PostCompact",
|
|
249
|
+
"Elicitation",
|
|
250
|
+
"ElicitationResult",
|
|
251
|
+
"SessionEnd",
|
|
252
|
+
]);
|
|
253
|
+
|
|
254
|
+
function checkHookEvents(facts: Facts): Finding[] {
|
|
255
|
+
const out: Finding[] = [];
|
|
256
|
+
const reported = new Set<string>();
|
|
257
|
+
for (const hook of facts.hooks) {
|
|
258
|
+
const event = hook.event ?? hook.name;
|
|
259
|
+
if (KNOWN_HOOK_EVENTS.has(event) || reported.has(event)) {
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
reported.add(event);
|
|
263
|
+
out.push(
|
|
264
|
+
make("hook.unknown-event", `hook:${event}`, {
|
|
265
|
+
action: "warn",
|
|
266
|
+
severity: "error",
|
|
267
|
+
message: `"${event}" is not a hook event that gets dispatched`,
|
|
268
|
+
reason:
|
|
269
|
+
"Hook events are matched by exact name. An unrecognised name never matches, so everything registered under it silently never runs — a typo here looks identical to a working hook.",
|
|
270
|
+
evidence: [
|
|
271
|
+
{ kind: "hook", value: `${event} @ ${hook.path}` },
|
|
272
|
+
{
|
|
273
|
+
kind: "known",
|
|
274
|
+
value: [...KNOWN_HOOK_EVENTS].join(", "),
|
|
275
|
+
},
|
|
276
|
+
],
|
|
277
|
+
suggest: `Fix the event name in ${hook.path} (or ignoreRules: ["hook.unknown-event"] if it is newly supported)`,
|
|
278
|
+
}),
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
return out;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function checkHooks(facts: Facts): Finding[] {
|
|
285
|
+
const out: Finding[] = [];
|
|
286
|
+
// One HookFact per command occurrence, so the same guard under two matchers —
|
|
287
|
+
// the canonical shape — produced two findings sharing one id, and `explain`
|
|
288
|
+
// could reach only the first.
|
|
289
|
+
const reported = new Set<string>();
|
|
290
|
+
for (const hook of facts.hooks) {
|
|
291
|
+
if (hook.scriptPath === undefined || hook.scriptExists !== false) {
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
const subject = `hook:${hook.event ?? hook.name}:${hook.scriptPath}`;
|
|
295
|
+
if (reported.has(subject)) {
|
|
296
|
+
continue;
|
|
297
|
+
}
|
|
298
|
+
reported.add(subject);
|
|
299
|
+
out.push(
|
|
300
|
+
make("hook.missing-script", subject, {
|
|
301
|
+
action: "warn",
|
|
302
|
+
severity: "error",
|
|
303
|
+
message: `${hook.event ?? hook.name} hook points at a script that does not exist: ${hook.scriptPath}`,
|
|
304
|
+
reason:
|
|
305
|
+
"The hook is registered but its script is missing, so it never runs. A guard hook that silently does nothing is worse than no hook — the config claims a protection that is not in effect.",
|
|
306
|
+
evidence: [
|
|
307
|
+
{ kind: "hook", value: `${hook.event ?? hook.name} @ ${hook.path}` },
|
|
308
|
+
{ kind: "script", value: hook.scriptPath },
|
|
309
|
+
],
|
|
310
|
+
suggest: `Restore ${hook.scriptPath} or remove the hook from ${hook.path}`,
|
|
311
|
+
}),
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
return out;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** Global skills live outside the repo; say so, or a reader cannot tell. */
|
|
318
|
+
function skillEvidence(skill: SkillFact, value: string): Finding["evidence"] {
|
|
319
|
+
const evidence: Finding["evidence"] = [{ kind: "skill", value }];
|
|
320
|
+
if (skill.source === "global") {
|
|
321
|
+
evidence.push({ kind: "source", value: "global" });
|
|
322
|
+
}
|
|
323
|
+
return evidence;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function checkSkillStructure(facts: Facts): Finding[] {
|
|
327
|
+
const out: Finding[] = [];
|
|
328
|
+
for (const skill of facts.skills) {
|
|
329
|
+
if (skill.unreadable === true) {
|
|
330
|
+
// Covers an unreadable SKILL.md and an unreadable skill directory alike.
|
|
331
|
+
// config.unreadable names it; every check below would otherwise make a
|
|
332
|
+
// claim about a file we never opened — "has no SKILL.md" for a directory
|
|
333
|
+
// whose SKILL.md is right there, or "no frontmatter" for a valid one.
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
if (!skill.hasSkillMd) {
|
|
338
|
+
out.push(
|
|
339
|
+
make("skill.missing-skill-md", `skill:${skill.id}`, {
|
|
340
|
+
action: "warn",
|
|
341
|
+
severity: "warning",
|
|
342
|
+
message: `Skill directory has no SKILL.md`,
|
|
343
|
+
reason:
|
|
344
|
+
"Without SKILL.md the directory is not a loadable skill; it is usually a leftover or a grouping folder that inflates the inventory.",
|
|
345
|
+
evidence: skillEvidence(skill, skill.path),
|
|
346
|
+
suggest: `Add ${skill.path}/SKILL.md or remove the directory`,
|
|
347
|
+
}),
|
|
348
|
+
);
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
if (skill.unparseableFrontmatter === true) {
|
|
353
|
+
// config.unreadable already names the parse failure; "no name" would be
|
|
354
|
+
// a guess about fields we could not read.
|
|
355
|
+
continue;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
if (!skill.hasFrontmatter) {
|
|
359
|
+
out.push(
|
|
360
|
+
make("skill.missing-frontmatter", `skill:${skill.id}`, {
|
|
361
|
+
action: "warn",
|
|
362
|
+
severity: "warning",
|
|
363
|
+
message: "SKILL.md has no YAML frontmatter block",
|
|
364
|
+
reason:
|
|
365
|
+
"Skill discovery reads `name` and `description` from frontmatter; without it the skill cannot be matched to a task.",
|
|
366
|
+
evidence: skillEvidence(skill, `${skill.path}/SKILL.md`),
|
|
367
|
+
suggest: "Add a --- delimited block with name and description",
|
|
368
|
+
}),
|
|
369
|
+
);
|
|
370
|
+
continue;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
const broken = skill.brokenReferences;
|
|
374
|
+
if (broken !== undefined && broken.length > 0) {
|
|
375
|
+
const shown = broken.slice(0, 3).join(", ");
|
|
376
|
+
const rest = broken.length - Math.min(3, broken.length);
|
|
377
|
+
out.push(
|
|
378
|
+
make("skill.broken-reference", `skill:${skill.id}`, {
|
|
379
|
+
action: "warn",
|
|
380
|
+
severity: "warning",
|
|
381
|
+
message: `SKILL.md points at ${broken.length} file${broken.length === 1 ? "" : "s"} that do not exist: ${shown}${rest > 0 ? ` (+${rest} more)` : ""}`,
|
|
382
|
+
reason:
|
|
383
|
+
"The skill tells the agent to read a bundled file that is not there. The agent follows the pointer, finds nothing, and the skill quietly does less than it says — the same failure as a hook whose script is missing.",
|
|
384
|
+
evidence: [
|
|
385
|
+
...skillEvidence(skill, skill.path),
|
|
386
|
+
...broken.map((r) => ({ kind: "missing", value: r })),
|
|
387
|
+
],
|
|
388
|
+
suggest: `Restore the missing files under ${skill.path}, or drop the references from SKILL.md`,
|
|
389
|
+
}),
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
if (skill.description === undefined) {
|
|
394
|
+
out.push(
|
|
395
|
+
make("skill.missing-description", `skill:${skill.id}`, {
|
|
396
|
+
action: "warn",
|
|
397
|
+
// The spec marks description "Recommended", not required — a skill
|
|
398
|
+
// without one still loads when invoked by name.
|
|
399
|
+
severity: "info",
|
|
400
|
+
message: "SKILL.md frontmatter has no `description`",
|
|
401
|
+
reason:
|
|
402
|
+
"The description is what Claude matches against when deciding to load a skill automatically. Without it the skill still works when invoked directly, but it will not be picked up on its own. The spec marks it Recommended, not required — see docs/spec/skills.md.",
|
|
403
|
+
evidence: skillEvidence(skill, `${skill.path}/SKILL.md`),
|
|
404
|
+
suggest: "Add a one-line description to the frontmatter",
|
|
405
|
+
}),
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
return out;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function checkLockIntegrity(facts: Facts, options: CheckOptions): Finding[] {
|
|
413
|
+
const out: Finding[] = [];
|
|
414
|
+
// A project lockfile cannot pin a skill that lives in the user's home
|
|
415
|
+
// directory, so judging global skills against it can only produce findings
|
|
416
|
+
// that are never true.
|
|
417
|
+
const projectSkills = facts.skills.filter(
|
|
418
|
+
(s) => s.source === "project" && s.hasSkillMd,
|
|
419
|
+
);
|
|
420
|
+
|
|
421
|
+
if (!facts.hasSkillsLock) {
|
|
422
|
+
if (options.requireLock === true && projectSkills.length > 0) {
|
|
423
|
+
out.push(
|
|
424
|
+
make("skill.no-lockfile", "skills:unpinned", {
|
|
425
|
+
action: "warn",
|
|
426
|
+
severity: "info",
|
|
427
|
+
message: `${projectSkills.length} skill${projectSkills.length === 1 ? "" : "s"} installed with no skills-lock.json`,
|
|
428
|
+
reason:
|
|
429
|
+
"Without a lockfile there is no record of where each skill came from or which version is pinned, so nothing can tell a managed install from a local edit.",
|
|
430
|
+
evidence: [{ kind: "count", value: `skills=${projectSkills.length}` }],
|
|
431
|
+
suggest: "Manage skills with a tool that writes skills-lock.json",
|
|
432
|
+
}),
|
|
433
|
+
);
|
|
434
|
+
}
|
|
435
|
+
return out;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
const onDisk = new Set(projectSkills.map((s) => s.id));
|
|
439
|
+
const locked = new Map(facts.lockedSkills.map((l) => [l.id, l]));
|
|
440
|
+
|
|
441
|
+
for (const skill of projectSkills) {
|
|
442
|
+
if (locked.has(skill.id)) {
|
|
443
|
+
continue;
|
|
444
|
+
}
|
|
445
|
+
out.push(
|
|
446
|
+
make("skill.not-in-lock", `skill:${skill.id}`, {
|
|
447
|
+
action: "warn",
|
|
448
|
+
severity: "info",
|
|
449
|
+
message: "Skill is not in skills-lock.json — local and unpinned",
|
|
450
|
+
reason:
|
|
451
|
+
"Every other skill here is a managed install pinned to an upstream source. This one is not tracked, so it will not be updated and cannot be reproduced on another machine.",
|
|
452
|
+
evidence: skillEvidence(skill, skill.path),
|
|
453
|
+
suggest:
|
|
454
|
+
"Keep it if it is intentionally project-local, otherwise install it through the skills tool so it gets pinned",
|
|
455
|
+
}),
|
|
456
|
+
);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
for (const entry of facts.lockedSkills) {
|
|
460
|
+
if (onDisk.has(entry.id)) {
|
|
461
|
+
continue;
|
|
462
|
+
}
|
|
463
|
+
const from = entry.source === undefined ? "" : ` (from ${entry.source})`;
|
|
464
|
+
out.push(
|
|
465
|
+
make("skill.locked-not-installed", `skill:${entry.id}`, {
|
|
466
|
+
action: "add",
|
|
467
|
+
severity: "warning",
|
|
468
|
+
message: `skills-lock.json pins "${entry.id}"${from} but it is not installed`,
|
|
469
|
+
reason:
|
|
470
|
+
"The lockfile and the working tree disagree: either the install was removed without updating the lock, or the install never completed.",
|
|
471
|
+
evidence: [
|
|
472
|
+
{ kind: "lock", value: entry.id },
|
|
473
|
+
...(entry.source === undefined
|
|
474
|
+
? []
|
|
475
|
+
: [{ kind: "source", value: entry.source }]),
|
|
476
|
+
],
|
|
477
|
+
suggest: "Reinstall the skill, or drop the entry from skills-lock.json",
|
|
478
|
+
}),
|
|
479
|
+
);
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
return out;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* Skills an agent cannot choose between.
|
|
487
|
+
*
|
|
488
|
+
* Claude picks a skill by reading its description. Two skills carrying the same
|
|
489
|
+
* one are indistinguishable at selection time: whichever is seen first wins and
|
|
490
|
+
* the other is dead weight that still costs context on every routing decision.
|
|
491
|
+
*
|
|
492
|
+
* Exact match after normalising whitespace and case — no similarity metric.
|
|
493
|
+
* A threshold needs tuning against real data, and an untuned threshold ships
|
|
494
|
+
* false positives into a report whose whole value is that everything in it is
|
|
495
|
+
* real. If exact matching proves too narrow, that is a finding to report, not a
|
|
496
|
+
* reason to guess.
|
|
497
|
+
*
|
|
498
|
+
* Project skills only: a collision between two global skills is not this
|
|
499
|
+
* project's to reconcile, the same scoping as the description budget.
|
|
500
|
+
*/
|
|
501
|
+
function checkDuplicateDescriptions(facts: Facts): Finding[] {
|
|
502
|
+
const groups = new Map<string, string[]>();
|
|
503
|
+
for (const skill of facts.skills) {
|
|
504
|
+
if (skill.source !== "project" || skill.description === undefined) {
|
|
505
|
+
continue;
|
|
506
|
+
}
|
|
507
|
+
const key = skill.description.trim().replace(/\s+/g, " ").toLowerCase();
|
|
508
|
+
if (key.length === 0) {
|
|
509
|
+
continue;
|
|
510
|
+
}
|
|
511
|
+
groups.set(key, [...(groups.get(key) ?? []), skill.id]);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
const out: Finding[] = [];
|
|
515
|
+
for (const ids of groups.values()) {
|
|
516
|
+
if (ids.length < 2) {
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
// Code-point sort, not localeCompare: ICU collation differs by locale, so
|
|
520
|
+
// an id copied from CI must still resolve locally.
|
|
521
|
+
const sorted = [...ids].sort();
|
|
522
|
+
out.push(
|
|
523
|
+
// A space cannot appear in a directory name; `+` can, and `{a+b, c}` and
|
|
524
|
+
// `{a, b+c}` produced one id.
|
|
525
|
+
make("skill.duplicate-description", `skills:${sorted.join(" ")}`, {
|
|
526
|
+
action: "warn",
|
|
527
|
+
severity: "warning",
|
|
528
|
+
message: `${sorted.length} skills share one description: ${sorted.join(", ")}`,
|
|
529
|
+
reason:
|
|
530
|
+
"Claude routes on the description. When two are identical the choice between those skills is arbitrary, and the one that loses is inert while still costing context every time.",
|
|
531
|
+
evidence: sorted.map((id) => ({ kind: "skill", value: id })),
|
|
532
|
+
suggest:
|
|
533
|
+
"Give each a description that says when to pick it over the other, or delete the redundant one",
|
|
534
|
+
}),
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
return out.sort((a, b) => (a.subject < b.subject ? -1 : 1));
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* Every skill's name and description is loaded at startup so Claude can decide
|
|
542
|
+
* what to reach for. That sits in a character budget of roughly 1-2% of the
|
|
543
|
+
* context window, shared across all of them — past it descriptions are
|
|
544
|
+
* truncated and the keywords Claude matches on are lost.
|
|
545
|
+
*
|
|
546
|
+
* This replaced a plain skill count, which measured the wrong thing: a project
|
|
547
|
+
* with 44 short-description skills sits comfortably under budget while one with
|
|
548
|
+
* 53 verbose ones is over.
|
|
549
|
+
*
|
|
550
|
+
* Evidence and confidence: docs/spec/thresholds.md
|
|
551
|
+
*/
|
|
552
|
+
function checkDescriptionBudget(
|
|
553
|
+
facts: Facts,
|
|
554
|
+
options: CheckOptions,
|
|
555
|
+
): Finding[] {
|
|
556
|
+
const ceiling = options.skillDescriptionBytes;
|
|
557
|
+
if (ceiling === undefined) {
|
|
558
|
+
return [];
|
|
559
|
+
}
|
|
560
|
+
// Only directories that load at startup, and only their descriptions: a
|
|
561
|
+
// folder with no SKILL.md contributes nothing, and the same report calls it
|
|
562
|
+
// "not a loadable skill".
|
|
563
|
+
const project = facts.skills.filter(
|
|
564
|
+
(s) => s.source === "project" && s.hasSkillMd && s.description !== undefined,
|
|
565
|
+
);
|
|
566
|
+
// Bytes, not UTF-16 units — the threshold and docs/spec/thresholds.md are
|
|
567
|
+
// both stated in bytes, and `.length` half-counts non-ASCII.
|
|
568
|
+
const bytes = project.reduce(
|
|
569
|
+
(sum, s) =>
|
|
570
|
+
sum + Buffer.byteLength(s.id) + Buffer.byteLength(s.description ?? ""),
|
|
571
|
+
0,
|
|
572
|
+
);
|
|
573
|
+
if (bytes <= ceiling) {
|
|
574
|
+
return [];
|
|
575
|
+
}
|
|
576
|
+
return [
|
|
577
|
+
make("skill.description-budget", "skills:description-budget", {
|
|
578
|
+
action: "warn",
|
|
579
|
+
severity: "info",
|
|
580
|
+
message: `Skill descriptions total ${bytes} bytes across ${project.length} skills (over ${ceiling})`,
|
|
581
|
+
reason:
|
|
582
|
+
"Names and descriptions for every skill are loaded at startup, within a budget of roughly 1-2% of the context window. Past it they are truncated, and a skill whose description is cut short stops being matched for the tasks it was written for.",
|
|
583
|
+
evidence: [
|
|
584
|
+
{ kind: "count", value: `descriptionBytes=${bytes}` },
|
|
585
|
+
{ kind: "threshold", value: String(ceiling) },
|
|
586
|
+
],
|
|
587
|
+
suggest:
|
|
588
|
+
"Shorten the longest descriptions, or remove skills this project does not use",
|
|
589
|
+
}),
|
|
590
|
+
];
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
/**
|
|
594
|
+
* Agent definitions, checked for structure only.
|
|
595
|
+
*
|
|
596
|
+
* Two checks, not four. `agent.unknown-model` and `agent.unknown-tool` were
|
|
597
|
+
* planned and cut: both would hardcode a list of valid values and report
|
|
598
|
+
* anything absent from it, which is exactly the shape of the hook-event list
|
|
599
|
+
* that held 9 of 31 names and called a working hook dead at severity error.
|
|
600
|
+
* Model ids change and MCP tool names are per-machine. See plans/003 for the
|
|
601
|
+
* conditions under which they may return.
|
|
602
|
+
*
|
|
603
|
+
* Frontmatter `name` is deliberately not compared to the filename — see
|
|
604
|
+
* docs/spec/skills.md for the same trap, and the regression test that guards it.
|
|
605
|
+
*/
|
|
606
|
+
function checkAgents(facts: Facts): Finding[] {
|
|
607
|
+
const out: Finding[] = [];
|
|
608
|
+
for (const agent of facts.agents) {
|
|
609
|
+
if (agent.unreadable === true || agent.unparseableFrontmatter === true) {
|
|
610
|
+
// config.unreadable names it. Anything else here would be a statement
|
|
611
|
+
// about a file the adjacent finding admits we could not read.
|
|
612
|
+
continue;
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
if (!agent.hasFrontmatter) {
|
|
616
|
+
out.push(
|
|
617
|
+
make("agent.missing-frontmatter", `agent:${agent.name}`, {
|
|
618
|
+
action: "warn",
|
|
619
|
+
severity: "warning",
|
|
620
|
+
message: "Agent definition has no YAML frontmatter block",
|
|
621
|
+
reason:
|
|
622
|
+
"An agent is selected by its declared description; without a frontmatter block there is nothing to select on, and the file is unlikely to load as an agent at all.",
|
|
623
|
+
evidence: [{ kind: "agent", value: agent.path }],
|
|
624
|
+
suggest: "Add a --- delimited block with a description",
|
|
625
|
+
}),
|
|
626
|
+
);
|
|
627
|
+
continue;
|
|
628
|
+
}
|
|
629
|
+
if (agent.description === undefined) {
|
|
630
|
+
out.push(
|
|
631
|
+
make("agent.missing-description", `agent:${agent.name}`, {
|
|
632
|
+
action: "warn",
|
|
633
|
+
severity: "info",
|
|
634
|
+
message: "Agent frontmatter has no `description`",
|
|
635
|
+
reason:
|
|
636
|
+
"The description is how the main session decides which agent to dispatch. Without one the agent can still be named explicitly, but it will not be chosen on its own.",
|
|
637
|
+
evidence: [{ kind: "agent", value: agent.path }],
|
|
638
|
+
suggest: "Add a one-line description saying when to dispatch it",
|
|
639
|
+
}),
|
|
640
|
+
);
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
return out;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
function checkMcp(facts: Facts): Finding[] {
|
|
647
|
+
const out: Finding[] = [];
|
|
648
|
+
for (const server of facts.mcp) {
|
|
649
|
+
if (!(server.hasCommand || server.hasUrl)) {
|
|
650
|
+
out.push(
|
|
651
|
+
make("mcp.no-launch", `mcp:${server.name}@${server.path}`, {
|
|
652
|
+
action: "warn",
|
|
653
|
+
severity: "error",
|
|
654
|
+
message: `MCP server "${server.name}" declares neither command nor url`,
|
|
655
|
+
reason:
|
|
656
|
+
"There is no way to start or reach this server, so its tools are never available — the entry is dead weight that still costs a config read.",
|
|
657
|
+
evidence: [{ kind: "mcp", value: `${server.name} @ ${server.path}` }],
|
|
658
|
+
suggest: `Add a command (or url) for "${server.name}", or remove it`,
|
|
659
|
+
}),
|
|
660
|
+
);
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
// Documented behaviour: an entry with a url and no `type` is read as a
|
|
664
|
+
// stdio server, fails, and is skipped — so its tools are silently absent.
|
|
665
|
+
if (server.hasUrl && server.transport === undefined) {
|
|
666
|
+
out.push(
|
|
667
|
+
make("mcp.url-without-type", `mcp:${server.name}@${server.path}`, {
|
|
668
|
+
action: "warn",
|
|
669
|
+
severity: "error",
|
|
670
|
+
message: `MCP server "${server.name}" has a url but no transport type`,
|
|
671
|
+
reason:
|
|
672
|
+
"An entry with no `type` is read as a stdio server, so a remote server declared this way fails to start and is skipped — its tools never appear, with no error in the report you are reading. See docs/spec/mcp.md.",
|
|
673
|
+
evidence: [{ kind: "mcp", value: `${server.name} @ ${server.path}` }],
|
|
674
|
+
suggest: `Add "type": "http" (or "sse" / "ws") to "${server.name}"`,
|
|
675
|
+
}),
|
|
676
|
+
);
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
const hit = SECRET_PATTERNS.find((p) => p.re.test(server.raw));
|
|
680
|
+
if (hit !== undefined) {
|
|
681
|
+
out.push(
|
|
682
|
+
make("mcp.hardcoded-secret", `mcp:${server.name}@${server.path}`, {
|
|
683
|
+
action: "warn",
|
|
684
|
+
severity: "error",
|
|
685
|
+
// The matched value is deliberately never echoed.
|
|
686
|
+
message: `MCP server "${server.name}" contains what looks like a hardcoded credential (${hit.label})`,
|
|
687
|
+
reason:
|
|
688
|
+
"Credentials in a config file get committed, shared and cached. Treat it as leaked: rotate the key, then reference it through an environment variable.",
|
|
689
|
+
evidence: [{ kind: "mcp", value: `${server.name} @ ${server.path}` }],
|
|
690
|
+
suggest:
|
|
691
|
+
"Rotate the credential, then replace the literal with ${ENV_VAR}",
|
|
692
|
+
}),
|
|
693
|
+
);
|
|
694
|
+
continue;
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
if (server.literalEnvKeys.length > 0) {
|
|
698
|
+
out.push(
|
|
699
|
+
make("mcp.literal-env", `mcp:${server.name}@${server.path}`, {
|
|
700
|
+
action: "warn",
|
|
701
|
+
severity: "warning",
|
|
702
|
+
message: `MCP server "${server.name}" has literal env values: ${server.literalEnvKeys.join(", ")}`,
|
|
703
|
+
reason:
|
|
704
|
+
"Long literal env values in config are usually secrets that should be indirected through ${VAR} so they are never committed.",
|
|
705
|
+
evidence: [
|
|
706
|
+
{ kind: "mcp", value: `${server.name} @ ${server.path}` },
|
|
707
|
+
{ kind: "env", value: server.literalEnvKeys.join(",") },
|
|
708
|
+
],
|
|
709
|
+
suggest: "Replace the literals with ${ENV_VAR} references",
|
|
710
|
+
}),
|
|
711
|
+
);
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
return out;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* Run every structural check. Order is deterministic; `check.ts` sorts anyway.
|
|
719
|
+
*/
|
|
720
|
+
export function runChecks(
|
|
721
|
+
facts: Facts,
|
|
722
|
+
options: CheckOptions = {},
|
|
723
|
+
): Finding[] {
|
|
724
|
+
return [
|
|
725
|
+
...checkConfigErrors(facts),
|
|
726
|
+
...checkHookEvents(facts),
|
|
727
|
+
...checkHooks(facts),
|
|
728
|
+
...checkSkillStructure(facts),
|
|
729
|
+
...checkLockIntegrity(facts, options),
|
|
730
|
+
...checkDuplicateDescriptions(facts),
|
|
731
|
+
...checkDescriptionBudget(facts, options),
|
|
732
|
+
...checkAgents(facts),
|
|
733
|
+
...checkMcp(facts),
|
|
734
|
+
...(options.budgets === undefined
|
|
735
|
+
? []
|
|
736
|
+
: runBudgets(facts, options.budgets)),
|
|
737
|
+
];
|
|
738
|
+
}
|