@dzhechkov/harness-core 0.3.23 → 0.3.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,134 @@
1
+ /**
2
+ * `dz mcp-scan --reconcile` — static capability reconciliation (Phase 3).
3
+ *
4
+ * Joins two things `dz` can read deterministically:
5
+ * - the project's GRANT surface (`scanMcp` of `.claude/settings*.json` + `.mcp.json`), and
6
+ * - the aggregate DECLARED capabilities of the installed skills (`.claude/skills/<id>/SKILL.md`),
7
+ * then REPORTS the gaps (under-grant / over-grant) and, optionally, EMITS a
8
+ * least-privilege advisory policy artifact for a host to consume.
9
+ *
10
+ * HONESTY: `dz` is build-time / static. It does NOT run agents and CANNOT block,
11
+ * time out, or rate-limit a tool call. The HOST (Claude Code enforcing
12
+ * settings.json allow/deny; or an MCP server consuming a policy.json) is the only
13
+ * thing that enforces at call time. Verbs here are REPORT / RECONCILE / EMIT /
14
+ * CANDIDATE — never BLOCK / DENIED / ENFORCED.
15
+ *
16
+ * @packageDocumentation
17
+ */
18
+ import { existsSync, readdirSync, readFileSync } from 'node:fs';
19
+ import { join } from 'node:path';
20
+ import { parseDeclaredCapabilities, parseDeclaredLimits } from './capability-vocab.js';
21
+ /** The honesty banner — repeated in --help, the report header, and the artifact $comment. */
22
+ export const RECONCILE_BANNER = 'dz is build-time/static: it REPORTS the grant-vs-declaration gap and may EMIT an advisory policy, ' +
23
+ 'but does NOT block, time out, or rate-limit anything. The HOST (Claude Code settings.json; or an MCP ' +
24
+ 'server consuming policy.json) is the only thing that enforces at call time.';
25
+ const AXES = ['shell', 'network', 'file-write'];
26
+ function readInstalled(skillsDir) {
27
+ if (!existsSync(skillsDir))
28
+ return [];
29
+ let entries;
30
+ try {
31
+ entries = readdirSync(skillsDir, { withFileTypes: true });
32
+ }
33
+ catch {
34
+ return [];
35
+ }
36
+ const out = [];
37
+ for (const e of entries) {
38
+ if (!e.isDirectory())
39
+ continue;
40
+ const md = join(skillsDir, e.name, 'SKILL.md');
41
+ if (!existsSync(md))
42
+ continue;
43
+ let content;
44
+ try {
45
+ content = readFileSync(md, 'utf-8');
46
+ }
47
+ catch {
48
+ continue;
49
+ }
50
+ out.push({ id: e.name, caps: parseDeclaredCapabilities(content), limits: parseDeclaredLimits(content) });
51
+ }
52
+ return out.sort((a, b) => a.id.localeCompare(b.id));
53
+ }
54
+ function grantFor(report, axis) {
55
+ const c = report.capabilities;
56
+ return axis === 'shell' ? c.shell : axis === 'network' ? c.network : c.fileWrite;
57
+ }
58
+ /**
59
+ * Statically reconcile a project's GRANT surface against the DECLARED needs of
60
+ * its installed skills. Pure: same inputs → same report. No execution, no writes.
61
+ */
62
+ export function reconcileCapabilities(report, skillsDir) {
63
+ const installed = readInstalled(skillsDir);
64
+ const axes = [];
65
+ const findings = [];
66
+ const allow = {};
67
+ const grants = {};
68
+ const declaredNeed = {};
69
+ const skillsByAxis = {};
70
+ for (const axis of AXES) {
71
+ const grant = grantFor(report, axis);
72
+ const needSkills = installed.filter((s) => s.caps[axis] === true).map((s) => s.id);
73
+ const need = needSkills.length > 0;
74
+ const silentCount = installed.filter((s) => s.caps[axis] === undefined).length;
75
+ axes.push({ axis, grant, need, needSkills, silentCount });
76
+ grants[axis] = grant;
77
+ declaredNeed[axis] = need;
78
+ if (need) {
79
+ skillsByAxis[axis] = needSkills;
80
+ allow[axis] = true;
81
+ }
82
+ if (need && !grant) {
83
+ findings.push({
84
+ id: `MS-UNDERGRANT-${axis.toUpperCase()}`,
85
+ kind: 'under-grant',
86
+ severity: 'medium',
87
+ axis,
88
+ detail: `${needSkills.length} installed skill(s) declare needing ${axis}, but the grant surface does not permit it — the host will starve them at runtime. Add the grant or remove the skill(s).`,
89
+ skills: needSkills,
90
+ });
91
+ }
92
+ else if (grant && !need) {
93
+ // over-grant is ALWAYS advisory; downgrade to info when any skill is silent
94
+ // (it may genuinely need the grant but didn't declare), or when there are
95
+ // no installed skills at all.
96
+ const downgrade = silentCount > 0 || installed.length === 0;
97
+ findings.push({
98
+ id: `MS-OVERGRANT-${axis.toUpperCase()}`,
99
+ kind: 'over-grant',
100
+ severity: downgrade ? 'info' : 'low',
101
+ axis,
102
+ detail: `The project permits ${axis} but no installed skill declares needing it${silentCount > 0 ? ` (${silentCount} skill(s) silent on ${axis})` : ''} — least-privilege CANDIDATE to revoke; may be for the operator or a non-skill MCP server.`,
103
+ skills: [],
104
+ });
105
+ }
106
+ }
107
+ // limits roll-up (tightest values) — INERT, never a gate
108
+ const withLimits = installed.filter((s) => Object.keys(s.limits).length > 0);
109
+ let limits = null;
110
+ if (withLimits.length > 0) {
111
+ const tts = withLimits.map((s) => s.limits.toolTimeoutMs).filter((n) => typeof n === 'number');
112
+ const mcs = withLimits.map((s) => s.limits.maxToolCallsPerTurn).filter((n) => typeof n === 'number');
113
+ const ras = withLimits.some((s) => s.limits.requireApprovalForDangerous === true);
114
+ limits = {
115
+ declaredBy: withLimits.length,
116
+ ...(tts.length > 0 ? { toolTimeoutMs: Math.min(...tts) } : {}),
117
+ ...(mcs.length > 0 ? { maxToolCallsPerTurn: Math.min(...mcs) } : {}),
118
+ ...(ras ? { requireApprovalForDangerous: true } : {}),
119
+ };
120
+ }
121
+ const policy = {
122
+ $comment: `ADVISORY. ${RECONCILE_BANNER} A host MUST consume this; dz does not.`,
123
+ version: 1,
124
+ defaultDeny: true,
125
+ allow,
126
+ ...(limits ? { limits } : {}),
127
+ derivedFrom: { grants, declaredNeed, skillsByAxis },
128
+ };
129
+ // stable ordering: under-grant (medium) first, then over-grant
130
+ const order = { medium: 0, low: 1, info: 2 };
131
+ findings.sort((a, b) => order[a.severity] - order[b.severity] || a.axis.localeCompare(b.axis));
132
+ return { skillsDir, installedCount: installed.length, axes, findings, limits, policy };
133
+ }
134
+ //# sourceMappingURL=reconcile.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reconcile.js","sourceRoot":"","sources":["../src/reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAuB,MAAM,uBAAuB,CAAC;AAG5G,6FAA6F;AAC7F,MAAM,CAAC,MAAM,gBAAgB,GAC3B,oGAAoG;IACpG,uGAAuG;IACvG,6EAA6E,CAAC;AAIhF,MAAM,IAAI,GAA6B,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;AAgE1E,SAAS,aAAa,CAAC,SAAiB;IACtC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,IAAI,OAAmC,CAAC;IACxC,IAAI,CAAC;QACH,OAAO,GAAG,WAAW,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,GAAG,GAAoB,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,SAAS;QAC/B,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAAE,SAAS;QAC9B,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,yBAAyB,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,mBAAmB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3G,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,QAAQ,CAAC,MAAqB,EAAE,IAAmB;IAC1D,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;IAC9B,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACnF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAqB,EAAE,SAAiB;IAC5E,MAAM,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAE3C,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAuB,EAAE,CAAC;IACxC,MAAM,KAAK,GAAyC,EAAE,CAAC;IACvD,MAAM,MAAM,GAAG,EAAoC,CAAC;IACpD,MAAM,YAAY,GAAG,EAAoC,CAAC;IAC1D,MAAM,YAAY,GAAsD,EAAE,CAAC;IAE3E,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;QACnC,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;QAC/E,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACrB,YAAY,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QAC1B,IAAI,IAAI,EAAE,CAAC;YACT,YAAY,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QACrB,CAAC;QAED,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC;gBACZ,EAAE,EAAE,iBAAiB,IAAI,CAAC,WAAW,EAAE,EAAE;gBACzC,IAAI,EAAE,aAAa;gBACnB,QAAQ,EAAE,QAAQ;gBAClB,IAAI;gBACJ,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,uCAAuC,IAAI,0HAA0H;gBACjM,MAAM,EAAE,UAAU;aACnB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1B,4EAA4E;YAC5E,0EAA0E;YAC1E,8BAA8B;YAC9B,MAAM,SAAS,GAAG,WAAW,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;YAC5D,QAAQ,CAAC,IAAI,CAAC;gBACZ,EAAE,EAAE,gBAAgB,IAAI,CAAC,WAAW,EAAE,EAAE;gBACxC,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;gBACpC,IAAI;gBACJ,MAAM,EAAE,uBAAuB,IAAI,8CAA8C,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,uBAAuB,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,4FAA4F;gBAClP,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7E,IAAI,MAAM,GAAwB,IAAI,CAAC;IACvC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;QAC5G,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;QAClH,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,2BAA2B,KAAK,IAAI,CAAC,CAAC;QAClF,MAAM,GAAG;YACP,UAAU,EAAE,UAAU,CAAC,MAAM;YAC7B,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,2BAA2B,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAmB;QAC7B,QAAQ,EAAE,aAAa,gBAAgB,yCAAyC;QAChF,OAAO,EAAE,CAAC;QACV,WAAW,EAAE,IAAI;QACjB,KAAK;QACL,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,WAAW,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE;KACpD,CAAC;IAEF,+DAA+D;IAC/D,MAAM,KAAK,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAW,CAAC;IACtD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAE/F,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACzF,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dzhechkov/harness-core",
3
- "version": "0.3.23",
3
+ "version": "0.3.24",
4
4
  "description": "Shared harness logic - skill loading, additive apply, and the init/sync/verify/doctor operations.",
5
5
  "type": "module",
6
6
  "license": "MIT",
package/src/benchmark.ts CHANGED
@@ -11,6 +11,7 @@
11
11
  import { existsSync, readFileSync, statSync } from 'node:fs';
12
12
  import { join } from 'node:path';
13
13
 
14
+ import { detectScriptCapabilities, parseDeclaredCapabilities } from './capability-vocab.js';
14
15
  import { estimateSkillCost, type CostEstimate } from './cost-scoring.js';
15
16
 
16
17
  /** A single benchmark check result. */
@@ -19,6 +20,12 @@ export interface BenchmarkCheck {
19
20
  readonly name: string;
20
21
  readonly passed: boolean;
21
22
  readonly detail?: string | undefined;
23
+ /**
24
+ * When true this check is informational ONLY — it is excluded from the
25
+ * pass-rate / grade math (so it can ship without regressing existing grades).
26
+ * Surfaced as a warning in output. Used by S16 (capability-declaration nudge).
27
+ */
28
+ readonly advisory?: boolean;
22
29
  }
23
30
 
24
31
  /** Score for a single skill benchmark. */
@@ -160,11 +167,52 @@ export function benchmarkSkill(skillDir: string, skillId: string): BenchmarkScor
160
167
  || /\*\*\s*PROHIBITED/i.test(content);
161
168
  checks.push({ id: 'S14', name: 'anti-patterns or self-check', passed: hasAntiPatterns });
162
169
 
163
- const passed = checks.filter((c) => c.passed).length;
164
- const passRate = Math.round((passed / checks.length) * 100);
170
+ // S15: capability declaration matches usage (advisory, contradiction-only).
171
+ // Phase 1 auto-detects network + shell from scripts/ only (never SKILL.md prose).
172
+ // PASS unless a declared `capabilities.<cap>: false` is contradicted by detected
173
+ // usage. Absent block, or under-declaration (uses it, declares nothing), PASS —
174
+ // so the ~all skills that declare nothing today stay green.
175
+ const declaredCaps = parseDeclaredCapabilities(content);
176
+ const detectedCaps = detectScriptCapabilities(skillDir);
177
+ const contradictions: string[] = [];
178
+ if (detectedCaps.network && declaredCaps.network === false) contradictions.push('network');
179
+ if (detectedCaps.shell && declaredCaps.shell === false) contradictions.push('shell');
180
+ checks.push({
181
+ id: 'S15',
182
+ name: 'capability declaration matches usage',
183
+ passed: contradictions.length === 0,
184
+ detail:
185
+ contradictions.length > 0
186
+ ? `scripts use ${contradictions.join(' + ')} but capabilities declare it false`
187
+ : undefined,
188
+ });
189
+
190
+ // S16: side-effecting skills should declare their capabilities (ADVISORY — not
191
+ // graded). Warns when scripts use network/shell but the capability is not
192
+ // declared at all (neither true nor false). The adoption nudge for the manifest;
193
+ // excluded from the grade so it never regresses an existing skill.
194
+ const undeclaredUsed: string[] = [];
195
+ if (detectedCaps.network && declaredCaps.network === undefined) undeclaredUsed.push('network');
196
+ if (detectedCaps.shell && declaredCaps.shell === undefined) undeclaredUsed.push('shell');
197
+ checks.push({
198
+ id: 'S16',
199
+ name: 'side-effects declared (advisory)',
200
+ passed: undeclaredUsed.length === 0,
201
+ advisory: true,
202
+ detail:
203
+ undeclaredUsed.length > 0
204
+ ? `scripts use ${undeclaredUsed.join(' + ')} — add a capabilities: block declaring it`
205
+ : undefined,
206
+ });
207
+
208
+ // grade math is over GRADED checks only — advisory checks (S16) are excluded
209
+ // so they ship without changing any existing grade.
210
+ const graded = checks.filter((c) => !c.advisory);
211
+ const passed = graded.filter((c) => c.passed).length;
212
+ const passRate = graded.length > 0 ? Math.round((passed / graded.length) * 100) : 0;
165
213
 
166
214
  return {
167
- skillId, skillDir, checks, passed, total: checks.length,
215
+ skillId, skillDir, checks, passed, total: graded.length,
168
216
  passRate, grade: gradeFromRate(passRate), cost: estimateSkillCost(content),
169
217
  };
170
218
  }
@@ -190,8 +238,12 @@ export function compareSkills(
190
238
  const skillA = benchmarkSkill(skillADir, skillAId);
191
239
  const skillB = benchmarkSkill(skillBDir, skillBId);
192
240
 
241
+ // deltaChecks is GRADED-ONLY — advisory checks (e.g. S16) never touch the grade,
242
+ // so surfacing them in the A/B diff would misrepresent an adoption nudge as a
243
+ // quality difference. Mirrors the graded filter in the grade math above.
193
244
  const deltaChecks: { id: string; aPass: boolean; bPass: boolean }[] = [];
194
245
  for (const checkA of skillA.checks) {
246
+ if (checkA.advisory) continue;
195
247
  const checkB = skillB.checks.find((c) => c.id === checkA.id);
196
248
  if (checkB && checkA.passed !== checkB.passed) {
197
249
  deltaChecks.push({ id: checkA.id, aPass: checkA.passed, bPass: checkB.passed });
@@ -0,0 +1,307 @@
1
+ /**
2
+ * Shared capability vocabulary — the single source of truth for how the harness
3
+ * classifies agent capabilities, used by BOTH `mcp-scan` (project settings audit)
4
+ * and `benchmark` (per-skill S15 capability-declaration check).
5
+ *
6
+ * Keeping these regexes/sets here (rather than private to mcp-scan) guarantees the
7
+ * project-level scan and the per-skill manifest speak ONE diffable vocabulary.
8
+ *
9
+ * @packageDocumentation
10
+ */
11
+
12
+ import { existsSync, lstatSync, readdirSync, readFileSync } from 'node:fs';
13
+ import { extname, join } from 'node:path';
14
+
15
+ /** The capability classes the harness recognises. */
16
+ export type CapabilityClass = 'shell' | 'network' | 'file-write' | 'secrets' | 'mcp' | 'policy';
17
+
18
+ // --- Claude permission-grammar tool sets (lowercase for case-insensitive tests) ---
19
+ export const SHELL_TOOLS = new Set(['bash', 'powershell', 'shell']);
20
+ export const NETWORK_TOOLS = new Set(['webfetch', 'websearch']);
21
+ export const WRITE_TOOLS = new Set(['write', 'edit', 'multiedit', 'notebookedit']);
22
+ export const READ_TOOLS = new Set(['read']);
23
+ /** Recognised + benign tools (so they aren't flagged "unknown"). */
24
+ export const SAFE_TOOLS = new Set([
25
+ 'glob', 'grep', 'task', 'bashoutput', 'killbash', 'todowrite',
26
+ 'notebookread', 'slashcommand', 'exitplanmode', 'ls',
27
+ ]);
28
+
29
+ /** Interpreter binaries that can run arbitrary code. */
30
+ export const INTERPRETER_RE = /^(bash|sh|zsh|fish|node|nodejs|python\d?|deno|ruby|perl|php)$/i;
31
+ /** Package runners that fetch + execute arbitrary remote code. */
32
+ export const PACKAGE_RUNNERS = new Set(['npx', 'npm', 'pnpm', 'yarn', 'uvx', 'uv', 'pipx', 'bunx', 'bun', 'deno']);
33
+ /** Inline-code argument flags. */
34
+ export const INLINE_CODE_ARGS = new Set(['-c', '-e', '-eval', '--eval', '-p']);
35
+
36
+ /** Binaries that imply outbound network. */
37
+ export const SHELL_NET_RE = /\b(curl|wget|nc|ncat|netcat|ssh|scp|sftp|telnet|ftp)\b/i;
38
+ /** Binaries / redirects that imply filesystem writes. */
39
+ export const SHELL_WRITE_RE = /\b(rm|mv|cp|tee|dd|truncate|chmod|chown|mkfifo)\b|>>?/;
40
+ /** Concrete secret-file location patterns (not bare "secret"/"credential" substrings). */
41
+ export const SECRET_FILE_RE =
42
+ /\.env\b|\.env\.|id_rsa|id_ed25519|id_ecdsa|\.ssh\/|\.aws\/|\.config\/gcloud|application_default_credentials|\.npmrc|\.netrc|\.git-credentials|\.kube\/config|\.pem\b|\.p12\b|\.pfx\b|\.key\b/i;
43
+
44
+ export const MAX_FILE_BYTES = 5 * 1024 * 1024;
45
+
46
+ // ---------------------------------------------------------------------------
47
+ // permission-grammar helpers
48
+ // ---------------------------------------------------------------------------
49
+
50
+ /** Parse a Claude permission grant like `Bash(git *)` → `{ tool, arg }`. Never throws. */
51
+ export function parseGrant(grant: unknown): { tool: string; arg: string | null } {
52
+ if (typeof grant !== 'string') return { tool: '', arg: null };
53
+ const s = grant.trim();
54
+ const m = /^([A-Za-z_][\w-]*)\s*\((.*)\)\s*$/.exec(s);
55
+ if (m) return { tool: m[1] ?? s, arg: m[2] ?? null };
56
+ const id = /^([A-Za-z_][\w-]*|\*)/.exec(s);
57
+ return { tool: id ? (id[1] ?? s) : s, arg: null };
58
+ }
59
+
60
+ export function isWildcard(arg: string | null): boolean {
61
+ return arg === null || arg.trim() === '' || arg.includes('*');
62
+ }
63
+
64
+ export function toolKind(tool: string): CapabilityClass | 'read' | 'safe' | 'unknown' {
65
+ const t = tool.toLowerCase();
66
+ if (t === '*') return 'shell';
67
+ if (SHELL_TOOLS.has(t)) return 'shell';
68
+ if (NETWORK_TOOLS.has(t)) return 'network';
69
+ if (WRITE_TOOLS.has(t)) return 'file-write';
70
+ if (READ_TOOLS.has(t)) return 'read';
71
+ if (t.startsWith('mcp__')) return 'mcp';
72
+ if (SAFE_TOOLS.has(t)) return 'safe';
73
+ return 'unknown';
74
+ }
75
+
76
+ // ---------------------------------------------------------------------------
77
+ // script capability detection (for the per-skill S15 check)
78
+ // ---------------------------------------------------------------------------
79
+
80
+ /** Self-declared capability surface parsed from a skill's `capabilities:` block. */
81
+ export interface DeclaredCapabilities {
82
+ network?: boolean;
83
+ shell?: boolean;
84
+ 'file-write'?: boolean;
85
+ dangerous?: boolean;
86
+ }
87
+
88
+ /** Capabilities statically detected in a skill's `scripts/`. P1: network + shell only. */
89
+ export interface DetectedCapabilities {
90
+ network: boolean;
91
+ shell: boolean;
92
+ }
93
+
94
+ /** Strip heredoc bodies (`<<EOF … EOF`, incl. `<<-'EOF'`) — unquoted free text. */
95
+ function stripHeredocs(text: string): string {
96
+ return text.replace(/<<-?\s*(['"]?)(\w+)\1[\s\S]*?\n[ \t]*\2\b/g, ' ');
97
+ }
98
+
99
+ /** Strip comments + heredocs (keeps string literals — curl inside an arg string is real). */
100
+ export function stripComments(text: string): string {
101
+ return stripHeredocs(text)
102
+ .replace(/\/\*[\s\S]*?\*\//g, ' ') // /* paired block */
103
+ .replace(/\/\*[\s\S]*$/g, ' ') // unterminated block → EOF
104
+ .replace(/(^|[^:])\/\/[^\n]*/g, '$1 ') // // line (not http://)
105
+ .replace(/(^|\s)#[^\n]*/g, '$1 '); // # shell/py
106
+ }
107
+
108
+ /**
109
+ * Strip comments, heredocs AND quoted string literals. Used for the IDENTIFIER
110
+ * pass (fetch/axios/execSync/http modules) where the token is real code, never a
111
+ * quoted search pattern — this is what kills FPs like `grep "writeFileSync"`.
112
+ */
113
+ export function stripCodeNoise(text: string): string {
114
+ return stripComments(text)
115
+ .replace(/'(?:[^'\\]|\\.)*'/g, ' ')
116
+ .replace(/"(?:[^"\\]|\\.)*"/g, ' ')
117
+ .replace(/`(?:[^`\\]|\\.)*`/g, ' ');
118
+ }
119
+
120
+ const SCRIPT_EXTS = new Set(['.sh', '.bash', '.zsh', '.js', '.mjs', '.cjs', '.ts', '.py', '.rb', '.pl', '.php']);
121
+
122
+ // command position = line start, after a shell separator, after `sudo`/`do`/`then`,
123
+ // inside `$( )`, or right after an opening quote (covers subprocess.run("curl …")).
124
+ const CMD_PREFIX = `(?:^|[;&|(\\n]|&&|\\|\\||\\$\\(|\\bsudo\\s+|\\bthen\\s+|\\bdo\\s+|["'\`])\\s*`;
125
+ // the next token must look like an argument/subcommand (a word/path/flag), not an
126
+ // operator — so `uv = coord` / `ssh = cfg` (assignments) and `grep "curl"` don't match.
127
+ const ARG_FOLLOWS = `(?=\\s+[\\w./~-])`;
128
+ // a network binary AT command position, followed by an argument.
129
+ const CMD_NET_RE = new RegExp(`${CMD_PREFIX}(curl|wget|nc|ncat|netcat|ssh|scp|sftp|telnet|ftp)\\b${ARG_FOLLOWS}`, 'i');
130
+ // a package-runner / interpreter AT command position, followed by an argument → shell.
131
+ const CMD_SHELL_RE = new RegExp(
132
+ `${CMD_PREFIX}(${[...PACKAGE_RUNNERS].join('|')}|python\\d?|node|nodejs|bash|sh|zsh|ruby|perl|php)\\b${ARG_FOLLOWS}`,
133
+ 'i',
134
+ );
135
+ // network via library identifiers (matched on noise-stripped code — never in a quote).
136
+ const NETWORK_CALL_RE =
137
+ /\bfetch\s*\(|\baxios\b|WebFetch\s*\(|\bWebSearch\b|\brequests\.|\burllib3?\b|\bhttpx\b|\baiohttp\b|\bparamiko\b|\bsmtplib\b|\bftplib\b|\bhttp\.client\b|\bsocket\.(?:socket|create_connection)\s*\(|\b(?:https?|net|dgram|tls|dns)\.(?:get|request|createConnection|connect|createSocket|resolve\w*)\s*\(|\bnew\s+WebSocket\b|(?:require\(|from\s+)['"](?:node:)?(?:http|https|net|dgram|tls|ws|node-fetch|got|undici)['"]/;
138
+ // shell exec via library identifiers (language-agnostic).
139
+ const EXEC_CALL_RE =
140
+ /child_process|execSync|execFileSync|spawnSync|\bspawn\s*\(|\bexec(?:File)?\s*\(|\bsubprocess\b|\bos\.system\b|\bos\.popen\b|\bPopen\b|\bOpen3\b|\bcommands\.get(?:status)?output\b|%x[([{]|(?:^|[^.\w])system\s*\(/m;
141
+
142
+ function isProbablyBinary(buf: string): boolean {
143
+ return /[\x00-\x08\x0E-\x1F]/.test(buf);
144
+ }
145
+
146
+ function collectScriptFiles(dir: string, depth: number, acc: string[]): void {
147
+ if (depth > 3 || acc.length >= 200) return;
148
+ let entries: string[];
149
+ try {
150
+ entries = readdirSync(dir);
151
+ } catch {
152
+ return;
153
+ }
154
+ for (const name of entries) {
155
+ if (acc.length >= 200) return;
156
+ const abs = join(dir, name);
157
+ let st;
158
+ try {
159
+ st = lstatSync(abs);
160
+ } catch {
161
+ continue;
162
+ }
163
+ if (st.isSymbolicLink()) continue;
164
+ if (st.isDirectory()) {
165
+ collectScriptFiles(abs, depth + 1, acc);
166
+ } else if (st.isFile() && st.size <= 256 * 1024) {
167
+ acc.push(abs);
168
+ }
169
+ }
170
+ }
171
+
172
+ /**
173
+ * Statically detect network + shell capability usage in a skill's `scripts/`.
174
+ * Reads only `scripts/` (never SKILL.md prose), recursively (bounded), skipping
175
+ * binary / symlinked / oversized files. Deterministic, no execution.
176
+ *
177
+ * Known scope (Phase 1, by design — documented, not bugs): file-write and
178
+ * `dangerous` are not auto-detected; dynamically-assembled commands
179
+ * (`$RUNNER install`, eval'd strings) and indirected calls evade static regexes.
180
+ * S15 is a best-effort self-consistency LINT, not a sandbox.
181
+ */
182
+ export function detectScriptCapabilities(skillDir: string): DetectedCapabilities {
183
+ const out: DetectedCapabilities = { network: false, shell: false };
184
+ const scriptsDir = join(skillDir, 'scripts');
185
+ if (!existsSync(scriptsDir)) return out;
186
+
187
+ const files: string[] = [];
188
+ collectScriptFiles(scriptsDir, 0, files);
189
+
190
+ for (const abs of files) {
191
+ const ext = extname(abs).toLowerCase();
192
+ let raw: string;
193
+ try {
194
+ raw = readFileSync(abs, 'utf-8');
195
+ } catch {
196
+ continue;
197
+ }
198
+ if (isProbablyBinary(raw)) continue;
199
+
200
+ const hasShebang = /^#!/.test(raw);
201
+ const isShellExt = ext === '.sh' || ext === '.bash' || ext === '.zsh';
202
+ // shell scripts (by extension, or extensionless with a shell shebang) ARE shell usage
203
+ if (isShellExt || (ext === '' && /^#![^\n]*\b(bash|sh|zsh)\b/.test(raw))) out.shell = true;
204
+ if (!SCRIPT_EXTS.has(ext) && !hasShebang) continue;
205
+
206
+ const commentless = stripComments(raw); // strings kept → curl in an arg string survives
207
+ const noiseless = stripCodeNoise(raw); // strings gone → identifier pass
208
+
209
+ if (CMD_NET_RE.test(commentless) || NETWORK_CALL_RE.test(noiseless)) out.network = true;
210
+ if (out.shell || CMD_SHELL_RE.test(commentless) || EXEC_CALL_RE.test(noiseless)) out.shell = true;
211
+ }
212
+ return out;
213
+ }
214
+
215
+ /**
216
+ * Parse the `capabilities:` block from a SKILL.md document. Reads only the
217
+ * frontmatter region (the first `---`-fenced block) and only DIRECT children of
218
+ * `capabilities:` (so a nested `limits.network` is never mistaken for a top-level
219
+ * declaration). Absent block or absent key → `undefined` ("not asserted").
220
+ */
221
+ export function parseDeclaredCapabilities(skillMd: string): DeclaredCapabilities {
222
+ const fmMatch = /^---\r?\n([\s\S]*?)\r?\n---/.exec(skillMd);
223
+ const fm = fmMatch ? (fmMatch[1] ?? '') : '';
224
+ const block = /^capabilities:[ \t]*\r?\n((?:[ \t]+.*\r?\n?)*)/m.exec(fm);
225
+ if (!block) return {};
226
+ const lines = (block[1] ?? '').split('\n');
227
+ const firstReal = lines.find((l) => l.trim() !== '');
228
+ if (!firstReal) return {};
229
+ const childIndent = (/^[ \t]*/.exec(firstReal)?.[0]) ?? '';
230
+
231
+ const out: DeclaredCapabilities = {};
232
+ const read = (key: string): boolean | undefined => {
233
+ const re = new RegExp(`^${childIndent}${key.replace('-', '\\-')}:[ \\t]*(true|false)\\b`);
234
+ for (const l of lines) {
235
+ const m = re.exec(l);
236
+ if (m) return m[1] === 'true';
237
+ }
238
+ return undefined;
239
+ };
240
+ const network = read('network');
241
+ const shell = read('shell');
242
+ const fileWrite = read('file-write');
243
+ const dangerous = read('dangerous');
244
+ if (network !== undefined) out.network = network;
245
+ if (shell !== undefined) out.shell = shell;
246
+ if (fileWrite !== undefined) out['file-write'] = fileWrite;
247
+ if (dangerous !== undefined) out.dangerous = dangerous;
248
+ return out;
249
+ }
250
+
251
+ /** Declared runtime limits (inert today — no enforcement home in Claude Code settings). */
252
+ export interface DeclaredLimits {
253
+ toolTimeoutMs?: number;
254
+ maxToolCallsPerTurn?: number;
255
+ requireApprovalForDangerous?: boolean;
256
+ }
257
+
258
+ /**
259
+ * Parse the nested `capabilities.limits` block from a SKILL.md frontmatter.
260
+ * Fail-open: a malformed/absent block yields `{}` (never throws). These values
261
+ * are INERT — Claude Code settings.json has no timeout/rate-limit field; they are
262
+ * only machine-actionable in an MCP host's policy.json.
263
+ */
264
+ export function parseDeclaredLimits(skillMd: string): DeclaredLimits {
265
+ const fmMatch = /^---\r?\n([\s\S]*?)\r?\n---/.exec(skillMd);
266
+ const fm = fmMatch ? (fmMatch[1] ?? '') : '';
267
+ // Anchor under capabilities: — only a `limits:` that is a DIRECT child of
268
+ // capabilities counts (not some unrelated indented `limits:` elsewhere).
269
+ const capBlock = /^capabilities:[ \t]*\r?\n((?:[ \t]+.*\r?\n?)*)/m.exec(fm);
270
+ if (!capBlock) return {};
271
+ const capLines = (capBlock[1] ?? '').split('\n');
272
+ const firstReal = capLines.find((l) => l.trim() !== '');
273
+ if (!firstReal) return {};
274
+ const childIndent = (/^[ \t]*/.exec(firstReal)?.[0]) ?? '';
275
+
276
+ // find the `limits:` line at the capabilities-child indent, then capture only
277
+ // its strictly-deeper-indented children.
278
+ const limitsIdx = capLines.findIndex((l) => new RegExp(`^${childIndent}limits:[ \\t]*$`).test(l));
279
+ if (limitsIdx === -1) return {};
280
+ const body: string[] = [];
281
+ for (let i = limitsIdx + 1; i < capLines.length; i++) {
282
+ const l = capLines[i] ?? '';
283
+ if (l.trim() === '') continue;
284
+ const indent = (/^[ \t]*/.exec(l)?.[0]) ?? '';
285
+ if (indent.length <= childIndent.length) break; // back to a sibling → end of limits
286
+ body.push(l);
287
+ }
288
+ const text = body.join('\n');
289
+ const out: DeclaredLimits = {};
290
+ const num = (key: string): number | undefined => {
291
+ const m = new RegExp(`^[ \\t]+${key}:[ \\t]*(\\d+)\\b`, 'm').exec(text);
292
+ if (!m) return undefined;
293
+ const n = Number(m[1]);
294
+ return Number.isSafeInteger(n) && n >= 0 ? n : undefined; // fail-open on absurd values
295
+ };
296
+ const bool = (key: string): boolean | undefined => {
297
+ const m = new RegExp(`^[ \\t]+${key}:[ \\t]*(true|false)\\b`, 'm').exec(text);
298
+ return m ? m[1] === 'true' : undefined;
299
+ };
300
+ const tt = num('toolTimeoutMs');
301
+ const mc = num('maxToolCallsPerTurn');
302
+ const ra = bool('requireApprovalForDangerous');
303
+ if (tt !== undefined) out.toolTimeoutMs = tt;
304
+ if (mc !== undefined) out.maxToolCallsPerTurn = mc;
305
+ if (ra !== undefined) out.requireApprovalForDangerous = ra;
306
+ return out;
307
+ }
package/src/index.ts CHANGED
@@ -56,5 +56,30 @@ export type {
56
56
  } from './cost-scoring.js';
57
57
  export { importEcc } from './import-ecc.js';
58
58
  export type { ImportEccReport, ImportEccOptions, ImportedSkill } from './import-ecc.js';
59
+ export { scanMcp, parseGrant } from './mcp-scan.js';
60
+ export {
61
+ detectScriptCapabilities,
62
+ parseDeclaredCapabilities,
63
+ parseDeclaredLimits,
64
+ stripCodeNoise,
65
+ toolKind,
66
+ } from './capability-vocab.js';
67
+ export type { CapabilityClass, DeclaredCapabilities, DetectedCapabilities, DeclaredLimits } from './capability-vocab.js';
68
+ export { reconcileCapabilities, RECONCILE_BANNER } from './reconcile.js';
69
+ export type {
70
+ ReconcileReport,
71
+ ReconcileFinding,
72
+ ReconcileAxis,
73
+ AxisState,
74
+ LimitsRollup,
75
+ PolicyArtifact,
76
+ } from './reconcile.js';
77
+ export type {
78
+ McpFinding,
79
+ McpScanReport,
80
+ McpVerdict,
81
+ McpSeverity,
82
+ McpCapability,
83
+ } from './mcp-scan.js';
59
84
  export type { RegistryEntry, Registry } from './registry.js';
60
85
  export type { BenchmarkCheck, BenchmarkScore, BenchmarkReport, CompareResult } from './benchmark.js';