@dzhechkov/harness-core 0.3.23 → 0.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark.d.ts +6 -0
- package/dist/benchmark.d.ts.map +1 -1
- package/dist/benchmark.js +50 -3
- package/dist/benchmark.js.map +1 -1
- package/dist/capability-vocab.d.ts +90 -0
- package/dist/capability-vocab.d.ts.map +1 -0
- package/dist/capability-vocab.js +286 -0
- package/dist/capability-vocab.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp-scan.d.ts +75 -0
- package/dist/mcp-scan.d.ts.map +1 -0
- package/dist/mcp-scan.js +383 -0
- package/dist/mcp-scan.js.map +1 -0
- package/dist/operations.d.ts.map +1 -1
- package/dist/operations.js +8 -4
- package/dist/operations.js.map +1 -1
- package/dist/reconcile.d.ts +79 -0
- package/dist/reconcile.d.ts.map +1 -0
- package/dist/reconcile.js +134 -0
- package/dist/reconcile.js.map +1 -0
- package/dist/targets.d.ts +2 -8
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +5 -4
- package/dist/targets.js.map +1 -1
- package/package.json +4 -3
- package/src/benchmark.ts +55 -3
- package/src/capability-vocab.ts +307 -0
- package/src/index.ts +25 -0
- package/src/mcp-scan.ts +489 -0
- package/src/operations.ts +8 -4
- package/src/reconcile.ts +207 -0
- package/src/targets.ts +10 -5
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `dz mcp-scan --reconcile` — static capability reconciliation (Phase 3).
|
|
3
|
+
*
|
|
4
|
+
* Joins two things `dz` can read deterministically:
|
|
5
|
+
* - the project's GRANT surface (`scanMcp` of `.claude/settings*.json` + `.mcp.json`), and
|
|
6
|
+
* - the aggregate DECLARED capabilities of the installed skills (`.claude/skills/<id>/SKILL.md`),
|
|
7
|
+
* then REPORTS the gaps (under-grant / over-grant) and, optionally, EMITS a
|
|
8
|
+
* least-privilege advisory policy artifact for a host to consume.
|
|
9
|
+
*
|
|
10
|
+
* HONESTY: `dz` is build-time / static. It does NOT run agents and CANNOT block,
|
|
11
|
+
* time out, or rate-limit a tool call. The HOST (Claude Code enforcing
|
|
12
|
+
* settings.json allow/deny; or an MCP server consuming a policy.json) is the only
|
|
13
|
+
* thing that enforces at call time. Verbs here are REPORT / RECONCILE / EMIT /
|
|
14
|
+
* CANDIDATE — never BLOCK / DENIED / ENFORCED.
|
|
15
|
+
*
|
|
16
|
+
* @packageDocumentation
|
|
17
|
+
*/
|
|
18
|
+
import type { McpScanReport } from './mcp-scan.js';
|
|
19
|
+
/** The honesty banner — repeated in --help, the report header, and the artifact $comment. */
|
|
20
|
+
export declare const RECONCILE_BANNER: string;
|
|
21
|
+
/** A reconcilable capability axis (the axes BOTH the grant surface and the manifest expose). */
|
|
22
|
+
export type ReconcileAxis = 'shell' | 'network' | 'file-write';
|
|
23
|
+
/** Per-axis reconciliation state. */
|
|
24
|
+
export interface AxisState {
|
|
25
|
+
readonly axis: ReconcileAxis;
|
|
26
|
+
/** Project permits this capability (from scanMcp). */
|
|
27
|
+
readonly grant: boolean;
|
|
28
|
+
/** At least one installed skill declares it needs this (declared === true). */
|
|
29
|
+
readonly need: boolean;
|
|
30
|
+
/** Skills that declared need (true) — for attribution. */
|
|
31
|
+
readonly needSkills: readonly string[];
|
|
32
|
+
/** Installed skills that are SILENT on this axis (declared === undefined). */
|
|
33
|
+
readonly silentCount: number;
|
|
34
|
+
}
|
|
35
|
+
/** A single reconciliation finding. */
|
|
36
|
+
export interface ReconcileFinding {
|
|
37
|
+
readonly id: string;
|
|
38
|
+
readonly kind: 'under-grant' | 'over-grant';
|
|
39
|
+
readonly severity: 'medium' | 'low' | 'info';
|
|
40
|
+
readonly axis: ReconcileAxis;
|
|
41
|
+
readonly detail: string;
|
|
42
|
+
readonly skills: readonly string[];
|
|
43
|
+
}
|
|
44
|
+
/** Aggregated declared limits across installed skills (tightest values; inert). */
|
|
45
|
+
export interface LimitsRollup {
|
|
46
|
+
readonly declaredBy: number;
|
|
47
|
+
readonly toolTimeoutMs?: number;
|
|
48
|
+
readonly maxToolCallsPerTurn?: number;
|
|
49
|
+
readonly requireApprovalForDangerous?: boolean;
|
|
50
|
+
}
|
|
51
|
+
/** The least-privilege advisory policy artifact (only written with --emit-policy). */
|
|
52
|
+
export interface PolicyArtifact {
|
|
53
|
+
readonly $comment: string;
|
|
54
|
+
readonly version: 1;
|
|
55
|
+
readonly defaultDeny: true;
|
|
56
|
+
/** axis → true ONLY where an installed skill declared need; absent otherwise. */
|
|
57
|
+
readonly allow: Partial<Record<ReconcileAxis, true>>;
|
|
58
|
+
readonly limits?: LimitsRollup;
|
|
59
|
+
readonly derivedFrom: {
|
|
60
|
+
readonly grants: Record<ReconcileAxis, boolean>;
|
|
61
|
+
readonly declaredNeed: Record<ReconcileAxis, boolean>;
|
|
62
|
+
readonly skillsByAxis: Partial<Record<ReconcileAxis, readonly string[]>>;
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** Result of {@link reconcileCapabilities}. */
|
|
66
|
+
export interface ReconcileReport {
|
|
67
|
+
readonly skillsDir: string;
|
|
68
|
+
readonly installedCount: number;
|
|
69
|
+
readonly axes: readonly AxisState[];
|
|
70
|
+
readonly findings: readonly ReconcileFinding[];
|
|
71
|
+
readonly limits: LimitsRollup | null;
|
|
72
|
+
readonly policy: PolicyArtifact;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Statically reconcile a project's GRANT surface against the DECLARED needs of
|
|
76
|
+
* its installed skills. Pure: same inputs → same report. No execution, no writes.
|
|
77
|
+
*/
|
|
78
|
+
export declare function reconcileCapabilities(report: McpScanReport, skillsDir: string): ReconcileReport;
|
|
79
|
+
//# sourceMappingURL=reconcile.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reconcile.d.ts","sourceRoot":"","sources":["../src/reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,6FAA6F;AAC7F,eAAO,MAAM,gBAAgB,QAGkD,CAAC;AAEhF,gGAAgG;AAChG,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,SAAS,GAAG,YAAY,CAAC;AAG/D,qCAAqC;AACrC,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,sDAAsD;IACtD,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,+EAA+E;IAC/E,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,0DAA0D;IAC1D,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,8EAA8E;IAC9E,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,aAAa,GAAG,YAAY,CAAC;IAC5C,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;IAC7C,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;CACpC;AAED,mFAAmF;AACnF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,OAAO,CAAC;CAChD;AAED,sFAAsF;AACtF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;IACpB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;IAC3B,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC;IACrD,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE;QACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QAChD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACtD,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC;KAC1E,CAAC;CACH;AAED,+CAA+C;AAC/C,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,SAAS,SAAS,EAAE,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAC/C,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;CACjC;AAqCD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,eAAe,CA6E/F"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `dz mcp-scan --reconcile` — static capability reconciliation (Phase 3).
|
|
3
|
+
*
|
|
4
|
+
* Joins two things `dz` can read deterministically:
|
|
5
|
+
* - the project's GRANT surface (`scanMcp` of `.claude/settings*.json` + `.mcp.json`), and
|
|
6
|
+
* - the aggregate DECLARED capabilities of the installed skills (`.claude/skills/<id>/SKILL.md`),
|
|
7
|
+
* then REPORTS the gaps (under-grant / over-grant) and, optionally, EMITS a
|
|
8
|
+
* least-privilege advisory policy artifact for a host to consume.
|
|
9
|
+
*
|
|
10
|
+
* HONESTY: `dz` is build-time / static. It does NOT run agents and CANNOT block,
|
|
11
|
+
* time out, or rate-limit a tool call. The HOST (Claude Code enforcing
|
|
12
|
+
* settings.json allow/deny; or an MCP server consuming a policy.json) is the only
|
|
13
|
+
* thing that enforces at call time. Verbs here are REPORT / RECONCILE / EMIT /
|
|
14
|
+
* CANDIDATE — never BLOCK / DENIED / ENFORCED.
|
|
15
|
+
*
|
|
16
|
+
* @packageDocumentation
|
|
17
|
+
*/
|
|
18
|
+
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
|
19
|
+
import { join } from 'node:path';
|
|
20
|
+
import { parseDeclaredCapabilities, parseDeclaredLimits } from './capability-vocab.js';
|
|
21
|
+
/** The honesty banner — repeated in --help, the report header, and the artifact $comment. */
|
|
22
|
+
export const RECONCILE_BANNER = 'dz is build-time/static: it REPORTS the grant-vs-declaration gap and may EMIT an advisory policy, ' +
|
|
23
|
+
'but does NOT block, time out, or rate-limit anything. The HOST (Claude Code settings.json; or an MCP ' +
|
|
24
|
+
'server consuming policy.json) is the only thing that enforces at call time.';
|
|
25
|
+
const AXES = ['shell', 'network', 'file-write'];
|
|
26
|
+
function readInstalled(skillsDir) {
|
|
27
|
+
if (!existsSync(skillsDir))
|
|
28
|
+
return [];
|
|
29
|
+
let entries;
|
|
30
|
+
try {
|
|
31
|
+
entries = readdirSync(skillsDir, { withFileTypes: true });
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
const out = [];
|
|
37
|
+
for (const e of entries) {
|
|
38
|
+
if (!e.isDirectory())
|
|
39
|
+
continue;
|
|
40
|
+
const md = join(skillsDir, e.name, 'SKILL.md');
|
|
41
|
+
if (!existsSync(md))
|
|
42
|
+
continue;
|
|
43
|
+
let content;
|
|
44
|
+
try {
|
|
45
|
+
content = readFileSync(md, 'utf-8');
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
out.push({ id: e.name, caps: parseDeclaredCapabilities(content), limits: parseDeclaredLimits(content) });
|
|
51
|
+
}
|
|
52
|
+
return out.sort((a, b) => a.id.localeCompare(b.id));
|
|
53
|
+
}
|
|
54
|
+
function grantFor(report, axis) {
|
|
55
|
+
const c = report.capabilities;
|
|
56
|
+
return axis === 'shell' ? c.shell : axis === 'network' ? c.network : c.fileWrite;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Statically reconcile a project's GRANT surface against the DECLARED needs of
|
|
60
|
+
* its installed skills. Pure: same inputs → same report. No execution, no writes.
|
|
61
|
+
*/
|
|
62
|
+
export function reconcileCapabilities(report, skillsDir) {
|
|
63
|
+
const installed = readInstalled(skillsDir);
|
|
64
|
+
const axes = [];
|
|
65
|
+
const findings = [];
|
|
66
|
+
const allow = {};
|
|
67
|
+
const grants = {};
|
|
68
|
+
const declaredNeed = {};
|
|
69
|
+
const skillsByAxis = {};
|
|
70
|
+
for (const axis of AXES) {
|
|
71
|
+
const grant = grantFor(report, axis);
|
|
72
|
+
const needSkills = installed.filter((s) => s.caps[axis] === true).map((s) => s.id);
|
|
73
|
+
const need = needSkills.length > 0;
|
|
74
|
+
const silentCount = installed.filter((s) => s.caps[axis] === undefined).length;
|
|
75
|
+
axes.push({ axis, grant, need, needSkills, silentCount });
|
|
76
|
+
grants[axis] = grant;
|
|
77
|
+
declaredNeed[axis] = need;
|
|
78
|
+
if (need) {
|
|
79
|
+
skillsByAxis[axis] = needSkills;
|
|
80
|
+
allow[axis] = true;
|
|
81
|
+
}
|
|
82
|
+
if (need && !grant) {
|
|
83
|
+
findings.push({
|
|
84
|
+
id: `MS-UNDERGRANT-${axis.toUpperCase()}`,
|
|
85
|
+
kind: 'under-grant',
|
|
86
|
+
severity: 'medium',
|
|
87
|
+
axis,
|
|
88
|
+
detail: `${needSkills.length} installed skill(s) declare needing ${axis}, but the grant surface does not permit it — the host will starve them at runtime. Add the grant or remove the skill(s).`,
|
|
89
|
+
skills: needSkills,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
else if (grant && !need) {
|
|
93
|
+
// over-grant is ALWAYS advisory; downgrade to info when any skill is silent
|
|
94
|
+
// (it may genuinely need the grant but didn't declare), or when there are
|
|
95
|
+
// no installed skills at all.
|
|
96
|
+
const downgrade = silentCount > 0 || installed.length === 0;
|
|
97
|
+
findings.push({
|
|
98
|
+
id: `MS-OVERGRANT-${axis.toUpperCase()}`,
|
|
99
|
+
kind: 'over-grant',
|
|
100
|
+
severity: downgrade ? 'info' : 'low',
|
|
101
|
+
axis,
|
|
102
|
+
detail: `The project permits ${axis} but no installed skill declares needing it${silentCount > 0 ? ` (${silentCount} skill(s) silent on ${axis})` : ''} — least-privilege CANDIDATE to revoke; may be for the operator or a non-skill MCP server.`,
|
|
103
|
+
skills: [],
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// limits roll-up (tightest values) — INERT, never a gate
|
|
108
|
+
const withLimits = installed.filter((s) => Object.keys(s.limits).length > 0);
|
|
109
|
+
let limits = null;
|
|
110
|
+
if (withLimits.length > 0) {
|
|
111
|
+
const tts = withLimits.map((s) => s.limits.toolTimeoutMs).filter((n) => typeof n === 'number');
|
|
112
|
+
const mcs = withLimits.map((s) => s.limits.maxToolCallsPerTurn).filter((n) => typeof n === 'number');
|
|
113
|
+
const ras = withLimits.some((s) => s.limits.requireApprovalForDangerous === true);
|
|
114
|
+
limits = {
|
|
115
|
+
declaredBy: withLimits.length,
|
|
116
|
+
...(tts.length > 0 ? { toolTimeoutMs: Math.min(...tts) } : {}),
|
|
117
|
+
...(mcs.length > 0 ? { maxToolCallsPerTurn: Math.min(...mcs) } : {}),
|
|
118
|
+
...(ras ? { requireApprovalForDangerous: true } : {}),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
const policy = {
|
|
122
|
+
$comment: `ADVISORY. ${RECONCILE_BANNER} A host MUST consume this; dz does not.`,
|
|
123
|
+
version: 1,
|
|
124
|
+
defaultDeny: true,
|
|
125
|
+
allow,
|
|
126
|
+
...(limits ? { limits } : {}),
|
|
127
|
+
derivedFrom: { grants, declaredNeed, skillsByAxis },
|
|
128
|
+
};
|
|
129
|
+
// stable ordering: under-grant (medium) first, then over-grant
|
|
130
|
+
const order = { medium: 0, low: 1, info: 2 };
|
|
131
|
+
findings.sort((a, b) => order[a.severity] - order[b.severity] || a.axis.localeCompare(b.axis));
|
|
132
|
+
return { skillsDir, installedCount: installed.length, axes, findings, limits, policy };
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=reconcile.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reconcile.js","sourceRoot":"","sources":["../src/reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAuB,MAAM,uBAAuB,CAAC;AAG5G,6FAA6F;AAC7F,MAAM,CAAC,MAAM,gBAAgB,GAC3B,oGAAoG;IACpG,uGAAuG;IACvG,6EAA6E,CAAC;AAIhF,MAAM,IAAI,GAA6B,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;AAgE1E,SAAS,aAAa,CAAC,SAAiB;IACtC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,IAAI,OAAmC,CAAC;IACxC,IAAI,CAAC;QACH,OAAO,GAAG,WAAW,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,GAAG,GAAoB,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,SAAS;QAC/B,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAAE,SAAS;QAC9B,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,yBAAyB,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,mBAAmB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3G,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,QAAQ,CAAC,MAAqB,EAAE,IAAmB;IAC1D,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;IAC9B,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACnF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAqB,EAAE,SAAiB;IAC5E,MAAM,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAE3C,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAuB,EAAE,CAAC;IACxC,MAAM,KAAK,GAAyC,EAAE,CAAC;IACvD,MAAM,MAAM,GAAG,EAAoC,CAAC;IACpD,MAAM,YAAY,GAAG,EAAoC,CAAC;IAC1D,MAAM,YAAY,GAAsD,EAAE,CAAC;IAE3E,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;QACnC,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;QAC/E,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACrB,YAAY,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QAC1B,IAAI,IAAI,EAAE,CAAC;YACT,YAAY,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QACrB,CAAC;QAED,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC;gBACZ,EAAE,EAAE,iBAAiB,IAAI,CAAC,WAAW,EAAE,EAAE;gBACzC,IAAI,EAAE,aAAa;gBACnB,QAAQ,EAAE,QAAQ;gBAClB,IAAI;gBACJ,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,uCAAuC,IAAI,0HAA0H;gBACjM,MAAM,EAAE,UAAU;aACnB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1B,4EAA4E;YAC5E,0EAA0E;YAC1E,8BAA8B;YAC9B,MAAM,SAAS,GAAG,WAAW,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;YAC5D,QAAQ,CAAC,IAAI,CAAC;gBACZ,EAAE,EAAE,gBAAgB,IAAI,CAAC,WAAW,EAAE,EAAE;gBACxC,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;gBACpC,IAAI;gBACJ,MAAM,EAAE,uBAAuB,IAAI,8CAA8C,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,uBAAuB,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,4FAA4F;gBAClP,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7E,IAAI,MAAM,GAAwB,IAAI,CAAC;IACvC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;QAC5G,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;QAClH,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,2BAA2B,KAAK,IAAI,CAAC,CAAC;QAClF,MAAM,GAAG;YACP,UAAU,EAAE,UAAU,CAAC,MAAM;YAC7B,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,2BAA2B,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAmB;QAC7B,QAAQ,EAAE,aAAa,gBAAgB,yCAAyC;QAChF,OAAO,EAAE,CAAC;QACV,WAAW,EAAE,IAAI;QACjB,KAAK;QACL,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,WAAW,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE;KACpD,CAAC;IAEF,+DAA+D;IAC/D,MAAM,KAAK,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAW,CAAC;IACtD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAE/F,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACzF,CAAC"}
|
package/dist/targets.d.ts
CHANGED
|
@@ -8,15 +8,9 @@ import type { Adapter } from '@dzhechkov/core';
|
|
|
8
8
|
* The targets the harness can initialise. The key is the CLI `--target` name
|
|
9
9
|
* (`claude-code`, not `claude`); the value is the adapter that emits for it.
|
|
10
10
|
*/
|
|
11
|
-
export declare const TARGETS: {
|
|
12
|
-
readonly 'claude-code': Adapter;
|
|
13
|
-
readonly codex: Adapter;
|
|
14
|
-
readonly opencode: Adapter;
|
|
15
|
-
readonly hermes: Adapter;
|
|
16
|
-
readonly openclaude: Adapter;
|
|
17
|
-
};
|
|
18
11
|
/** A valid `--target` name. */
|
|
19
|
-
export type TargetName =
|
|
12
|
+
export type TargetName = 'claude-code' | 'codex' | 'opencode' | 'hermes' | 'openclaude' | 'copilot';
|
|
13
|
+
export declare const TARGETS: Record<TargetName, Adapter>;
|
|
20
14
|
/** Every supported `--target` name. */
|
|
21
15
|
export declare const TARGET_NAMES: TargetName[];
|
|
22
16
|
/** Type guard: is `value` a supported `--target` name? */
|
package/dist/targets.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"targets.d.ts","sourceRoot":"","sources":["../src/targets.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"targets.d.ts","sourceRoot":"","sources":["../src/targets.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C;;;GAGG;AACH,+BAA+B;AAC/B,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,UAAU,GAAG,QAAQ,GAAG,YAAY,GAAG,SAAS,CAAC;AAKpG,eAAO,MAAM,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE,OAAO,CAO/C,CAAC;AAEF,uCAAuC;AACvC,eAAO,MAAM,YAAY,EAA2B,UAAU,EAAE,CAAC;AAEjE,0DAA0D;AAC1D,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,UAAU,CAE/D"}
|
package/dist/targets.js
CHANGED
|
@@ -5,19 +5,20 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { claudeAdapter } from '@dzhechkov/adapter-claude';
|
|
7
7
|
import { codexAdapter } from '@dzhechkov/adapter-codex';
|
|
8
|
+
import { copilotAdapter } from '@dzhechkov/adapter-copilot';
|
|
8
9
|
import { hermesAdapter } from '@dzhechkov/adapter-hermes';
|
|
9
10
|
import { opencodeAdapter } from '@dzhechkov/adapter-opencode';
|
|
10
11
|
import { openclaudeAdapter } from '@dzhechkov/adapter-openclaude';
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
*/
|
|
12
|
+
// Explicitly annotated (not `as const satisfies`) so the type is portable across
|
|
13
|
+
// the workspace — adapters compiled against slightly different `@dzhechkov/core`
|
|
14
|
+
// versions would otherwise make the inferred type un-nameable (TS2742).
|
|
15
15
|
export const TARGETS = {
|
|
16
16
|
'claude-code': claudeAdapter,
|
|
17
17
|
codex: codexAdapter,
|
|
18
18
|
opencode: opencodeAdapter,
|
|
19
19
|
hermes: hermesAdapter,
|
|
20
20
|
openclaude: openclaudeAdapter,
|
|
21
|
+
copilot: copilotAdapter,
|
|
21
22
|
};
|
|
22
23
|
/** Every supported `--target` name. */
|
|
23
24
|
export const TARGET_NAMES = Object.keys(TARGETS);
|
package/dist/targets.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"targets.js","sourceRoot":"","sources":["../src/targets.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;
|
|
1
|
+
{"version":3,"file":"targets.js","sourceRoot":"","sources":["../src/targets.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAUlE,iFAAiF;AACjF,iFAAiF;AACjF,wEAAwE;AACxE,MAAM,CAAC,MAAM,OAAO,GAAgC;IAClD,aAAa,EAAE,aAAa;IAC5B,KAAK,EAAE,YAAY;IACnB,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,UAAU,EAAE,iBAAiB;IAC7B,OAAO,EAAE,cAAc;CACxB,CAAC;AAEF,uCAAuC;AACvC,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAiB,CAAC;AAEjE,0DAA0D;AAC1D,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,OAAO,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;AAC9D,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dzhechkov/harness-core",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.25",
|
|
4
4
|
"description": "Shared harness logic - skill loading, additive apply, and the init/sync/verify/doctor operations.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -29,8 +29,9 @@
|
|
|
29
29
|
"@dzhechkov/adapter-hermes": "^0.2.0",
|
|
30
30
|
"@dzhechkov/adapter-openclaude": "^0.1.0",
|
|
31
31
|
"@dzhechkov/adapter-opencode": "^0.2.0",
|
|
32
|
-
"
|
|
33
|
-
"
|
|
32
|
+
"yaml": "^2.0.0",
|
|
33
|
+
"@dzhechkov/adapter-copilot": "0.1.0",
|
|
34
|
+
"@dzhechkov/core": "0.2.3"
|
|
34
35
|
},
|
|
35
36
|
"devDependencies": {
|
|
36
37
|
"@types/node": "^25.6.0",
|
package/src/benchmark.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
import { existsSync, readFileSync, statSync } from 'node:fs';
|
|
12
12
|
import { join } from 'node:path';
|
|
13
13
|
|
|
14
|
+
import { detectScriptCapabilities, parseDeclaredCapabilities } from './capability-vocab.js';
|
|
14
15
|
import { estimateSkillCost, type CostEstimate } from './cost-scoring.js';
|
|
15
16
|
|
|
16
17
|
/** A single benchmark check result. */
|
|
@@ -19,6 +20,12 @@ export interface BenchmarkCheck {
|
|
|
19
20
|
readonly name: string;
|
|
20
21
|
readonly passed: boolean;
|
|
21
22
|
readonly detail?: string | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* When true this check is informational ONLY — it is excluded from the
|
|
25
|
+
* pass-rate / grade math (so it can ship without regressing existing grades).
|
|
26
|
+
* Surfaced as a warning in output. Used by S16 (capability-declaration nudge).
|
|
27
|
+
*/
|
|
28
|
+
readonly advisory?: boolean;
|
|
22
29
|
}
|
|
23
30
|
|
|
24
31
|
/** Score for a single skill benchmark. */
|
|
@@ -160,11 +167,52 @@ export function benchmarkSkill(skillDir: string, skillId: string): BenchmarkScor
|
|
|
160
167
|
|| /\*\*\s*PROHIBITED/i.test(content);
|
|
161
168
|
checks.push({ id: 'S14', name: 'anti-patterns or self-check', passed: hasAntiPatterns });
|
|
162
169
|
|
|
163
|
-
|
|
164
|
-
|
|
170
|
+
// S15: capability declaration matches usage (advisory, contradiction-only).
|
|
171
|
+
// Phase 1 auto-detects network + shell from scripts/ only (never SKILL.md prose).
|
|
172
|
+
// PASS unless a declared `capabilities.<cap>: false` is contradicted by detected
|
|
173
|
+
// usage. Absent block, or under-declaration (uses it, declares nothing), PASS —
|
|
174
|
+
// so the ~all skills that declare nothing today stay green.
|
|
175
|
+
const declaredCaps = parseDeclaredCapabilities(content);
|
|
176
|
+
const detectedCaps = detectScriptCapabilities(skillDir);
|
|
177
|
+
const contradictions: string[] = [];
|
|
178
|
+
if (detectedCaps.network && declaredCaps.network === false) contradictions.push('network');
|
|
179
|
+
if (detectedCaps.shell && declaredCaps.shell === false) contradictions.push('shell');
|
|
180
|
+
checks.push({
|
|
181
|
+
id: 'S15',
|
|
182
|
+
name: 'capability declaration matches usage',
|
|
183
|
+
passed: contradictions.length === 0,
|
|
184
|
+
detail:
|
|
185
|
+
contradictions.length > 0
|
|
186
|
+
? `scripts use ${contradictions.join(' + ')} but capabilities declare it false`
|
|
187
|
+
: undefined,
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
// S16: side-effecting skills should declare their capabilities (ADVISORY — not
|
|
191
|
+
// graded). Warns when scripts use network/shell but the capability is not
|
|
192
|
+
// declared at all (neither true nor false). The adoption nudge for the manifest;
|
|
193
|
+
// excluded from the grade so it never regresses an existing skill.
|
|
194
|
+
const undeclaredUsed: string[] = [];
|
|
195
|
+
if (detectedCaps.network && declaredCaps.network === undefined) undeclaredUsed.push('network');
|
|
196
|
+
if (detectedCaps.shell && declaredCaps.shell === undefined) undeclaredUsed.push('shell');
|
|
197
|
+
checks.push({
|
|
198
|
+
id: 'S16',
|
|
199
|
+
name: 'side-effects declared (advisory)',
|
|
200
|
+
passed: undeclaredUsed.length === 0,
|
|
201
|
+
advisory: true,
|
|
202
|
+
detail:
|
|
203
|
+
undeclaredUsed.length > 0
|
|
204
|
+
? `scripts use ${undeclaredUsed.join(' + ')} — add a capabilities: block declaring it`
|
|
205
|
+
: undefined,
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
// grade math is over GRADED checks only — advisory checks (S16) are excluded
|
|
209
|
+
// so they ship without changing any existing grade.
|
|
210
|
+
const graded = checks.filter((c) => !c.advisory);
|
|
211
|
+
const passed = graded.filter((c) => c.passed).length;
|
|
212
|
+
const passRate = graded.length > 0 ? Math.round((passed / graded.length) * 100) : 0;
|
|
165
213
|
|
|
166
214
|
return {
|
|
167
|
-
skillId, skillDir, checks, passed, total:
|
|
215
|
+
skillId, skillDir, checks, passed, total: graded.length,
|
|
168
216
|
passRate, grade: gradeFromRate(passRate), cost: estimateSkillCost(content),
|
|
169
217
|
};
|
|
170
218
|
}
|
|
@@ -190,8 +238,12 @@ export function compareSkills(
|
|
|
190
238
|
const skillA = benchmarkSkill(skillADir, skillAId);
|
|
191
239
|
const skillB = benchmarkSkill(skillBDir, skillBId);
|
|
192
240
|
|
|
241
|
+
// deltaChecks is GRADED-ONLY — advisory checks (e.g. S16) never touch the grade,
|
|
242
|
+
// so surfacing them in the A/B diff would misrepresent an adoption nudge as a
|
|
243
|
+
// quality difference. Mirrors the graded filter in the grade math above.
|
|
193
244
|
const deltaChecks: { id: string; aPass: boolean; bPass: boolean }[] = [];
|
|
194
245
|
for (const checkA of skillA.checks) {
|
|
246
|
+
if (checkA.advisory) continue;
|
|
195
247
|
const checkB = skillB.checks.find((c) => c.id === checkA.id);
|
|
196
248
|
if (checkB && checkA.passed !== checkB.passed) {
|
|
197
249
|
deltaChecks.push({ id: checkA.id, aPass: checkA.passed, bPass: checkB.passed });
|