@lifeaitools/rdc-skills 0.24.27 → 0.24.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/git-sha.json +1 -1
- package/package.json +2 -2
- package/scripts/lib/manifest-schema.mjs +140 -6
- package/skills/tests/MATRIX.md +7 -7
- package/skills/tests/rdc-deploy.test.json +15 -0
- package/skills/tests/rdc-help.test.json +14 -0
- package/skills/tests/rdc-release.test.json +14 -0
- package/skills/tests/rdc-self-test.test.json +13 -0
- package/skills/tests/rdc-status.test.json +14 -0
- package/skills/tests/rdc-terminal-config.test.json +14 -0
- package/tests/curl-surface.test.mjs +20 -0
- package/tests/mcp.test.mjs +4 -0
- package/tests/skill-test-matrix.test.mjs +59 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rdc",
|
|
3
|
-
"version": "0.24.
|
|
3
|
+
"version": "0.24.29",
|
|
4
4
|
"description": "RDC typed-agent dispatch skill suite for Claude Code — plan, build, review, overnight unattended builds with work-item tracking and TDD enforcement.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "LIFEAI",
|
package/git-sha.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lifeaitools/rdc-skills",
|
|
3
|
-
"version": "0.24.
|
|
3
|
+
"version": "0.24.29",
|
|
4
4
|
"description": "RDC typed-agent dispatch skill suite for Claude Code - plan, build, review, overnight builds",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"rdc-design": "node scripts/rdc-design-cli.mjs",
|
|
36
36
|
"test:hooks": "node scripts/test-rdc-hooks.mjs",
|
|
37
37
|
"test:truth-gate": "node tests/run-evidence-gate.test.mjs && node tests/work-item-exit-gate-l2.test.mjs && node tests/work-item-exit-gate-l3.test.mjs && node tests/require-work-item-on-commit.test.mjs && node tests/harness-gates.test.mjs",
|
|
38
|
-
"test:acceptance": "node tests/acceptance.test.mjs && node tests/install-rdc-skills.test.mjs && node tests/help-surface.test.mjs && node tests/manifest-contract-fields.test.mjs && node tests/curl-surface.test.mjs",
|
|
38
|
+
"test:acceptance": "node tests/acceptance.test.mjs && node tests/install-rdc-skills.test.mjs && node tests/help-surface.test.mjs && node tests/manifest-contract-fields.test.mjs && node tests/skill-test-matrix.test.mjs && node tests/curl-surface.test.mjs",
|
|
39
39
|
"acceptance": "node scripts/acceptance.mjs --changed",
|
|
40
40
|
"test:mcp": "node tests/mcp.test.mjs",
|
|
41
41
|
"test:mcp:remote": "node tests/mcp.test.mjs --remote",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
//
|
|
3
3
|
// Source of truth: .rdc/plans/skill-self-test-tier-2.md decision D3.
|
|
4
4
|
//
|
|
5
|
-
// A manifest describes one behavioral test for an
|
|
5
|
+
// A manifest describes one behavioral test for an installed skill. The Tier 2
|
|
6
6
|
// runner loads a manifest, validates it here, then uses it to drive a headless
|
|
7
7
|
// Claude invocation inside a sandbox and assert on real artifacts.
|
|
8
8
|
//
|
|
@@ -55,8 +55,16 @@ const ASSERTION_FIELDS = new Set([
|
|
|
55
55
|
const WIC_FIELDS = new Set(["min", "max", "status", "labels_include"]);
|
|
56
56
|
const COMMITS_FIELDS = new Set(["min", "message_matches"]);
|
|
57
57
|
const TEARDOWN_FIELDS = new Set(["reset_branch"]);
|
|
58
|
+
const ACCEPTANCE_FIELDS = new Set([
|
|
59
|
+
"output_contains",
|
|
60
|
+
"output_not_contains",
|
|
61
|
+
"tool_calls_include_any",
|
|
62
|
+
"tool_calls_include_all",
|
|
63
|
+
"tool_calls_argument_matches",
|
|
64
|
+
]);
|
|
65
|
+
const TOOL_CALL_MATCHER_FIELDS = new Set(["tools", "pattern"]);
|
|
58
66
|
|
|
59
|
-
const SKILL_RE = /^rdc:[a-z][a-z0-9-]*$/;
|
|
67
|
+
const SKILL_RE = /^(rdc:)?[a-z][a-z0-9-]*$/;
|
|
60
68
|
|
|
61
69
|
function isPlainObject(v) {
|
|
62
70
|
return v !== null && typeof v === "object" && !Array.isArray(v);
|
|
@@ -92,12 +100,12 @@ function validateFixture(fx, errors, warnings) {
|
|
|
92
100
|
// prompt
|
|
93
101
|
if (typeof fx.prompt !== "string" || fx.prompt.trim().length === 0) {
|
|
94
102
|
err(errors, "fixture.prompt", "missing", "fixture.prompt must be a non-empty string");
|
|
95
|
-
} else if (!fx.prompt.
|
|
103
|
+
} else if (!SKILL_RE.test(fx.prompt.split(/\s+/, 1)[0] || "")) {
|
|
96
104
|
err(
|
|
97
105
|
errors,
|
|
98
106
|
"fixture.prompt",
|
|
99
|
-
"prompt-not-
|
|
100
|
-
`fixture.prompt must start with
|
|
107
|
+
"prompt-not-skill",
|
|
108
|
+
`fixture.prompt must start with a valid skill name (got "${fx.prompt.slice(0, 20)}...")`,
|
|
101
109
|
);
|
|
102
110
|
}
|
|
103
111
|
|
|
@@ -372,6 +380,78 @@ function validateTeardown(t, errors, warnings) {
|
|
|
372
380
|
}
|
|
373
381
|
}
|
|
374
382
|
|
|
383
|
+
function validateStringArray(value, path, errors) {
|
|
384
|
+
if (!Array.isArray(value)) {
|
|
385
|
+
err(errors, path, "type", `${path} must be an array`);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
value.forEach((s, i) => {
|
|
389
|
+
if (typeof s !== "string" || s.length === 0) {
|
|
390
|
+
err(errors, `${path}[${i}]`, "type", "entry must be a non-empty string");
|
|
391
|
+
}
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function validateAcceptance(a, errors, warnings) {
|
|
396
|
+
if (!isPlainObject(a)) {
|
|
397
|
+
err(errors, "acceptance", "type", "acceptance must be an object");
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
if (a.output_contains !== undefined) {
|
|
402
|
+
validateStringArray(a.output_contains, "acceptance.output_contains", errors);
|
|
403
|
+
}
|
|
404
|
+
if (a.output_not_contains !== undefined) {
|
|
405
|
+
validateStringArray(a.output_not_contains, "acceptance.output_not_contains", errors);
|
|
406
|
+
}
|
|
407
|
+
if (a.tool_calls_include_any !== undefined) {
|
|
408
|
+
validateStringArray(a.tool_calls_include_any, "acceptance.tool_calls_include_any", errors);
|
|
409
|
+
}
|
|
410
|
+
if (a.tool_calls_include_all !== undefined) {
|
|
411
|
+
validateStringArray(a.tool_calls_include_all, "acceptance.tool_calls_include_all", errors);
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
if (a.tool_calls_argument_matches !== undefined) {
|
|
415
|
+
if (!Array.isArray(a.tool_calls_argument_matches)) {
|
|
416
|
+
err(
|
|
417
|
+
errors,
|
|
418
|
+
"acceptance.tool_calls_argument_matches",
|
|
419
|
+
"type",
|
|
420
|
+
"tool_calls_argument_matches must be an array",
|
|
421
|
+
);
|
|
422
|
+
} else {
|
|
423
|
+
a.tool_calls_argument_matches.forEach((matcher, i) => {
|
|
424
|
+
const p = `acceptance.tool_calls_argument_matches[${i}]`;
|
|
425
|
+
if (!isPlainObject(matcher)) {
|
|
426
|
+
err(errors, p, "type", "matcher must be an object");
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
validateStringArray(matcher.tools, `${p}.tools`, errors);
|
|
430
|
+
if (typeof matcher.pattern !== "string" || matcher.pattern.length === 0) {
|
|
431
|
+
err(errors, `${p}.pattern`, "type", "pattern must be a non-empty regex string");
|
|
432
|
+
} else {
|
|
433
|
+
try {
|
|
434
|
+
new RegExp(matcher.pattern);
|
|
435
|
+
} catch (e) {
|
|
436
|
+
err(errors, `${p}.pattern`, "invalid-regex", `invalid regex: ${e.message}`);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
for (const k of Object.keys(matcher)) {
|
|
440
|
+
if (!TOOL_CALL_MATCHER_FIELDS.has(k)) {
|
|
441
|
+
warn(warnings, `${p}.${k}`, "unknown-field", `unknown matcher field "${k}"`);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
for (const k of Object.keys(a)) {
|
|
449
|
+
if (!ACCEPTANCE_FIELDS.has(k)) {
|
|
450
|
+
warn(warnings, `acceptance.${k}`, "unknown-field", `unknown acceptance field "${k}"`);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
375
455
|
export function validateManifest(raw) {
|
|
376
456
|
const errors = [];
|
|
377
457
|
const warnings = [];
|
|
@@ -405,7 +485,7 @@ export function validateManifest(raw) {
|
|
|
405
485
|
errors,
|
|
406
486
|
"skill",
|
|
407
487
|
"skill-invalid",
|
|
408
|
-
`skill must
|
|
488
|
+
`skill must be a slash skill like "rdc:build" or bare skill like "lifeai-brochure-author" (got ${JSON.stringify(raw.skill)})`,
|
|
409
489
|
);
|
|
410
490
|
}
|
|
411
491
|
|
|
@@ -430,6 +510,10 @@ export function validateManifest(raw) {
|
|
|
430
510
|
validateAssertions(raw.assertions, errors, warnings);
|
|
431
511
|
}
|
|
432
512
|
|
|
513
|
+
if (raw.acceptance !== undefined) {
|
|
514
|
+
validateAcceptance(raw.acceptance, errors, warnings);
|
|
515
|
+
}
|
|
516
|
+
|
|
433
517
|
// teardown (optional)
|
|
434
518
|
if (raw.teardown !== undefined) {
|
|
435
519
|
validateTeardown(raw.teardown, errors, warnings);
|
|
@@ -544,6 +628,17 @@ if (__isMain) {
|
|
|
544
628
|
expect: (r) => !r.ok && r.errors.some((e) => e.code === "rdc-test-not-set"),
|
|
545
629
|
expectStr: "error code=rdc-test-not-set",
|
|
546
630
|
},
|
|
631
|
+
{
|
|
632
|
+
n: 4.5,
|
|
633
|
+
desc: "valid bare installed skill",
|
|
634
|
+
input: {
|
|
635
|
+
...validMinimal,
|
|
636
|
+
skill: "lifeai-brochure-author",
|
|
637
|
+
fixture: { ...validMinimal.fixture, prompt: "lifeai-brochure-author review fixture" },
|
|
638
|
+
},
|
|
639
|
+
expect: (r) => r.ok && r.errors.length === 0,
|
|
640
|
+
expectStr: "ok=true, 0 errors",
|
|
641
|
+
},
|
|
547
642
|
{
|
|
548
643
|
n: 5,
|
|
549
644
|
desc: "invalid regex in commits_made.message_matches",
|
|
@@ -591,6 +686,45 @@ if (__isMain) {
|
|
|
591
686
|
expect: (r) => !r.ok && r.errors.some((e) => e.code === "path-not-relative"),
|
|
592
687
|
expectStr: "error code=path-not-relative",
|
|
593
688
|
},
|
|
689
|
+
{
|
|
690
|
+
n: 9,
|
|
691
|
+
desc: "valid acceptance tool matcher",
|
|
692
|
+
input: {
|
|
693
|
+
...validMinimal,
|
|
694
|
+
acceptance: {
|
|
695
|
+
output_contains: ["PASS"],
|
|
696
|
+
output_not_contains: ["fabricated"],
|
|
697
|
+
tool_calls_include_any: ["Grep", "Glob"],
|
|
698
|
+
tool_calls_argument_matches: [{ tools: ["Grep"], pattern: "global-corpus|web-search" }],
|
|
699
|
+
},
|
|
700
|
+
},
|
|
701
|
+
expect: (r) => r.ok && r.errors.length === 0,
|
|
702
|
+
expectStr: "ok=true, 0 errors",
|
|
703
|
+
},
|
|
704
|
+
{
|
|
705
|
+
n: 10,
|
|
706
|
+
desc: "invalid acceptance matcher regex",
|
|
707
|
+
input: {
|
|
708
|
+
...validMinimal,
|
|
709
|
+
acceptance: {
|
|
710
|
+
tool_calls_argument_matches: [{ tools: ["Grep"], pattern: "[unclosed" }],
|
|
711
|
+
},
|
|
712
|
+
},
|
|
713
|
+
expect: (r) => !r.ok && r.errors.some((e) => e.path.endsWith(".pattern") && e.code === "invalid-regex"),
|
|
714
|
+
expectStr: "error code=invalid-regex",
|
|
715
|
+
},
|
|
716
|
+
{
|
|
717
|
+
n: 11,
|
|
718
|
+
desc: "unknown acceptance field",
|
|
719
|
+
input: {
|
|
720
|
+
...validMinimal,
|
|
721
|
+
acceptance: { made_up: true },
|
|
722
|
+
},
|
|
723
|
+
expect: (r) =>
|
|
724
|
+
r.ok &&
|
|
725
|
+
r.warnings.some((w) => w.path === "acceptance.made_up" && w.code === "unknown-field"),
|
|
726
|
+
expectStr: "ok=true, warning code=unknown-field",
|
|
727
|
+
},
|
|
594
728
|
];
|
|
595
729
|
|
|
596
730
|
const pad = (s, n) => String(s) + " ".repeat(Math.max(0, n - String(s).length));
|
package/skills/tests/MATRIX.md
CHANGED
|
@@ -4,7 +4,7 @@ Current coverage: 29 manifests for 29 skill directories.
|
|
|
4
4
|
|
|
5
5
|
The manifest layer verifies each skill can be started from a realistic caller prompt in an isolated `RDC_TEST=1` sandbox. The acceptance harness records the engine stream, extracted tool calls, stdout/stderr artifacts, rendered assistant output, failures, lessons learned, and next build optimizations under `.rdc/reports/`.
|
|
6
6
|
|
|
7
|
-
`rdc:channel-formatter` currently has the strongest content acceptance fixture: it asserts source-grounded social-pack output, forbidden-claim absence, and observable corpus or web-search tool routing. `rdc:
|
|
7
|
+
`rdc:channel-formatter` currently has the strongest content acceptance fixture: it asserts source-grounded social-pack output, forbidden-claim absence, and observable corpus or web-search tool routing. `rdc:help` asserts the public MCP/curl caller surface and structured `format:"json"` discovery path. `rdc:self-test` validates the test-tier/evidence language. `rdc:release`, `rdc:deploy`, `rdc:terminal-config`, `rdc:status`, and `rdc:watch` add safety negative checks for publish/deploy/window/read-only behavior. The remaining manifests are basic behavioral smoke tests and should gain deeper acceptance assertions as each skill is touched.
|
|
8
8
|
|
|
9
9
|
| Skill | Manifest | Fixture prompt class | Assertions | Acceptance depth |
|
|
10
10
|
|---|---|---|---|---|
|
|
@@ -14,12 +14,12 @@ The manifest layer verifies each skill can be started from a realistic caller pr
|
|
|
14
14
|
| `rdc:co-develop` | `rdc-co-develop.test.json` | Coordination status | `commits_made`, `exit_code`, `stdout_contains` | Basic manifest |
|
|
15
15
|
| `rdc:collab` | `rdc-collab.test.json` | Claude session relay fixture | `commits_made`, `exit_code`, `stdout_contains` | Basic manifest |
|
|
16
16
|
| `rdc:convert` | `rdc-convert.test.json` | Markdown-to-Word conversion fixture | `exit_code`, `stdout_contains` | Basic manifest |
|
|
17
|
-
| `rdc:deploy` | `rdc-deploy.test.json` | Deployment diagnosis | `commits_made`, `exit_code`, `stdout_contains` |
|
|
17
|
+
| `rdc:deploy` | `rdc-deploy.test.json` | Deployment diagnosis | `commits_made`, `exit_code`, `stdout_contains` | Read-only diagnose output and destructive deploy/DNS negative checks |
|
|
18
18
|
| `rdc:design` | `rdc-design.test.json` | Studio palette audit | `exit_code`, `stdout_contains` | Basic manifest |
|
|
19
19
|
| `rdc:fixit` | `rdc-fixit.test.json` | Tiny sandbox typo fix | `commits_made`, `exit_code`, `stdout_contains` | Basic manifest |
|
|
20
20
|
| `rdc:fs-mcp` | `rdc-fs-mcp.test.json` | File-system bridge read fixture | `commits_made`, `exit_code`, `stdout_contains` | Basic manifest |
|
|
21
21
|
| `rdc:handoff` | `rdc-handoff.test.json` | Stub work handoff | `exit_code`, `stdout_contains` | Basic manifest |
|
|
22
|
-
| `rdc:help` | `rdc-help.test.json` | Help menu rendering | `commits_made`, `exit_code`, `stdout_contains` |
|
|
22
|
+
| `rdc:help` | `rdc-help.test.json` | Help menu rendering | `commits_made`, `exit_code`, `stdout_contains` | MCP/curl output, structured JSON fetch, dev-endpoint negative checks |
|
|
23
23
|
| `rdc:housekeeping` | `rdc-housekeeping.test.json` | Read-only housekeeping audit | `commits_made`, `exit_code`, `stdout_contains` | Basic manifest |
|
|
24
24
|
| `lifeai-brochure-author` | `rdc-lifeai-brochure-author.test.json` | JSX compliance review fixture | `exit_code`, `stdout_contains` | Basic manifest |
|
|
25
25
|
| `rdc:overnight` | `rdc-overnight.test.json` | Label-based overnight queue drain | `exit_code`, `stdout_contains` | Basic manifest |
|
|
@@ -28,13 +28,13 @@ The manifest layer verifies each skill can be started from a realistic caller pr
|
|
|
28
28
|
| `rdc:prototype` | `rdc-prototype.test.json` | Tiny component prototype prompt | `exit_code`, `stdout_contains` | Basic manifest |
|
|
29
29
|
| `rdc:brochurify` | `rdc-rdc-brochurify.test.json` | Read-only markdown Brochurify fixture | `exit_code`, `stdout_contains` | Sandbox output and no-follow-up negative checks |
|
|
30
30
|
| `rdc:extract-verifier-rules` | `rdc-rdc-extract-verifier-rules.test.json` | Enhancement-log verifier fixture | `exit_code`, `stdout_contains` | Basic manifest |
|
|
31
|
-
| `rdc:release` | `rdc-release.test.json` | Dry-run package release | `commits_made`, `exit_code`, `stdout_contains` |
|
|
31
|
+
| `rdc:release` | `rdc-release.test.json` | Dry-run package release | `commits_made`, `exit_code`, `stdout_contains` | Dry-run release checklist and force/bypass negative checks |
|
|
32
32
|
| `rdc:report` | `rdc-report.test.json` | Unattended report generation | `exit_code`, `stdout_contains` | Basic manifest |
|
|
33
33
|
| `rdc:review` | `rdc-review.test.json` | Unattended review gate | `exit_code`, `stdout_contains` | Basic manifest |
|
|
34
34
|
| `rdc:rpms-filemap` | `rdc-rpms-filemap.test.json` | Canonical location lookup | `commits_made`, `exit_code`, `stdout_contains` | Basic manifest |
|
|
35
|
-
| `rdc:self-test` | `rdc-self-test.test.json` | Strict self-test prompt | `commits_made`, `exit_code`, `stdout_contains` |
|
|
36
|
-
| `rdc:status` | `rdc-status.test.json` | Read-only status snapshot | `commits_made`, `exit_code`, `stdout_contains` |
|
|
37
|
-
| `rdc:terminal-config` | `rdc-terminal-config.test.json` | Hidden-window launch policy audit | `commits_made`, `exit_code`, `stdout_contains` |
|
|
35
|
+
| `rdc:self-test` | `rdc-self-test.test.json` | Strict self-test prompt | `commits_made`, `exit_code`, `stdout_contains` | Tier/evidence output checks and no-relaxed-linter negative check |
|
|
36
|
+
| `rdc:status` | `rdc-status.test.json` | Read-only status snapshot | `commits_made`, `exit_code`, `stdout_contains` | Read-only status output and raw-MCP/write negative checks |
|
|
37
|
+
| `rdc:terminal-config` | `rdc-terminal-config.test.json` | Hidden-window launch policy audit | `commits_made`, `exit_code`, `stdout_contains` | Hidden-window output and focus/collapse API negative checks |
|
|
38
38
|
| `rdc:watch` | `rdc-watch.test.json` | Watcher initialization prompt | `exit_code`, `stdout_contains` | Test-mode no-open output, tool-call routing, and focus-event negative checks |
|
|
39
39
|
| `rdc:workitems` | `rdc-workitems.test.json` | Work-item epic listing | `commits_made`, `exit_code`, `stdout_contains` | Basic manifest |
|
|
40
40
|
|
|
@@ -11,5 +11,20 @@
|
|
|
11
11
|
"commits_made": { "min": 0 },
|
|
12
12
|
"stdout_contains": ["Verdict"]
|
|
13
13
|
},
|
|
14
|
+
"acceptance": {
|
|
15
|
+
"output_contains": [
|
|
16
|
+
"diagnose",
|
|
17
|
+
"read-only",
|
|
18
|
+
"Registry",
|
|
19
|
+
"health",
|
|
20
|
+
"Verdict"
|
|
21
|
+
],
|
|
22
|
+
"output_not_contains": [
|
|
23
|
+
"Deploy triggered",
|
|
24
|
+
"Cloudflare cache purged",
|
|
25
|
+
"PR/admin-merge to main",
|
|
26
|
+
"DNS write"
|
|
27
|
+
]
|
|
28
|
+
},
|
|
14
29
|
"teardown": { "reset_branch": true }
|
|
15
30
|
}
|
|
@@ -11,5 +11,19 @@
|
|
|
11
11
|
"commits_made": { "min": 0 },
|
|
12
12
|
"stdout_contains": ["rdc:help", "Decision tree"]
|
|
13
13
|
},
|
|
14
|
+
"acceptance": {
|
|
15
|
+
"output_contains": [
|
|
16
|
+
"Direct MCP / curl Access",
|
|
17
|
+
"https://rdc-skills.regendevcorp.com/mcp",
|
|
18
|
+
"rdc_skill_list",
|
|
19
|
+
"rdc_skill_search",
|
|
20
|
+
"rdc_skill_get",
|
|
21
|
+
"\"format\":\"json\""
|
|
22
|
+
],
|
|
23
|
+
"output_not_contains": [
|
|
24
|
+
"https://rdc-skills.dev.regendevcorp.com/mcp",
|
|
25
|
+
"All user-invocable skills become available as slash commands"
|
|
26
|
+
]
|
|
27
|
+
},
|
|
14
28
|
"teardown": { "reset_branch": true }
|
|
15
29
|
}
|
|
@@ -11,5 +11,19 @@
|
|
|
11
11
|
"commits_made": { "min": 0 },
|
|
12
12
|
"stdout_contains": ["Verdict"]
|
|
13
13
|
},
|
|
14
|
+
"acceptance": {
|
|
15
|
+
"output_contains": [
|
|
16
|
+
"dry-run",
|
|
17
|
+
"Tests/self-test",
|
|
18
|
+
"CI/publish",
|
|
19
|
+
"Installed/runtime version",
|
|
20
|
+
"Smoke test"
|
|
21
|
+
],
|
|
22
|
+
"output_not_contains": [
|
|
23
|
+
"git push --force",
|
|
24
|
+
"bypass hooks",
|
|
25
|
+
"raw git/npm/CI dumps"
|
|
26
|
+
]
|
|
27
|
+
},
|
|
14
28
|
"teardown": { "reset_branch": true }
|
|
15
29
|
}
|
|
@@ -11,5 +11,18 @@
|
|
|
11
11
|
"commits_made": { "min": 0 },
|
|
12
12
|
"stdout_contains": ["PASS", "strict"]
|
|
13
13
|
},
|
|
14
|
+
"acceptance": {
|
|
15
|
+
"output_contains": [
|
|
16
|
+
"Tier 1",
|
|
17
|
+
"Tier 2",
|
|
18
|
+
"29 manifests",
|
|
19
|
+
"JSONL",
|
|
20
|
+
"tool calls"
|
|
21
|
+
],
|
|
22
|
+
"output_not_contains": [
|
|
23
|
+
"skip findings by relaxing the linter",
|
|
24
|
+
"raw runner dumps"
|
|
25
|
+
]
|
|
26
|
+
},
|
|
14
27
|
"teardown": { "reset_branch": true }
|
|
15
28
|
}
|
|
@@ -11,5 +11,19 @@
|
|
|
11
11
|
"commits_made": { "min": 0 },
|
|
12
12
|
"stdout_contains": ["Recommended Next"]
|
|
13
13
|
},
|
|
14
|
+
"acceptance": {
|
|
15
|
+
"output_contains": [
|
|
16
|
+
"Recommended Next",
|
|
17
|
+
"read-only",
|
|
18
|
+
"work items",
|
|
19
|
+
"health"
|
|
20
|
+
],
|
|
21
|
+
"output_not_contains": [
|
|
22
|
+
"raw MCP",
|
|
23
|
+
"UUID",
|
|
24
|
+
"UPDATE work_items",
|
|
25
|
+
"INSERT"
|
|
26
|
+
]
|
|
27
|
+
},
|
|
14
28
|
"teardown": { "reset_branch": true }
|
|
15
29
|
}
|
|
@@ -11,5 +11,19 @@
|
|
|
11
11
|
"commits_made": { "min": 0 },
|
|
12
12
|
"stdout_contains": ["hidden-window", "No settings written"]
|
|
13
13
|
},
|
|
14
|
+
"acceptance": {
|
|
15
|
+
"output_contains": [
|
|
16
|
+
"hidden-window",
|
|
17
|
+
"No settings written",
|
|
18
|
+
"read before edit",
|
|
19
|
+
"JSON/TOML syntax"
|
|
20
|
+
],
|
|
21
|
+
"output_not_contains": [
|
|
22
|
+
"SetForegroundWindow",
|
|
23
|
+
"ShowWindow",
|
|
24
|
+
"collapse all windows",
|
|
25
|
+
"restore all windows"
|
|
26
|
+
]
|
|
27
|
+
},
|
|
14
28
|
"teardown": { "reset_branch": true }
|
|
15
29
|
}
|
|
@@ -226,6 +226,26 @@ async function main() {
|
|
|
226
226
|
check('rdc_skill_get format=json returns skill metadata', getJsonBody.skill?.slash === 'rdc:build' && getJsonBody.skill?.codeflow_required === true);
|
|
227
227
|
check('rdc_skill_get format=json returns rendered body', /rdc:build/i.test(getJsonBody.body || ''));
|
|
228
228
|
|
|
229
|
+
const getJsonUnknown = postMcp({
|
|
230
|
+
jsonrpc: '2.0',
|
|
231
|
+
id: 6,
|
|
232
|
+
method: 'tools/call',
|
|
233
|
+
params: {
|
|
234
|
+
name: 'rdc_skill_get',
|
|
235
|
+
arguments: { name: 'rdc:not-real', format: 'json' },
|
|
236
|
+
},
|
|
237
|
+
}, 'rdc_skill_get_json_unknown');
|
|
238
|
+
check('rdc_skill_get unknown format=json curl exits 0', getJsonUnknown.status === 0, getJsonUnknown.stderr);
|
|
239
|
+
let getJsonUnknownBody = {};
|
|
240
|
+
const getJsonUnknownText = resultText(latestEnvelope(getJsonUnknown.stdout));
|
|
241
|
+
try {
|
|
242
|
+
getJsonUnknownBody = JSON.parse(getJsonUnknownText);
|
|
243
|
+
} catch {
|
|
244
|
+
check('rdc_skill_get unknown format=json returns parseable JSON', false, getJsonUnknownText || getJsonUnknown.stdout.slice(0, 500));
|
|
245
|
+
}
|
|
246
|
+
check('rdc_skill_get unknown format=json returns machine error', getJsonUnknownBody.error === 'unknown_skill');
|
|
247
|
+
check('rdc_skill_get unknown format=json returns alternatives', getJsonUnknownBody.valid_count >= 29 && getJsonUnknownBody.valid?.some((s) => s.slash === 'rdc:help'));
|
|
248
|
+
|
|
229
249
|
const getMcp = curlWithStatus([
|
|
230
250
|
'-s',
|
|
231
251
|
`${TARGET}/mcp`,
|
package/tests/mcp.test.mjs
CHANGED
|
@@ -195,6 +195,10 @@ async function sweep(url, label, { compareSource }) {
|
|
|
195
195
|
// error path
|
|
196
196
|
const unk = await mcp(url, { jsonrpc: '2.0', id: 20, method: 'tools/call', params: { name: 'rdc_skill_get', arguments: { name: '___nope___' } } });
|
|
197
197
|
check(`${label}: unknown skill returns helpful error`, /unknown skill/i.test(callText(unk.json)));
|
|
198
|
+
const unkJson = await mcp(url, { jsonrpc: '2.0', id: 24, method: 'tools/call', params: { name: 'rdc_skill_get', arguments: { name: '___nope___', format: 'json' } } });
|
|
199
|
+
let unkJsonBody; try { unkJsonBody = JSON.parse(callText(unkJson.json)); } catch { unkJsonBody = {}; }
|
|
200
|
+
check(`${label}: unknown skill format=json returns machine-readable error`, unkJsonBody.error === 'unknown_skill');
|
|
201
|
+
check(`${label}: unknown skill format=json includes valid alternatives`, unkJsonBody.valid_count === names.length && Array.isArray(unkJsonBody.valid) && unkJsonBody.valid.some((s) => s.slash === 'rdc:help'));
|
|
198
202
|
|
|
199
203
|
// Alias path: callers naturally copy the slash form from rdc_skill_list.
|
|
200
204
|
const aliasGet = await mcp(url, { jsonrpc: '2.0', id: 22, method: 'tools/call', params: { name: 'rdc_skill_get', arguments: { name: 'rdc:brochurify', variant: 'cli' } } });
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
4
|
+
import { join, resolve } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { dirname } from 'node:path';
|
|
7
|
+
|
|
8
|
+
import { loadAllManifests } from '../scripts/lib/manifest-schema.mjs';
|
|
9
|
+
|
|
10
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
11
|
+
const root = resolve(__dirname, '..');
|
|
12
|
+
const testsDir = join(root, 'skills', 'tests');
|
|
13
|
+
const matrixPath = join(testsDir, 'MATRIX.md');
|
|
14
|
+
const readmePath = join(testsDir, 'README.md');
|
|
15
|
+
|
|
16
|
+
const manifestFiles = readdirSync(testsDir).filter((name) => name.endsWith('.test.json')).sort();
|
|
17
|
+
const manifests = loadAllManifests(testsDir);
|
|
18
|
+
const valid = manifests.filter((entry) => entry.ok && entry.manifest);
|
|
19
|
+
const matrix = readFileSync(matrixPath, 'utf8');
|
|
20
|
+
const readme = readFileSync(readmePath, 'utf8');
|
|
21
|
+
|
|
22
|
+
assert.equal(valid.length, manifestFiles.length, 'all manifest files must validate');
|
|
23
|
+
assert.match(matrix, new RegExp(`Current coverage: ${manifestFiles.length} manifests for ${manifestFiles.length} skill directories`));
|
|
24
|
+
assert.match(readme, new RegExp(`There are currently ${manifestFiles.length} manifests for ${manifestFiles.length} skill directories`));
|
|
25
|
+
|
|
26
|
+
const rows = new Map();
|
|
27
|
+
for (const line of matrix.split(/\r?\n/)) {
|
|
28
|
+
const m = line.match(/^\| `([^`]+)` \| `([^`]+)` \| ([^|]+) \| ([^|]+) \| ([^|]+) \|$/);
|
|
29
|
+
if (!m) continue;
|
|
30
|
+
rows.set(m[1], {
|
|
31
|
+
manifest: m[2],
|
|
32
|
+
promptClass: m[3].trim(),
|
|
33
|
+
assertions: m[4].replace(/`/g, '').trim(),
|
|
34
|
+
acceptanceDepth: m[5].trim(),
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
assert.equal(rows.size, manifestFiles.length, 'MATRIX.md must list every manifest exactly once');
|
|
39
|
+
|
|
40
|
+
for (const entry of valid) {
|
|
41
|
+
const manifest = entry.manifest;
|
|
42
|
+
const filename = entry.file.replace(/^skills\/tests\//, '');
|
|
43
|
+
const row = rows.get(manifest.skill);
|
|
44
|
+
assert.ok(row, `${manifest.skill} missing from MATRIX.md`);
|
|
45
|
+
assert.equal(row.manifest, filename, `${manifest.skill} matrix filename mismatch`);
|
|
46
|
+
|
|
47
|
+
for (const key of Object.keys(manifest.assertions || {}).sort()) {
|
|
48
|
+
assert.match(row.assertions, new RegExp(`(^|, )${key}(,|$)`), `${manifest.skill} matrix must mention assertion ${key}`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const hasAcceptance = manifest.acceptance && Object.keys(manifest.acceptance).length > 0;
|
|
52
|
+
if (hasAcceptance) {
|
|
53
|
+
assert.notEqual(row.acceptanceDepth, 'Basic manifest', `${manifest.skill} has acceptance checks but matrix says Basic manifest`);
|
|
54
|
+
} else {
|
|
55
|
+
assert.equal(row.acceptanceDepth, 'Basic manifest', `${manifest.skill} has no acceptance block and should be documented as Basic manifest`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
console.log('skill test matrix tests — PASS');
|