@sun-asterisk/sungen 3.0.0-beta.80 → 3.0.0-beta.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +8 -0
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/eval.d.ts +3 -0
- package/dist/cli/commands/eval.d.ts.map +1 -0
- package/dist/cli/commands/eval.js +37 -0
- package/dist/cli/commands/eval.js.map +1 -0
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/harness/audit.d.ts +2 -0
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +7 -1
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/eval/skill-lint.d.ts +16 -0
- package/dist/harness/eval/skill-lint.d.ts.map +1 -0
- package/dist/harness/eval/skill-lint.js +129 -0
- package/dist/harness/eval/skill-lint.js.map +1 -0
- package/dist/harness/viewpoint-ledger.d.ts +23 -0
- package/dist/harness/viewpoint-ledger.d.ts.map +1 -0
- package/dist/harness/viewpoint-ledger.js +118 -0
- package/dist/harness/viewpoint-ledger.js.map +1 -0
- package/package.json +2 -2
- package/src/cli/commands/audit.ts +6 -0
- package/src/cli/commands/eval.ts +28 -0
- package/src/cli/index.ts +2 -0
- package/src/harness/audit.ts +11 -4
- package/src/harness/eval/skill-lint.ts +87 -0
- package/src/harness/viewpoint-ledger.ts +80 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audit.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/audit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"audit.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/audit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA+EpC,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAmC3D"}
|
|
@@ -111,6 +111,14 @@ function render(r) {
|
|
|
111
111
|
L(' ✓ every MUST FR + per-constraint trigger covered');
|
|
112
112
|
L('');
|
|
113
113
|
}
|
|
114
|
+
if (r.ledger.hasViewpoint && r.ledger.total > 0) {
|
|
115
|
+
L(` ⑧ Viewpoint atomic coverage — ${r.ledger.covered}/${r.ledger.total} items (${(r.ledger.ratio * 100).toFixed(0)}%)`);
|
|
116
|
+
for (const m of r.ledger.missing.slice(0, 8))
|
|
117
|
+
L(` ○ missing: ${m.id ? m.id + ' — ' : ''}${m.text.slice(0, 70)}`);
|
|
118
|
+
if (r.ledger.missing.length > 8)
|
|
119
|
+
L(` … +${r.ledger.missing.length - 8} more`);
|
|
120
|
+
L('');
|
|
121
|
+
}
|
|
114
122
|
L(' ── Findings (Repair targets) ──');
|
|
115
123
|
if (r.findings.length === 0)
|
|
116
124
|
L(' ✓ none — output passes the harness');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../../src/cli/commands/audit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../../src/cli/commands/audit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+EA,oDAmCC;AAjHD,2CAA6B;AAC7B,uCAAyB;AACzB,+CAA4D;AAE5D,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IAC/D,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC3D,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;IACpC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IACpC,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,MAAM,CAAC,CAAc;IAC5B,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC;IACtB,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,CAAC,CAAC,sBAAsB,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,aAAa,iBAAiB,CAAC,CAAC;IACvE,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,CAAC,CAAC,wCAAwC,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC;IACpG,CAAC,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAC,KAAK,sBAAsB,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9G,CAAC,CAAC,wBAAwB,CAAC,CAAC,UAAU,CAAC,aAAa,cAAc,CAAC,CAAC,UAAU,CAAC,WAAW,6CAA6C,CAAC,CAAC;IACzI,CAAC,CAAC,qBAAqB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACxF,CAAC,CAAC,qBAAqB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IACtI,CAAC,CAAC,qBAAqB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IACtH,CAAC,CAAC,qBAAqB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACtF,CAAC,CAAC,qBAAqB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAChG,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;IAC9B,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,CAAC,CAAC,mCAAmC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,SAAS,aAAa,CAAC,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,UAAU,CAAC,CAAC;IACpI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI;YAAE,CAAC,CAAC,WAAW,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;;QAC3H,CAAC,CAAC,4DAA4D,CAAC,CAAC;IACrE,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,CAAC,CAAC,yBAAyB,CAAC,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,CAAC,KAAK,CAAC,qBAAqB,gEAAgE,CAAC,CAAC;IAC7J,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,CAAC,CAAC,WAAW,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACnG,IAAI,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC,MAAM,GAAG,CAAC;QAAE,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,CAAC;IACjH,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,CAAC,CAAC,qBAAqB,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,UAAU,oCAAoC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IACjI,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,CAAC,CAAC,wFAAwF,CAAC,CAAC;IAC/H,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACvI,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,CAAC;IACnF,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,CAAC,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,cAAc,EAAE,CAAC,CAAC;IACtJ,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,OAAO,aAAa,CAAC,CAAC,SAAS,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;IACpI,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,CAAC,CAAC,0BAA0B,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC;IACpF,CAAC,CAAC,kBAAkB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClG,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,CAAC,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,2BAA2B,CAAC,CAAC,UAAU,CAAC,mBAAmB,sBAAsB,CAAC,CAAC;IACpI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAClD,CAAC,CAAC,SAAS,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACvK,CAAC;IACD,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,0CAA0C,CAAC,CAAC;IAC1G,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3B,CAAC,CAAC,EAAE,CAAC,CAAC;IACN,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,EAAE,CAAC;QACzG,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,cAAc,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAC7G,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW;YAAE,CAAC,CAAC,+BAA+B,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,eAAe,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChP,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAAE,CAAC,CAAC,2BAA2B,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;QAC5G,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM;YAAE,CAAC,CAAC,wDAAwD,CAAC,CAAC;QAC5H,CAAC,CAAC,EAAE,CAAC,CAAC;IACR,CAAC;IACD,IAAI,CAAC,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QAChD,CAAC,CAAC,mCAAmC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzH,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,CAAC;QACnF,CAAC,CAAC,EAAE,CAAC,CAAC;IACR,CAAC;IACD,CAAC,CAAC,mCAAmC,CAAC,CAAC;IACvC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,CAAC,CAAC,0CAA0C,CAAC,CAAC;IAC3E,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ;QAAE,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,EAAE,CAAC,CAAC;AACR,CAAC;AAED,SAAgB,oBAAoB,CAAC,OAAgB;IACnD,OAAO;SACJ,OAAO,CAAC,OAAO,CAAC;SAChB,WAAW,CAAC,iGAAiG,CAAC;SAC9G,MAAM,CAAC,qBAAqB,EAAE,8BAA8B,CAAC;SAC7D,MAAM,CAAC,QAAQ,EAAE,4BAA4B,CAAC;SAC9C,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;QAClB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;YAC5B,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACtD,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,gBAAgB,IAAI,EAAE,CAAC,CAAC;YAE3F,MAAM,MAAM,GAAG,IAAA,gBAAQ,EAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAEnC,wCAAwC;YACxC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YAC9D,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,aAAa,CAAC,CAAC;YACxD,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAEpE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,MAAM,CAAC,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;gBAClE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAClB,CAAC;YACD,kEAAkE;YAClE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAwB1D"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.registerEvalCommand = registerEvalCommand;
|
|
4
|
+
const skill_lint_1 = require("../../harness/eval/skill-lint");
|
|
5
|
+
function registerEvalCommand(program) {
|
|
6
|
+
program
|
|
7
|
+
.command('eval')
|
|
8
|
+
.description('Eval harness: quality checks on Sungen\'s own skills/instructions (dev/CI)')
|
|
9
|
+
.option('--skills', 'Static skill-lint: frontmatter, line budget, claude↔github sync, registration')
|
|
10
|
+
.option('--dir <path>', 'Templates dir to lint (default: bundled ai-instructions)')
|
|
11
|
+
.option('--json', 'Output the raw findings JSON')
|
|
12
|
+
.action((options) => {
|
|
13
|
+
try {
|
|
14
|
+
if (!options.skills)
|
|
15
|
+
throw new Error('Provide --skills (the only eval mode today)');
|
|
16
|
+
const dir = options.dir || (0, skill_lint_1.defaultSkillDir)();
|
|
17
|
+
const r = (0, skill_lint_1.lintSkills)(dir);
|
|
18
|
+
if (options.json) {
|
|
19
|
+
console.log(JSON.stringify(r, null, 2));
|
|
20
|
+
process.exit(r.errors > 0 ? 2 : 0);
|
|
21
|
+
}
|
|
22
|
+
console.log('');
|
|
23
|
+
console.log(`━━━ Skill-lint: ${r.checked} skill template(s) ━━━`);
|
|
24
|
+
if (!r.findings.length)
|
|
25
|
+
console.log(' ✓ all skills pass (frontmatter · line-budget · variant-sync · registration)');
|
|
26
|
+
for (const f of r.findings)
|
|
27
|
+
console.log(` ${f.level === 'error' ? '✗' : '⚠'} [${f.rule}] ${f.file} — ${f.detail}`);
|
|
28
|
+
console.log('');
|
|
29
|
+
process.exit(r.errors > 0 ? 2 : 0);
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
console.error('Error:', error instanceof Error ? error.message : error);
|
|
33
|
+
process.exit(1);
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=eval.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/cli/commands/eval.ts"],"names":[],"mappings":";;AAGA,kDAwBC;AA1BD,8DAA4E;AAE5E,SAAgB,mBAAmB,CAAC,OAAgB;IAClD,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,4EAA4E,CAAC;SACzF,MAAM,CAAC,UAAU,EAAE,+EAA+E,CAAC;SACnG,MAAM,CAAC,cAAc,EAAE,0DAA0D,CAAC;SAClF,MAAM,CAAC,QAAQ,EAAE,8BAA8B,CAAC;SAChD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;QAClB,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACpF,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAA,4BAAe,GAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,IAAA,uBAAU,EAAC,GAAG,CAAC,CAAC;YAC1B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAAC,CAAC;YAClG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,OAAO,wBAAwB,CAAC,CAAC;YAClE,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM;gBAAE,OAAO,CAAC,GAAG,CAAC,+EAA+E,CAAC,CAAC;YACrH,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ;gBAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YACpH,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
package/dist/cli/index.js
CHANGED
|
@@ -17,6 +17,7 @@ const add_flow_1 = require("./commands/add-flow");
|
|
|
17
17
|
const dashboard_1 = require("./commands/dashboard");
|
|
18
18
|
const audit_1 = require("./commands/audit");
|
|
19
19
|
const ingest_1 = require("./commands/ingest");
|
|
20
|
+
const eval_1 = require("./commands/eval");
|
|
20
21
|
const manifest_1 = require("./commands/manifest");
|
|
21
22
|
const ledger_1 = require("./commands/ledger");
|
|
22
23
|
const feedback_1 = require("./commands/feedback");
|
|
@@ -58,6 +59,7 @@ async function main() {
|
|
|
58
59
|
(0, capability_1.registerCapabilityCommand)(program);
|
|
59
60
|
(0, flow_check_1.registerFlowCheckCommand)(program);
|
|
60
61
|
(0, ingest_1.registerIngestCommand)(program);
|
|
62
|
+
(0, eval_1.registerEvalCommand)(program);
|
|
61
63
|
await program.parseAsync(process.argv);
|
|
62
64
|
}
|
|
63
65
|
main().catch((error) => {
|
package/dist/cli/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";;AACA;;;GAGG;;AAEH,yCAAoC;AACpC,0CAAsD;AACtD,wCAAoD;AACpD,kDAA8D;AAC9D,kDAA8D;AAC9D,8CAA0D;AAC1D,kDAA8D;AAC9D,4CAAwD;AACxD,kDAA6D;AAC7D,oDAAgE;AAChE,4CAAwD;AACxD,8CAA0D;AAC1D,kDAA8D;AAC9D,8CAA0D;AAC1D,kDAA8D;AAC9D,0DAAqE;AACrE,4CAAwD;AACxD,oDAAgE;AAChE,oDAAgE;AAChE,sDAAkE;AAClE,sDAAiE;AAEjE,wFAAwF;AACxF,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAwB,CAAC;AAEzE,KAAK,UAAU,IAAI;IACjB,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,IAAI,CAAC,QAAQ,CAAC;SACd,WAAW,CAAC,oEAAoE,CAAC;SACjF,OAAO,CAAC,OAAO,CAAC,CAAC;IAEpB,iBAAiB;IACjB,OAAO;SACJ,MAAM,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAC;IAErD,wBAAwB;IACxB,IAAA,0BAAmB,EAAC,OAAO,CAAC,CAAC;IAC7B,IAAA,wBAAkB,EAAC,OAAO,CAAC,CAAC;IAC5B,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;IAC/B,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,4BAAoB,EAAC,OAAO,CAAC,CAAC;IAC9B,IAAA,iCAAsB,EAAC,OAAO,CAAC,CAAC;IAChC,IAAA,oCAAwB,EAAC,OAAO,CAAC,CAAC;IAClC,IAAA,4BAAoB,EAAC,OAAO,CAAC,CAAC;IAC9B,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;IAC/B,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,yCAA0B,EAAC,OAAO,CAAC,CAAC;IACpC,IAAA,4BAAoB,EAAC,OAAO,CAAC,CAAC;IAC9B,IAAA,oCAAwB,EAAC,OAAO,CAAC,CAAC;IAClC,IAAA,oCAAwB,EAAC,OAAO,CAAC,CAAC;IAClC,IAAA,sCAAyB,EAAC,OAAO,CAAC,CAAC;IACnC,IAAA,qCAAwB,EAAC,OAAO,CAAC,CAAC;IAClC,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";;AACA;;;GAGG;;AAEH,yCAAoC;AACpC,0CAAsD;AACtD,wCAAoD;AACpD,kDAA8D;AAC9D,kDAA8D;AAC9D,8CAA0D;AAC1D,kDAA8D;AAC9D,4CAAwD;AACxD,kDAA6D;AAC7D,oDAAgE;AAChE,4CAAwD;AACxD,8CAA0D;AAC1D,0CAAsD;AACtD,kDAA8D;AAC9D,8CAA0D;AAC1D,kDAA8D;AAC9D,0DAAqE;AACrE,4CAAwD;AACxD,oDAAgE;AAChE,oDAAgE;AAChE,sDAAkE;AAClE,sDAAiE;AAEjE,wFAAwF;AACxF,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAwB,CAAC;AAEzE,KAAK,UAAU,IAAI;IACjB,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,IAAI,CAAC,QAAQ,CAAC;SACd,WAAW,CAAC,oEAAoE,CAAC;SACjF,OAAO,CAAC,OAAO,CAAC,CAAC;IAEpB,iBAAiB;IACjB,OAAO;SACJ,MAAM,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAC;IAErD,wBAAwB;IACxB,IAAA,0BAAmB,EAAC,OAAO,CAAC,CAAC;IAC7B,IAAA,wBAAkB,EAAC,OAAO,CAAC,CAAC;IAC5B,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;IAC/B,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,4BAAoB,EAAC,OAAO,CAAC,CAAC;IAC9B,IAAA,iCAAsB,EAAC,OAAO,CAAC,CAAC;IAChC,IAAA,oCAAwB,EAAC,OAAO,CAAC,CAAC;IAClC,IAAA,4BAAoB,EAAC,OAAO,CAAC,CAAC;IAC9B,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;IAC/B,IAAA,kCAAuB,EAAC,OAAO,CAAC,CAAC;IACjC,IAAA,yCAA0B,EAAC,OAAO,CAAC,CAAC;IACpC,IAAA,4BAAoB,EAAC,OAAO,CAAC,CAAC;IAC9B,IAAA,oCAAwB,EAAC,OAAO,CAAC,CAAC;IAClC,IAAA,oCAAwB,EAAC,OAAO,CAAC,CAAC;IAClC,IAAA,sCAAyB,EAAC,OAAO,CAAC,CAAC;IACnC,IAAA,qCAAwB,EAAC,OAAO,CAAC,CAAC;IAClC,IAAA,8BAAqB,EAAC,OAAO,CAAC,CAAC;IAC/B,IAAA,0BAAmB,EAAC,OAAO,CAAC,CAAC;IAE7B,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/harness/audit.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { IntentProfile } from './intent';
|
|
|
3
3
|
import { Provenance } from './provenance';
|
|
4
4
|
import { SpecCoverageResult } from './spec-coverage';
|
|
5
5
|
import { DownstreamResult, ManualOracleResult } from './quality-gates';
|
|
6
|
+
import { LedgerResult } from './viewpoint-ledger';
|
|
6
7
|
export interface AuditReport {
|
|
7
8
|
screen: string;
|
|
8
9
|
scenarioCount: number;
|
|
@@ -16,6 +17,7 @@ export interface AuditReport {
|
|
|
16
17
|
taxonomyMismatch: boolean;
|
|
17
18
|
downstream: DownstreamResult;
|
|
18
19
|
manualOracle: ManualOracleResult;
|
|
20
|
+
ledger: LedgerResult;
|
|
19
21
|
score: {
|
|
20
22
|
overall: number;
|
|
21
23
|
coverage: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audit.d.ts","sourceRoot":"","sources":["../../src/harness/audit.ts"],"names":[],"mappings":"AAWA,OAAO,EAEL,UAAU,EAAE,WAAW,EAAE,aAAa,EAAE,eAAe,EAAE,WAAW,EAAE,gBAAgB,EAAE,cAAc,EACvG,MAAM,WAAW,CAAC;AACnB,OAAO,EAAwC,aAAa,EAAE,MAAM,UAAU,CAAC;AAC/E,OAAO,EAAiB,UAAU,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAgB,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAA2C,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"audit.d.ts","sourceRoot":"","sources":["../../src/harness/audit.ts"],"names":[],"mappings":"AAWA,OAAO,EAEL,UAAU,EAAE,WAAW,EAAE,aAAa,EAAE,eAAe,EAAE,WAAW,EAAE,gBAAgB,EAAE,cAAc,EACvG,MAAM,WAAW,CAAC;AACnB,OAAO,EAAwC,aAAa,EAAE,MAAM,UAAU,CAAC;AAC/E,OAAO,EAAiB,UAAU,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAgB,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAA2C,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAChH,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEnE,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,EAAE,WAAW,CAAC;IACnB,KAAK,EAAE,gBAAgB,CAAC;IACxB,QAAQ,EAAE,cAAc,CAAC;IACzB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,EAAE,eAAe,CAAC;IAC5B,KAAK,EAAE,WAAW,CAAC;IACnB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,UAAU,EAAE,gBAAgB,CAAC;IAC7B,YAAY,EAAE,kBAAkB,CAAC;IACjC,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,aAAa,EAAE,MAAM,CAAC;QACtB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,EAAE,UAAU,CAAC;IACvB,IAAI,EAAE,kBAAkB,CAAC;CAC1B;AAED,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,WAAW,CAkH3E"}
|
package/dist/harness/audit.js
CHANGED
|
@@ -50,6 +50,7 @@ const intent_1 = require("./intent");
|
|
|
50
50
|
const provenance_1 = require("./provenance");
|
|
51
51
|
const spec_coverage_1 = require("./spec-coverage");
|
|
52
52
|
const quality_gates_1 = require("./quality-gates");
|
|
53
|
+
const viewpoint_ledger_1 = require("./viewpoint-ledger");
|
|
53
54
|
function runAudit(screenDir, screenName) {
|
|
54
55
|
const featurePath = path.join(screenDir, 'features', `${screenName}.feature`);
|
|
55
56
|
const viewpointPath = path.join(screenDir, 'requirements', 'test-viewpoint.md');
|
|
@@ -73,6 +74,7 @@ function runAudit(screenDir, screenName) {
|
|
|
73
74
|
// #2 downstream-scope + #4 manual-oracle
|
|
74
75
|
const downstream = (0, quality_gates_1.downstreamScope)((0, quality_gates_1.readText)(specPath), scenarios);
|
|
75
76
|
const manualOracleResult = (0, quality_gates_1.manualOracle)(featureText);
|
|
77
|
+
const ledger = (0, viewpoint_ledger_1.viewpointLedger)(viewpointPath, scenarios, featureText);
|
|
76
78
|
// Sub-scores
|
|
77
79
|
const coverage = gate.coverageRatio;
|
|
78
80
|
const businessDepth = depth.bcDepthRatio;
|
|
@@ -128,6 +130,10 @@ function runAudit(screenDir, screenName) {
|
|
|
128
130
|
for (const m of manualOracleResult.insufficient.slice(0, 8)) {
|
|
129
131
|
findings.push(`MANUAL-STEPS-INSUFFICIENT: "${m}" — a @manual scenario needs setup · action · observable expected · oracle/tool (not just a one-line note).`);
|
|
130
132
|
}
|
|
133
|
+
if (ledger.hasViewpoint && ledger.missing.length) {
|
|
134
|
+
const sample = ledger.missing.slice(0, 6).map((m) => m.id || `"${m.text}"`).join(', ');
|
|
135
|
+
findings.push(`VIEWPOINT-ITEM-MISSING: ${ledger.missing.length}/${ledger.total} atomic viewpoint items have no covering scenario (${(ledger.ratio * 100).toFixed(0)}% covered) — e.g. ${sample}. Cover each item or mark it deferred/spec-gap.`);
|
|
136
|
+
}
|
|
131
137
|
// Gate spans coverage (viewpoint themes), depth, claim-proof, spec-clause coverage,
|
|
132
138
|
// AND taxonomy-match (scenarios must use the project's viewpoint IDs when defined).
|
|
133
139
|
const gateStatus = gate.gaps.length === 0 && depth.verdict !== 'fail' && claim.verdict !== 'fail' && spec.verdict !== 'fail' && !taxonomyMismatch ? 'PASS' : 'FAIL';
|
|
@@ -135,7 +141,7 @@ function runAudit(screenDir, screenName) {
|
|
|
135
141
|
screen: screenName,
|
|
136
142
|
scenarioCount: scenarios.length,
|
|
137
143
|
gate, depth, claim, taxonomy, balance, duplicates, trace, spec,
|
|
138
|
-
taxonomyMismatch, downstream, manualOracle: manualOracleResult,
|
|
144
|
+
taxonomyMismatch, downstream, manualOracle: manualOracleResult, ledger,
|
|
139
145
|
score: {
|
|
140
146
|
overall: Math.round(overall * 10) / 10,
|
|
141
147
|
coverage: Math.round(coverage * 100) / 100,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../src/harness/audit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../src/harness/audit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,4BAkHC;AApKD;;;;;;;GAOG;AACH,2CAA6B;AAC7B,uCAAyB;AACzB,mCAA8F;AAC9F,uCAGmB;AACnB,qCAA+E;AAC/E,6CAAyD;AACzD,mDAAmE;AACnE,mDAAgH;AAChH,yDAAmE;AA+BnE,SAAgB,QAAQ,CAAC,SAAiB,EAAE,UAAkB;IAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,GAAG,UAAU,UAAU,CAAC,CAAC;IAC9E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,mBAAmB,CAAC,CAAC;IAEhF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE5F,MAAM,SAAS,GAAmB,IAAA,qBAAa,EAAC,WAAW,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAqB,IAAA,8BAAsB,EAAC,aAAa,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,IAAA,qBAAW,GAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,IAAA,4BAAY,EAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAE5D,MAAM,IAAI,GAAG,IAAA,uBAAa,EAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAC3D,6EAA6E;IAC7E,MAAM,MAAM,GAAG,IAAA,mBAAU,EAAC,IAAA,iCAAwB,EAAC,SAAS,CAAC,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,IAAA,wBAAc,EAAC,SAAS,EAAE,IAAA,uBAAa,EAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAG,IAAA,oBAAU,EAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAA,sBAAY,EAAC,SAAS,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,IAAA,yBAAe,EAAC,SAAS,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,IAAA,2BAAiB,EAAC,SAAS,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAA,sBAAY,EAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAClD,2FAA2F;IAC3F,MAAM,gBAAgB,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,IAAI,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC;IAClG,yCAAyC;IACzC,MAAM,UAAU,GAAG,IAAA,+BAAe,EAAC,IAAA,wBAAQ,EAAC,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;IAClE,MAAM,kBAAkB,GAAG,IAAA,4BAAY,EAAC,WAAW,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,IAAA,kCAAe,EAAC,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAEtE,aAAa;IACb,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC;IACpC,MAAM,aAAa,GAAG,KAAK,CAAC,YAAY,CAAC;IACzC,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,GAAG,OAAO,CAAC,cAAc,GAAG,CAAC;QACjE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,UAAU,GAAG,GAAG,GAAG,KAAK,CAAC,eAAe,GAAG,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC;IAEzE,wDAAwD;IACxD,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,QAAQ,GAAG,GAAG,GAAG,aAAa,GAAG,IAAI,GAAG,YAAY,GAAG,IAAI,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC;IAEtG,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,KAAK,kKAAkK,CAAC,CAAC;QACpN,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,KAAK,oBAAoB,IAAI,CAAC,QAAQ,oFAAoF,CAAC,CAAC;QACvK,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,uBAAuB,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC;QACxG,QAAQ,CAAC,IAAI,CACX,GAAG,GAAG,KAAK,KAAK,CAAC,uBAAuB,IAAI,KAAK,CAAC,qBAAqB,qDAAqD;YAC5H,UAAU,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,KAAK,CAAC,KAAK,OAAO;YAClH,yIAAyI,CAC1I,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,YAAY,CAAC;QACpE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,IAAI,qBAAqB,CAAC,CAAC,KAAK,oBAAoB,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,iBAAiB,CAAC,CAAC,OAAO,iBAAiB,CAAC,CAAC,SAAS,cAAc,CAAC,CAAC,MAAM,kBAAkB,CAAC,CAAC,SAAS,2CAA2C,CAAC,CAAC;IAC7L,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC,YAAY,OAAO,CAAC,IAAI,2EAA2E,CAAC,CAAC;IACrH,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,GAAG,GAAG,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC,gCAAgC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1G,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,aAAa,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;IAC1T,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,EAAE,uCAAuC,CAAC,CAAC,IAAI,uCAAuC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACpI,CAAC;IACD,IAAI,gBAAgB,EAAE,CAAC;QACrB,QAAQ,CAAC,IAAI,CAAC,8BAA8B,CAAC,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,8LAA8L,CAAC,CAAC;IAClR,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;QACxC,QAAQ,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC,KAAK,0IAA0I,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC;IAC7M,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,kBAAkB,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC5D,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,6GAA6G,CAAC,CAAC;IAC/J,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvF,QAAQ,CAAC,IAAI,CAAC,2BAA2B,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,sDAAsD,CAAC,MAAM,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,MAAM,iDAAiD,CAAC,CAAC;IACnP,CAAC;IAED,oFAAoF;IACpF,oFAAoF;IACpF,MAAM,UAAU,GACd,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,KAAK,MAAM,IAAI,KAAK,CAAC,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEnJ,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,SAAS,CAAC,MAAM;QAC/B,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI;QAC9D,gBAAgB,EAAE,UAAU,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM;QACtE,KAAK,EAAE;YACL,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,EAAE;YACtC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,GAAG;YAC1C,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC,GAAG,GAAG;YACpD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,GAAG,GAAG;YAC7C,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,GAAG;YAChD,OAAO,EAAE,sFAAsF;SAChG;QACD,UAAU;QACV,QAAQ;QACR,MAAM;QACN,UAAU,EAAE,IAAA,0BAAa,GAAE;KAC5B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface SkillLintFinding {
|
|
2
|
+
level: 'error' | 'warn';
|
|
3
|
+
file: string;
|
|
4
|
+
rule: string;
|
|
5
|
+
detail: string;
|
|
6
|
+
}
|
|
7
|
+
export interface SkillLintResult {
|
|
8
|
+
checked: number;
|
|
9
|
+
findings: SkillLintFinding[];
|
|
10
|
+
errors: number;
|
|
11
|
+
}
|
|
12
|
+
/** Lint the AI-instruction templates in `dir` (default: the sungen source templates). */
|
|
13
|
+
export declare function lintSkills(dir: string): SkillLintResult;
|
|
14
|
+
/** Default templates dir, resolved relative to this module (works from src via tsx and dist). */
|
|
15
|
+
export declare function defaultSkillDir(): string;
|
|
16
|
+
//# sourceMappingURL=skill-lint.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"skill-lint.d.ts","sourceRoot":"","sources":["../../../src/harness/eval/skill-lint.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,gBAAgB;IAAG,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACzG,MAAM,WAAW,eAAe;IAAG,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AAWlG,yFAAyF;AACzF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAkDvD;AAED,iGAAiG;AACjG,wBAAgB,eAAe,IAAI,MAAM,CAGxC"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.lintSkills = lintSkills;
|
|
37
|
+
exports.defaultSkillDir = defaultSkillDir;
|
|
38
|
+
/**
|
|
39
|
+
* Static skill-lint (Eval Harness L1) — deterministic quality checks on Sungen's OWN
|
|
40
|
+
* AI-instruction templates, so a broken / unregistered / oversized skill fails before it
|
|
41
|
+
* ships. Learned (generically) from the "static validations" tier of an agent-kit evals
|
|
42
|
+
* layer. No project data — this lints the sungen package's own templates.
|
|
43
|
+
*
|
|
44
|
+
* Design note: the checks are MAPPING-DRIVEN. `AI_RULES_FILE_MAPPING` is the source of
|
|
45
|
+
* truth for what each template installs as, so the lint uses the install target (does it
|
|
46
|
+
* end in `/SKILL.md`?) to tell a top-level skill from a sub-content fragment — instead of
|
|
47
|
+
* guessing from filenames. We deliberately do NOT enforce claude↔github body parity: the
|
|
48
|
+
* two variants are hand-tuned per platform and intentionally diverge in wording and even
|
|
49
|
+
* structure, so byte/heading equality would be pure false positives.
|
|
50
|
+
*/
|
|
51
|
+
const fs = __importStar(require("fs"));
|
|
52
|
+
const path = __importStar(require("path"));
|
|
53
|
+
const ai_rules_updater_1 = require("../../orchestrator/ai-rules-updater");
|
|
54
|
+
const LINE_BUDGET = 700; // a skill much larger than this is a context-cost smell (warn)
|
|
55
|
+
const SKILL_RE = /^(claude|github)-skill-/;
|
|
56
|
+
function stripFrontmatter(text) {
|
|
57
|
+
const m = text.match(/^---\n([\s\S]*?)\n---\n?/);
|
|
58
|
+
if (!m)
|
|
59
|
+
return { fm: null, body: text };
|
|
60
|
+
return { fm: m[1], body: text.slice(m[0].length) };
|
|
61
|
+
}
|
|
62
|
+
/** Lint the AI-instruction templates in `dir` (default: the sungen source templates). */
|
|
63
|
+
function lintSkills(dir) {
|
|
64
|
+
const findings = [];
|
|
65
|
+
const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith('.md')) : [];
|
|
66
|
+
const skillFiles = files.filter((f) => SKILL_RE.test(f));
|
|
67
|
+
// mapping: template file -> install target (source of truth for "is this a top-level skill")
|
|
68
|
+
const target = new Map(ai_rules_updater_1.AI_RULES_FILE_MAPPING.map(([tpl, dst]) => [tpl, dst]));
|
|
69
|
+
const isTopLevelSkill = (f) => (target.get(f) || '').endsWith('/SKILL.md');
|
|
70
|
+
// 1) registration integrity (bidirectional) — the highest-value check:
|
|
71
|
+
// a skill file missing from the mapping never installs; a mapping to a missing file
|
|
72
|
+
// ships a broken/empty skill.
|
|
73
|
+
for (const f of skillFiles) {
|
|
74
|
+
if (!target.has(f))
|
|
75
|
+
findings.push({ level: 'error', file: f, rule: 'unregistered', detail: 'skill template not in AI_RULES_FILE_MAPPING (it would never be installed)' });
|
|
76
|
+
}
|
|
77
|
+
for (const [tpl] of ai_rules_updater_1.AI_RULES_FILE_MAPPING) {
|
|
78
|
+
if (!fs.existsSync(path.join(dir, tpl)))
|
|
79
|
+
findings.push({ level: 'error', file: tpl, rule: 'mapped-missing', detail: 'AI_RULES_FILE_MAPPING points to a template that does not exist' });
|
|
80
|
+
}
|
|
81
|
+
// 2) frontmatter (name + description) — ONLY for top-level skills (SKILL.md targets).
|
|
82
|
+
// Sub-content fragments (mode-*.md, group-*.md) are loaded by their parent router
|
|
83
|
+
// and legitimately carry no frontmatter.
|
|
84
|
+
for (const f of skillFiles) {
|
|
85
|
+
if (!isTopLevelSkill(f))
|
|
86
|
+
continue;
|
|
87
|
+
const text = fs.readFileSync(path.join(dir, f), 'utf8');
|
|
88
|
+
const { fm } = stripFrontmatter(text);
|
|
89
|
+
if (!fm) {
|
|
90
|
+
findings.push({ level: 'error', file: f, rule: 'frontmatter', detail: 'top-level skill (SKILL.md) is missing --- frontmatter --- (Claude/Copilot will not load it)' });
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (!/\bname\s*:/.test(fm))
|
|
94
|
+
findings.push({ level: 'error', file: f, rule: 'frontmatter-name', detail: 'no `name:` in frontmatter' });
|
|
95
|
+
if (!/\bdescription\s*:/.test(fm))
|
|
96
|
+
findings.push({ level: 'error', file: f, rule: 'frontmatter-description', detail: 'no `description:` in frontmatter' });
|
|
97
|
+
}
|
|
98
|
+
// 3) line budget — context-cost smell (advisory).
|
|
99
|
+
for (const f of skillFiles) {
|
|
100
|
+
const lines = fs.readFileSync(path.join(dir, f), 'utf8').split('\n').length;
|
|
101
|
+
if (lines > LINE_BUDGET)
|
|
102
|
+
findings.push({ level: 'warn', file: f, rule: 'line-budget', detail: `${lines} lines > ${LINE_BUDGET} (context-cost smell)` });
|
|
103
|
+
}
|
|
104
|
+
// 4) variant PRESENCE (not body equality) — every top-level skill should ship for both
|
|
105
|
+
// platforms. Catches "added a Claude skill but forgot the Copilot variant". Advisory.
|
|
106
|
+
const skillName = (dst) => { const m = dst.match(/\/(sungen-[^/]+)\/SKILL\.md$/); return m ? m[1] : null; };
|
|
107
|
+
const claudeSkills = new Set(), githubSkills = new Set();
|
|
108
|
+
for (const f of skillFiles) {
|
|
109
|
+
if (!isTopLevelSkill(f))
|
|
110
|
+
continue;
|
|
111
|
+
const name = skillName(target.get(f));
|
|
112
|
+
if (!name)
|
|
113
|
+
continue;
|
|
114
|
+
(f.startsWith('claude-') ? claudeSkills : githubSkills).add(name);
|
|
115
|
+
}
|
|
116
|
+
for (const n of claudeSkills)
|
|
117
|
+
if (!githubSkills.has(n))
|
|
118
|
+
findings.push({ level: 'warn', file: `claude .../${n}/SKILL.md`, rule: 'variant-missing', detail: `Claude skill "${n}" has no GitHub (Copilot) variant` });
|
|
119
|
+
for (const n of githubSkills)
|
|
120
|
+
if (!claudeSkills.has(n))
|
|
121
|
+
findings.push({ level: 'warn', file: `github .../${n}/SKILL.md`, rule: 'variant-missing', detail: `GitHub skill "${n}" has no Claude variant` });
|
|
122
|
+
return { checked: skillFiles.length, findings, errors: findings.filter((f) => f.level === 'error').length };
|
|
123
|
+
}
|
|
124
|
+
/** Default templates dir, resolved relative to this module (works from src via tsx and dist). */
|
|
125
|
+
function defaultSkillDir() {
|
|
126
|
+
// src/harness/eval → src/orchestrator/... | dist/harness/eval → dist/orchestrator/...
|
|
127
|
+
return path.resolve(__dirname, '..', '..', 'orchestrator', 'templates', 'ai-instructions');
|
|
128
|
+
}
|
|
129
|
+
//# sourceMappingURL=skill-lint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"skill-lint.js","sourceRoot":"","sources":["../../../src/harness/eval/skill-lint.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,gCAkDC;AAGD,0CAGC;AAtFD;;;;;;;;;;;;GAYG;AACH,uCAAyB;AACzB,2CAA6B;AAC7B,0EAA4E;AAK5E,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,+DAA+D;AACxF,MAAM,QAAQ,GAAG,yBAAyB,CAAC;AAE3C,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IACjD,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACxC,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AACrD,CAAC;AAED,yFAAyF;AACzF,SAAgB,UAAU,CAAC,GAAW;IACpC,MAAM,QAAQ,GAAuB,EAAE,CAAC;IACxC,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7F,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzD,6FAA6F;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,CAAiB,wCAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAC9F,MAAM,eAAe,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAEnF,uEAAuE;IACvE,uFAAuF;IACvF,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,2EAA2E,EAAE,CAAC,CAAC;IAC5K,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,wCAAqB,EAAE,CAAC;QAC1C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,gEAAgE,EAAE,CAAC,CAAC;IAC1L,CAAC;IAED,sFAAsF;IACtF,qFAAqF;IACrF,4CAA4C;IAC5C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;YAAE,SAAS;QAClC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,EAAE,EAAE,EAAE,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,EAAE,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,6FAA6F,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAC9L,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,2BAA2B,EAAE,CAAC,CAAC;QACtI,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,kCAAkC,EAAE,CAAC,CAAC;IAC7J,CAAC;IAED,kDAAkD;IAClD,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC5E,IAAI,KAAK,GAAG,WAAW;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,KAAK,YAAY,WAAW,uBAAuB,EAAE,CAAC,CAAC;IAC1J,CAAC;IAED,uFAAuF;IACvF,yFAAyF;IACzF,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,EAAE,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,EAAE,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACzE,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;YAAE,SAAS;QAClC,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC;QAAC,IAAI,CAAC,IAAI;YAAE,SAAS;QAC5D,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpE,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,YAAY;QAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,WAAW,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,iBAAiB,CAAC,mCAAmC,EAAE,CAAC,CAAC;IACnN,KAAK,MAAM,CAAC,IAAI,YAAY;QAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,WAAW,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,iBAAiB,CAAC,yBAAyB,EAAE,CAAC,CAAC;IAEzM,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;AAC9G,CAAC;AAED,iGAAiG;AACjG,SAAgB,eAAe;IAC7B,wFAAwF;IACxF,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,iBAAiB,CAAC,CAAC;AAC7F,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { ScenarioInfo } from './parse';
|
|
2
|
+
export interface LedgerItem {
|
|
3
|
+
id?: string;
|
|
4
|
+
text: string;
|
|
5
|
+
covered: boolean;
|
|
6
|
+
}
|
|
7
|
+
export interface LedgerResult {
|
|
8
|
+
hasViewpoint: boolean;
|
|
9
|
+
total: number;
|
|
10
|
+
covered: number;
|
|
11
|
+
ratio: number;
|
|
12
|
+
missing: {
|
|
13
|
+
id?: string;
|
|
14
|
+
text: string;
|
|
15
|
+
}[];
|
|
16
|
+
}
|
|
17
|
+
/** Extract atomic checklist items from a viewpoint file (format-tolerant). */
|
|
18
|
+
export declare function parseViewpointItems(viewpointPath: string): {
|
|
19
|
+
id?: string;
|
|
20
|
+
text: string;
|
|
21
|
+
}[];
|
|
22
|
+
export declare function viewpointLedger(viewpointPath: string, scenarios: ScenarioInfo[], featureText: string): LedgerResult;
|
|
23
|
+
//# sourceMappingURL=viewpoint-ledger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"viewpoint-ledger.d.ts","sourceRoot":"","sources":["../../src/harness/viewpoint-ledger.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,MAAM,WAAW,UAAU;IAAG,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE;AAE3E,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,OAAO,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC1C;AAKD,8EAA8E;AAC9E,wBAAgB,mBAAmB,CAAC,aAAa,EAAE,MAAM,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,CA4B1F;AAED,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,WAAW,EAAE,MAAM,GAAG,YAAY,CAsBnH"}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.parseViewpointItems = parseViewpointItems;
|
|
37
|
+
exports.viewpointLedger = viewpointLedger;
|
|
38
|
+
/**
|
|
39
|
+
* Viewpoint Atomic Coverage Ledger (harness #2).
|
|
40
|
+
*
|
|
41
|
+
* The project's `test-viewpoint.md` IS the coverage contract. This parses it into ATOMIC
|
|
42
|
+
* items (each bullet / table row / ID-prefixed line) and reports the status of EACH —
|
|
43
|
+
* covered / missing — instead of the coarse "viewpoint mentioned" signal. It is fully
|
|
44
|
+
* project-driven (works on any project's viewpoint file, any domain), which is why it
|
|
45
|
+
* scales where a hardcoded domain catalog does not. Advisory: it surfaces the per-item
|
|
46
|
+
* gaps that inflate a "looks-covered" score; it does not fail the gate.
|
|
47
|
+
*/
|
|
48
|
+
const fs = __importStar(require("fs"));
|
|
49
|
+
const ID_RE = /\b([A-Z]{1,5}\d{0,2}(?:[.\-][A-Za-z0-9]+)*-?\d{0,3})\b/; // VP0.Title, VP7-002, MS-HP-001, TV-01
|
|
50
|
+
const GENERIC = new Set(['display', 'shown', 'value', 'field', 'input', 'page', 'screen', 'button', 'link', 'text', 'check', 'verify', 'should', 'with', 'when', 'then', 'user', 'this', 'that', 'each', 'item', 'items']);
|
|
51
|
+
/** Extract atomic checklist items from a viewpoint file (format-tolerant). */
|
|
52
|
+
function parseViewpointItems(viewpointPath) {
|
|
53
|
+
if (!fs.existsSync(viewpointPath))
|
|
54
|
+
return [];
|
|
55
|
+
const lines = fs.readFileSync(viewpointPath, 'utf-8').split('\n');
|
|
56
|
+
const items = [];
|
|
57
|
+
let inFence = false;
|
|
58
|
+
for (const raw of lines) {
|
|
59
|
+
const line = raw.trim();
|
|
60
|
+
if (line.startsWith('```')) {
|
|
61
|
+
inFence = !inFence;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (inFence || !line)
|
|
65
|
+
continue;
|
|
66
|
+
if (/^#{1,6}\s/.test(line))
|
|
67
|
+
continue; // markdown heading
|
|
68
|
+
let text = '';
|
|
69
|
+
const bullet = line.match(/^(?:[-*+]|\d+[.)])\s+(.*)$/);
|
|
70
|
+
if (bullet)
|
|
71
|
+
text = bullet[1];
|
|
72
|
+
else if (line.startsWith('|')) { // table data row
|
|
73
|
+
if (/^\|[\s|:-]+\|?$/.test(line))
|
|
74
|
+
continue; // separator
|
|
75
|
+
const cells = line.split('|').map((c) => c.trim()).filter(Boolean);
|
|
76
|
+
if (/^(vp|id|viewpoint|priority|reason|no\.?|category|item|trigger|#|pattern|applicable|notes|field|constraint|code|description|status)$/i.test(cells[0] || ''))
|
|
77
|
+
continue; // header
|
|
78
|
+
text = cells.join(' — ');
|
|
79
|
+
}
|
|
80
|
+
else
|
|
81
|
+
continue;
|
|
82
|
+
text = text.replace(/[*`]/g, '').trim();
|
|
83
|
+
if (!text)
|
|
84
|
+
continue;
|
|
85
|
+
const idM = text.match(ID_RE);
|
|
86
|
+
const id = idM && /\d/.test(idM[1]) ? idM[1] : undefined; // require a digit so prose words aren't IDs
|
|
87
|
+
const words = (text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w));
|
|
88
|
+
if (!id && words.length < 2)
|
|
89
|
+
continue; // not substantive enough to track
|
|
90
|
+
items.push({ id, text: text.slice(0, 100) });
|
|
91
|
+
}
|
|
92
|
+
return items;
|
|
93
|
+
}
|
|
94
|
+
function viewpointLedger(viewpointPath, scenarios, featureText) {
|
|
95
|
+
const items = parseViewpointItems(viewpointPath);
|
|
96
|
+
if (!fs.existsSync(viewpointPath) || items.length === 0) {
|
|
97
|
+
return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [] };
|
|
98
|
+
}
|
|
99
|
+
const featLower = featureText.toLowerCase();
|
|
100
|
+
const missing = [];
|
|
101
|
+
let covered = 0;
|
|
102
|
+
for (const item of items) {
|
|
103
|
+
let isCovered = false;
|
|
104
|
+
if (item.id && featLower.includes(item.id.toLowerCase()))
|
|
105
|
+
isCovered = true;
|
|
106
|
+
else {
|
|
107
|
+
const words = [...new Set((item.text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w)))];
|
|
108
|
+
const need = Math.min(2, words.length);
|
|
109
|
+
isCovered = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= need);
|
|
110
|
+
}
|
|
111
|
+
if (isCovered)
|
|
112
|
+
covered++;
|
|
113
|
+
else
|
|
114
|
+
missing.push({ id: item.id, text: item.text });
|
|
115
|
+
}
|
|
116
|
+
return { hasViewpoint: true, total: items.length, covered, ratio: items.length ? covered / items.length : 1, missing };
|
|
117
|
+
}
|
|
118
|
+
//# sourceMappingURL=viewpoint-ledger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"viewpoint-ledger.js","sourceRoot":"","sources":["../../src/harness/viewpoint-ledger.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BA,kDA4BC;AAED,0CAsBC;AA/ED;;;;;;;;;GASG;AACH,uCAAyB;AAazB,MAAM,KAAK,GAAG,wDAAwD,CAAC,CAAC,uCAAuC;AAC/G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE3N,8EAA8E;AAC9E,SAAgB,mBAAmB,CAAC,aAAqB;IACvD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,EAAE,CAAC;IAC7C,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClE,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,OAAO,GAAG,CAAC,OAAO,CAAC;YAAC,SAAS;QAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,CAAC,IAAI;YAAE,SAAS;QAC/B,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS,CAAiB,mBAAmB;QACzE,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACxD,IAAI,MAAM;YAAE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;aACxB,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAuB,iBAAiB;YACtE,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS,CAAW,YAAY;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,sIAAsI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAAE,SAAS,CAAC,SAAS;YACpL,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;;YAAM,SAAS;QAChB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,4CAA4C;QACtG,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,IAAI,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS,CAAiB,kCAAkC;QACzF,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAgB,eAAe,CAAC,aAAqB,EAAE,SAAyB,EAAE,WAAmB;IACnG,MAAM,KAAK,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAC;IACjD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,OAAO,EAAE,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACrG,CAAC;IACD,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAoC,EAAE,CAAC;IACpD,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC;aACtE,CAAC;YACJ,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACvC,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;QACpH,CAAC;QACD,IAAI,SAAS;YAAE,OAAO,EAAE,CAAC;;YACpB,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AACzH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sun-asterisk/sungen",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.82",
|
|
4
4
|
"description": "Deterministic E2E Test Compiler - Gherkin + Selectors → Playwright tests",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
"copy-templates": "mkdir -p dist/generators/test-generator/adapters/playwright/templates/steps && mkdir -p dist/generators/test-generator/templates && mkdir -p dist/orchestrator/templates && mkdir -p dist/dashboard/templates && cp -r src/generators/test-generator/adapters/playwright/templates/*.hbs dist/generators/test-generator/adapters/playwright/templates/ 2>/dev/null || true && cp -r src/generators/test-generator/adapters/playwright/templates/steps dist/generators/test-generator/adapters/playwright/templates/ && cp src/generators/test-generator/templates/*.hbs dist/generators/test-generator/templates/ 2>/dev/null || true && cp -r src/orchestrator/templates/* dist/orchestrator/templates/ && cp src/dashboard/templates/index.html dist/dashboard/templates/index.html && mkdir -p dist/harness/catalog && cp src/harness/catalog/*.yaml dist/harness/catalog/",
|
|
13
13
|
"build:dashboard": "cd dashboard && npm install --silent && npm run build && cd .. && cp dashboard/dist/index.html src/dashboard/templates/index.html",
|
|
14
14
|
"dev": "tsx src/cli/index.ts",
|
|
15
|
-
"test": "tsx tests/golden/run.ts && tsx tests/audit/run.ts && tsx tests/ingest/run.ts",
|
|
15
|
+
"test": "tsx tests/golden/run.ts && tsx tests/audit/run.ts && tsx tests/ingest/run.ts && tsx tests/eval/run.ts",
|
|
16
16
|
"test:update": "tsx tests/golden/run.ts --update && tsx tests/audit/run.ts --update && tsx tests/ingest/run.ts --update",
|
|
17
17
|
"prepublishOnly": "npm run build:dashboard && npm run build"
|
|
18
18
|
},
|
|
@@ -65,6 +65,12 @@ function render(r: AuditReport): void {
|
|
|
65
65
|
if (!r.spec.triggerGaps.length && !r.spec.uncoveredMust.length) L(' ✓ every MUST FR + per-constraint trigger covered');
|
|
66
66
|
L('');
|
|
67
67
|
}
|
|
68
|
+
if (r.ledger.hasViewpoint && r.ledger.total > 0) {
|
|
69
|
+
L(` ⑧ Viewpoint atomic coverage — ${r.ledger.covered}/${r.ledger.total} items (${(r.ledger.ratio * 100).toFixed(0)}%)`);
|
|
70
|
+
for (const m of r.ledger.missing.slice(0, 8)) L(` ○ missing: ${m.id ? m.id + ' — ' : ''}${m.text.slice(0, 70)}`);
|
|
71
|
+
if (r.ledger.missing.length > 8) L(` … +${r.ledger.missing.length - 8} more`);
|
|
72
|
+
L('');
|
|
73
|
+
}
|
|
68
74
|
L(' ── Findings (Repair targets) ──');
|
|
69
75
|
if (r.findings.length === 0) L(' ✓ none — output passes the harness');
|
|
70
76
|
for (const f of r.findings) L(` • ${f}`);
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
import { lintSkills, defaultSkillDir } from '../../harness/eval/skill-lint';
|
|
3
|
+
|
|
4
|
+
export function registerEvalCommand(program: Command): void {
|
|
5
|
+
program
|
|
6
|
+
.command('eval')
|
|
7
|
+
.description('Eval harness: quality checks on Sungen\'s own skills/instructions (dev/CI)')
|
|
8
|
+
.option('--skills', 'Static skill-lint: frontmatter, line budget, claude↔github sync, registration')
|
|
9
|
+
.option('--dir <path>', 'Templates dir to lint (default: bundled ai-instructions)')
|
|
10
|
+
.option('--json', 'Output the raw findings JSON')
|
|
11
|
+
.action((options) => {
|
|
12
|
+
try {
|
|
13
|
+
if (!options.skills) throw new Error('Provide --skills (the only eval mode today)');
|
|
14
|
+
const dir = options.dir || defaultSkillDir();
|
|
15
|
+
const r = lintSkills(dir);
|
|
16
|
+
if (options.json) { console.log(JSON.stringify(r, null, 2)); process.exit(r.errors > 0 ? 2 : 0); }
|
|
17
|
+
console.log('');
|
|
18
|
+
console.log(`━━━ Skill-lint: ${r.checked} skill template(s) ━━━`);
|
|
19
|
+
if (!r.findings.length) console.log(' ✓ all skills pass (frontmatter · line-budget · variant-sync · registration)');
|
|
20
|
+
for (const f of r.findings) console.log(` ${f.level === 'error' ? '✗' : '⚠'} [${f.rule}] ${f.file} — ${f.detail}`);
|
|
21
|
+
console.log('');
|
|
22
|
+
process.exit(r.errors > 0 ? 2 : 0);
|
|
23
|
+
} catch (error) {
|
|
24
|
+
console.error('Error:', error instanceof Error ? error.message : error);
|
|
25
|
+
process.exit(1);
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
}
|
package/src/cli/index.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { registerAddFlowCommand } from './commands/add-flow';
|
|
|
16
16
|
import { registerDashboardCommand } from './commands/dashboard';
|
|
17
17
|
import { registerAuditCommand } from './commands/audit';
|
|
18
18
|
import { registerIngestCommand } from './commands/ingest';
|
|
19
|
+
import { registerEvalCommand } from './commands/eval';
|
|
19
20
|
import { registerManifestCommand } from './commands/manifest';
|
|
20
21
|
import { registerLedgerCommand } from './commands/ledger';
|
|
21
22
|
import { registerFeedbackCommand } from './commands/feedback';
|
|
@@ -62,6 +63,7 @@ async function main() {
|
|
|
62
63
|
registerCapabilityCommand(program);
|
|
63
64
|
registerFlowCheckCommand(program);
|
|
64
65
|
registerIngestCommand(program);
|
|
66
|
+
registerEvalCommand(program);
|
|
65
67
|
|
|
66
68
|
await program.parseAsync(process.argv);
|
|
67
69
|
}
|
package/src/harness/audit.ts
CHANGED
|
@@ -17,6 +17,7 @@ import { readIntent, projectRootFromScreenDir, IntentProfile } from './intent';
|
|
|
17
17
|
import { getProvenance, Provenance } from './provenance';
|
|
18
18
|
import { specCoverage, SpecCoverageResult } from './spec-coverage';
|
|
19
19
|
import { downstreamScope, manualOracle, readText, DownstreamResult, ManualOracleResult } from './quality-gates';
|
|
20
|
+
import { viewpointLedger, LedgerResult } from './viewpoint-ledger';
|
|
20
21
|
|
|
21
22
|
export interface AuditReport {
|
|
22
23
|
screen: string;
|
|
@@ -28,9 +29,10 @@ export interface AuditReport {
|
|
|
28
29
|
balance: BalanceResult;
|
|
29
30
|
duplicates: DuplicateResult;
|
|
30
31
|
trace: TraceResult;
|
|
31
|
-
taxonomyMismatch: boolean; //
|
|
32
|
-
downstream: DownstreamResult; //
|
|
33
|
-
manualOracle: ManualOracleResult; //
|
|
32
|
+
taxonomyMismatch: boolean; // scenarios use IDs not in the project's test-viewpoint.md
|
|
33
|
+
downstream: DownstreamResult; // downstream screens referenced but under-covered
|
|
34
|
+
manualOracle: ManualOracleResult; // @manual scenarios lacking setup/action/oracle
|
|
35
|
+
ledger: LedgerResult; // atomic viewpoint-item coverage (per-bullet status)
|
|
34
36
|
score: {
|
|
35
37
|
overall: number; // 0..10, business-weighted
|
|
36
38
|
coverage: number; // 0..1
|
|
@@ -72,6 +74,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
72
74
|
// #2 downstream-scope + #4 manual-oracle
|
|
73
75
|
const downstream = downstreamScope(readText(specPath), scenarios);
|
|
74
76
|
const manualOracleResult = manualOracle(featureText);
|
|
77
|
+
const ledger = viewpointLedger(viewpointPath, scenarios, featureText);
|
|
75
78
|
|
|
76
79
|
// Sub-scores
|
|
77
80
|
const coverage = gate.coverageRatio;
|
|
@@ -131,6 +134,10 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
131
134
|
for (const m of manualOracleResult.insufficient.slice(0, 8)) {
|
|
132
135
|
findings.push(`MANUAL-STEPS-INSUFFICIENT: "${m}" — a @manual scenario needs setup · action · observable expected · oracle/tool (not just a one-line note).`);
|
|
133
136
|
}
|
|
137
|
+
if (ledger.hasViewpoint && ledger.missing.length) {
|
|
138
|
+
const sample = ledger.missing.slice(0, 6).map((m) => m.id || `"${m.text}"`).join(', ');
|
|
139
|
+
findings.push(`VIEWPOINT-ITEM-MISSING: ${ledger.missing.length}/${ledger.total} atomic viewpoint items have no covering scenario (${(ledger.ratio * 100).toFixed(0)}% covered) — e.g. ${sample}. Cover each item or mark it deferred/spec-gap.`);
|
|
140
|
+
}
|
|
134
141
|
|
|
135
142
|
// Gate spans coverage (viewpoint themes), depth, claim-proof, spec-clause coverage,
|
|
136
143
|
// AND taxonomy-match (scenarios must use the project's viewpoint IDs when defined).
|
|
@@ -141,7 +148,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
141
148
|
screen: screenName,
|
|
142
149
|
scenarioCount: scenarios.length,
|
|
143
150
|
gate, depth, claim, taxonomy, balance, duplicates, trace, spec,
|
|
144
|
-
taxonomyMismatch, downstream, manualOracle: manualOracleResult,
|
|
151
|
+
taxonomyMismatch, downstream, manualOracle: manualOracleResult, ledger,
|
|
145
152
|
score: {
|
|
146
153
|
overall: Math.round(overall * 10) / 10,
|
|
147
154
|
coverage: Math.round(coverage * 100) / 100,
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Static skill-lint (Eval Harness L1) — deterministic quality checks on Sungen's OWN
|
|
3
|
+
* AI-instruction templates, so a broken / unregistered / oversized skill fails before it
|
|
4
|
+
* ships. Learned (generically) from the "static validations" tier of an agent-kit evals
|
|
5
|
+
* layer. No project data — this lints the sungen package's own templates.
|
|
6
|
+
*
|
|
7
|
+
* Design note: the checks are MAPPING-DRIVEN. `AI_RULES_FILE_MAPPING` is the source of
|
|
8
|
+
* truth for what each template installs as, so the lint uses the install target (does it
|
|
9
|
+
* end in `/SKILL.md`?) to tell a top-level skill from a sub-content fragment — instead of
|
|
10
|
+
* guessing from filenames. We deliberately do NOT enforce claude↔github body parity: the
|
|
11
|
+
* two variants are hand-tuned per platform and intentionally diverge in wording and even
|
|
12
|
+
* structure, so byte/heading equality would be pure false positives.
|
|
13
|
+
*/
|
|
14
|
+
import * as fs from 'fs';
|
|
15
|
+
import * as path from 'path';
|
|
16
|
+
import { AI_RULES_FILE_MAPPING } from '../../orchestrator/ai-rules-updater';
|
|
17
|
+
|
|
18
|
+
export interface SkillLintFinding { level: 'error' | 'warn'; file: string; rule: string; detail: string }
|
|
19
|
+
export interface SkillLintResult { checked: number; findings: SkillLintFinding[]; errors: number }
|
|
20
|
+
|
|
21
|
+
const LINE_BUDGET = 700; // a skill much larger than this is a context-cost smell (warn)
|
|
22
|
+
const SKILL_RE = /^(claude|github)-skill-/;
|
|
23
|
+
|
|
24
|
+
function stripFrontmatter(text: string): { fm: string | null; body: string } {
|
|
25
|
+
const m = text.match(/^---\n([\s\S]*?)\n---\n?/);
|
|
26
|
+
if (!m) return { fm: null, body: text };
|
|
27
|
+
return { fm: m[1], body: text.slice(m[0].length) };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Lint the AI-instruction templates in `dir` (default: the sungen source templates). */
|
|
31
|
+
export function lintSkills(dir: string): SkillLintResult {
|
|
32
|
+
const findings: SkillLintFinding[] = [];
|
|
33
|
+
const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith('.md')) : [];
|
|
34
|
+
const skillFiles = files.filter((f) => SKILL_RE.test(f));
|
|
35
|
+
|
|
36
|
+
// mapping: template file -> install target (source of truth for "is this a top-level skill")
|
|
37
|
+
const target = new Map<string, string>(AI_RULES_FILE_MAPPING.map(([tpl, dst]) => [tpl, dst]));
|
|
38
|
+
const isTopLevelSkill = (f: string) => (target.get(f) || '').endsWith('/SKILL.md');
|
|
39
|
+
|
|
40
|
+
// 1) registration integrity (bidirectional) — the highest-value check:
|
|
41
|
+
// a skill file missing from the mapping never installs; a mapping to a missing file
|
|
42
|
+
// ships a broken/empty skill.
|
|
43
|
+
for (const f of skillFiles) {
|
|
44
|
+
if (!target.has(f)) findings.push({ level: 'error', file: f, rule: 'unregistered', detail: 'skill template not in AI_RULES_FILE_MAPPING (it would never be installed)' });
|
|
45
|
+
}
|
|
46
|
+
for (const [tpl] of AI_RULES_FILE_MAPPING) {
|
|
47
|
+
if (!fs.existsSync(path.join(dir, tpl))) findings.push({ level: 'error', file: tpl, rule: 'mapped-missing', detail: 'AI_RULES_FILE_MAPPING points to a template that does not exist' });
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// 2) frontmatter (name + description) — ONLY for top-level skills (SKILL.md targets).
|
|
51
|
+
// Sub-content fragments (mode-*.md, group-*.md) are loaded by their parent router
|
|
52
|
+
// and legitimately carry no frontmatter.
|
|
53
|
+
for (const f of skillFiles) {
|
|
54
|
+
if (!isTopLevelSkill(f)) continue;
|
|
55
|
+
const text = fs.readFileSync(path.join(dir, f), 'utf8');
|
|
56
|
+
const { fm } = stripFrontmatter(text);
|
|
57
|
+
if (!fm) { findings.push({ level: 'error', file: f, rule: 'frontmatter', detail: 'top-level skill (SKILL.md) is missing --- frontmatter --- (Claude/Copilot will not load it)' }); continue; }
|
|
58
|
+
if (!/\bname\s*:/.test(fm)) findings.push({ level: 'error', file: f, rule: 'frontmatter-name', detail: 'no `name:` in frontmatter' });
|
|
59
|
+
if (!/\bdescription\s*:/.test(fm)) findings.push({ level: 'error', file: f, rule: 'frontmatter-description', detail: 'no `description:` in frontmatter' });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// 3) line budget — context-cost smell (advisory).
|
|
63
|
+
for (const f of skillFiles) {
|
|
64
|
+
const lines = fs.readFileSync(path.join(dir, f), 'utf8').split('\n').length;
|
|
65
|
+
if (lines > LINE_BUDGET) findings.push({ level: 'warn', file: f, rule: 'line-budget', detail: `${lines} lines > ${LINE_BUDGET} (context-cost smell)` });
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// 4) variant PRESENCE (not body equality) — every top-level skill should ship for both
|
|
69
|
+
// platforms. Catches "added a Claude skill but forgot the Copilot variant". Advisory.
|
|
70
|
+
const skillName = (dst: string) => { const m = dst.match(/\/(sungen-[^/]+)\/SKILL\.md$/); return m ? m[1] : null; };
|
|
71
|
+
const claudeSkills = new Set<string>(), githubSkills = new Set<string>();
|
|
72
|
+
for (const f of skillFiles) {
|
|
73
|
+
if (!isTopLevelSkill(f)) continue;
|
|
74
|
+
const name = skillName(target.get(f)!); if (!name) continue;
|
|
75
|
+
(f.startsWith('claude-') ? claudeSkills : githubSkills).add(name);
|
|
76
|
+
}
|
|
77
|
+
for (const n of claudeSkills) if (!githubSkills.has(n)) findings.push({ level: 'warn', file: `claude .../${n}/SKILL.md`, rule: 'variant-missing', detail: `Claude skill "${n}" has no GitHub (Copilot) variant` });
|
|
78
|
+
for (const n of githubSkills) if (!claudeSkills.has(n)) findings.push({ level: 'warn', file: `github .../${n}/SKILL.md`, rule: 'variant-missing', detail: `GitHub skill "${n}" has no Claude variant` });
|
|
79
|
+
|
|
80
|
+
return { checked: skillFiles.length, findings, errors: findings.filter((f) => f.level === 'error').length };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Default templates dir, resolved relative to this module (works from src via tsx and dist). */
|
|
84
|
+
export function defaultSkillDir(): string {
|
|
85
|
+
// src/harness/eval → src/orchestrator/... | dist/harness/eval → dist/orchestrator/...
|
|
86
|
+
return path.resolve(__dirname, '..', '..', 'orchestrator', 'templates', 'ai-instructions');
|
|
87
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Viewpoint Atomic Coverage Ledger (harness #2).
|
|
3
|
+
*
|
|
4
|
+
* The project's `test-viewpoint.md` IS the coverage contract. This parses it into ATOMIC
|
|
5
|
+
* items (each bullet / table row / ID-prefixed line) and reports the status of EACH —
|
|
6
|
+
* covered / missing — instead of the coarse "viewpoint mentioned" signal. It is fully
|
|
7
|
+
* project-driven (works on any project's viewpoint file, any domain), which is why it
|
|
8
|
+
* scales where a hardcoded domain catalog does not. Advisory: it surfaces the per-item
|
|
9
|
+
* gaps that inflate a "looks-covered" score; it does not fail the gate.
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from 'fs';
|
|
12
|
+
import { ScenarioInfo } from './parse';
|
|
13
|
+
|
|
14
|
+
export interface LedgerItem { id?: string; text: string; covered: boolean }
|
|
15
|
+
|
|
16
|
+
export interface LedgerResult {
|
|
17
|
+
hasViewpoint: boolean;
|
|
18
|
+
total: number;
|
|
19
|
+
covered: number;
|
|
20
|
+
ratio: number;
|
|
21
|
+
missing: { id?: string; text: string }[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const ID_RE = /\b([A-Z]{1,5}\d{0,2}(?:[.\-][A-Za-z0-9]+)*-?\d{0,3})\b/; // VP0.Title, VP7-002, MS-HP-001, TV-01
|
|
25
|
+
const GENERIC = new Set(['display', 'shown', 'value', 'field', 'input', 'page', 'screen', 'button', 'link', 'text', 'check', 'verify', 'should', 'with', 'when', 'then', 'user', 'this', 'that', 'each', 'item', 'items']);
|
|
26
|
+
|
|
27
|
+
/** Extract atomic checklist items from a viewpoint file (format-tolerant). */
|
|
28
|
+
export function parseViewpointItems(viewpointPath: string): { id?: string; text: string }[] {
|
|
29
|
+
if (!fs.existsSync(viewpointPath)) return [];
|
|
30
|
+
const lines = fs.readFileSync(viewpointPath, 'utf-8').split('\n');
|
|
31
|
+
const items: { id?: string; text: string }[] = [];
|
|
32
|
+
let inFence = false;
|
|
33
|
+
for (const raw of lines) {
|
|
34
|
+
const line = raw.trim();
|
|
35
|
+
if (line.startsWith('```')) { inFence = !inFence; continue; }
|
|
36
|
+
if (inFence || !line) continue;
|
|
37
|
+
if (/^#{1,6}\s/.test(line)) continue; // markdown heading
|
|
38
|
+
let text = '';
|
|
39
|
+
const bullet = line.match(/^(?:[-*+]|\d+[.)])\s+(.*)$/);
|
|
40
|
+
if (bullet) text = bullet[1];
|
|
41
|
+
else if (line.startsWith('|')) { // table data row
|
|
42
|
+
if (/^\|[\s|:-]+\|?$/.test(line)) continue; // separator
|
|
43
|
+
const cells = line.split('|').map((c) => c.trim()).filter(Boolean);
|
|
44
|
+
if (/^(vp|id|viewpoint|priority|reason|no\.?|category|item|trigger|#|pattern|applicable|notes|field|constraint|code|description|status)$/i.test(cells[0] || '')) continue; // header
|
|
45
|
+
text = cells.join(' — ');
|
|
46
|
+
} else continue;
|
|
47
|
+
text = text.replace(/[*`]/g, '').trim();
|
|
48
|
+
if (!text) continue;
|
|
49
|
+
const idM = text.match(ID_RE);
|
|
50
|
+
const id = idM && /\d/.test(idM[1]) ? idM[1] : undefined; // require a digit so prose words aren't IDs
|
|
51
|
+
const words = (text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w));
|
|
52
|
+
if (!id && words.length < 2) continue; // not substantive enough to track
|
|
53
|
+
items.push({ id, text: text.slice(0, 100) });
|
|
54
|
+
}
|
|
55
|
+
return items;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function viewpointLedger(viewpointPath: string, scenarios: ScenarioInfo[], featureText: string): LedgerResult {
|
|
59
|
+
const items = parseViewpointItems(viewpointPath);
|
|
60
|
+
if (!fs.existsSync(viewpointPath) || items.length === 0) {
|
|
61
|
+
return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [] };
|
|
62
|
+
}
|
|
63
|
+
const featLower = featureText.toLowerCase();
|
|
64
|
+
const missing: { id?: string; text: string }[] = [];
|
|
65
|
+
let covered = 0;
|
|
66
|
+
|
|
67
|
+
for (const item of items) {
|
|
68
|
+
let isCovered = false;
|
|
69
|
+
if (item.id && featLower.includes(item.id.toLowerCase())) isCovered = true;
|
|
70
|
+
else {
|
|
71
|
+
const words = [...new Set((item.text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w)))];
|
|
72
|
+
const need = Math.min(2, words.length);
|
|
73
|
+
isCovered = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= need);
|
|
74
|
+
}
|
|
75
|
+
if (isCovered) covered++;
|
|
76
|
+
else missing.push({ id: item.id, text: item.text });
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return { hasViewpoint: true, total: items.length, covered, ratio: items.length ? covered / items.length : 1, missing };
|
|
80
|
+
}
|