@sun-asterisk/sungen 3.0.0-beta.84 → 3.0.0-beta.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +0 -14
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/index.js +0 -2
- package/dist/cli/index.js.map +1 -1
- package/dist/harness/audit.d.ts +0 -14
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +3 -56
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/parse.d.ts +0 -6
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +3 -18
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +6 -85
- package/dist/harness/sensors.js.map +1 -1
- package/dist/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +0 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +1 -25
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +7 -44
- package/package.json +2 -2
- package/src/cli/commands/audit.ts +0 -12
- package/src/cli/index.ts +0 -2
- package/src/harness/audit.ts +4 -68
- package/src/harness/parse.ts +3 -19
- package/src/harness/sensors.ts +7 -84
- package/src/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +0 -1
- package/src/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +1 -25
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +7 -44
- package/dist/cli/commands/eval.d.ts +0 -3
- package/dist/cli/commands/eval.d.ts.map +0 -1
- package/dist/cli/commands/eval.js +0 -37
- package/dist/cli/commands/eval.js.map +0 -1
- package/dist/harness/eval/skill-lint.d.ts +0 -16
- package/dist/harness/eval/skill-lint.d.ts.map +0 -1
- package/dist/harness/eval/skill-lint.js +0 -129
- package/dist/harness/eval/skill-lint.js.map +0 -1
- package/dist/harness/quality-gates.d.ts +0 -29
- package/dist/harness/quality-gates.d.ts.map +0 -1
- package/dist/harness/quality-gates.js +0 -183
- package/dist/harness/quality-gates.js.map +0 -1
- package/dist/harness/viewpoint-ledger.d.ts +0 -23
- package/dist/harness/viewpoint-ledger.d.ts.map +0 -1
- package/dist/harness/viewpoint-ledger.js +0 -118
- package/dist/harness/viewpoint-ledger.js.map +0 -1
- package/src/cli/commands/eval.ts +0 -28
- package/src/harness/eval/skill-lint.ts +0 -87
- package/src/harness/quality-gates.ts +0 -152
- package/src/harness/viewpoint-ledger.ts +0 -80
|
@@ -1,118 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.parseViewpointItems = parseViewpointItems;
|
|
37
|
-
exports.viewpointLedger = viewpointLedger;
|
|
38
|
-
/**
|
|
39
|
-
* Viewpoint Atomic Coverage Ledger (harness #2).
|
|
40
|
-
*
|
|
41
|
-
* The project's `test-viewpoint.md` IS the coverage contract. This parses it into ATOMIC
|
|
42
|
-
* items (each bullet / table row / ID-prefixed line) and reports the status of EACH —
|
|
43
|
-
* covered / missing — instead of the coarse "viewpoint mentioned" signal. It is fully
|
|
44
|
-
* project-driven (works on any project's viewpoint file, any domain), which is why it
|
|
45
|
-
* scales where a hardcoded domain catalog does not. Advisory: it surfaces the per-item
|
|
46
|
-
* gaps that inflate a "looks-covered" score; it does not fail the gate.
|
|
47
|
-
*/
|
|
48
|
-
const fs = __importStar(require("fs"));
|
|
49
|
-
const ID_RE = /\b([A-Z]{1,5}\d{0,2}(?:[.\-][A-Za-z0-9]+)*-?\d{0,3})\b/; // VP0.Title, VP7-002, MS-HP-001, TV-01
|
|
50
|
-
const GENERIC = new Set(['display', 'shown', 'value', 'field', 'input', 'page', 'screen', 'button', 'link', 'text', 'check', 'verify', 'should', 'with', 'when', 'then', 'user', 'this', 'that', 'each', 'item', 'items']);
|
|
51
|
-
/** Extract atomic checklist items from a viewpoint file (format-tolerant). */
|
|
52
|
-
function parseViewpointItems(viewpointPath) {
|
|
53
|
-
if (!fs.existsSync(viewpointPath))
|
|
54
|
-
return [];
|
|
55
|
-
const lines = fs.readFileSync(viewpointPath, 'utf-8').split('\n');
|
|
56
|
-
const items = [];
|
|
57
|
-
let inFence = false;
|
|
58
|
-
for (const raw of lines) {
|
|
59
|
-
const line = raw.trim();
|
|
60
|
-
if (line.startsWith('```')) {
|
|
61
|
-
inFence = !inFence;
|
|
62
|
-
continue;
|
|
63
|
-
}
|
|
64
|
-
if (inFence || !line)
|
|
65
|
-
continue;
|
|
66
|
-
if (/^#{1,6}\s/.test(line))
|
|
67
|
-
continue; // markdown heading
|
|
68
|
-
let text = '';
|
|
69
|
-
const bullet = line.match(/^(?:[-*+]|\d+[.)])\s+(.*)$/);
|
|
70
|
-
if (bullet)
|
|
71
|
-
text = bullet[1];
|
|
72
|
-
else if (line.startsWith('|')) { // table data row
|
|
73
|
-
if (/^\|[\s|:-]+\|?$/.test(line))
|
|
74
|
-
continue; // separator
|
|
75
|
-
const cells = line.split('|').map((c) => c.trim()).filter(Boolean);
|
|
76
|
-
if (/^(vp|id|viewpoint|priority|reason|no\.?|category|item|trigger|#|pattern|applicable|notes|field|constraint|code|description|status)$/i.test(cells[0] || ''))
|
|
77
|
-
continue; // header
|
|
78
|
-
text = cells.join(' — ');
|
|
79
|
-
}
|
|
80
|
-
else
|
|
81
|
-
continue;
|
|
82
|
-
text = text.replace(/[*`]/g, '').trim();
|
|
83
|
-
if (!text)
|
|
84
|
-
continue;
|
|
85
|
-
const idM = text.match(ID_RE);
|
|
86
|
-
const id = idM && /\d/.test(idM[1]) ? idM[1] : undefined; // require a digit so prose words aren't IDs
|
|
87
|
-
const words = (text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w));
|
|
88
|
-
if (!id && words.length < 2)
|
|
89
|
-
continue; // not substantive enough to track
|
|
90
|
-
items.push({ id, text: text.slice(0, 100) });
|
|
91
|
-
}
|
|
92
|
-
return items;
|
|
93
|
-
}
|
|
94
|
-
function viewpointLedger(viewpointPath, scenarios, featureText) {
|
|
95
|
-
const items = parseViewpointItems(viewpointPath);
|
|
96
|
-
if (!fs.existsSync(viewpointPath) || items.length === 0) {
|
|
97
|
-
return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [] };
|
|
98
|
-
}
|
|
99
|
-
const featLower = featureText.toLowerCase();
|
|
100
|
-
const missing = [];
|
|
101
|
-
let covered = 0;
|
|
102
|
-
for (const item of items) {
|
|
103
|
-
let isCovered = false;
|
|
104
|
-
if (item.id && featLower.includes(item.id.toLowerCase()))
|
|
105
|
-
isCovered = true;
|
|
106
|
-
else {
|
|
107
|
-
const words = [...new Set((item.text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w)))];
|
|
108
|
-
const need = Math.min(2, words.length);
|
|
109
|
-
isCovered = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= need);
|
|
110
|
-
}
|
|
111
|
-
if (isCovered)
|
|
112
|
-
covered++;
|
|
113
|
-
else
|
|
114
|
-
missing.push({ id: item.id, text: item.text });
|
|
115
|
-
}
|
|
116
|
-
return { hasViewpoint: true, total: items.length, covered, ratio: items.length ? covered / items.length : 1, missing };
|
|
117
|
-
}
|
|
118
|
-
//# sourceMappingURL=viewpoint-ledger.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"viewpoint-ledger.js","sourceRoot":"","sources":["../../src/harness/viewpoint-ledger.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BA,kDA4BC;AAED,0CAsBC;AA/ED;;;;;;;;;GASG;AACH,uCAAyB;AAazB,MAAM,KAAK,GAAG,wDAAwD,CAAC,CAAC,uCAAuC;AAC/G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE3N,8EAA8E;AAC9E,SAAgB,mBAAmB,CAAC,aAAqB;IACvD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,EAAE,CAAC;IAC7C,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClE,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,OAAO,GAAG,CAAC,OAAO,CAAC;YAAC,SAAS;QAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,CAAC,IAAI;YAAE,SAAS;QAC/B,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS,CAAiB,mBAAmB;QACzE,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACxD,IAAI,MAAM;YAAE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;aACxB,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAuB,iBAAiB;YACtE,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS,CAAW,YAAY;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,sIAAsI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAAE,SAAS,CAAC,SAAS;YACpL,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;;YAAM,SAAS;QAChB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,4CAA4C;QACtG,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,IAAI,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS,CAAiB,kCAAkC;QACzF,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAgB,eAAe,CAAC,aAAqB,EAAE,SAAyB,EAAE,WAAmB;IACnG,MAAM,KAAK,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAC;IACjD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,OAAO,EAAE,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACrG,CAAC;IACD,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAoC,EAAE,CAAC;IACpD,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC;aACtE,CAAC;YACJ,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACvC,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;QACpH,CAAC;QACD,IAAI,SAAS;YAAE,OAAO,EAAE,CAAC;;YACpB,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AACzH,CAAC"}
|
package/src/cli/commands/eval.ts
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { Command } from 'commander';
|
|
2
|
-
import { lintSkills, defaultSkillDir } from '../../harness/eval/skill-lint';
|
|
3
|
-
|
|
4
|
-
export function registerEvalCommand(program: Command): void {
|
|
5
|
-
program
|
|
6
|
-
.command('eval')
|
|
7
|
-
.description('Eval harness: quality checks on Sungen\'s own skills/instructions (dev/CI)')
|
|
8
|
-
.option('--skills', 'Static skill-lint: frontmatter, line budget, claude↔github sync, registration')
|
|
9
|
-
.option('--dir <path>', 'Templates dir to lint (default: bundled ai-instructions)')
|
|
10
|
-
.option('--json', 'Output the raw findings JSON')
|
|
11
|
-
.action((options) => {
|
|
12
|
-
try {
|
|
13
|
-
if (!options.skills) throw new Error('Provide --skills (the only eval mode today)');
|
|
14
|
-
const dir = options.dir || defaultSkillDir();
|
|
15
|
-
const r = lintSkills(dir);
|
|
16
|
-
if (options.json) { console.log(JSON.stringify(r, null, 2)); process.exit(r.errors > 0 ? 2 : 0); }
|
|
17
|
-
console.log('');
|
|
18
|
-
console.log(`━━━ Skill-lint: ${r.checked} skill template(s) ━━━`);
|
|
19
|
-
if (!r.findings.length) console.log(' ✓ all skills pass (frontmatter · line-budget · variant-sync · registration)');
|
|
20
|
-
for (const f of r.findings) console.log(` ${f.level === 'error' ? '✗' : '⚠'} [${f.rule}] ${f.file} — ${f.detail}`);
|
|
21
|
-
console.log('');
|
|
22
|
-
process.exit(r.errors > 0 ? 2 : 0);
|
|
23
|
-
} catch (error) {
|
|
24
|
-
console.error('Error:', error instanceof Error ? error.message : error);
|
|
25
|
-
process.exit(1);
|
|
26
|
-
}
|
|
27
|
-
});
|
|
28
|
-
}
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Static skill-lint (Eval Harness L1) — deterministic quality checks on Sungen's OWN
|
|
3
|
-
* AI-instruction templates, so a broken / unregistered / oversized skill fails before it
|
|
4
|
-
* ships. Learned (generically) from the "static validations" tier of an agent-kit evals
|
|
5
|
-
* layer. No project data — this lints the sungen package's own templates.
|
|
6
|
-
*
|
|
7
|
-
* Design note: the checks are MAPPING-DRIVEN. `AI_RULES_FILE_MAPPING` is the source of
|
|
8
|
-
* truth for what each template installs as, so the lint uses the install target (does it
|
|
9
|
-
* end in `/SKILL.md`?) to tell a top-level skill from a sub-content fragment — instead of
|
|
10
|
-
* guessing from filenames. We deliberately do NOT enforce claude↔github body parity: the
|
|
11
|
-
* two variants are hand-tuned per platform and intentionally diverge in wording and even
|
|
12
|
-
* structure, so byte/heading equality would be pure false positives.
|
|
13
|
-
*/
|
|
14
|
-
import * as fs from 'fs';
|
|
15
|
-
import * as path from 'path';
|
|
16
|
-
import { AI_RULES_FILE_MAPPING } from '../../orchestrator/ai-rules-updater';
|
|
17
|
-
|
|
18
|
-
export interface SkillLintFinding { level: 'error' | 'warn'; file: string; rule: string; detail: string }
|
|
19
|
-
export interface SkillLintResult { checked: number; findings: SkillLintFinding[]; errors: number }
|
|
20
|
-
|
|
21
|
-
const LINE_BUDGET = 700; // a skill much larger than this is a context-cost smell (warn)
|
|
22
|
-
const SKILL_RE = /^(claude|github)-skill-/;
|
|
23
|
-
|
|
24
|
-
function stripFrontmatter(text: string): { fm: string | null; body: string } {
|
|
25
|
-
const m = text.match(/^---\n([\s\S]*?)\n---\n?/);
|
|
26
|
-
if (!m) return { fm: null, body: text };
|
|
27
|
-
return { fm: m[1], body: text.slice(m[0].length) };
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/** Lint the AI-instruction templates in `dir` (default: the sungen source templates). */
|
|
31
|
-
export function lintSkills(dir: string): SkillLintResult {
|
|
32
|
-
const findings: SkillLintFinding[] = [];
|
|
33
|
-
const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith('.md')) : [];
|
|
34
|
-
const skillFiles = files.filter((f) => SKILL_RE.test(f));
|
|
35
|
-
|
|
36
|
-
// mapping: template file -> install target (source of truth for "is this a top-level skill")
|
|
37
|
-
const target = new Map<string, string>(AI_RULES_FILE_MAPPING.map(([tpl, dst]) => [tpl, dst]));
|
|
38
|
-
const isTopLevelSkill = (f: string) => (target.get(f) || '').endsWith('/SKILL.md');
|
|
39
|
-
|
|
40
|
-
// 1) registration integrity (bidirectional) — the highest-value check:
|
|
41
|
-
// a skill file missing from the mapping never installs; a mapping to a missing file
|
|
42
|
-
// ships a broken/empty skill.
|
|
43
|
-
for (const f of skillFiles) {
|
|
44
|
-
if (!target.has(f)) findings.push({ level: 'error', file: f, rule: 'unregistered', detail: 'skill template not in AI_RULES_FILE_MAPPING (it would never be installed)' });
|
|
45
|
-
}
|
|
46
|
-
for (const [tpl] of AI_RULES_FILE_MAPPING) {
|
|
47
|
-
if (!fs.existsSync(path.join(dir, tpl))) findings.push({ level: 'error', file: tpl, rule: 'mapped-missing', detail: 'AI_RULES_FILE_MAPPING points to a template that does not exist' });
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
// 2) frontmatter (name + description) — ONLY for top-level skills (SKILL.md targets).
|
|
51
|
-
// Sub-content fragments (mode-*.md, group-*.md) are loaded by their parent router
|
|
52
|
-
// and legitimately carry no frontmatter.
|
|
53
|
-
for (const f of skillFiles) {
|
|
54
|
-
if (!isTopLevelSkill(f)) continue;
|
|
55
|
-
const text = fs.readFileSync(path.join(dir, f), 'utf8');
|
|
56
|
-
const { fm } = stripFrontmatter(text);
|
|
57
|
-
if (!fm) { findings.push({ level: 'error', file: f, rule: 'frontmatter', detail: 'top-level skill (SKILL.md) is missing --- frontmatter --- (Claude/Copilot will not load it)' }); continue; }
|
|
58
|
-
if (!/\bname\s*:/.test(fm)) findings.push({ level: 'error', file: f, rule: 'frontmatter-name', detail: 'no `name:` in frontmatter' });
|
|
59
|
-
if (!/\bdescription\s*:/.test(fm)) findings.push({ level: 'error', file: f, rule: 'frontmatter-description', detail: 'no `description:` in frontmatter' });
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// 3) line budget — context-cost smell (advisory).
|
|
63
|
-
for (const f of skillFiles) {
|
|
64
|
-
const lines = fs.readFileSync(path.join(dir, f), 'utf8').split('\n').length;
|
|
65
|
-
if (lines > LINE_BUDGET) findings.push({ level: 'warn', file: f, rule: 'line-budget', detail: `${lines} lines > ${LINE_BUDGET} (context-cost smell)` });
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// 4) variant PRESENCE (not body equality) — every top-level skill should ship for both
|
|
69
|
-
// platforms. Catches "added a Claude skill but forgot the Copilot variant". Advisory.
|
|
70
|
-
const skillName = (dst: string) => { const m = dst.match(/\/(sungen-[^/]+)\/SKILL\.md$/); return m ? m[1] : null; };
|
|
71
|
-
const claudeSkills = new Set<string>(), githubSkills = new Set<string>();
|
|
72
|
-
for (const f of skillFiles) {
|
|
73
|
-
if (!isTopLevelSkill(f)) continue;
|
|
74
|
-
const name = skillName(target.get(f)!); if (!name) continue;
|
|
75
|
-
(f.startsWith('claude-') ? claudeSkills : githubSkills).add(name);
|
|
76
|
-
}
|
|
77
|
-
for (const n of claudeSkills) if (!githubSkills.has(n)) findings.push({ level: 'warn', file: `claude .../${n}/SKILL.md`, rule: 'variant-missing', detail: `Claude skill "${n}" has no GitHub (Copilot) variant` });
|
|
78
|
-
for (const n of githubSkills) if (!claudeSkills.has(n)) findings.push({ level: 'warn', file: `github .../${n}/SKILL.md`, rule: 'variant-missing', detail: `GitHub skill "${n}" has no Claude variant` });
|
|
79
|
-
|
|
80
|
-
return { checked: skillFiles.length, findings, errors: findings.filter((f) => f.level === 'error').length };
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
/** Default templates dir, resolved relative to this module (works from src via tsx and dist). */
|
|
84
|
-
export function defaultSkillDir(): string {
|
|
85
|
-
// src/harness/eval → src/orchestrator/... | dist/harness/eval → dist/orchestrator/...
|
|
86
|
-
return path.resolve(__dirname, '..', '..', 'orchestrator', 'templates', 'ai-instructions');
|
|
87
|
-
}
|
|
@@ -1,152 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Quality gates (batch): downstream-scope + manual-oracle + negative-side-effect +
|
|
3
|
-
* cross-artifact ownership + source-backed strictness.
|
|
4
|
-
* Generic — read the project's own spec.md / feature text / sibling flows; no project data.
|
|
5
|
-
*/
|
|
6
|
-
import * as fs from 'fs';
|
|
7
|
-
import * as path from 'path';
|
|
8
|
-
import { ScenarioInfo, loadScenarios, idPrefix } from './parse';
|
|
9
|
-
|
|
10
|
-
// ---------- #2 Downstream-scope ----------
|
|
11
|
-
|
|
12
|
-
export interface DownstreamResult {
|
|
13
|
-
downstreamRoutes: string[]; // success/navigation targets ≠ own route
|
|
14
|
-
underCovered: { route: string; slug: string }[]; // referenced only by a bare page-nav
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
/** Routes the spec hands off to (Navigation Flow / success), other than the screen's own route. */
|
|
18
|
-
function downstreamRoutes(specText: string): string[] {
|
|
19
|
-
const ownRoute = (specText.match(/\*\*Route\*\*\s*:\s*`?(\/[^\s`]+)/) || [])[1] || '';
|
|
20
|
-
const routes = new Set<string>();
|
|
21
|
-
for (const line of specText.split('\n')) {
|
|
22
|
-
if (!/success|navigat|to \(|→/i.test(line)) continue;
|
|
23
|
-
for (const m of line.matchAll(/`?(\/[a-z][a-z0-9/_-]+)`?/gi)) {
|
|
24
|
-
const r = m[1];
|
|
25
|
-
if (r !== ownRoute && r.split('/').length > ownRoute.split('/').length - 0) routes.add(r);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
// keep only routes that extend beyond the own route (a distinct downstream surface)
|
|
29
|
-
return [...routes].filter((r) => r !== ownRoute && (!ownRoute || r.startsWith(ownRoute + '/') || r.split('/').length >= 3));
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
export function downstreamScope(specText: string, scenarios: ScenarioInfo[]): DownstreamResult {
|
|
33
|
-
const routes = downstreamRoutes(specText);
|
|
34
|
-
const underCovered: { route: string; slug: string }[] = [];
|
|
35
|
-
for (const route of routes) {
|
|
36
|
-
const slug = (route.split('/').filter(Boolean).pop() || route).toLowerCase();
|
|
37
|
-
const refs = scenarios.filter((s) => s.haystack.includes(slug) || s.haystack.includes(route.toLowerCase()));
|
|
38
|
-
if (!refs.length) continue; // not referenced at all — out of this screen's scope entirely
|
|
39
|
-
// Substantively covered only if some scenario OPERATES on the downstream — i.e. it
|
|
40
|
-
// starts there (`is on [<downstream>]`) — not merely navigates to it as a terminal
|
|
41
|
-
// `see [<downstream>] page` assertion. The latter just proves the transition.
|
|
42
|
-
const opensOn = new RegExp(`\\bis on \\[[^\\]]*${slug}`, 'i');
|
|
43
|
-
const contentCovered = refs.some((s) => opensOn.test(s.haystack));
|
|
44
|
-
if (!contentCovered) underCovered.push({ route, slug });
|
|
45
|
-
}
|
|
46
|
-
return { downstreamRoutes: routes, underCovered };
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// ---------- #4 Manual-oracle ----------
|
|
50
|
-
|
|
51
|
-
export interface ManualOracleResult {
|
|
52
|
-
manualTotal: number;
|
|
53
|
-
insufficient: string[]; // @manual scenarios lacking setup/action/oracle
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
function blocks(featureText: string): string[] {
|
|
57
|
-
return featureText.split(/\n\s*\n/).filter((b) => /\bScenario:/.test(b));
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export function manualOracle(featureText: string): ManualOracleResult {
|
|
61
|
-
const insufficient: string[] = [];
|
|
62
|
-
let manualTotal = 0;
|
|
63
|
-
for (const b of blocks(featureText)) {
|
|
64
|
-
if (!/@manual\b/.test(b)) continue;
|
|
65
|
-
manualTotal++;
|
|
66
|
-
const commentLines = b.split('\n').filter((l) => /^\s*#/.test(l));
|
|
67
|
-
const hasOracle = /tester verifies|oracle\s*:|requires|verify that|expected\s*:|steps?\s*:/i.test(b);
|
|
68
|
-
const hasNumberedSteps = /^\s*#?\s*\d+\.\s/m.test(b);
|
|
69
|
-
// sufficient = an oracle/steps marker, OR a substantive comment block (≥3 comment lines)
|
|
70
|
-
if (!(hasOracle || hasNumberedSteps || commentLines.length >= 3)) {
|
|
71
|
-
const name = (b.match(/Scenario:\s*(.+)/) || [])[1] || '(unnamed)';
|
|
72
|
-
insufficient.push(name.trim().slice(0, 80));
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
return { manualTotal, insufficient };
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
// ---------- #4 Negative side-effect ----------
|
|
79
|
-
|
|
80
|
-
const NEG_TITLE = /\b(does not|doesn't|no second|not dispatch|not sent|without submitting|no leak|single request|exactly one|count is 1|only one request|no duplicate|not create)\b/i;
|
|
81
|
-
|
|
82
|
-
/** Titles asserting an ABSENCE must prove it (count / negative / @manual+oracle), not just a happy outcome. */
|
|
83
|
-
export function negativeSideEffect(scenarios: ScenarioInfo[]): string[] {
|
|
84
|
-
const flagged: string[] = [];
|
|
85
|
-
for (const s of scenarios) {
|
|
86
|
-
if (s.manual) continue; // @manual is a legitimate deferral (oracle checked by #4 manual-oracle)
|
|
87
|
-
if (!NEG_TITLE.test(s.name)) continue;
|
|
88
|
-
const proven = /\bcount\b|tohavecount|table with|is hidden|are hidden|not complete|message is hidden/.test(s.stepsText);
|
|
89
|
-
if (!proven) flagged.push(s.name.slice(0, 80));
|
|
90
|
-
}
|
|
91
|
-
return flagged;
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
// ---------- #7 Source-backed strictness ----------
|
|
95
|
-
|
|
96
|
-
/** A scenario should trace to a source: a viewpoint ID (its own scheme), an FR id, or a
|
|
97
|
-
* viewpoint item (keyword overlap). ID match is language-agnostic and primary. */
|
|
98
|
-
export function sourceBacked(scenarios: ScenarioInfo[], frIds: string[], viewpointItems: string[], viewpointIds: string[], featureText: string): string[] {
|
|
99
|
-
if (!frIds.length && !viewpointItems.length && !viewpointIds.length) return []; // no contract
|
|
100
|
-
const vpIds = viewpointIds.map((s) => s.toUpperCase());
|
|
101
|
-
const itemWords = viewpointItems.map((t) => new Set((t.toLowerCase().match(/[a-z][a-z-]{4,}/g) || [])));
|
|
102
|
-
// per-scenario blocks (INCLUDING comments) so an FR cited in a comment counts as a source
|
|
103
|
-
const blockOf = new Map<string, string>();
|
|
104
|
-
for (const b of featureText.split(/\n\s*\n/)) {
|
|
105
|
-
const m = b.match(/Scenario:\s*(.+)/);
|
|
106
|
-
if (m) blockOf.set(m[1].trim().toLowerCase(), b.toLowerCase());
|
|
107
|
-
}
|
|
108
|
-
const unsourced: string[] = [];
|
|
109
|
-
for (const s of scenarios) {
|
|
110
|
-
const id = (s.vpId || s.vpCode || '').toUpperCase();
|
|
111
|
-
const mapsId = !!id && vpIds.some((v) => id === v || id.startsWith(v) || v.startsWith(idPrefix(id)));
|
|
112
|
-
const block = blockOf.get(s.name.trim().toLowerCase()) || s.haystack;
|
|
113
|
-
const citesFr = frIds.some((fid) => block.includes(fid.toLowerCase()));
|
|
114
|
-
const sWords = new Set((s.haystack.match(/[a-z][a-z-]{4,}/g) || []));
|
|
115
|
-
const mapsItem = itemWords.some((iw) => { let hits = 0; for (const w of iw) if (sWords.has(w)) hits++; return hits >= 2; });
|
|
116
|
-
if (!mapsId && !citesFr && !mapsItem) unsourced.push(s.name.slice(0, 80));
|
|
117
|
-
}
|
|
118
|
-
return unsourced;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
// ---------- #6 Cross-artifact ownership ----------
|
|
122
|
-
|
|
123
|
-
export interface OwnershipResult { duplicates: { scenario: string; flow: string }[] }
|
|
124
|
-
|
|
125
|
-
/** Scenarios whose step-skeleton also appears in a sibling flow feature → duplicate ownership. */
|
|
126
|
-
export function crossArtifactOwnership(screenDir: string, scenarios: ScenarioInfo[]): OwnershipResult {
|
|
127
|
-
const duplicates: { scenario: string; flow: string }[] = [];
|
|
128
|
-
// screenDir = <root>/qa/screens/<name>; flows live at <root>/qa/flows/*/features/*.feature
|
|
129
|
-
const flowsRoot = path.resolve(screenDir, '..', '..', 'flows');
|
|
130
|
-
if (!fs.existsSync(flowsRoot)) return { duplicates };
|
|
131
|
-
const bySkeleton = new Map<string, string>();
|
|
132
|
-
for (const flow of fs.readdirSync(flowsRoot)) {
|
|
133
|
-
const fdir = path.join(flowsRoot, flow, 'features');
|
|
134
|
-
if (!fs.existsSync(fdir)) continue;
|
|
135
|
-
for (const f of fs.readdirSync(fdir).filter((x) => x.endsWith('.feature'))) {
|
|
136
|
-
for (const fs2 of loadScenarios(path.join(fdir, f))) {
|
|
137
|
-
if (fs2.stepSkeleton && fs2.stepSkeleton.length > 20) bySkeleton.set(fs2.stepSkeleton, flow);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
if (!bySkeleton.size) return { duplicates };
|
|
142
|
-
for (const s of scenarios) {
|
|
143
|
-
const flow = s.stepSkeleton && s.stepSkeleton.length > 20 ? bySkeleton.get(s.stepSkeleton) : undefined;
|
|
144
|
-
if (flow) duplicates.push({ scenario: s.name.slice(0, 70), flow });
|
|
145
|
-
}
|
|
146
|
-
return { duplicates };
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
// convenience reader
|
|
150
|
-
export function readText(p: string): string {
|
|
151
|
-
return fs.existsSync(p) ? fs.readFileSync(p, 'utf-8') : '';
|
|
152
|
-
}
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Viewpoint Atomic Coverage Ledger (harness #2).
|
|
3
|
-
*
|
|
4
|
-
* The project's `test-viewpoint.md` IS the coverage contract. This parses it into ATOMIC
|
|
5
|
-
* items (each bullet / table row / ID-prefixed line) and reports the status of EACH —
|
|
6
|
-
* covered / missing — instead of the coarse "viewpoint mentioned" signal. It is fully
|
|
7
|
-
* project-driven (works on any project's viewpoint file, any domain), which is why it
|
|
8
|
-
* scales where a hardcoded domain catalog does not. Advisory: it surfaces the per-item
|
|
9
|
-
* gaps that inflate a "looks-covered" score; it does not fail the gate.
|
|
10
|
-
*/
|
|
11
|
-
import * as fs from 'fs';
|
|
12
|
-
import { ScenarioInfo } from './parse';
|
|
13
|
-
|
|
14
|
-
export interface LedgerItem { id?: string; text: string; covered: boolean }
|
|
15
|
-
|
|
16
|
-
export interface LedgerResult {
|
|
17
|
-
hasViewpoint: boolean;
|
|
18
|
-
total: number;
|
|
19
|
-
covered: number;
|
|
20
|
-
ratio: number;
|
|
21
|
-
missing: { id?: string; text: string }[];
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
const ID_RE = /\b([A-Z]{1,5}\d{0,2}(?:[.\-][A-Za-z0-9]+)*-?\d{0,3})\b/; // VP0.Title, VP7-002, MS-HP-001, TV-01
|
|
25
|
-
const GENERIC = new Set(['display', 'shown', 'value', 'field', 'input', 'page', 'screen', 'button', 'link', 'text', 'check', 'verify', 'should', 'with', 'when', 'then', 'user', 'this', 'that', 'each', 'item', 'items']);
|
|
26
|
-
|
|
27
|
-
/** Extract atomic checklist items from a viewpoint file (format-tolerant). */
|
|
28
|
-
export function parseViewpointItems(viewpointPath: string): { id?: string; text: string }[] {
|
|
29
|
-
if (!fs.existsSync(viewpointPath)) return [];
|
|
30
|
-
const lines = fs.readFileSync(viewpointPath, 'utf-8').split('\n');
|
|
31
|
-
const items: { id?: string; text: string }[] = [];
|
|
32
|
-
let inFence = false;
|
|
33
|
-
for (const raw of lines) {
|
|
34
|
-
const line = raw.trim();
|
|
35
|
-
if (line.startsWith('```')) { inFence = !inFence; continue; }
|
|
36
|
-
if (inFence || !line) continue;
|
|
37
|
-
if (/^#{1,6}\s/.test(line)) continue; // markdown heading
|
|
38
|
-
let text = '';
|
|
39
|
-
const bullet = line.match(/^(?:[-*+]|\d+[.)])\s+(.*)$/);
|
|
40
|
-
if (bullet) text = bullet[1];
|
|
41
|
-
else if (line.startsWith('|')) { // table data row
|
|
42
|
-
if (/^\|[\s|:-]+\|?$/.test(line)) continue; // separator
|
|
43
|
-
const cells = line.split('|').map((c) => c.trim()).filter(Boolean);
|
|
44
|
-
if (/^(vp|id|viewpoint|priority|reason|no\.?|category|item|trigger|#|pattern|applicable|notes|field|constraint|code|description|status)$/i.test(cells[0] || '')) continue; // header
|
|
45
|
-
text = cells.join(' — ');
|
|
46
|
-
} else continue;
|
|
47
|
-
text = text.replace(/[*`]/g, '').trim();
|
|
48
|
-
if (!text) continue;
|
|
49
|
-
const idM = text.match(ID_RE);
|
|
50
|
-
const id = idM && /\d/.test(idM[1]) ? idM[1] : undefined; // require a digit so prose words aren't IDs
|
|
51
|
-
const words = (text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w));
|
|
52
|
-
if (!id && words.length < 2) continue; // not substantive enough to track
|
|
53
|
-
items.push({ id, text: text.slice(0, 100) });
|
|
54
|
-
}
|
|
55
|
-
return items;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
export function viewpointLedger(viewpointPath: string, scenarios: ScenarioInfo[], featureText: string): LedgerResult {
|
|
59
|
-
const items = parseViewpointItems(viewpointPath);
|
|
60
|
-
if (!fs.existsSync(viewpointPath) || items.length === 0) {
|
|
61
|
-
return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [] };
|
|
62
|
-
}
|
|
63
|
-
const featLower = featureText.toLowerCase();
|
|
64
|
-
const missing: { id?: string; text: string }[] = [];
|
|
65
|
-
let covered = 0;
|
|
66
|
-
|
|
67
|
-
for (const item of items) {
|
|
68
|
-
let isCovered = false;
|
|
69
|
-
if (item.id && featLower.includes(item.id.toLowerCase())) isCovered = true;
|
|
70
|
-
else {
|
|
71
|
-
const words = [...new Set((item.text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w)))];
|
|
72
|
-
const need = Math.min(2, words.length);
|
|
73
|
-
isCovered = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= need);
|
|
74
|
-
}
|
|
75
|
-
if (isCovered) covered++;
|
|
76
|
-
else missing.push({ id: item.id, text: item.text });
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
return { hasViewpoint: true, total: items.length, covered, ratio: items.length ? covered / items.length : 1, missing };
|
|
80
|
-
}
|