@sun-asterisk/sungen 3.0.0-beta.84 → 3.0.0-beta.92

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/cli/commands/audit.d.ts.map +1 -1
  2. package/dist/cli/commands/audit.js +0 -14
  3. package/dist/cli/commands/audit.js.map +1 -1
  4. package/dist/cli/index.js +0 -2
  5. package/dist/cli/index.js.map +1 -1
  6. package/dist/harness/audit.d.ts +0 -14
  7. package/dist/harness/audit.d.ts.map +1 -1
  8. package/dist/harness/audit.js +3 -56
  9. package/dist/harness/audit.js.map +1 -1
  10. package/dist/harness/parse.d.ts +0 -6
  11. package/dist/harness/parse.d.ts.map +1 -1
  12. package/dist/harness/parse.js +3 -18
  13. package/dist/harness/parse.js.map +1 -1
  14. package/dist/harness/sensors.d.ts.map +1 -1
  15. package/dist/harness/sensors.js +6 -85
  16. package/dist/harness/sensors.js.map +1 -1
  17. package/dist/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +0 -1
  18. package/dist/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +1 -25
  19. package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +7 -44
  20. package/package.json +2 -2
  21. package/src/cli/commands/audit.ts +0 -12
  22. package/src/cli/index.ts +0 -2
  23. package/src/harness/audit.ts +4 -68
  24. package/src/harness/parse.ts +3 -19
  25. package/src/harness/sensors.ts +7 -84
  26. package/src/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +0 -1
  27. package/src/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +1 -25
  28. package/src/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +7 -44
  29. package/dist/cli/commands/eval.d.ts +0 -3
  30. package/dist/cli/commands/eval.d.ts.map +0 -1
  31. package/dist/cli/commands/eval.js +0 -37
  32. package/dist/cli/commands/eval.js.map +0 -1
  33. package/dist/harness/eval/skill-lint.d.ts +0 -16
  34. package/dist/harness/eval/skill-lint.d.ts.map +0 -1
  35. package/dist/harness/eval/skill-lint.js +0 -129
  36. package/dist/harness/eval/skill-lint.js.map +0 -1
  37. package/dist/harness/quality-gates.d.ts +0 -29
  38. package/dist/harness/quality-gates.d.ts.map +0 -1
  39. package/dist/harness/quality-gates.js +0 -183
  40. package/dist/harness/quality-gates.js.map +0 -1
  41. package/dist/harness/viewpoint-ledger.d.ts +0 -23
  42. package/dist/harness/viewpoint-ledger.d.ts.map +0 -1
  43. package/dist/harness/viewpoint-ledger.js +0 -118
  44. package/dist/harness/viewpoint-ledger.js.map +0 -1
  45. package/src/cli/commands/eval.ts +0 -28
  46. package/src/harness/eval/skill-lint.ts +0 -87
  47. package/src/harness/quality-gates.ts +0 -152
  48. package/src/harness/viewpoint-ledger.ts +0 -80
@@ -1,118 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.parseViewpointItems = parseViewpointItems;
37
- exports.viewpointLedger = viewpointLedger;
38
- /**
39
- * Viewpoint Atomic Coverage Ledger (harness #2).
40
- *
41
- * The project's `test-viewpoint.md` IS the coverage contract. This parses it into ATOMIC
42
- * items (each bullet / table row / ID-prefixed line) and reports the status of EACH —
43
- * covered / missing — instead of the coarse "viewpoint mentioned" signal. It is fully
44
- * project-driven (works on any project's viewpoint file, any domain), which is why it
45
- * scales where a hardcoded domain catalog does not. Advisory: it surfaces the per-item
46
- * gaps that inflate a "looks-covered" score; it does not fail the gate.
47
- */
48
- const fs = __importStar(require("fs"));
49
- const ID_RE = /\b([A-Z]{1,5}\d{0,2}(?:[.\-][A-Za-z0-9]+)*-?\d{0,3})\b/; // VP0.Title, VP7-002, MS-HP-001, TV-01
50
- const GENERIC = new Set(['display', 'shown', 'value', 'field', 'input', 'page', 'screen', 'button', 'link', 'text', 'check', 'verify', 'should', 'with', 'when', 'then', 'user', 'this', 'that', 'each', 'item', 'items']);
51
- /** Extract atomic checklist items from a viewpoint file (format-tolerant). */
52
- function parseViewpointItems(viewpointPath) {
53
- if (!fs.existsSync(viewpointPath))
54
- return [];
55
- const lines = fs.readFileSync(viewpointPath, 'utf-8').split('\n');
56
- const items = [];
57
- let inFence = false;
58
- for (const raw of lines) {
59
- const line = raw.trim();
60
- if (line.startsWith('```')) {
61
- inFence = !inFence;
62
- continue;
63
- }
64
- if (inFence || !line)
65
- continue;
66
- if (/^#{1,6}\s/.test(line))
67
- continue; // markdown heading
68
- let text = '';
69
- const bullet = line.match(/^(?:[-*+]|\d+[.)])\s+(.*)$/);
70
- if (bullet)
71
- text = bullet[1];
72
- else if (line.startsWith('|')) { // table data row
73
- if (/^\|[\s|:-]+\|?$/.test(line))
74
- continue; // separator
75
- const cells = line.split('|').map((c) => c.trim()).filter(Boolean);
76
- if (/^(vp|id|viewpoint|priority|reason|no\.?|category|item|trigger|#|pattern|applicable|notes|field|constraint|code|description|status)$/i.test(cells[0] || ''))
77
- continue; // header
78
- text = cells.join(' — ');
79
- }
80
- else
81
- continue;
82
- text = text.replace(/[*`]/g, '').trim();
83
- if (!text)
84
- continue;
85
- const idM = text.match(ID_RE);
86
- const id = idM && /\d/.test(idM[1]) ? idM[1] : undefined; // require a digit so prose words aren't IDs
87
- const words = (text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w));
88
- if (!id && words.length < 2)
89
- continue; // not substantive enough to track
90
- items.push({ id, text: text.slice(0, 100) });
91
- }
92
- return items;
93
- }
94
- function viewpointLedger(viewpointPath, scenarios, featureText) {
95
- const items = parseViewpointItems(viewpointPath);
96
- if (!fs.existsSync(viewpointPath) || items.length === 0) {
97
- return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [] };
98
- }
99
- const featLower = featureText.toLowerCase();
100
- const missing = [];
101
- let covered = 0;
102
- for (const item of items) {
103
- let isCovered = false;
104
- if (item.id && featLower.includes(item.id.toLowerCase()))
105
- isCovered = true;
106
- else {
107
- const words = [...new Set((item.text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w)))];
108
- const need = Math.min(2, words.length);
109
- isCovered = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= need);
110
- }
111
- if (isCovered)
112
- covered++;
113
- else
114
- missing.push({ id: item.id, text: item.text });
115
- }
116
- return { hasViewpoint: true, total: items.length, covered, ratio: items.length ? covered / items.length : 1, missing };
117
- }
118
- //# sourceMappingURL=viewpoint-ledger.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"viewpoint-ledger.js","sourceRoot":"","sources":["../../src/harness/viewpoint-ledger.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BA,kDA4BC;AAED,0CAsBC;AA/ED;;;;;;;;;GASG;AACH,uCAAyB;AAazB,MAAM,KAAK,GAAG,wDAAwD,CAAC,CAAC,uCAAuC;AAC/G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE3N,8EAA8E;AAC9E,SAAgB,mBAAmB,CAAC,aAAqB;IACvD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,EAAE,CAAC;IAC7C,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClE,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,OAAO,GAAG,CAAC,OAAO,CAAC;YAAC,SAAS;QAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,CAAC,IAAI;YAAE,SAAS;QAC/B,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS,CAAiB,mBAAmB;QACzE,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACxD,IAAI,MAAM;YAAE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;aACxB,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAuB,iBAAiB;YACtE,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS,CAAW,YAAY;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,sIAAsI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAAE,SAAS,CAAC,SAAS;YACpL,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;;YAAM,SAAS;QAChB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,4CAA4C;QACtG,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,IAAI,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS,CAAiB,kCAAkC;QACzF,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAgB,eAAe,CAAC,aAAqB,EAAE,SAAyB,EAAE,WAAmB;IACnG,MAAM,KAAK,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAC;IACjD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,OAAO,EAAE,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACrG,CAAC;IACD,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAoC,EAAE,CAAC;IACpD,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC;aACtE,CAAC;YACJ,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACvC,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;QACpH,CAAC;QACD,IAAI,SAAS;YAAE,OAAO,EAAE,CAAC;;YACpB,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AACzH,CAAC"}
@@ -1,28 +0,0 @@
1
- import { Command } from 'commander';
2
- import { lintSkills, defaultSkillDir } from '../../harness/eval/skill-lint';
3
-
4
- export function registerEvalCommand(program: Command): void {
5
- program
6
- .command('eval')
7
- .description('Eval harness: quality checks on Sungen\'s own skills/instructions (dev/CI)')
8
- .option('--skills', 'Static skill-lint: frontmatter, line budget, claude↔github sync, registration')
9
- .option('--dir <path>', 'Templates dir to lint (default: bundled ai-instructions)')
10
- .option('--json', 'Output the raw findings JSON')
11
- .action((options) => {
12
- try {
13
- if (!options.skills) throw new Error('Provide --skills (the only eval mode today)');
14
- const dir = options.dir || defaultSkillDir();
15
- const r = lintSkills(dir);
16
- if (options.json) { console.log(JSON.stringify(r, null, 2)); process.exit(r.errors > 0 ? 2 : 0); }
17
- console.log('');
18
- console.log(`━━━ Skill-lint: ${r.checked} skill template(s) ━━━`);
19
- if (!r.findings.length) console.log(' ✓ all skills pass (frontmatter · line-budget · variant-sync · registration)');
20
- for (const f of r.findings) console.log(` ${f.level === 'error' ? '✗' : '⚠'} [${f.rule}] ${f.file} — ${f.detail}`);
21
- console.log('');
22
- process.exit(r.errors > 0 ? 2 : 0);
23
- } catch (error) {
24
- console.error('Error:', error instanceof Error ? error.message : error);
25
- process.exit(1);
26
- }
27
- });
28
- }
@@ -1,87 +0,0 @@
1
- /**
2
- * Static skill-lint (Eval Harness L1) — deterministic quality checks on Sungen's OWN
3
- * AI-instruction templates, so a broken / unregistered / oversized skill fails before it
4
- * ships. Learned (generically) from the "static validations" tier of an agent-kit evals
5
- * layer. No project data — this lints the sungen package's own templates.
6
- *
7
- * Design note: the checks are MAPPING-DRIVEN. `AI_RULES_FILE_MAPPING` is the source of
8
- * truth for what each template installs as, so the lint uses the install target (does it
9
- * end in `/SKILL.md`?) to tell a top-level skill from a sub-content fragment — instead of
10
- * guessing from filenames. We deliberately do NOT enforce claude↔github body parity: the
11
- * two variants are hand-tuned per platform and intentionally diverge in wording and even
12
- * structure, so byte/heading equality would be pure false positives.
13
- */
14
- import * as fs from 'fs';
15
- import * as path from 'path';
16
- import { AI_RULES_FILE_MAPPING } from '../../orchestrator/ai-rules-updater';
17
-
18
- export interface SkillLintFinding { level: 'error' | 'warn'; file: string; rule: string; detail: string }
19
- export interface SkillLintResult { checked: number; findings: SkillLintFinding[]; errors: number }
20
-
21
- const LINE_BUDGET = 700; // a skill much larger than this is a context-cost smell (warn)
22
- const SKILL_RE = /^(claude|github)-skill-/;
23
-
24
- function stripFrontmatter(text: string): { fm: string | null; body: string } {
25
- const m = text.match(/^---\n([\s\S]*?)\n---\n?/);
26
- if (!m) return { fm: null, body: text };
27
- return { fm: m[1], body: text.slice(m[0].length) };
28
- }
29
-
30
- /** Lint the AI-instruction templates in `dir` (default: the sungen source templates). */
31
- export function lintSkills(dir: string): SkillLintResult {
32
- const findings: SkillLintFinding[] = [];
33
- const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith('.md')) : [];
34
- const skillFiles = files.filter((f) => SKILL_RE.test(f));
35
-
36
- // mapping: template file -> install target (source of truth for "is this a top-level skill")
37
- const target = new Map<string, string>(AI_RULES_FILE_MAPPING.map(([tpl, dst]) => [tpl, dst]));
38
- const isTopLevelSkill = (f: string) => (target.get(f) || '').endsWith('/SKILL.md');
39
-
40
- // 1) registration integrity (bidirectional) — the highest-value check:
41
- // a skill file missing from the mapping never installs; a mapping to a missing file
42
- // ships a broken/empty skill.
43
- for (const f of skillFiles) {
44
- if (!target.has(f)) findings.push({ level: 'error', file: f, rule: 'unregistered', detail: 'skill template not in AI_RULES_FILE_MAPPING (it would never be installed)' });
45
- }
46
- for (const [tpl] of AI_RULES_FILE_MAPPING) {
47
- if (!fs.existsSync(path.join(dir, tpl))) findings.push({ level: 'error', file: tpl, rule: 'mapped-missing', detail: 'AI_RULES_FILE_MAPPING points to a template that does not exist' });
48
- }
49
-
50
- // 2) frontmatter (name + description) — ONLY for top-level skills (SKILL.md targets).
51
- // Sub-content fragments (mode-*.md, group-*.md) are loaded by their parent router
52
- // and legitimately carry no frontmatter.
53
- for (const f of skillFiles) {
54
- if (!isTopLevelSkill(f)) continue;
55
- const text = fs.readFileSync(path.join(dir, f), 'utf8');
56
- const { fm } = stripFrontmatter(text);
57
- if (!fm) { findings.push({ level: 'error', file: f, rule: 'frontmatter', detail: 'top-level skill (SKILL.md) is missing --- frontmatter --- (Claude/Copilot will not load it)' }); continue; }
58
- if (!/\bname\s*:/.test(fm)) findings.push({ level: 'error', file: f, rule: 'frontmatter-name', detail: 'no `name:` in frontmatter' });
59
- if (!/\bdescription\s*:/.test(fm)) findings.push({ level: 'error', file: f, rule: 'frontmatter-description', detail: 'no `description:` in frontmatter' });
60
- }
61
-
62
- // 3) line budget — context-cost smell (advisory).
63
- for (const f of skillFiles) {
64
- const lines = fs.readFileSync(path.join(dir, f), 'utf8').split('\n').length;
65
- if (lines > LINE_BUDGET) findings.push({ level: 'warn', file: f, rule: 'line-budget', detail: `${lines} lines > ${LINE_BUDGET} (context-cost smell)` });
66
- }
67
-
68
- // 4) variant PRESENCE (not body equality) — every top-level skill should ship for both
69
- // platforms. Catches "added a Claude skill but forgot the Copilot variant". Advisory.
70
- const skillName = (dst: string) => { const m = dst.match(/\/(sungen-[^/]+)\/SKILL\.md$/); return m ? m[1] : null; };
71
- const claudeSkills = new Set<string>(), githubSkills = new Set<string>();
72
- for (const f of skillFiles) {
73
- if (!isTopLevelSkill(f)) continue;
74
- const name = skillName(target.get(f)!); if (!name) continue;
75
- (f.startsWith('claude-') ? claudeSkills : githubSkills).add(name);
76
- }
77
- for (const n of claudeSkills) if (!githubSkills.has(n)) findings.push({ level: 'warn', file: `claude .../${n}/SKILL.md`, rule: 'variant-missing', detail: `Claude skill "${n}" has no GitHub (Copilot) variant` });
78
- for (const n of githubSkills) if (!claudeSkills.has(n)) findings.push({ level: 'warn', file: `github .../${n}/SKILL.md`, rule: 'variant-missing', detail: `GitHub skill "${n}" has no Claude variant` });
79
-
80
- return { checked: skillFiles.length, findings, errors: findings.filter((f) => f.level === 'error').length };
81
- }
82
-
83
- /** Default templates dir, resolved relative to this module (works from src via tsx and dist). */
84
- export function defaultSkillDir(): string {
85
- // src/harness/eval → src/orchestrator/... | dist/harness/eval → dist/orchestrator/...
86
- return path.resolve(__dirname, '..', '..', 'orchestrator', 'templates', 'ai-instructions');
87
- }
@@ -1,152 +0,0 @@
1
- /**
2
- * Quality gates (batch): downstream-scope + manual-oracle + negative-side-effect +
3
- * cross-artifact ownership + source-backed strictness.
4
- * Generic — read the project's own spec.md / feature text / sibling flows; no project data.
5
- */
6
- import * as fs from 'fs';
7
- import * as path from 'path';
8
- import { ScenarioInfo, loadScenarios, idPrefix } from './parse';
9
-
10
- // ---------- #2 Downstream-scope ----------
11
-
12
- export interface DownstreamResult {
13
- downstreamRoutes: string[]; // success/navigation targets ≠ own route
14
- underCovered: { route: string; slug: string }[]; // referenced only by a bare page-nav
15
- }
16
-
17
- /** Routes the spec hands off to (Navigation Flow / success), other than the screen's own route. */
18
- function downstreamRoutes(specText: string): string[] {
19
- const ownRoute = (specText.match(/\*\*Route\*\*\s*:\s*`?(\/[^\s`]+)/) || [])[1] || '';
20
- const routes = new Set<string>();
21
- for (const line of specText.split('\n')) {
22
- if (!/success|navigat|to \(|→/i.test(line)) continue;
23
- for (const m of line.matchAll(/`?(\/[a-z][a-z0-9/_-]+)`?/gi)) {
24
- const r = m[1];
25
- if (r !== ownRoute && r.split('/').length > ownRoute.split('/').length - 0) routes.add(r);
26
- }
27
- }
28
- // keep only routes that extend beyond the own route (a distinct downstream surface)
29
- return [...routes].filter((r) => r !== ownRoute && (!ownRoute || r.startsWith(ownRoute + '/') || r.split('/').length >= 3));
30
- }
31
-
32
- export function downstreamScope(specText: string, scenarios: ScenarioInfo[]): DownstreamResult {
33
- const routes = downstreamRoutes(specText);
34
- const underCovered: { route: string; slug: string }[] = [];
35
- for (const route of routes) {
36
- const slug = (route.split('/').filter(Boolean).pop() || route).toLowerCase();
37
- const refs = scenarios.filter((s) => s.haystack.includes(slug) || s.haystack.includes(route.toLowerCase()));
38
- if (!refs.length) continue; // not referenced at all — out of this screen's scope entirely
39
- // Substantively covered only if some scenario OPERATES on the downstream — i.e. it
40
- // starts there (`is on [<downstream>]`) — not merely navigates to it as a terminal
41
- // `see [<downstream>] page` assertion. The latter just proves the transition.
42
- const opensOn = new RegExp(`\\bis on \\[[^\\]]*${slug}`, 'i');
43
- const contentCovered = refs.some((s) => opensOn.test(s.haystack));
44
- if (!contentCovered) underCovered.push({ route, slug });
45
- }
46
- return { downstreamRoutes: routes, underCovered };
47
- }
48
-
49
- // ---------- #4 Manual-oracle ----------
50
-
51
- export interface ManualOracleResult {
52
- manualTotal: number;
53
- insufficient: string[]; // @manual scenarios lacking setup/action/oracle
54
- }
55
-
56
- function blocks(featureText: string): string[] {
57
- return featureText.split(/\n\s*\n/).filter((b) => /\bScenario:/.test(b));
58
- }
59
-
60
- export function manualOracle(featureText: string): ManualOracleResult {
61
- const insufficient: string[] = [];
62
- let manualTotal = 0;
63
- for (const b of blocks(featureText)) {
64
- if (!/@manual\b/.test(b)) continue;
65
- manualTotal++;
66
- const commentLines = b.split('\n').filter((l) => /^\s*#/.test(l));
67
- const hasOracle = /tester verifies|oracle\s*:|requires|verify that|expected\s*:|steps?\s*:/i.test(b);
68
- const hasNumberedSteps = /^\s*#?\s*\d+\.\s/m.test(b);
69
- // sufficient = an oracle/steps marker, OR a substantive comment block (≥3 comment lines)
70
- if (!(hasOracle || hasNumberedSteps || commentLines.length >= 3)) {
71
- const name = (b.match(/Scenario:\s*(.+)/) || [])[1] || '(unnamed)';
72
- insufficient.push(name.trim().slice(0, 80));
73
- }
74
- }
75
- return { manualTotal, insufficient };
76
- }
77
-
78
- // ---------- #4 Negative side-effect ----------
79
-
80
- const NEG_TITLE = /\b(does not|doesn't|no second|not dispatch|not sent|without submitting|no leak|single request|exactly one|count is 1|only one request|no duplicate|not create)\b/i;
81
-
82
- /** Titles asserting an ABSENCE must prove it (count / negative / @manual+oracle), not just a happy outcome. */
83
- export function negativeSideEffect(scenarios: ScenarioInfo[]): string[] {
84
- const flagged: string[] = [];
85
- for (const s of scenarios) {
86
- if (s.manual) continue; // @manual is a legitimate deferral (oracle checked by #4 manual-oracle)
87
- if (!NEG_TITLE.test(s.name)) continue;
88
- const proven = /\bcount\b|tohavecount|table with|is hidden|are hidden|not complete|message is hidden/.test(s.stepsText);
89
- if (!proven) flagged.push(s.name.slice(0, 80));
90
- }
91
- return flagged;
92
- }
93
-
94
- // ---------- #7 Source-backed strictness ----------
95
-
96
- /** A scenario should trace to a source: a viewpoint ID (its own scheme), an FR id, or a
97
- * viewpoint item (keyword overlap). ID match is language-agnostic and primary. */
98
- export function sourceBacked(scenarios: ScenarioInfo[], frIds: string[], viewpointItems: string[], viewpointIds: string[], featureText: string): string[] {
99
- if (!frIds.length && !viewpointItems.length && !viewpointIds.length) return []; // no contract
100
- const vpIds = viewpointIds.map((s) => s.toUpperCase());
101
- const itemWords = viewpointItems.map((t) => new Set((t.toLowerCase().match(/[a-z][a-z-]{4,}/g) || [])));
102
- // per-scenario blocks (INCLUDING comments) so an FR cited in a comment counts as a source
103
- const blockOf = new Map<string, string>();
104
- for (const b of featureText.split(/\n\s*\n/)) {
105
- const m = b.match(/Scenario:\s*(.+)/);
106
- if (m) blockOf.set(m[1].trim().toLowerCase(), b.toLowerCase());
107
- }
108
- const unsourced: string[] = [];
109
- for (const s of scenarios) {
110
- const id = (s.vpId || s.vpCode || '').toUpperCase();
111
- const mapsId = !!id && vpIds.some((v) => id === v || id.startsWith(v) || v.startsWith(idPrefix(id)));
112
- const block = blockOf.get(s.name.trim().toLowerCase()) || s.haystack;
113
- const citesFr = frIds.some((fid) => block.includes(fid.toLowerCase()));
114
- const sWords = new Set((s.haystack.match(/[a-z][a-z-]{4,}/g) || []));
115
- const mapsItem = itemWords.some((iw) => { let hits = 0; for (const w of iw) if (sWords.has(w)) hits++; return hits >= 2; });
116
- if (!mapsId && !citesFr && !mapsItem) unsourced.push(s.name.slice(0, 80));
117
- }
118
- return unsourced;
119
- }
120
-
121
- // ---------- #6 Cross-artifact ownership ----------
122
-
123
- export interface OwnershipResult { duplicates: { scenario: string; flow: string }[] }
124
-
125
- /** Scenarios whose step-skeleton also appears in a sibling flow feature → duplicate ownership. */
126
- export function crossArtifactOwnership(screenDir: string, scenarios: ScenarioInfo[]): OwnershipResult {
127
- const duplicates: { scenario: string; flow: string }[] = [];
128
- // screenDir = <root>/qa/screens/<name>; flows live at <root>/qa/flows/*/features/*.feature
129
- const flowsRoot = path.resolve(screenDir, '..', '..', 'flows');
130
- if (!fs.existsSync(flowsRoot)) return { duplicates };
131
- const bySkeleton = new Map<string, string>();
132
- for (const flow of fs.readdirSync(flowsRoot)) {
133
- const fdir = path.join(flowsRoot, flow, 'features');
134
- if (!fs.existsSync(fdir)) continue;
135
- for (const f of fs.readdirSync(fdir).filter((x) => x.endsWith('.feature'))) {
136
- for (const fs2 of loadScenarios(path.join(fdir, f))) {
137
- if (fs2.stepSkeleton && fs2.stepSkeleton.length > 20) bySkeleton.set(fs2.stepSkeleton, flow);
138
- }
139
- }
140
- }
141
- if (!bySkeleton.size) return { duplicates };
142
- for (const s of scenarios) {
143
- const flow = s.stepSkeleton && s.stepSkeleton.length > 20 ? bySkeleton.get(s.stepSkeleton) : undefined;
144
- if (flow) duplicates.push({ scenario: s.name.slice(0, 70), flow });
145
- }
146
- return { duplicates };
147
- }
148
-
149
- // convenience reader
150
- export function readText(p: string): string {
151
- return fs.existsSync(p) ? fs.readFileSync(p, 'utf-8') : '';
152
- }
@@ -1,80 +0,0 @@
1
- /**
2
- * Viewpoint Atomic Coverage Ledger (harness #2).
3
- *
4
- * The project's `test-viewpoint.md` IS the coverage contract. This parses it into ATOMIC
5
- * items (each bullet / table row / ID-prefixed line) and reports the status of EACH —
6
- * covered / missing — instead of the coarse "viewpoint mentioned" signal. It is fully
7
- * project-driven (works on any project's viewpoint file, any domain), which is why it
8
- * scales where a hardcoded domain catalog does not. Advisory: it surfaces the per-item
9
- * gaps that inflate a "looks-covered" score; it does not fail the gate.
10
- */
11
- import * as fs from 'fs';
12
- import { ScenarioInfo } from './parse';
13
-
14
- export interface LedgerItem { id?: string; text: string; covered: boolean }
15
-
16
- export interface LedgerResult {
17
- hasViewpoint: boolean;
18
- total: number;
19
- covered: number;
20
- ratio: number;
21
- missing: { id?: string; text: string }[];
22
- }
23
-
24
- const ID_RE = /\b([A-Z]{1,5}\d{0,2}(?:[.\-][A-Za-z0-9]+)*-?\d{0,3})\b/; // VP0.Title, VP7-002, MS-HP-001, TV-01
25
- const GENERIC = new Set(['display', 'shown', 'value', 'field', 'input', 'page', 'screen', 'button', 'link', 'text', 'check', 'verify', 'should', 'with', 'when', 'then', 'user', 'this', 'that', 'each', 'item', 'items']);
26
-
27
- /** Extract atomic checklist items from a viewpoint file (format-tolerant). */
28
- export function parseViewpointItems(viewpointPath: string): { id?: string; text: string }[] {
29
- if (!fs.existsSync(viewpointPath)) return [];
30
- const lines = fs.readFileSync(viewpointPath, 'utf-8').split('\n');
31
- const items: { id?: string; text: string }[] = [];
32
- let inFence = false;
33
- for (const raw of lines) {
34
- const line = raw.trim();
35
- if (line.startsWith('```')) { inFence = !inFence; continue; }
36
- if (inFence || !line) continue;
37
- if (/^#{1,6}\s/.test(line)) continue; // markdown heading
38
- let text = '';
39
- const bullet = line.match(/^(?:[-*+]|\d+[.)])\s+(.*)$/);
40
- if (bullet) text = bullet[1];
41
- else if (line.startsWith('|')) { // table data row
42
- if (/^\|[\s|:-]+\|?$/.test(line)) continue; // separator
43
- const cells = line.split('|').map((c) => c.trim()).filter(Boolean);
44
- if (/^(vp|id|viewpoint|priority|reason|no\.?|category|item|trigger|#|pattern|applicable|notes|field|constraint|code|description|status)$/i.test(cells[0] || '')) continue; // header
45
- text = cells.join(' — ');
46
- } else continue;
47
- text = text.replace(/[*`]/g, '').trim();
48
- if (!text) continue;
49
- const idM = text.match(ID_RE);
50
- const id = idM && /\d/.test(idM[1]) ? idM[1] : undefined; // require a digit so prose words aren't IDs
51
- const words = (text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w));
52
- if (!id && words.length < 2) continue; // not substantive enough to track
53
- items.push({ id, text: text.slice(0, 100) });
54
- }
55
- return items;
56
- }
57
-
58
- export function viewpointLedger(viewpointPath: string, scenarios: ScenarioInfo[], featureText: string): LedgerResult {
59
- const items = parseViewpointItems(viewpointPath);
60
- if (!fs.existsSync(viewpointPath) || items.length === 0) {
61
- return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [] };
62
- }
63
- const featLower = featureText.toLowerCase();
64
- const missing: { id?: string; text: string }[] = [];
65
- let covered = 0;
66
-
67
- for (const item of items) {
68
- let isCovered = false;
69
- if (item.id && featLower.includes(item.id.toLowerCase())) isCovered = true;
70
- else {
71
- const words = [...new Set((item.text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w)))];
72
- const need = Math.min(2, words.length);
73
- isCovered = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= need);
74
- }
75
- if (isCovered) covered++;
76
- else missing.push({ id: item.id, text: item.text });
77
- }
78
-
79
- return { hasViewpoint: true, total: items.length, covered, ratio: items.length ? covered / items.length : 1, missing };
80
- }