@sun-asterisk/sungen 3.0.0-beta.83 → 3.0.0-beta.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +0 -14
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/index.js +0 -2
- package/dist/cli/index.js.map +1 -1
- package/dist/harness/audit.d.ts +0 -14
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +3 -56
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/parse.d.ts +0 -6
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +3 -18
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +5 -53
- package/dist/harness/sensors.js.map +1 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +1 -24
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +7 -43
- package/package.json +2 -2
- package/src/cli/commands/audit.ts +0 -12
- package/src/cli/index.ts +0 -2
- package/src/harness/audit.ts +4 -68
- package/src/harness/parse.ts +3 -19
- package/src/harness/sensors.ts +6 -52
- package/src/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +1 -24
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +7 -43
- package/dist/cli/commands/eval.d.ts +0 -3
- package/dist/cli/commands/eval.d.ts.map +0 -1
- package/dist/cli/commands/eval.js +0 -37
- package/dist/cli/commands/eval.js.map +0 -1
- package/dist/harness/eval/skill-lint.d.ts +0 -16
- package/dist/harness/eval/skill-lint.d.ts.map +0 -1
- package/dist/harness/eval/skill-lint.js +0 -129
- package/dist/harness/eval/skill-lint.js.map +0 -1
- package/dist/harness/quality-gates.d.ts +0 -29
- package/dist/harness/quality-gates.d.ts.map +0 -1
- package/dist/harness/quality-gates.js +0 -183
- package/dist/harness/quality-gates.js.map +0 -1
- package/dist/harness/viewpoint-ledger.d.ts +0 -23
- package/dist/harness/viewpoint-ledger.d.ts.map +0 -1
- package/dist/harness/viewpoint-ledger.js +0 -118
- package/dist/harness/viewpoint-ledger.js.map +0 -1
- package/src/cli/commands/eval.ts +0 -28
- package/src/harness/eval/skill-lint.ts +0 -87
- package/src/harness/quality-gates.ts +0 -152
- package/src/harness/viewpoint-ledger.ts +0 -80
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.lintSkills = lintSkills;
|
|
37
|
-
exports.defaultSkillDir = defaultSkillDir;
|
|
38
|
-
/**
|
|
39
|
-
* Static skill-lint (Eval Harness L1) — deterministic quality checks on Sungen's OWN
|
|
40
|
-
* AI-instruction templates, so a broken / unregistered / oversized skill fails before it
|
|
41
|
-
* ships. Learned (generically) from the "static validations" tier of an agent-kit evals
|
|
42
|
-
* layer. No project data — this lints the sungen package's own templates.
|
|
43
|
-
*
|
|
44
|
-
* Design note: the checks are MAPPING-DRIVEN. `AI_RULES_FILE_MAPPING` is the source of
|
|
45
|
-
* truth for what each template installs as, so the lint uses the install target (does it
|
|
46
|
-
* end in `/SKILL.md`?) to tell a top-level skill from a sub-content fragment — instead of
|
|
47
|
-
* guessing from filenames. We deliberately do NOT enforce claude↔github body parity: the
|
|
48
|
-
* two variants are hand-tuned per platform and intentionally diverge in wording and even
|
|
49
|
-
* structure, so byte/heading equality would be pure false positives.
|
|
50
|
-
*/
|
|
51
|
-
const fs = __importStar(require("fs"));
|
|
52
|
-
const path = __importStar(require("path"));
|
|
53
|
-
const ai_rules_updater_1 = require("../../orchestrator/ai-rules-updater");
|
|
54
|
-
const LINE_BUDGET = 700; // a skill much larger than this is a context-cost smell (warn)
|
|
55
|
-
const SKILL_RE = /^(claude|github)-skill-/;
|
|
56
|
-
function stripFrontmatter(text) {
|
|
57
|
-
const m = text.match(/^---\n([\s\S]*?)\n---\n?/);
|
|
58
|
-
if (!m)
|
|
59
|
-
return { fm: null, body: text };
|
|
60
|
-
return { fm: m[1], body: text.slice(m[0].length) };
|
|
61
|
-
}
|
|
62
|
-
/** Lint the AI-instruction templates in `dir` (default: the sungen source templates). */
|
|
63
|
-
function lintSkills(dir) {
|
|
64
|
-
const findings = [];
|
|
65
|
-
const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith('.md')) : [];
|
|
66
|
-
const skillFiles = files.filter((f) => SKILL_RE.test(f));
|
|
67
|
-
// mapping: template file -> install target (source of truth for "is this a top-level skill")
|
|
68
|
-
const target = new Map(ai_rules_updater_1.AI_RULES_FILE_MAPPING.map(([tpl, dst]) => [tpl, dst]));
|
|
69
|
-
const isTopLevelSkill = (f) => (target.get(f) || '').endsWith('/SKILL.md');
|
|
70
|
-
// 1) registration integrity (bidirectional) — the highest-value check:
|
|
71
|
-
// a skill file missing from the mapping never installs; a mapping to a missing file
|
|
72
|
-
// ships a broken/empty skill.
|
|
73
|
-
for (const f of skillFiles) {
|
|
74
|
-
if (!target.has(f))
|
|
75
|
-
findings.push({ level: 'error', file: f, rule: 'unregistered', detail: 'skill template not in AI_RULES_FILE_MAPPING (it would never be installed)' });
|
|
76
|
-
}
|
|
77
|
-
for (const [tpl] of ai_rules_updater_1.AI_RULES_FILE_MAPPING) {
|
|
78
|
-
if (!fs.existsSync(path.join(dir, tpl)))
|
|
79
|
-
findings.push({ level: 'error', file: tpl, rule: 'mapped-missing', detail: 'AI_RULES_FILE_MAPPING points to a template that does not exist' });
|
|
80
|
-
}
|
|
81
|
-
// 2) frontmatter (name + description) — ONLY for top-level skills (SKILL.md targets).
|
|
82
|
-
// Sub-content fragments (mode-*.md, group-*.md) are loaded by their parent router
|
|
83
|
-
// and legitimately carry no frontmatter.
|
|
84
|
-
for (const f of skillFiles) {
|
|
85
|
-
if (!isTopLevelSkill(f))
|
|
86
|
-
continue;
|
|
87
|
-
const text = fs.readFileSync(path.join(dir, f), 'utf8');
|
|
88
|
-
const { fm } = stripFrontmatter(text);
|
|
89
|
-
if (!fm) {
|
|
90
|
-
findings.push({ level: 'error', file: f, rule: 'frontmatter', detail: 'top-level skill (SKILL.md) is missing --- frontmatter --- (Claude/Copilot will not load it)' });
|
|
91
|
-
continue;
|
|
92
|
-
}
|
|
93
|
-
if (!/\bname\s*:/.test(fm))
|
|
94
|
-
findings.push({ level: 'error', file: f, rule: 'frontmatter-name', detail: 'no `name:` in frontmatter' });
|
|
95
|
-
if (!/\bdescription\s*:/.test(fm))
|
|
96
|
-
findings.push({ level: 'error', file: f, rule: 'frontmatter-description', detail: 'no `description:` in frontmatter' });
|
|
97
|
-
}
|
|
98
|
-
// 3) line budget — context-cost smell (advisory).
|
|
99
|
-
for (const f of skillFiles) {
|
|
100
|
-
const lines = fs.readFileSync(path.join(dir, f), 'utf8').split('\n').length;
|
|
101
|
-
if (lines > LINE_BUDGET)
|
|
102
|
-
findings.push({ level: 'warn', file: f, rule: 'line-budget', detail: `${lines} lines > ${LINE_BUDGET} (context-cost smell)` });
|
|
103
|
-
}
|
|
104
|
-
// 4) variant PRESENCE (not body equality) — every top-level skill should ship for both
|
|
105
|
-
// platforms. Catches "added a Claude skill but forgot the Copilot variant". Advisory.
|
|
106
|
-
const skillName = (dst) => { const m = dst.match(/\/(sungen-[^/]+)\/SKILL\.md$/); return m ? m[1] : null; };
|
|
107
|
-
const claudeSkills = new Set(), githubSkills = new Set();
|
|
108
|
-
for (const f of skillFiles) {
|
|
109
|
-
if (!isTopLevelSkill(f))
|
|
110
|
-
continue;
|
|
111
|
-
const name = skillName(target.get(f));
|
|
112
|
-
if (!name)
|
|
113
|
-
continue;
|
|
114
|
-
(f.startsWith('claude-') ? claudeSkills : githubSkills).add(name);
|
|
115
|
-
}
|
|
116
|
-
for (const n of claudeSkills)
|
|
117
|
-
if (!githubSkills.has(n))
|
|
118
|
-
findings.push({ level: 'warn', file: `claude .../${n}/SKILL.md`, rule: 'variant-missing', detail: `Claude skill "${n}" has no GitHub (Copilot) variant` });
|
|
119
|
-
for (const n of githubSkills)
|
|
120
|
-
if (!claudeSkills.has(n))
|
|
121
|
-
findings.push({ level: 'warn', file: `github .../${n}/SKILL.md`, rule: 'variant-missing', detail: `GitHub skill "${n}" has no Claude variant` });
|
|
122
|
-
return { checked: skillFiles.length, findings, errors: findings.filter((f) => f.level === 'error').length };
|
|
123
|
-
}
|
|
124
|
-
/** Default templates dir, resolved relative to this module (works from src via tsx and dist). */
|
|
125
|
-
function defaultSkillDir() {
|
|
126
|
-
// src/harness/eval → src/orchestrator/... | dist/harness/eval → dist/orchestrator/...
|
|
127
|
-
return path.resolve(__dirname, '..', '..', 'orchestrator', 'templates', 'ai-instructions');
|
|
128
|
-
}
|
|
129
|
-
//# sourceMappingURL=skill-lint.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"skill-lint.js","sourceRoot":"","sources":["../../../src/harness/eval/skill-lint.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,gCAkDC;AAGD,0CAGC;AAtFD;;;;;;;;;;;;GAYG;AACH,uCAAyB;AACzB,2CAA6B;AAC7B,0EAA4E;AAK5E,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,+DAA+D;AACxF,MAAM,QAAQ,GAAG,yBAAyB,CAAC;AAE3C,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IACjD,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACxC,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AACrD,CAAC;AAED,yFAAyF;AACzF,SAAgB,UAAU,CAAC,GAAW;IACpC,MAAM,QAAQ,GAAuB,EAAE,CAAC;IACxC,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7F,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzD,6FAA6F;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,CAAiB,wCAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAC9F,MAAM,eAAe,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAEnF,uEAAuE;IACvE,uFAAuF;IACvF,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,2EAA2E,EAAE,CAAC,CAAC;IAC5K,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,wCAAqB,EAAE,CAAC;QAC1C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,gEAAgE,EAAE,CAAC,CAAC;IAC1L,CAAC;IAED,sFAAsF;IACtF,qFAAqF;IACrF,4CAA4C;IAC5C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;YAAE,SAAS;QAClC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,EAAE,EAAE,EAAE,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,EAAE,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,6FAA6F,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAC9L,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,2BAA2B,EAAE,CAAC,CAAC;QACtI,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,kCAAkC,EAAE,CAAC,CAAC;IAC7J,CAAC;IAED,kDAAkD;IAClD,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC5E,IAAI,KAAK,GAAG,WAAW;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,KAAK,YAAY,WAAW,uBAAuB,EAAE,CAAC,CAAC;IAC1J,CAAC;IAED,uFAAuF;IACvF,yFAAyF;IACzF,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,EAAE,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,EAAE,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACzE,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;YAAE,SAAS;QAClC,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC;QAAC,IAAI,CAAC,IAAI;YAAE,SAAS;QAC5D,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpE,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,YAAY;QAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,WAAW,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,iBAAiB,CAAC,mCAAmC,EAAE,CAAC,CAAC;IACnN,KAAK,MAAM,CAAC,IAAI,YAAY;QAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,WAAW,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,iBAAiB,CAAC,yBAAyB,EAAE,CAAC,CAAC;IAEzM,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;AAC9G,CAAC;AAED,iGAAiG;AACjG,SAAgB,eAAe;IAC7B,wFAAwF;IACxF,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,iBAAiB,CAAC,CAAC;AAC7F,CAAC"}
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import { ScenarioInfo } from './parse';
|
|
2
|
-
export interface DownstreamResult {
|
|
3
|
-
downstreamRoutes: string[];
|
|
4
|
-
underCovered: {
|
|
5
|
-
route: string;
|
|
6
|
-
slug: string;
|
|
7
|
-
}[];
|
|
8
|
-
}
|
|
9
|
-
export declare function downstreamScope(specText: string, scenarios: ScenarioInfo[]): DownstreamResult;
|
|
10
|
-
export interface ManualOracleResult {
|
|
11
|
-
manualTotal: number;
|
|
12
|
-
insufficient: string[];
|
|
13
|
-
}
|
|
14
|
-
export declare function manualOracle(featureText: string): ManualOracleResult;
|
|
15
|
-
/** Titles asserting an ABSENCE must prove it (count / negative / @manual+oracle), not just a happy outcome. */
|
|
16
|
-
export declare function negativeSideEffect(scenarios: ScenarioInfo[]): string[];
|
|
17
|
-
/** A scenario should trace to a source: a viewpoint ID (its own scheme), an FR id, or a
|
|
18
|
-
* viewpoint item (keyword overlap). ID match is language-agnostic and primary. */
|
|
19
|
-
export declare function sourceBacked(scenarios: ScenarioInfo[], frIds: string[], viewpointItems: string[], viewpointIds: string[], featureText: string): string[];
|
|
20
|
-
export interface OwnershipResult {
|
|
21
|
-
duplicates: {
|
|
22
|
-
scenario: string;
|
|
23
|
-
flow: string;
|
|
24
|
-
}[];
|
|
25
|
-
}
|
|
26
|
-
/** Scenarios whose step-skeleton also appears in a sibling flow feature → duplicate ownership. */
|
|
27
|
-
export declare function crossArtifactOwnership(screenDir: string, scenarios: ScenarioInfo[]): OwnershipResult;
|
|
28
|
-
export declare function readText(p: string): string;
|
|
29
|
-
//# sourceMappingURL=quality-gates.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"quality-gates.d.ts","sourceRoot":"","sources":["../../src/harness/quality-gates.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAA2B,MAAM,SAAS,CAAC;AAIhE,MAAM,WAAW,gBAAgB;IAC/B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACjD;AAiBD,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,gBAAgB,CAe7F;AAID,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAMD,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,kBAAkB,CAgBpE;AAMD,+GAA+G;AAC/G,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,EAAE,CAStE;AAID;kFACkF;AAClF,wBAAgB,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAqBxJ;AAID,MAAM,WAAW,eAAe;IAAG,UAAU,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAAE;AAErF,kGAAkG;AAClG,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,eAAe,CAqBpG;AAGD,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAE1C"}
|
|
@@ -1,183 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.downstreamScope = downstreamScope;
|
|
37
|
-
exports.manualOracle = manualOracle;
|
|
38
|
-
exports.negativeSideEffect = negativeSideEffect;
|
|
39
|
-
exports.sourceBacked = sourceBacked;
|
|
40
|
-
exports.crossArtifactOwnership = crossArtifactOwnership;
|
|
41
|
-
exports.readText = readText;
|
|
42
|
-
/**
|
|
43
|
-
* Quality gates (batch): downstream-scope + manual-oracle + negative-side-effect +
|
|
44
|
-
* cross-artifact ownership + source-backed strictness.
|
|
45
|
-
* Generic — read the project's own spec.md / feature text / sibling flows; no project data.
|
|
46
|
-
*/
|
|
47
|
-
const fs = __importStar(require("fs"));
|
|
48
|
-
const path = __importStar(require("path"));
|
|
49
|
-
const parse_1 = require("./parse");
|
|
50
|
-
/** Routes the spec hands off to (Navigation Flow / success), other than the screen's own route. */
|
|
51
|
-
function downstreamRoutes(specText) {
|
|
52
|
-
const ownRoute = (specText.match(/\*\*Route\*\*\s*:\s*`?(\/[^\s`]+)/) || [])[1] || '';
|
|
53
|
-
const routes = new Set();
|
|
54
|
-
for (const line of specText.split('\n')) {
|
|
55
|
-
if (!/success|navigat|to \(|→/i.test(line))
|
|
56
|
-
continue;
|
|
57
|
-
for (const m of line.matchAll(/`?(\/[a-z][a-z0-9/_-]+)`?/gi)) {
|
|
58
|
-
const r = m[1];
|
|
59
|
-
if (r !== ownRoute && r.split('/').length > ownRoute.split('/').length - 0)
|
|
60
|
-
routes.add(r);
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
// keep only routes that extend beyond the own route (a distinct downstream surface)
|
|
64
|
-
return [...routes].filter((r) => r !== ownRoute && (!ownRoute || r.startsWith(ownRoute + '/') || r.split('/').length >= 3));
|
|
65
|
-
}
|
|
66
|
-
function downstreamScope(specText, scenarios) {
|
|
67
|
-
const routes = downstreamRoutes(specText);
|
|
68
|
-
const underCovered = [];
|
|
69
|
-
for (const route of routes) {
|
|
70
|
-
const slug = (route.split('/').filter(Boolean).pop() || route).toLowerCase();
|
|
71
|
-
const refs = scenarios.filter((s) => s.haystack.includes(slug) || s.haystack.includes(route.toLowerCase()));
|
|
72
|
-
if (!refs.length)
|
|
73
|
-
continue; // not referenced at all — out of this screen's scope entirely
|
|
74
|
-
// Substantively covered only if some scenario OPERATES on the downstream — i.e. it
|
|
75
|
-
// starts there (`is on [<downstream>]`) — not merely navigates to it as a terminal
|
|
76
|
-
// `see [<downstream>] page` assertion. The latter just proves the transition.
|
|
77
|
-
const opensOn = new RegExp(`\\bis on \\[[^\\]]*${slug}`, 'i');
|
|
78
|
-
const contentCovered = refs.some((s) => opensOn.test(s.haystack));
|
|
79
|
-
if (!contentCovered)
|
|
80
|
-
underCovered.push({ route, slug });
|
|
81
|
-
}
|
|
82
|
-
return { downstreamRoutes: routes, underCovered };
|
|
83
|
-
}
|
|
84
|
-
function blocks(featureText) {
|
|
85
|
-
return featureText.split(/\n\s*\n/).filter((b) => /\bScenario:/.test(b));
|
|
86
|
-
}
|
|
87
|
-
function manualOracle(featureText) {
|
|
88
|
-
const insufficient = [];
|
|
89
|
-
let manualTotal = 0;
|
|
90
|
-
for (const b of blocks(featureText)) {
|
|
91
|
-
if (!/@manual\b/.test(b))
|
|
92
|
-
continue;
|
|
93
|
-
manualTotal++;
|
|
94
|
-
const commentLines = b.split('\n').filter((l) => /^\s*#/.test(l));
|
|
95
|
-
const hasOracle = /tester verifies|oracle\s*:|requires|verify that|expected\s*:|steps?\s*:/i.test(b);
|
|
96
|
-
const hasNumberedSteps = /^\s*#?\s*\d+\.\s/m.test(b);
|
|
97
|
-
// sufficient = an oracle/steps marker, OR a substantive comment block (≥3 comment lines)
|
|
98
|
-
if (!(hasOracle || hasNumberedSteps || commentLines.length >= 3)) {
|
|
99
|
-
const name = (b.match(/Scenario:\s*(.+)/) || [])[1] || '(unnamed)';
|
|
100
|
-
insufficient.push(name.trim().slice(0, 80));
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
return { manualTotal, insufficient };
|
|
104
|
-
}
|
|
105
|
-
// ---------- #4 Negative side-effect ----------
|
|
106
|
-
const NEG_TITLE = /\b(does not|doesn't|no second|not dispatch|not sent|without submitting|no leak|single request|exactly one|count is 1|only one request|no duplicate|not create)\b/i;
|
|
107
|
-
/** Titles asserting an ABSENCE must prove it (count / negative / @manual+oracle), not just a happy outcome. */
|
|
108
|
-
function negativeSideEffect(scenarios) {
|
|
109
|
-
const flagged = [];
|
|
110
|
-
for (const s of scenarios) {
|
|
111
|
-
if (s.manual)
|
|
112
|
-
continue; // @manual is a legitimate deferral (oracle checked by #4 manual-oracle)
|
|
113
|
-
if (!NEG_TITLE.test(s.name))
|
|
114
|
-
continue;
|
|
115
|
-
const proven = /\bcount\b|tohavecount|table with|is hidden|are hidden|not complete|message is hidden/.test(s.stepsText);
|
|
116
|
-
if (!proven)
|
|
117
|
-
flagged.push(s.name.slice(0, 80));
|
|
118
|
-
}
|
|
119
|
-
return flagged;
|
|
120
|
-
}
|
|
121
|
-
// ---------- #7 Source-backed strictness ----------
|
|
122
|
-
/** A scenario should trace to a source: a viewpoint ID (its own scheme), an FR id, or a
|
|
123
|
-
* viewpoint item (keyword overlap). ID match is language-agnostic and primary. */
|
|
124
|
-
function sourceBacked(scenarios, frIds, viewpointItems, viewpointIds, featureText) {
|
|
125
|
-
if (!frIds.length && !viewpointItems.length && !viewpointIds.length)
|
|
126
|
-
return []; // no contract
|
|
127
|
-
const vpIds = viewpointIds.map((s) => s.toUpperCase());
|
|
128
|
-
const itemWords = viewpointItems.map((t) => new Set((t.toLowerCase().match(/[a-z][a-z-]{4,}/g) || [])));
|
|
129
|
-
// per-scenario blocks (INCLUDING comments) so an FR cited in a comment counts as a source
|
|
130
|
-
const blockOf = new Map();
|
|
131
|
-
for (const b of featureText.split(/\n\s*\n/)) {
|
|
132
|
-
const m = b.match(/Scenario:\s*(.+)/);
|
|
133
|
-
if (m)
|
|
134
|
-
blockOf.set(m[1].trim().toLowerCase(), b.toLowerCase());
|
|
135
|
-
}
|
|
136
|
-
const unsourced = [];
|
|
137
|
-
for (const s of scenarios) {
|
|
138
|
-
const id = (s.vpId || s.vpCode || '').toUpperCase();
|
|
139
|
-
const mapsId = !!id && vpIds.some((v) => id === v || id.startsWith(v) || v.startsWith((0, parse_1.idPrefix)(id)));
|
|
140
|
-
const block = blockOf.get(s.name.trim().toLowerCase()) || s.haystack;
|
|
141
|
-
const citesFr = frIds.some((fid) => block.includes(fid.toLowerCase()));
|
|
142
|
-
const sWords = new Set((s.haystack.match(/[a-z][a-z-]{4,}/g) || []));
|
|
143
|
-
const mapsItem = itemWords.some((iw) => { let hits = 0; for (const w of iw)
|
|
144
|
-
if (sWords.has(w))
|
|
145
|
-
hits++; return hits >= 2; });
|
|
146
|
-
if (!mapsId && !citesFr && !mapsItem)
|
|
147
|
-
unsourced.push(s.name.slice(0, 80));
|
|
148
|
-
}
|
|
149
|
-
return unsourced;
|
|
150
|
-
}
|
|
151
|
-
/** Scenarios whose step-skeleton also appears in a sibling flow feature → duplicate ownership. */
|
|
152
|
-
function crossArtifactOwnership(screenDir, scenarios) {
|
|
153
|
-
const duplicates = [];
|
|
154
|
-
// screenDir = <root>/qa/screens/<name>; flows live at <root>/qa/flows/*/features/*.feature
|
|
155
|
-
const flowsRoot = path.resolve(screenDir, '..', '..', 'flows');
|
|
156
|
-
if (!fs.existsSync(flowsRoot))
|
|
157
|
-
return { duplicates };
|
|
158
|
-
const bySkeleton = new Map();
|
|
159
|
-
for (const flow of fs.readdirSync(flowsRoot)) {
|
|
160
|
-
const fdir = path.join(flowsRoot, flow, 'features');
|
|
161
|
-
if (!fs.existsSync(fdir))
|
|
162
|
-
continue;
|
|
163
|
-
for (const f of fs.readdirSync(fdir).filter((x) => x.endsWith('.feature'))) {
|
|
164
|
-
for (const fs2 of (0, parse_1.loadScenarios)(path.join(fdir, f))) {
|
|
165
|
-
if (fs2.stepSkeleton && fs2.stepSkeleton.length > 20)
|
|
166
|
-
bySkeleton.set(fs2.stepSkeleton, flow);
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
if (!bySkeleton.size)
|
|
171
|
-
return { duplicates };
|
|
172
|
-
for (const s of scenarios) {
|
|
173
|
-
const flow = s.stepSkeleton && s.stepSkeleton.length > 20 ? bySkeleton.get(s.stepSkeleton) : undefined;
|
|
174
|
-
if (flow)
|
|
175
|
-
duplicates.push({ scenario: s.name.slice(0, 70), flow });
|
|
176
|
-
}
|
|
177
|
-
return { duplicates };
|
|
178
|
-
}
|
|
179
|
-
// convenience reader
|
|
180
|
-
function readText(p) {
|
|
181
|
-
return fs.existsSync(p) ? fs.readFileSync(p, 'utf-8') : '';
|
|
182
|
-
}
|
|
183
|
-
//# sourceMappingURL=quality-gates.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"quality-gates.js","sourceRoot":"","sources":["../../src/harness/quality-gates.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BA,0CAeC;AAaD,oCAgBC;AAOD,gDASC;AAMD,oCAqBC;AAOD,wDAqBC;AAGD,4BAEC;AAvJD;;;;GAIG;AACH,uCAAyB;AACzB,2CAA6B;AAC7B,mCAAgE;AAShE,mGAAmG;AACnG,SAAS,gBAAgB,CAAC,QAAgB;IACxC,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,mCAAmC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACtF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QACrD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE,CAAC;YAC7D,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACf,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IACD,oFAAoF;IACpF,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;AAC9H,CAAC;AAED,SAAgB,eAAe,CAAC,QAAgB,EAAE,SAAyB;IACzE,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAsC,EAAE,CAAC;IAC3D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;QAC7E,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5G,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAS,CAAC,8DAA8D;QAC1F,mFAAmF;QACnF,mFAAmF;QACnF,8EAA8E;QAC9E,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,sBAAsB,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QAC9D,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,cAAc;YAAE,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AACpD,CAAC;AASD,SAAS,MAAM,CAAC,WAAmB;IACjC,OAAO,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,SAAgB,YAAY,CAAC,WAAmB;IAC9C,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,SAAS;QACnC,WAAW,EAAE,CAAC;QACd,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,0EAA0E,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrG,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrD,yFAAyF;QACzF,IAAI,CAAC,CAAC,SAAS,IAAI,gBAAgB,IAAI,YAAY,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC;YACnE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;AACvC,CAAC;AAED,gDAAgD;AAEhD,MAAM,SAAS,GAAG,mKAAmK,CAAC;AAEtL,+GAA+G;AAC/G,SAAgB,kBAAkB,CAAC,SAAyB;IAC1D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,MAAM;YAAE,SAAS,CAAiB,wEAAwE;QAChH,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,SAAS;QACtC,MAAM,MAAM,GAAG,sFAAsF,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACxH,IAAI,CAAC,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,oDAAoD;AAEpD;kFACkF;AAClF,SAAgB,YAAY,CAAC,SAAyB,EAAE,KAAe,EAAE,cAAwB,EAAE,YAAsB,EAAE,WAAmB;IAC5I,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC,CAAC,cAAc;IAC9F,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACxG,0FAA0F;IAC1F,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,IAAI,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,IAAA,gBAAQ,EAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACrG,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;QACrE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,EAAE;YAAE,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,IAAI,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5H,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ;YAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAMD,kGAAkG;AAClG,SAAgB,sBAAsB,CAAC,SAAiB,EAAE,SAAyB;IACjF,MAAM,UAAU,GAAyC,EAAE,CAAC;IAC5D,2FAA2F;IAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,CAAC;IACrD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QACnC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;YAC3E,KAAK,MAAM,GAAG,IAAI,IAAA,qBAAa,EAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,IAAI,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,EAAE;oBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,IAAI;QAAE,OAAO,EAAE,UAAU,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACvG,IAAI,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,CAAC;AACxB,CAAC;AAED,qBAAqB;AACrB,SAAgB,QAAQ,CAAC,CAAS;IAChC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC7D,CAAC"}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { ScenarioInfo } from './parse';
|
|
2
|
-
export interface LedgerItem {
|
|
3
|
-
id?: string;
|
|
4
|
-
text: string;
|
|
5
|
-
covered: boolean;
|
|
6
|
-
}
|
|
7
|
-
export interface LedgerResult {
|
|
8
|
-
hasViewpoint: boolean;
|
|
9
|
-
total: number;
|
|
10
|
-
covered: number;
|
|
11
|
-
ratio: number;
|
|
12
|
-
missing: {
|
|
13
|
-
id?: string;
|
|
14
|
-
text: string;
|
|
15
|
-
}[];
|
|
16
|
-
}
|
|
17
|
-
/** Extract atomic checklist items from a viewpoint file (format-tolerant). */
|
|
18
|
-
export declare function parseViewpointItems(viewpointPath: string): {
|
|
19
|
-
id?: string;
|
|
20
|
-
text: string;
|
|
21
|
-
}[];
|
|
22
|
-
export declare function viewpointLedger(viewpointPath: string, scenarios: ScenarioInfo[], featureText: string): LedgerResult;
|
|
23
|
-
//# sourceMappingURL=viewpoint-ledger.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"viewpoint-ledger.d.ts","sourceRoot":"","sources":["../../src/harness/viewpoint-ledger.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,MAAM,WAAW,UAAU;IAAG,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE;AAE3E,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,OAAO,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC1C;AAKD,8EAA8E;AAC9E,wBAAgB,mBAAmB,CAAC,aAAa,EAAE,MAAM,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,CA4B1F;AAED,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,WAAW,EAAE,MAAM,GAAG,YAAY,CAsBnH"}
|
|
@@ -1,118 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.parseViewpointItems = parseViewpointItems;
|
|
37
|
-
exports.viewpointLedger = viewpointLedger;
|
|
38
|
-
/**
|
|
39
|
-
* Viewpoint Atomic Coverage Ledger (harness #2).
|
|
40
|
-
*
|
|
41
|
-
* The project's `test-viewpoint.md` IS the coverage contract. This parses it into ATOMIC
|
|
42
|
-
* items (each bullet / table row / ID-prefixed line) and reports the status of EACH —
|
|
43
|
-
* covered / missing — instead of the coarse "viewpoint mentioned" signal. It is fully
|
|
44
|
-
* project-driven (works on any project's viewpoint file, any domain), which is why it
|
|
45
|
-
* scales where a hardcoded domain catalog does not. Advisory: it surfaces the per-item
|
|
46
|
-
* gaps that inflate a "looks-covered" score; it does not fail the gate.
|
|
47
|
-
*/
|
|
48
|
-
const fs = __importStar(require("fs"));
|
|
49
|
-
const ID_RE = /\b([A-Z]{1,5}\d{0,2}(?:[.\-][A-Za-z0-9]+)*-?\d{0,3})\b/; // VP0.Title, VP7-002, MS-HP-001, TV-01
|
|
50
|
-
const GENERIC = new Set(['display', 'shown', 'value', 'field', 'input', 'page', 'screen', 'button', 'link', 'text', 'check', 'verify', 'should', 'with', 'when', 'then', 'user', 'this', 'that', 'each', 'item', 'items']);
|
|
51
|
-
/** Extract atomic checklist items from a viewpoint file (format-tolerant). */
|
|
52
|
-
function parseViewpointItems(viewpointPath) {
|
|
53
|
-
if (!fs.existsSync(viewpointPath))
|
|
54
|
-
return [];
|
|
55
|
-
const lines = fs.readFileSync(viewpointPath, 'utf-8').split('\n');
|
|
56
|
-
const items = [];
|
|
57
|
-
let inFence = false;
|
|
58
|
-
for (const raw of lines) {
|
|
59
|
-
const line = raw.trim();
|
|
60
|
-
if (line.startsWith('```')) {
|
|
61
|
-
inFence = !inFence;
|
|
62
|
-
continue;
|
|
63
|
-
}
|
|
64
|
-
if (inFence || !line)
|
|
65
|
-
continue;
|
|
66
|
-
if (/^#{1,6}\s/.test(line))
|
|
67
|
-
continue; // markdown heading
|
|
68
|
-
let text = '';
|
|
69
|
-
const bullet = line.match(/^(?:[-*+]|\d+[.)])\s+(.*)$/);
|
|
70
|
-
if (bullet)
|
|
71
|
-
text = bullet[1];
|
|
72
|
-
else if (line.startsWith('|')) { // table data row
|
|
73
|
-
if (/^\|[\s|:-]+\|?$/.test(line))
|
|
74
|
-
continue; // separator
|
|
75
|
-
const cells = line.split('|').map((c) => c.trim()).filter(Boolean);
|
|
76
|
-
if (/^(vp|id|viewpoint|priority|reason|no\.?|category|item|trigger|#|pattern|applicable|notes|field|constraint|code|description|status)$/i.test(cells[0] || ''))
|
|
77
|
-
continue; // header
|
|
78
|
-
text = cells.join(' — ');
|
|
79
|
-
}
|
|
80
|
-
else
|
|
81
|
-
continue;
|
|
82
|
-
text = text.replace(/[*`]/g, '').trim();
|
|
83
|
-
if (!text)
|
|
84
|
-
continue;
|
|
85
|
-
const idM = text.match(ID_RE);
|
|
86
|
-
const id = idM && /\d/.test(idM[1]) ? idM[1] : undefined; // require a digit so prose words aren't IDs
|
|
87
|
-
const words = (text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w));
|
|
88
|
-
if (!id && words.length < 2)
|
|
89
|
-
continue; // not substantive enough to track
|
|
90
|
-
items.push({ id, text: text.slice(0, 100) });
|
|
91
|
-
}
|
|
92
|
-
return items;
|
|
93
|
-
}
|
|
94
|
-
function viewpointLedger(viewpointPath, scenarios, featureText) {
|
|
95
|
-
const items = parseViewpointItems(viewpointPath);
|
|
96
|
-
if (!fs.existsSync(viewpointPath) || items.length === 0) {
|
|
97
|
-
return { hasViewpoint: fs.existsSync(viewpointPath), total: 0, covered: 0, ratio: 1, missing: [] };
|
|
98
|
-
}
|
|
99
|
-
const featLower = featureText.toLowerCase();
|
|
100
|
-
const missing = [];
|
|
101
|
-
let covered = 0;
|
|
102
|
-
for (const item of items) {
|
|
103
|
-
let isCovered = false;
|
|
104
|
-
if (item.id && featLower.includes(item.id.toLowerCase()))
|
|
105
|
-
isCovered = true;
|
|
106
|
-
else {
|
|
107
|
-
const words = [...new Set((item.text.toLowerCase().match(/[a-z][a-z-]{3,}/g) || []).filter((w) => !GENERIC.has(w)))];
|
|
108
|
-
const need = Math.min(2, words.length);
|
|
109
|
-
isCovered = words.length > 0 && scenarios.some((s) => words.filter((w) => s.haystack.includes(w)).length >= need);
|
|
110
|
-
}
|
|
111
|
-
if (isCovered)
|
|
112
|
-
covered++;
|
|
113
|
-
else
|
|
114
|
-
missing.push({ id: item.id, text: item.text });
|
|
115
|
-
}
|
|
116
|
-
return { hasViewpoint: true, total: items.length, covered, ratio: items.length ? covered / items.length : 1, missing };
|
|
117
|
-
}
|
|
118
|
-
//# sourceMappingURL=viewpoint-ledger.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"viewpoint-ledger.js","sourceRoot":"","sources":["../../src/harness/viewpoint-ledger.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BA,kDA4BC;AAED,0CAsBC;AA/ED;;;;;;;;;GASG;AACH,uCAAyB;AAazB,MAAM,KAAK,GAAG,wDAAwD,CAAC,CAAC,uCAAuC;AAC/G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE3N,8EAA8E;AAC9E,SAAgB,mBAAmB,CAAC,aAAqB;IACvD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,EAAE,CAAC;IAC7C,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClE,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,OAAO,GAAG,CAAC,OAAO,CAAC;YAAC,SAAS;QAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,CAAC,IAAI;YAAE,SAAS;QAC/B,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS,CAAiB,mBAAmB;QACzE,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACxD,IAAI,MAAM;YAAE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;aACxB,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAuB,iBAAiB;YACtE,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS,CAAW,YAAY;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,sIAAsI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAAE,SAAS,CAAC,SAAS;YACpL,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;;YAAM,SAAS;QAChB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,4CAA4C;QACtG,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,IAAI,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS,CAAiB,kCAAkC;QACzF,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAgB,eAAe,CAAC,aAAqB,EAAE,SAAyB,EAAE,WAAmB;IACnG,MAAM,KAAK,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAC;IACjD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,OAAO,EAAE,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACrG,CAAC;IACD,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAoC,EAAE,CAAC;IACpD,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC;aACtE,CAAC;YACJ,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACvC,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;QACpH,CAAC;QACD,IAAI,SAAS;YAAE,OAAO,EAAE,CAAC;;YACpB,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AACzH,CAAC"}
|
package/src/cli/commands/eval.ts
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { Command } from 'commander';
|
|
2
|
-
import { lintSkills, defaultSkillDir } from '../../harness/eval/skill-lint';
|
|
3
|
-
|
|
4
|
-
export function registerEvalCommand(program: Command): void {
|
|
5
|
-
program
|
|
6
|
-
.command('eval')
|
|
7
|
-
.description('Eval harness: quality checks on Sungen\'s own skills/instructions (dev/CI)')
|
|
8
|
-
.option('--skills', 'Static skill-lint: frontmatter, line budget, claude↔github sync, registration')
|
|
9
|
-
.option('--dir <path>', 'Templates dir to lint (default: bundled ai-instructions)')
|
|
10
|
-
.option('--json', 'Output the raw findings JSON')
|
|
11
|
-
.action((options) => {
|
|
12
|
-
try {
|
|
13
|
-
if (!options.skills) throw new Error('Provide --skills (the only eval mode today)');
|
|
14
|
-
const dir = options.dir || defaultSkillDir();
|
|
15
|
-
const r = lintSkills(dir);
|
|
16
|
-
if (options.json) { console.log(JSON.stringify(r, null, 2)); process.exit(r.errors > 0 ? 2 : 0); }
|
|
17
|
-
console.log('');
|
|
18
|
-
console.log(`━━━ Skill-lint: ${r.checked} skill template(s) ━━━`);
|
|
19
|
-
if (!r.findings.length) console.log(' ✓ all skills pass (frontmatter · line-budget · variant-sync · registration)');
|
|
20
|
-
for (const f of r.findings) console.log(` ${f.level === 'error' ? '✗' : '⚠'} [${f.rule}] ${f.file} — ${f.detail}`);
|
|
21
|
-
console.log('');
|
|
22
|
-
process.exit(r.errors > 0 ? 2 : 0);
|
|
23
|
-
} catch (error) {
|
|
24
|
-
console.error('Error:', error instanceof Error ? error.message : error);
|
|
25
|
-
process.exit(1);
|
|
26
|
-
}
|
|
27
|
-
});
|
|
28
|
-
}
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Static skill-lint (Eval Harness L1) — deterministic quality checks on Sungen's OWN
|
|
3
|
-
* AI-instruction templates, so a broken / unregistered / oversized skill fails before it
|
|
4
|
-
* ships. Learned (generically) from the "static validations" tier of an agent-kit evals
|
|
5
|
-
* layer. No project data — this lints the sungen package's own templates.
|
|
6
|
-
*
|
|
7
|
-
* Design note: the checks are MAPPING-DRIVEN. `AI_RULES_FILE_MAPPING` is the source of
|
|
8
|
-
* truth for what each template installs as, so the lint uses the install target (does it
|
|
9
|
-
* end in `/SKILL.md`?) to tell a top-level skill from a sub-content fragment — instead of
|
|
10
|
-
* guessing from filenames. We deliberately do NOT enforce claude↔github body parity: the
|
|
11
|
-
* two variants are hand-tuned per platform and intentionally diverge in wording and even
|
|
12
|
-
* structure, so byte/heading equality would be pure false positives.
|
|
13
|
-
*/
|
|
14
|
-
import * as fs from 'fs';
|
|
15
|
-
import * as path from 'path';
|
|
16
|
-
import { AI_RULES_FILE_MAPPING } from '../../orchestrator/ai-rules-updater';
|
|
17
|
-
|
|
18
|
-
export interface SkillLintFinding { level: 'error' | 'warn'; file: string; rule: string; detail: string }
|
|
19
|
-
export interface SkillLintResult { checked: number; findings: SkillLintFinding[]; errors: number }
|
|
20
|
-
|
|
21
|
-
const LINE_BUDGET = 700; // a skill much larger than this is a context-cost smell (warn)
|
|
22
|
-
const SKILL_RE = /^(claude|github)-skill-/;
|
|
23
|
-
|
|
24
|
-
function stripFrontmatter(text: string): { fm: string | null; body: string } {
|
|
25
|
-
const m = text.match(/^---\n([\s\S]*?)\n---\n?/);
|
|
26
|
-
if (!m) return { fm: null, body: text };
|
|
27
|
-
return { fm: m[1], body: text.slice(m[0].length) };
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/** Lint the AI-instruction templates in `dir` (default: the sungen source templates). */
|
|
31
|
-
export function lintSkills(dir: string): SkillLintResult {
|
|
32
|
-
const findings: SkillLintFinding[] = [];
|
|
33
|
-
const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith('.md')) : [];
|
|
34
|
-
const skillFiles = files.filter((f) => SKILL_RE.test(f));
|
|
35
|
-
|
|
36
|
-
// mapping: template file -> install target (source of truth for "is this a top-level skill")
|
|
37
|
-
const target = new Map<string, string>(AI_RULES_FILE_MAPPING.map(([tpl, dst]) => [tpl, dst]));
|
|
38
|
-
const isTopLevelSkill = (f: string) => (target.get(f) || '').endsWith('/SKILL.md');
|
|
39
|
-
|
|
40
|
-
// 1) registration integrity (bidirectional) — the highest-value check:
|
|
41
|
-
// a skill file missing from the mapping never installs; a mapping to a missing file
|
|
42
|
-
// ships a broken/empty skill.
|
|
43
|
-
for (const f of skillFiles) {
|
|
44
|
-
if (!target.has(f)) findings.push({ level: 'error', file: f, rule: 'unregistered', detail: 'skill template not in AI_RULES_FILE_MAPPING (it would never be installed)' });
|
|
45
|
-
}
|
|
46
|
-
for (const [tpl] of AI_RULES_FILE_MAPPING) {
|
|
47
|
-
if (!fs.existsSync(path.join(dir, tpl))) findings.push({ level: 'error', file: tpl, rule: 'mapped-missing', detail: 'AI_RULES_FILE_MAPPING points to a template that does not exist' });
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
// 2) frontmatter (name + description) — ONLY for top-level skills (SKILL.md targets).
|
|
51
|
-
// Sub-content fragments (mode-*.md, group-*.md) are loaded by their parent router
|
|
52
|
-
// and legitimately carry no frontmatter.
|
|
53
|
-
for (const f of skillFiles) {
|
|
54
|
-
if (!isTopLevelSkill(f)) continue;
|
|
55
|
-
const text = fs.readFileSync(path.join(dir, f), 'utf8');
|
|
56
|
-
const { fm } = stripFrontmatter(text);
|
|
57
|
-
if (!fm) { findings.push({ level: 'error', file: f, rule: 'frontmatter', detail: 'top-level skill (SKILL.md) is missing --- frontmatter --- (Claude/Copilot will not load it)' }); continue; }
|
|
58
|
-
if (!/\bname\s*:/.test(fm)) findings.push({ level: 'error', file: f, rule: 'frontmatter-name', detail: 'no `name:` in frontmatter' });
|
|
59
|
-
if (!/\bdescription\s*:/.test(fm)) findings.push({ level: 'error', file: f, rule: 'frontmatter-description', detail: 'no `description:` in frontmatter' });
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// 3) line budget — context-cost smell (advisory).
|
|
63
|
-
for (const f of skillFiles) {
|
|
64
|
-
const lines = fs.readFileSync(path.join(dir, f), 'utf8').split('\n').length;
|
|
65
|
-
if (lines > LINE_BUDGET) findings.push({ level: 'warn', file: f, rule: 'line-budget', detail: `${lines} lines > ${LINE_BUDGET} (context-cost smell)` });
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// 4) variant PRESENCE (not body equality) — every top-level skill should ship for both
|
|
69
|
-
// platforms. Catches "added a Claude skill but forgot the Copilot variant". Advisory.
|
|
70
|
-
const skillName = (dst: string) => { const m = dst.match(/\/(sungen-[^/]+)\/SKILL\.md$/); return m ? m[1] : null; };
|
|
71
|
-
const claudeSkills = new Set<string>(), githubSkills = new Set<string>();
|
|
72
|
-
for (const f of skillFiles) {
|
|
73
|
-
if (!isTopLevelSkill(f)) continue;
|
|
74
|
-
const name = skillName(target.get(f)!); if (!name) continue;
|
|
75
|
-
(f.startsWith('claude-') ? claudeSkills : githubSkills).add(name);
|
|
76
|
-
}
|
|
77
|
-
for (const n of claudeSkills) if (!githubSkills.has(n)) findings.push({ level: 'warn', file: `claude .../${n}/SKILL.md`, rule: 'variant-missing', detail: `Claude skill "${n}" has no GitHub (Copilot) variant` });
|
|
78
|
-
for (const n of githubSkills) if (!claudeSkills.has(n)) findings.push({ level: 'warn', file: `github .../${n}/SKILL.md`, rule: 'variant-missing', detail: `GitHub skill "${n}" has no Claude variant` });
|
|
79
|
-
|
|
80
|
-
return { checked: skillFiles.length, findings, errors: findings.filter((f) => f.level === 'error').length };
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
/** Default templates dir, resolved relative to this module (works from src via tsx and dist). */
|
|
84
|
-
export function defaultSkillDir(): string {
|
|
85
|
-
// src/harness/eval → src/orchestrator/... | dist/harness/eval → dist/orchestrator/...
|
|
86
|
-
return path.resolve(__dirname, '..', '..', 'orchestrator', 'templates', 'ai-instructions');
|
|
87
|
-
}
|