@sun-asterisk/sungen 3.0.0-beta.81 → 3.0.0-beta.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +6 -0
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/eval.d.ts +3 -0
- package/dist/cli/commands/eval.d.ts.map +1 -0
- package/dist/cli/commands/eval.js +37 -0
- package/dist/cli/commands/eval.js.map +1 -0
- package/dist/cli/index.js +2 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/harness/audit.d.ts +8 -0
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +32 -1
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/eval/skill-lint.d.ts +16 -0
- package/dist/harness/eval/skill-lint.d.ts.map +1 -0
- package/dist/harness/eval/skill-lint.js +129 -0
- package/dist/harness/eval/skill-lint.js.map +1 -0
- package/dist/harness/parse.d.ts +6 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +18 -3
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/quality-gates.d.ts +13 -0
- package/dist/harness/quality-gates.d.ts.map +1 -1
- package/dist/harness/quality-gates.js +82 -2
- package/dist/harness/quality-gates.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +13 -5
- package/dist/harness/sensors.js.map +1 -1
- package/package.json +2 -2
- package/src/cli/commands/audit.ts +6 -0
- package/src/cli/commands/eval.ts +28 -0
- package/src/cli/index.ts +2 -0
- package/src/harness/audit.ts +42 -4
- package/src/harness/eval/skill-lint.ts +87 -0
- package/src/harness/parse.ts +19 -3
- package/src/harness/quality-gates.ts +76 -3
- package/src/harness/sensors.ts +12 -6
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Quality gates (batch): downstream-scope
|
|
3
|
-
*
|
|
2
|
+
* Quality gates (batch): downstream-scope + manual-oracle + negative-side-effect +
|
|
3
|
+
* cross-artifact ownership + source-backed strictness.
|
|
4
|
+
* Generic — read the project's own spec.md / feature text / sibling flows; no project data.
|
|
4
5
|
*/
|
|
5
6
|
import * as fs from 'fs';
|
|
6
|
-
import
|
|
7
|
+
import * as path from 'path';
|
|
8
|
+
import { ScenarioInfo, loadScenarios, idPrefix } from './parse';
|
|
7
9
|
|
|
8
10
|
// ---------- #2 Downstream-scope ----------
|
|
9
11
|
|
|
@@ -73,6 +75,77 @@ export function manualOracle(featureText: string): ManualOracleResult {
|
|
|
73
75
|
return { manualTotal, insufficient };
|
|
74
76
|
}
|
|
75
77
|
|
|
78
|
+
// ---------- #4 Negative side-effect ----------
|
|
79
|
+
|
|
80
|
+
const NEG_TITLE = /\b(does not|doesn't|no second|not dispatch|not sent|without submitting|no leak|single request|exactly one|count is 1|only one request|no duplicate|not create)\b/i;
|
|
81
|
+
|
|
82
|
+
/** Titles asserting an ABSENCE must prove it (count / negative / @manual+oracle), not just a happy outcome. */
|
|
83
|
+
export function negativeSideEffect(scenarios: ScenarioInfo[]): string[] {
|
|
84
|
+
const flagged: string[] = [];
|
|
85
|
+
for (const s of scenarios) {
|
|
86
|
+
if (s.manual) continue; // @manual is a legitimate deferral (oracle checked by #4 manual-oracle)
|
|
87
|
+
if (!NEG_TITLE.test(s.name)) continue;
|
|
88
|
+
const proven = /\bcount\b|tohavecount|table with|is hidden|are hidden|not complete|message is hidden/.test(s.stepsText);
|
|
89
|
+
if (!proven) flagged.push(s.name.slice(0, 80));
|
|
90
|
+
}
|
|
91
|
+
return flagged;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ---------- #7 Source-backed strictness ----------
|
|
95
|
+
|
|
96
|
+
/** A scenario should trace to a source: a viewpoint ID (its own scheme), an FR id, or a
|
|
97
|
+
* viewpoint item (keyword overlap). ID match is language-agnostic and primary. */
|
|
98
|
+
export function sourceBacked(scenarios: ScenarioInfo[], frIds: string[], viewpointItems: string[], viewpointIds: string[], featureText: string): string[] {
|
|
99
|
+
if (!frIds.length && !viewpointItems.length && !viewpointIds.length) return []; // no contract
|
|
100
|
+
const vpIds = viewpointIds.map((s) => s.toUpperCase());
|
|
101
|
+
const itemWords = viewpointItems.map((t) => new Set((t.toLowerCase().match(/[a-z][a-z-]{4,}/g) || [])));
|
|
102
|
+
// per-scenario blocks (INCLUDING comments) so an FR cited in a comment counts as a source
|
|
103
|
+
const blockOf = new Map<string, string>();
|
|
104
|
+
for (const b of featureText.split(/\n\s*\n/)) {
|
|
105
|
+
const m = b.match(/Scenario:\s*(.+)/);
|
|
106
|
+
if (m) blockOf.set(m[1].trim().toLowerCase(), b.toLowerCase());
|
|
107
|
+
}
|
|
108
|
+
const unsourced: string[] = [];
|
|
109
|
+
for (const s of scenarios) {
|
|
110
|
+
const id = (s.vpId || s.vpCode || '').toUpperCase();
|
|
111
|
+
const mapsId = !!id && vpIds.some((v) => id === v || id.startsWith(v) || v.startsWith(idPrefix(id)));
|
|
112
|
+
const block = blockOf.get(s.name.trim().toLowerCase()) || s.haystack;
|
|
113
|
+
const citesFr = frIds.some((fid) => block.includes(fid.toLowerCase()));
|
|
114
|
+
const sWords = new Set((s.haystack.match(/[a-z][a-z-]{4,}/g) || []));
|
|
115
|
+
const mapsItem = itemWords.some((iw) => { let hits = 0; for (const w of iw) if (sWords.has(w)) hits++; return hits >= 2; });
|
|
116
|
+
if (!mapsId && !citesFr && !mapsItem) unsourced.push(s.name.slice(0, 80));
|
|
117
|
+
}
|
|
118
|
+
return unsourced;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// ---------- #6 Cross-artifact ownership ----------
|
|
122
|
+
|
|
123
|
+
export interface OwnershipResult { duplicates: { scenario: string; flow: string }[] }
|
|
124
|
+
|
|
125
|
+
/** Scenarios whose step-skeleton also appears in a sibling flow feature → duplicate ownership. */
|
|
126
|
+
export function crossArtifactOwnership(screenDir: string, scenarios: ScenarioInfo[]): OwnershipResult {
|
|
127
|
+
const duplicates: { scenario: string; flow: string }[] = [];
|
|
128
|
+
// screenDir = <root>/qa/screens/<name>; flows live at <root>/qa/flows/*/features/*.feature
|
|
129
|
+
const flowsRoot = path.resolve(screenDir, '..', '..', 'flows');
|
|
130
|
+
if (!fs.existsSync(flowsRoot)) return { duplicates };
|
|
131
|
+
const bySkeleton = new Map<string, string>();
|
|
132
|
+
for (const flow of fs.readdirSync(flowsRoot)) {
|
|
133
|
+
const fdir = path.join(flowsRoot, flow, 'features');
|
|
134
|
+
if (!fs.existsSync(fdir)) continue;
|
|
135
|
+
for (const f of fs.readdirSync(fdir).filter((x) => x.endsWith('.feature'))) {
|
|
136
|
+
for (const fs2 of loadScenarios(path.join(fdir, f))) {
|
|
137
|
+
if (fs2.stepSkeleton && fs2.stepSkeleton.length > 20) bySkeleton.set(fs2.stepSkeleton, flow);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
if (!bySkeleton.size) return { duplicates };
|
|
142
|
+
for (const s of scenarios) {
|
|
143
|
+
const flow = s.stepSkeleton && s.stepSkeleton.length > 20 ? bySkeleton.get(s.stepSkeleton) : undefined;
|
|
144
|
+
if (flow) duplicates.push({ scenario: s.name.slice(0, 70), flow });
|
|
145
|
+
}
|
|
146
|
+
return { duplicates };
|
|
147
|
+
}
|
|
148
|
+
|
|
76
149
|
// convenience reader
|
|
77
150
|
export function readText(p: string): string {
|
|
78
151
|
return fs.existsSync(p) ? fs.readFileSync(p, 'utf-8') : '';
|
package/src/harness/sensors.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import * as fs from 'fs';
|
|
10
10
|
import * as path from 'path';
|
|
11
11
|
import { parse as parseYaml } from 'yaml';
|
|
12
|
-
import { ScenarioInfo, ViewpointEntry } from './parse';
|
|
12
|
+
import { ScenarioInfo, ViewpointEntry, idPrefix } from './parse';
|
|
13
13
|
|
|
14
14
|
// Business-critical category codes (project VP-<CAT> prefixes). Configurable later.
|
|
15
15
|
const BUSINESS_CRITICAL_CATS = ['LIST', 'CART', 'PRODUCT', 'FILTER', 'CHECKOUT', 'ORDER'];
|
|
@@ -263,17 +263,23 @@ export interface TraceResult {
|
|
|
263
263
|
|
|
264
264
|
export function traceability(scenarios: ScenarioInfo[], viewpoints: ViewpointEntry[]): TraceResult {
|
|
265
265
|
const overviewIds = new Set(viewpoints.map((v) => v.id.toUpperCase()));
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
266
|
+
// A scenario carries an ID if it has a project-scheme leading ID (vpId) or a VP-CAT code.
|
|
267
|
+
const withCode = scenarios.filter((s) => s.vpId || s.vpCode);
|
|
268
|
+
// Maps to overview if the scenario's ID, its sequence-stripped prefix, or its VP-CAT code
|
|
269
|
+
// matches a declared viewpoint ID (format-tolerant: VP0-001↔VP0, MS-HP-001↔MS-HP-001).
|
|
270
|
+
const mapped = withCode.filter((s) => {
|
|
271
|
+
const id = (s.vpId || s.vpCode || '').toUpperCase();
|
|
272
|
+
if (overviewIds.has(id) || overviewIds.has(idPrefix(id))) return true;
|
|
273
|
+
return [...overviewIds].some((oid) => id.startsWith(oid) || oid.startsWith(idPrefix(id)) || (!!s.category && oid.includes(s.category)));
|
|
274
|
+
});
|
|
269
275
|
return {
|
|
270
276
|
total: scenarios.length,
|
|
271
277
|
withVpCode: withCode.length,
|
|
272
278
|
mappedToOverview: mapped.length,
|
|
273
279
|
withVpCodeRatio: scenarios.length ? withCode.length / scenarios.length : 0,
|
|
274
280
|
mappedRatio: scenarios.length ? mapped.length / scenarios.length : 0,
|
|
275
|
-
note: mapped.length < withCode.length * 0.5
|
|
276
|
-
? '
|
|
281
|
+
note: withCode.length && mapped.length < withCode.length * 0.5
|
|
282
|
+
? 'Scenario IDs do not match the viewpoint-overview ids (weak traceability — re-tag to the project viewpoint IDs).'
|
|
277
283
|
: 'Traceable.',
|
|
278
284
|
};
|
|
279
285
|
}
|