@sun-asterisk/sungen 3.2.21-beta.2 → 3.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +0 -8
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +0 -13
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/trace.d.ts.map +1 -1
- package/dist/cli/commands/trace.js +0 -9
- package/dist/cli/commands/trace.js.map +1 -1
- package/dist/exporters/matrix/build.d.ts +0 -10
- package/dist/exporters/matrix/build.d.ts.map +1 -1
- package/dist/exporters/matrix/build.js +0 -37
- package/dist/exporters/matrix/build.js.map +1 -1
- package/dist/exporters/matrix/export.d.ts.map +1 -1
- package/dist/exporters/matrix/export.js +0 -11
- package/dist/exporters/matrix/export.js.map +1 -1
- package/dist/exporters/matrix/render-xlsx.d.ts.map +1 -1
- package/dist/exporters/matrix/render-xlsx.js +1 -44
- package/dist/exporters/matrix/render-xlsx.js.map +1 -1
- package/dist/exporters/matrix/types.d.ts +0 -8
- package/dist/exporters/matrix/types.d.ts.map +1 -1
- package/dist/exporters/matrix/types.js.map +1 -1
- package/dist/exporters/playwright-report-parser.d.ts.map +1 -1
- package/dist/exporters/playwright-report-parser.js +0 -1
- package/dist/exporters/playwright-report-parser.js.map +1 -1
- package/dist/exporters/types.d.ts +0 -2
- package/dist/exporters/types.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.d.ts +1 -6
- package/dist/generators/test-generator/diagnostics.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.js +0 -5
- package/dist/generators/test-generator/diagnostics.js.map +1 -1
- package/dist/generators/test-generator/patterns/index.d.ts.map +1 -1
- package/dist/generators/test-generator/patterns/index.js +19 -27
- package/dist/generators/test-generator/patterns/index.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.d.ts.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +0 -22
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/harness/audit.d.ts +0 -2
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +9 -91
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-plan.d.ts +0 -3
- package/dist/harness/flow-plan.d.ts.map +1 -1
- package/dist/harness/flow-plan.js +2 -6
- package/dist/harness/flow-plan.js.map +1 -1
- package/dist/harness/parse.d.ts +0 -5
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +1 -29
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +1 -13
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/spec-coverage.d.ts.map +1 -1
- package/dist/harness/spec-coverage.js +5 -29
- package/dist/harness/spec-coverage.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/add-flow.md +3 -46
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +0 -10
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +16 -52
- package/dist/orchestrator/templates/qa-context.md +1 -14
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +0 -8
- package/src/cli/commands/delivery.ts +0 -12
- package/src/cli/commands/trace.ts +0 -9
- package/src/exporters/matrix/build.ts +0 -39
- package/src/exporters/matrix/export.ts +0 -11
- package/src/exporters/matrix/render-xlsx.ts +1 -45
- package/src/exporters/matrix/types.ts +0 -8
- package/src/exporters/playwright-report-parser.ts +0 -2
- package/src/exporters/types.ts +0 -2
- package/src/generators/test-generator/diagnostics.ts +1 -6
- package/src/generators/test-generator/patterns/index.ts +24 -30
- package/src/generators/test-generator/step-mapper.ts +0 -22
- package/src/harness/audit.ts +10 -94
- package/src/harness/flow-plan.ts +3 -10
- package/src/harness/parse.ts +1 -31
- package/src/harness/sensors.ts +1 -13
- package/src/harness/spec-coverage.ts +4 -26
- package/src/orchestrator/templates/ai-src/commands/add-flow.md +3 -46
- package/src/orchestrator/templates/ai-src/commands/create-test.md +0 -10
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +16 -52
- package/src/orchestrator/templates/qa-context.md +1 -14
- package/dist/harness/flow-contract.d.ts +0 -71
- package/dist/harness/flow-contract.d.ts.map +0 -1
- package/dist/harness/flow-contract.js +0 -235
- package/dist/harness/flow-contract.js.map +0 -1
- package/dist/harness/perf.d.ts +0 -40
- package/dist/harness/perf.d.ts.map +0 -1
- package/dist/harness/perf.js +0 -136
- package/dist/harness/perf.js.map +0 -1
- package/src/harness/flow-contract.ts +0 -229
- package/src/harness/perf.ts +0 -112
|
@@ -224,7 +224,7 @@ function addMatrixSheet(wb: ExcelJS.Workbook, model: MatrixModel, sungenVersion?
|
|
|
224
224
|
// parent's steps + these read as one continuous procedure.
|
|
225
225
|
numbered(item.triggerDeltas[v.ref] ?? [], item.trigger.length + 1),
|
|
226
226
|
expectedWithVerification(v.oracle, v.verification),
|
|
227
|
-
v.mode === 'manual' ? `Manual${v.manualReason ? ` (${v.manualReason})` : ''}` :
|
|
227
|
+
v.mode === 'manual' ? `Manual${v.manualReason ? ` (${v.manualReason})` : ''}` : 'Auto',
|
|
228
228
|
v.traces.join(', '),
|
|
229
229
|
variantState(v),
|
|
230
230
|
isoDate(v.result?.startTime),
|
|
@@ -315,50 +315,6 @@ function addCoverageSheet(wb: ExcelJS.Workbook, model: MatrixModel, sungenVersio
|
|
|
315
315
|
|
|
316
316
|
let rowIdx = 7;
|
|
317
317
|
|
|
318
|
-
// --- Performance budget (#569) — advisory: config (qa/perf.yaml) vs measured runs.
|
|
319
|
-
if (model.perf) {
|
|
320
|
-
const p = model.perf;
|
|
321
|
-
dataCell(ws.getRow(rowIdx).getCell(2), 'Performance budget', { bold: true });
|
|
322
|
-
rowIdx++;
|
|
323
|
-
const r = ws.getRow(rowIdx);
|
|
324
|
-
dataCell(r.getCell(2), `p${p.percentile} ${p.metric.replace('_ms', ' time')}`);
|
|
325
|
-
dataCell(r.getCell(3), p.pass ? 'PASS' : 'OVER BUDGET', { bold: true });
|
|
326
|
-
dataCell(r.getCell(4), `measured ${p.measuredMs}ms`);
|
|
327
|
-
dataCell(r.getCell(5), `budget ${p.budgetMs}ms`);
|
|
328
|
-
dataCell(r.getCell(6), `${p.samples} runs`);
|
|
329
|
-
if (!p.pass && p.slowest.length) dataCell(r.getCell(8), `slowest: ${p.slowest.map((x) => `${x.ms}ms ${x.title.slice(0, 40)}`).join(' · ')}`);
|
|
330
|
-
rowIdx += 2;
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
// --- External Dependency Tracking (#580, ISTQB system-test practice): every case that
|
|
334
|
-
// ran against a MOCK is listed centrally. Its Result stays in the canonical vocabulary
|
|
335
|
-
// (the roll-up formulas depend on it); THIS table is where "Pass (Mocked) ≠ Pass (Real)"
|
|
336
|
-
// is visible — a case verified only against mocks is not Done until re-verified real.
|
|
337
|
-
{
|
|
338
|
-
const mockedVars = model.items.flatMap((it) => it.variants.filter((v) => v.mocked).map((v) => ({ it, v })));
|
|
339
|
-
if (mockedVars.length > 0) {
|
|
340
|
-
dataCell(ws.getRow(rowIdx).getCell(2), 'External Dependency Tracking — mock-verified cases (Pass here = Pass (Mocked), not Pass (Real))', { bold: true });
|
|
341
|
-
rowIdx++;
|
|
342
|
-
const hd = ws.getRow(rowIdx);
|
|
343
|
-
dataCell(hd.getCell(2), 'Variant');
|
|
344
|
-
dataCell(hd.getCell(3), 'Mocked dependency');
|
|
345
|
-
dataCell(hd.getCell(4), 'Delivery item');
|
|
346
|
-
dataCell(hd.getCell(5), 'Result');
|
|
347
|
-
dataCell(hd.getCell(8), 'Real-environment re-verify');
|
|
348
|
-
rowIdx++;
|
|
349
|
-
for (const { it, v } of mockedVars) {
|
|
350
|
-
const r = ws.getRow(rowIdx);
|
|
351
|
-
dataCell(r.getCell(2), v.ref);
|
|
352
|
-
dataCell(r.getCell(3), (v.mockDeps ?? []).join(', ') || '(scenario-level @mock)');
|
|
353
|
-
dataCell(r.getCell(4), it.id);
|
|
354
|
-
dataCell(r.getCell(5), variantState(v));
|
|
355
|
-
dataCell(r.getCell(8), '');
|
|
356
|
-
rowIdx++;
|
|
357
|
-
}
|
|
358
|
-
rowIdx++;
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
|
|
362
318
|
// --- Requirement coverage (review §6): every requirement id gets ONE explicit status.
|
|
363
319
|
if (model.requirements.length > 0) {
|
|
364
320
|
dataCell(ws.getRow(rowIdx).getCell(2), 'Requirement coverage — every id has an explicit status', { bold: true });
|
|
@@ -92,12 +92,6 @@ export interface CoverageVariant {
|
|
|
92
92
|
/** 'auto' | 'manual'; manual keeps its @manual:Mx reason. */
|
|
93
93
|
mode: 'auto' | 'manual';
|
|
94
94
|
manualReason?: string;
|
|
95
|
-
/** The scenario runs against MOCKED network responses (@mock / `[x] mock is active`) —
|
|
96
|
-
* a pass verifies the handling logic, not the real dependency (Pass (Mocked) ≠ Pass (Real),
|
|
97
|
-
* #580: ISTQB external-dependency tracking). */
|
|
98
|
-
mocked?: boolean;
|
|
99
|
-
/** Named mocked dependencies (`[name] mock is active` refs), for the tracking table. */
|
|
100
|
-
mockDeps?: string[];
|
|
101
95
|
layers: MatrixLayer[];
|
|
102
96
|
/** @spec:* requirement traces. */
|
|
103
97
|
traces: string[];
|
|
@@ -231,8 +225,6 @@ export interface MatrixModel {
|
|
|
231
225
|
requirements: RequirementCoverage[];
|
|
232
226
|
findings: MatrixFinding[];
|
|
233
227
|
manifest: MatrixManifest;
|
|
234
|
-
/** Perf verdict (#569) — present when qa/perf.yaml sets a budget AND the run produced durations. */
|
|
235
|
-
perf?: import('../../harness/perf').PerfVerdict | null;
|
|
236
228
|
}
|
|
237
229
|
|
|
238
230
|
/** Complexity warning threshold (rules draft Gate I; raised for compact grouping) — echoed in the manifest. */
|
|
@@ -54,7 +54,6 @@ interface PlaywrightSpec {
|
|
|
54
54
|
projectId?: string;
|
|
55
55
|
results?: Array<{
|
|
56
56
|
status?: string; // 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'
|
|
57
|
-
duration?: number; // ms — Playwright emits this on every result (#569 perf budgets)
|
|
58
57
|
startTime?: string;
|
|
59
58
|
error?: { message?: string; stack?: string };
|
|
60
59
|
attachments?: Array<{ name?: string; path?: string; contentType?: string }>;
|
|
@@ -121,7 +120,6 @@ export function loadPlaywrightReport(reportPath: string): Map<string, Playwright
|
|
|
121
120
|
result.set(stripped, {
|
|
122
121
|
testTitle: stripped,
|
|
123
122
|
status,
|
|
124
|
-
duration: typeof res?.duration === 'number' ? res.duration : undefined,
|
|
125
123
|
startTime: res?.startTime,
|
|
126
124
|
error: errorMsg,
|
|
127
125
|
tracePath: trace,
|
package/src/exporters/types.ts
CHANGED
|
@@ -121,8 +121,6 @@ export interface SpecFileData {
|
|
|
121
121
|
export interface PlaywrightResult {
|
|
122
122
|
testTitle: string;
|
|
123
123
|
status: 'passed' | 'failed' | 'skipped' | 'timedOut' | 'interrupted' | 'unknown';
|
|
124
|
-
/** Wall-clock run time in ms (from Playwright's results[].duration) — perf budgets (#569). */
|
|
125
|
-
duration?: number;
|
|
126
124
|
startTime?: string;
|
|
127
125
|
error?: string;
|
|
128
126
|
tracePath?: string;
|
|
@@ -35,17 +35,12 @@
|
|
|
35
35
|
* nothing for the query half to assert, so the step checks the PATH only. Not an
|
|
36
36
|
* error (the value may legitimately vary per environment), but the step asserts
|
|
37
37
|
* strictly less than it appears to
|
|
38
|
-
* SG-W015 — `see [X] card/item with {{n}}` where the value is a bare number: this compiles
|
|
39
|
-
* to a TEXT assertion (toHaveText('3')), NOT a count — an author reaching for
|
|
40
|
-
* "3 cards are shown" gets an assertion that can only pass if the card's text
|
|
41
|
-
* IS the digit. Count semantics exist on row/table ("see [T] table with {{n}}",
|
|
42
|
-
* "see [Ref] row with {{n}}") and lists ("should have count") only
|
|
43
38
|
*
|
|
44
39
|
* Diagnostics never block generation by themselves; `sungen generate --strict`
|
|
45
40
|
* turns any collected diagnostic into a non-zero exit.
|
|
46
41
|
*/
|
|
47
42
|
|
|
48
|
-
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007' | 'SG-W008' | 'SG-W009' | 'SG-W010' | 'SG-W011' | 'SG-W012' | 'SG-W013' | 'SG-W014'
|
|
43
|
+
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007' | 'SG-W008' | 'SG-W009' | 'SG-W010' | 'SG-W011' | 'SG-W012' | 'SG-W013' | 'SG-W014';
|
|
49
44
|
|
|
50
45
|
export interface Diagnostic {
|
|
51
46
|
code: DiagnosticCode;
|
|
@@ -73,40 +73,34 @@ export class PatternRegistry {
|
|
|
73
73
|
* Prefers resolver (returns template data) over generator (renders directly).
|
|
74
74
|
*/
|
|
75
75
|
generateStep(step: ParsedStep, context: PatternContext): MappedStep | null {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
const matchStep: ParsedStep = { ...step, text: maskRefsForMatching(step.text) };
|
|
81
|
-
for (const pattern of this.patterns) {
|
|
82
|
-
if (!this.matchesPattern(matchStep, pattern.matcher)) continue;
|
|
83
|
-
|
|
84
|
-
// Prefer resolver (framework-agnostic) over generator (legacy)
|
|
85
|
-
if (pattern.resolver) {
|
|
86
|
-
const resolved = pattern.resolver(step, context);
|
|
87
|
-
if (!resolved) continue; // resolver declined — fall through
|
|
88
|
-
|
|
89
|
-
// Auto-inject parent scoping if step has parentRef
|
|
90
|
-
// Skip for table-* patterns — they resolve the table name internally from step text
|
|
91
|
-
if (step.parentRef && step.parentType && !pattern.name.startsWith('table-')) {
|
|
92
|
-
resolved.data.parentLocator = PatternRegistry.resolveParentLocator(
|
|
93
|
-
step.parentRef, step.parentType, context
|
|
94
|
-
);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
const code = context.templateEngine.renderStep(resolved.templateName, resolved.data);
|
|
98
|
-
return {
|
|
99
|
-
code,
|
|
100
|
-
comment: resolved.comment,
|
|
101
|
-
};
|
|
102
|
-
}
|
|
76
|
+
const pattern = this.findPattern(step);
|
|
77
|
+
if (!pattern) {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
103
80
|
|
|
104
|
-
|
|
105
|
-
|
|
81
|
+
// Prefer resolver (framework-agnostic) over generator (legacy)
|
|
82
|
+
if (pattern.resolver) {
|
|
83
|
+
const resolved = pattern.resolver(step, context);
|
|
84
|
+
|
|
85
|
+
// Auto-inject parent scoping if step has parentRef
|
|
86
|
+
// Skip for table-* patterns — they resolve the table name internally from step text
|
|
87
|
+
if (step.parentRef && step.parentType && !pattern.name.startsWith('table-')) {
|
|
88
|
+
resolved.data.parentLocator = PatternRegistry.resolveParentLocator(
|
|
89
|
+
step.parentRef, step.parentType, context
|
|
90
|
+
);
|
|
106
91
|
}
|
|
107
92
|
|
|
108
|
-
|
|
93
|
+
const code = context.templateEngine.renderStep(resolved.templateName, resolved.data);
|
|
94
|
+
return {
|
|
95
|
+
code,
|
|
96
|
+
comment: resolved.comment,
|
|
97
|
+
};
|
|
109
98
|
}
|
|
99
|
+
|
|
100
|
+
if (pattern.generator) {
|
|
101
|
+
return pattern.generator(step, context);
|
|
102
|
+
}
|
|
103
|
+
|
|
110
104
|
return null;
|
|
111
105
|
}
|
|
112
106
|
|
|
@@ -175,28 +175,6 @@ export class StepMapper {
|
|
|
175
175
|
this.lastPrimaryKeyword = keyword;
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
-
// SG-W015 — `see [X] card with {{n}}` where n is a bare number compiles to a TEXT
|
|
179
|
-
// assertion (toHaveText('3')), not a count. An author reaching for "3 cards are shown"
|
|
180
|
-
// gets an assertion that can only pass if the card's TEXT is the digit — a wrong test
|
|
181
|
-
// with no error. Count semantics live on row/table ("see [T] table with {{n}}",
|
|
182
|
-
// "see [Ref] row with {{n}}") and lists ("should have count"); on the container types
|
|
183
|
-
// below, a bare-number text expectation is almost never what the author meant. A text
|
|
184
|
-
// element asserting a numeric readout ("[Row Quantity] text with 4") stays silent.
|
|
185
|
-
const COUNT_MISREAD_TYPES = new Set(['card', 'item', 'items', 'cards', 'option', 'options']);
|
|
186
|
-
if (this.diagnostics && step.dataRef && step.selectorRef &&
|
|
187
|
-
COUNT_MISREAD_TYPES.has((step.elementType || '').toLowerCase()) &&
|
|
188
|
-
/\bsees?\s+\[/.test(step.text) && /\bwith\b/.test(step.text)) {
|
|
189
|
-
// peekBaseValue reads the literal from test-data even in runtime-data mode,
|
|
190
|
-
// where resolveData returns an opaque marker (never numeric-looking).
|
|
191
|
-
let resolvedValue: string | undefined;
|
|
192
|
-
try { resolvedValue = this.dataResolver.peekBaseValue(step.dataRef, this.featureName); } catch { /* unresolved at compile time → cannot judge */ }
|
|
193
|
-
if (resolvedValue !== undefined && /^\d+$/.test(resolvedValue.trim())) {
|
|
194
|
-
this.diagnostics.add('SG-W015',
|
|
195
|
-
`\`see [${step.selectorRef}] ${step.elementType} with {{${step.dataRef}}}\` compiles to a TEXT assertion toHaveText('${resolvedValue.trim()}'), NOT a count of ${step.elementType}s. For a count use \`see [Ref] row with {{n}}\` / \`see [T] table with {{n}}\` (or a list "should have count"); if '${resolvedValue.trim()}' really is the ${step.elementType}'s text, assert it on a text/label element instead`,
|
|
196
|
-
{ feature: this.featureName, step: step.text });
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
|
|
200
178
|
// SG-W007 — a two-step-form browser-alert handler (`click [OK] alert`) written AFTER
|
|
201
179
|
// an action step registers its page.once('dialog') listener too late: a dialog
|
|
202
180
|
// triggered by the earlier action auto-dismisses with no error (issue #472).
|
package/src/harness/audit.ts
CHANGED
|
@@ -10,11 +10,10 @@ import * as path from 'path';
|
|
|
10
10
|
import * as fs from 'fs';
|
|
11
11
|
import { loadUnitScenarios, readUnitFeatureText, parseViewpointOverview, ScenarioInfo, ViewpointEntry } from './parse';
|
|
12
12
|
import { checkViewpointBaseline, ViewpointBaseline } from './viewpoint-baseline';
|
|
13
|
-
import { flowQuality, statefulDepthFor, FlowQualityResult } from './flow-contract';
|
|
14
13
|
import { featureFilesFor } from './unit-paths';
|
|
15
14
|
import {
|
|
16
15
|
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, depthThresholdFor, coverageBalance, duplicateClusters, traceability, claimProof, taxonomyLint,
|
|
17
|
-
automatableManual, flowCoveredThemes, flowRegressionDepth, oracleStrength,
|
|
16
|
+
automatableManual, flowCoveredThemes, flowRegressionDepth, oracleStrength,
|
|
18
17
|
GateResult, DepthResult, BalanceResult, DuplicateResult, TraceResult, ClaimProofResult, TaxonomyResult, Catalog, AutomatableManualResult, FlowDepthResult, OracleStrengthResult,
|
|
19
18
|
} from './sensors';
|
|
20
19
|
import { loadFlowScenarios } from './flow-check';
|
|
@@ -49,7 +48,6 @@ export interface AuditReport {
|
|
|
49
48
|
oracle: OracleStrengthResult; // H4 — facet-oracle strength (weak name-substring vs title/detail/API/DB)
|
|
50
49
|
ledger: LedgerResult; // atomic viewpoint-item coverage (per-bullet status)
|
|
51
50
|
viewpointBaseline: ViewpointBaseline; // is the yardstick still the accepted one? (#557)
|
|
52
|
-
flowQuality?: FlowQualityResult; // #569 — flow contract verification (flows only)
|
|
53
51
|
calibration: { // #8 — multi-axis score so a high overall can't hide a weak axis
|
|
54
52
|
axes: Record<string, number>;
|
|
55
53
|
weakest: { axis: string; value: number };
|
|
@@ -156,28 +154,15 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
156
154
|
// back to the in-core functions if the capability provides none.
|
|
157
155
|
// A capability gate may need project context (the API gate resolves endpoint methods from the
|
|
158
156
|
// catalog) + the focus depth threshold (so it scores depth with the SAME bar as the UI gate).
|
|
157
|
+
const capGate = scoringCap?.gateProvider as
|
|
158
|
+
((i: { scenarios: ScenarioInfo[]; viewpoints: ViewpointEntry[]; catalog: Catalog; focus: typeof intent.focus; cwd: string; screenName: string; threshold: number; businessCriticalMethods?: string[] }) => { gate: GateResult; depth: DepthResult }) | undefined;
|
|
159
|
+
const provided = capGate?.({ scenarios, viewpoints, catalog, focus: intent.focus, cwd: projectRootFromScreenDir(screenDir), screenName: catalogScreenName, threshold: depthThresholdFor(intent.focus), businessCriticalMethods: intent.businessCriticalMethods });
|
|
159
160
|
// Is the declaration this suite is measured against still the one a human accepted?
|
|
160
161
|
const viewpointBaseline = checkViewpointBaseline(
|
|
161
162
|
projectRootFromScreenDir(screenDir), catalogScreenName, viewpointPath);
|
|
162
163
|
const viewpointText = fs.existsSync(viewpointPath) ? readTextFile(viewpointPath) : '';
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
// flow judged as a `form` screen scored coverage 0%), so for contract-holding flows the
|
|
166
|
-
// page-type machinery only applies when the project DECLARED a page type explicitly.
|
|
167
|
-
// The stripped catalog feeds the CAPABILITY gate too — driver-ui provides the gate on
|
|
168
|
-
// real projects, and handing it the unstripped catalog re-introduced the force-fit.
|
|
169
|
-
const isUiFlowUnit = /^flows\//.test(catalogScreenName);
|
|
170
|
-
const flowQ: FlowQualityResult = isUiFlowUnit
|
|
171
|
-
? flowQuality(screenDir, scenarios)
|
|
172
|
-
: { hasContract: false, errors: [], outcomeProven: false, outcomeManualOnly: false, offGoal: [], offGoalRatio: 0, offGoalCategories: [], phases: [], phaseRatio: 1, handoffs: { total: 0, asserted: 0, ratio: 1 } };
|
|
173
|
-
const gateCatalog = (flowQ.hasContract && !declaredPageType(viewpointText, catalog))
|
|
174
|
-
? { ...catalog, page_types: {} } as Catalog
|
|
175
|
-
: catalog;
|
|
176
|
-
const capGate = scoringCap?.gateProvider as
|
|
177
|
-
((i: { scenarios: ScenarioInfo[]; viewpoints: ViewpointEntry[]; catalog: Catalog; focus: typeof intent.focus; cwd: string; screenName: string; threshold: number; businessCriticalMethods?: string[] }) => { gate: GateResult; depth: DepthResult }) | undefined;
|
|
178
|
-
const provided = capGate?.({ scenarios, viewpoints, catalog: gateCatalog, focus: intent.focus, cwd: projectRootFromScreenDir(screenDir), screenName: catalogScreenName, threshold: depthThresholdFor(intent.focus), businessCriticalMethods: intent.businessCriticalMethods });
|
|
179
|
-
const gate = provided?.gate ?? viewpointGate(scenarios, viewpoints, gateCatalog, platform === 'mobile', viewpointText);
|
|
180
|
-
const depth = provided?.depth ?? assertionDepth(scenarios, dataThemesFor(gateCatalog, gate.pageType), intent.focus);
|
|
164
|
+
const gate = provided?.gate ?? viewpointGate(scenarios, viewpoints, catalog, platform === 'mobile', viewpointText);
|
|
165
|
+
const depth = provided?.depth ?? assertionDepth(scenarios, dataThemesFor(catalog, gate.pageType), intent.focus);
|
|
181
166
|
|
|
182
167
|
// TQ-4 — deferral-aware coverage credit: an inherently cross-screen theme (cart / detail /
|
|
183
168
|
// filter correctness) belongs in a FLOW, not on the screen. When a flow deeply covers a screen
|
|
@@ -215,15 +200,8 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
215
200
|
// collection, the regression dimensions (count/quantity proof · teardown · multi-source) cap the
|
|
216
201
|
// businessDepth headroom: it can reach 1.0 only when all three are exercised, so a present-but-
|
|
217
202
|
// shallow flow can't claim a perfect score (floor 0.5 — assertion depth still dominates).
|
|
218
|
-
const isUiFlow =
|
|
219
|
-
|
|
220
|
-
// Contract-declared statefulness generalizes the cart-hardcoded vocabulary: a flow that
|
|
221
|
-
// mutates ANY named collection (order, application, submission …) gets the same three
|
|
222
|
-
// regression dims. The legacy cart detection stays for contract-less flows (snapshots).
|
|
223
|
-
if (isUiFlow && !flowDepth.stateful && flowQ.contract?.stateful) {
|
|
224
|
-
const d = statefulDepthFor(flowQ.contract.stateful, scenarios);
|
|
225
|
-
flowDepth = { stateful: true, countProof: d.countProof, teardown: d.teardown, multiSource: d.multiSource, ratio: d.ratio, missing: d.missing };
|
|
226
|
-
}
|
|
203
|
+
const isUiFlow = /^flows\//.test(catalogScreenName);
|
|
204
|
+
const flowDepth = isUiFlow ? flowRegressionDepth(scenarios) : { stateful: false, countProof: false, teardown: false, multiSource: false, ratio: 1, missing: [] } as FlowDepthResult;
|
|
227
205
|
const FLOW_DEPTH_FLOOR = 0.5;
|
|
228
206
|
// H4 — oracle strength: a weak facet oracle (name-substring "proves" category/brand membership)
|
|
229
207
|
// caps businessDepth the same way (floor 0.5). A suite with no facet claim, or a strong oracle, is
|
|
@@ -233,14 +211,10 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
233
211
|
|
|
234
212
|
// Sub-scores
|
|
235
213
|
const coverage = gate.coverageRatio;
|
|
236
|
-
// #569 — handoff integrity: a flow's depth is also whether each cross-screen transition
|
|
237
|
-
// is followed by an assertion on the carried state (same floor discipline as the other caps).
|
|
238
|
-
const HANDOFF_FLOOR = 0.5;
|
|
239
214
|
const businessDepth = Math.min(
|
|
240
215
|
depth.bcDepthRatio,
|
|
241
216
|
flowDepth.stateful ? FLOW_DEPTH_FLOOR + (1 - FLOW_DEPTH_FLOOR) * flowDepth.ratio : 1,
|
|
242
217
|
oracle.weak.length ? ORACLE_FLOOR + (1 - ORACLE_FLOOR) * oracle.ratio : 1,
|
|
243
|
-
flowQ.hasContract && flowQ.handoffs.total > 0 ? HANDOFF_FLOOR + (1 - HANDOFF_FLOOR) * flowQ.handoffs.ratio : 1,
|
|
244
218
|
);
|
|
245
219
|
// When the taxonomy drifted (most scenarios unclassified), the balance axis is unreliable — cap it
|
|
246
220
|
// at 0.5 instead of awarding a vacuous 1.0 so a stale taxonomy fails loudly, not silently (H1).
|
|
@@ -272,15 +246,8 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
272
246
|
// what a create-test run did, silently dropping the performance viewpoint on
|
|
273
247
|
// the way. While the declaration is unconfirmed, neither axis is evidence.
|
|
274
248
|
const viewpointMoved = viewpointBaseline.status === 'changed';
|
|
275
|
-
// 4. A flow with a contract is measured on ITS coverage: journey phases (HP/ER/EH
|
|
276
|
-
// automated) — not a screen's page-type themes. Same weight, same criticality,
|
|
277
|
-
// so a flow can finally EARN the coverage axis instead of always losing it to
|
|
278
|
-
// PAGE-TYPE-UNDETERMINED (or worse, being judged against a form checklist).
|
|
279
|
-
const flowScored = flowQ.hasContract;
|
|
280
249
|
const axisDefs: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean }> = [
|
|
281
|
-
|
|
282
|
-
? { key: 'flowCoverage', value: flowQ.phaseRatio, weight: 0.22, applicable: true, critical: true }
|
|
283
|
-
: { key: 'coverage', value: coverage, weight: 0.22, applicable: !!gate.pageType && gate.themesTotal > 0, critical: true },
|
|
250
|
+
{ key: 'coverage', value: coverage, weight: 0.22, applicable: !!gate.pageType && gate.themesTotal > 0, critical: true },
|
|
284
251
|
{ key: 'specFR', value: specRatio, weight: 0.15, applicable: spec.hasSpec && spec.frTotal > 0, critical: true },
|
|
285
252
|
{ key: 'atomicLedger', value: ledger.ratio, weight: 0.13, applicable: ledger.hasViewpoint && ledger.total > 0 && !viewpointMoved, critical: true },
|
|
286
253
|
{ key: 'businessDepth', value: businessDepth, weight: 0.20, applicable: true, critical: true },
|
|
@@ -330,40 +297,9 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
330
297
|
].filter(Boolean).join(', ');
|
|
331
298
|
findings.push(`VIEWPOINT-BASELINE-CHANGED: test-viewpoint.md no longer declares what it did when this unit was last accepted — ${moved || 'the declared ids were reordered or replaced'}. atomicLedger + traceability measure the suite AGAINST this file, so they are excluded from the score until the change is confirmed: a generator that rewrites the declaration scores both 100% by construction, and a viewpoint dropped from the file stops being missing from anything. Review the diff (a removed id means that coverage is now unclaimed), then run \`sungen audit --screen ${screenName} --accept-viewpoint\`.`);
|
|
332
299
|
}
|
|
333
|
-
|
|
334
|
-
if (gate.pageTypeSource === 'undetermined' && !flowScored) {
|
|
300
|
+
if (gate.pageTypeSource === 'undetermined') {
|
|
335
301
|
findings.push(`PAGE-TYPE-UNDETERMINED: no page type fits this screen with enough confidence (best ${gate.pageTypeEvidence?.hits ?? 0} keyword hit(s)), so NO critical themes were demanded and the coverage axis is excluded from the score — declare it in test-viewpoint.md (\`page-type: <id>\`) to have the theme checklist applied.`);
|
|
336
302
|
}
|
|
337
|
-
// ---- #569: flow-quality findings -----------------------------------------
|
|
338
|
-
if (isUiFlow && !flowQ.hasContract) {
|
|
339
|
-
const why = flowQ.errors.length ? ` (${flowQ.errors.join('; ')})` : '';
|
|
340
|
-
findings.push(`FLOW-CONTRACT-MISSING: this flow declares no contract${why} — without \`requirements/flow-contract.yaml\` (goal · trigger · outcome.screen · phases) the flow cannot be scored on ITS coverage (journey phases) and falls back to screen machinery. Author it with the boundary checklist: one business goal, a clear trigger, ONE observable outcome valuable to the actor, name = "Verb + outcome"; split anything that forms an independently valuable flow.`);
|
|
341
|
-
}
|
|
342
|
-
if (flowQ.hasContract) {
|
|
343
|
-
if (!flowQ.outcomeProven) {
|
|
344
|
-
findings.push(flowQ.outcomeManualOnly
|
|
345
|
-
? `FLOW-OUTCOME-UNPROVEN: the goal "${flowQ.contract!.goal}" is covered only by @manual scenarios — the flow's own outcome (\`${flowQ.contract!.outcome.screen}\`) has no AUTOMATED data assertion, so a regression run cannot prove the journey still works. Automate the happy-path proof first; it is the reason this flow exists.`
|
|
346
|
-
: `FLOW-OUTCOME-UNPROVEN: no scenario asserts data on the outcome screen \`${flowQ.contract!.outcome.screen}\` — the flow never proves its own goal ("${flowQ.contract!.goal}"). Add the happy-path scenario that ends in the declared final assertion.`);
|
|
347
|
-
}
|
|
348
|
-
if (flowQ.offGoalRatio > 0.35) {
|
|
349
|
-
findings.push(`FLOW-SCOPE-CREEP: ${flowQ.offGoal.length}/${scenarios.length} scenarios never touch the outcome screen \`${flowQ.contract!.outcome.screen}\` and are not guards/error-recovery (categories: ${flowQ.offGoalCategories.join(', ')}) — evidence of a SECOND business goal in this flow. Split them into their own flow (one flow = one goal, one observable outcome).`);
|
|
350
|
-
}
|
|
351
|
-
for (const ph of flowQ.phases.filter((p) => !p.covered || !p.automated)) {
|
|
352
|
-
findings.push(`FLOW-PHASE-${ph.phase}-MISSING: journey phase ${ph.phase} (${ph.phase === 'HP' ? 'happy path proving the outcome' : ph.phase === 'ER' ? 'error recovery — validation must not trap the journey' : 'guards — direct access / back / refresh'}) is ${ph.covered ? 'covered only by @manual' : 'not covered'} → it does not count toward flowCoverage until automated.`);
|
|
353
|
-
}
|
|
354
|
-
if (flowQ.handoffs.total > 0 && flowQ.handoffs.ratio < 0.7) {
|
|
355
|
-
findings.push(`FLOW-HANDOFF-SHALLOW: only ${flowQ.handoffs.asserted}/${flowQ.handoffs.total} cross-screen transitions are followed by an assertion in the new screen — carried state (the thing a flow exists to verify) is unproven at the handoff. businessDepth is capped until asserted.`);
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
if (isUiFlow) {
|
|
359
|
-
// Flows defer more than screens, so an UNCODED @manual is costlier here: the capability
|
|
360
|
-
// planner cannot route it to the driver that would automate it (password-reset: 11/22
|
|
361
|
-
// manual, most of them M5-inbox — invisible while tagged bare).
|
|
362
|
-
const bareManual = (featureText.match(/@manual(?![:\w])/g) || []).length;
|
|
363
|
-
if (bareManual > 0) {
|
|
364
|
-
findings.push(`MANUAL-CODE-MISSING: ${bareManual} @manual tag(s) carry no reason code — tag @manual:M1..M9 (M5 = external dependency e.g. inbox, M3 = mock/network, M7 = environment) so the capability planner can route what a driver would automate.`);
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
303
|
for (const c of flowCredits) {
|
|
368
304
|
findings.push(`COVERED-VIA-FLOW: critical theme "${c.theme}" is not covered on this screen but is deeply covered by flow "${c.flow}" — cross-screen depth correctly owned by the flow, so it is credited (not a screen gap). Verify with \`sungen flow-check\`.`);
|
|
369
305
|
}
|
|
@@ -440,13 +376,6 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
440
376
|
const shown = spec.inferredOnly.slice(0, 8).join(', ');
|
|
441
377
|
findings.push(`SPEC-TRACE-IMPLICIT: ${spec.inferredOnly.length} requirement(s) are covered by keyword inference only (${shown}${spec.inferredOnly.length > 8 ? ', …' : ''}) — add \`@spec:<id>\` to the scenario that proves each so the trace survives edits and delivery can report it.`);
|
|
442
378
|
}
|
|
443
|
-
// Safety net for the parser (#572): the feature CITES requirement ids the spec parser
|
|
444
|
-
// could not find a declaration for. Silence here would repeat the original failure —
|
|
445
|
-
// "nothing found" quietly excluding the specFR axis while the evidence sits in the file.
|
|
446
|
-
if (spec.hasSpec && spec.frTotal === 0 && /@spec:/i.test(featureText)) {
|
|
447
|
-
const cited = [...new Set([...featureText.matchAll(/@spec:([A-Za-z0-9.-]+)/g)].map((m) => m[1].toUpperCase()))];
|
|
448
|
-
findings.push(`SPEC-IDS-UNPARSED: the feature cites ${cited.length} requirement id(s) via @spec: (${cited.slice(0, 5).join(', ')}${cited.length > 5 ? ', …' : ''}) but requirements/spec.md yielded ZERO parseable requirement declarations — the spec likely declares ids in a scheme the parser cannot read. Declare each requirement on its own line as \`<ID>: <clause>\` (bullet/bold/backtick all fine; the id must end in a number) so the MUST-coverage gate and the specFR axis can run.`);
|
|
449
|
-
}
|
|
450
379
|
if (taxonomyMismatch) {
|
|
451
380
|
findings.push(`VP-TAXONOMY-MISMATCH: only ${(trace.mappedRatio * 100).toFixed(0)}% of scenarios use the viewpoint IDs declared in test-viewpoint.md — scenarios invented a generic VP-<CAT> scheme. Re-tag to the project's viewpoint IDs so the coverage matrix is accurate.`);
|
|
452
381
|
}
|
|
@@ -500,18 +429,6 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
500
429
|
for (const d of ownership.duplicates.slice(0, 6)) {
|
|
501
430
|
findings.push(`DUPLICATE-FLOW-OWNERSHIP: "${d.scenario}" has the same shape as a scenario in flow "${d.flow}" — keep one owner (screen-local vs flow); the other should only reference/set up.`);
|
|
502
431
|
}
|
|
503
|
-
// #580 — the flow→screen half of the ownership rule (ISTQB system-test design:
|
|
504
|
-
// representative, not exhaustive). A flow's cases take ONE representative input per
|
|
505
|
-
// branch — one valid value on the main flow, one error value that triggers the
|
|
506
|
-
// exception flow. An EP/data-family cluster (≥3 same-shape scenarios differing only
|
|
507
|
-
// in data) inside a FLOW is the screen's validation matrix being re-run where it
|
|
508
|
-
// does not belong: the screen already proved the field rejects each class; the flow
|
|
509
|
-
// only has to prove the journey survives one of them. Advisory — never a gate.
|
|
510
|
-
if (isUiFlow) {
|
|
511
|
-
for (const c of duplicates.clusters.filter((x) => !x.sameDataLikely && x.scenarios.length >= 3).slice(0, 4)) {
|
|
512
|
-
findings.push(`FLOW-EXHAUSTIVE-INPUTS: ${c.scenarios.length} same-shape scenarios differ only in data (${c.scenarios.slice(0, 3).join(' | ')}${c.scenarios.length > 3 ? ' …' : ''}) — a system-test flow uses REPRESENTATIVE inputs (one valid + one that triggers the exception branch); the full EP/boundary matrix belongs to the owning screen (or a @cases dataset there). Keep one representative per branch here.`);
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
432
|
for (const u of unsourced.slice(0, 6)) {
|
|
516
433
|
findings.push(`UNSOURCEABLE-SCENARIO: "${u}" doesn't trace to any FR / viewpoint item — link it to a source, or tag it @exploration (not part of the official suite).`);
|
|
517
434
|
}
|
|
@@ -568,7 +485,6 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
568
485
|
scenarioCount: scenarios.length,
|
|
569
486
|
gate, depth, claim, taxonomy, balance, duplicates, trace, spec,
|
|
570
487
|
taxonomyMismatch, downstream, manualOracle: manualOracleResult, automatableManual: autoManual, flowDepth, oracle, ledger, viewpointBaseline, calibration,
|
|
571
|
-
...(isUiFlow ? { flowQuality: flowQ } : {}),
|
|
572
488
|
score: {
|
|
573
489
|
overall: Math.round(overall * 10) / 10,
|
|
574
490
|
coverage: Math.round(coverage * 100) / 100,
|
package/src/harness/flow-plan.ts
CHANGED
|
@@ -71,9 +71,6 @@ export interface FlowPlan {
|
|
|
71
71
|
legs: LegPlan[];
|
|
72
72
|
byReason: Record<string, number>;
|
|
73
73
|
capabilityManual: number;
|
|
74
|
-
/** @manual whose reason is "cross-screen → automate via flow" (class XS) — inside a flow this
|
|
75
|
-
* usually means the scenario should simply BE automated here. (#569) */
|
|
76
|
-
crossScreenManual: number;
|
|
77
74
|
judgmentManual: number;
|
|
78
75
|
contracts: Contract[];
|
|
79
76
|
readiness: 'ready' | 'not-ready';
|
|
@@ -90,18 +87,14 @@ export function buildFlowPlan(cwd: string, flow: string): FlowPlan {
|
|
|
90
87
|
// Legs = distinct screen namespaces.
|
|
91
88
|
const legMap = new Map<string, { scenarios: Set<string>; refs: Set<string>; automated: boolean }>();
|
|
92
89
|
const byReason: Record<string, number> = {};
|
|
93
|
-
let capabilityManual = 0, judgmentManual = 0
|
|
90
|
+
let capabilityManual = 0, judgmentManual = 0;
|
|
94
91
|
|
|
95
92
|
for (const sc of scenarios) {
|
|
96
93
|
if (sc.manual) {
|
|
97
94
|
const { code } = inferReasonCode(sc.tags, sc.reason);
|
|
98
95
|
byReason[code] = (byReason[code] || 0) + 1;
|
|
99
96
|
const cls = MANUAL_REASONS[code]?.cls;
|
|
100
|
-
|
|
101
|
-
// dropped from both counters, understating the plan's manual load. (#569)
|
|
102
|
-
if (cls === 'capability') capabilityManual++;
|
|
103
|
-
else if (cls === 'keep') judgmentManual++;
|
|
104
|
-
else if (cls === 'flow') crossScreenManual++;
|
|
97
|
+
if (cls === 'capability') capabilityManual++; else if (cls === 'keep') judgmentManual++;
|
|
105
98
|
}
|
|
106
99
|
for (const r of sc.refs) {
|
|
107
100
|
const leg = r.screen.toLowerCase();
|
|
@@ -140,5 +133,5 @@ export function buildFlowPlan(cwd: string, flow: string): FlowPlan {
|
|
|
140
133
|
}
|
|
141
134
|
if (readiness === 'ready') plan.unshift('Selectors present for every automated leg — ready to compile + run.');
|
|
142
135
|
|
|
143
|
-
return { flow, total: scenarios.length, legs, byReason, capabilityManual, judgmentManual,
|
|
136
|
+
return { flow, total: scenarios.length, legs, byReason, capabilityManual, judgmentManual, contracts, readiness, missingLegs, plan };
|
|
144
137
|
}
|
package/src/harness/parse.ts
CHANGED
|
@@ -38,8 +38,6 @@ export interface ScenarioInfo {
|
|
|
38
38
|
requiresCaps?: string[]; // @requires:<cap> — automation-ready but needs an opt-in driver (TQ-11)
|
|
39
39
|
deferredToFlow?: boolean; // @deferred:flow — owned by a flow, not automated on this screen (H6)
|
|
40
40
|
ownedByFlow?: string; // @owned-by:<flow> — the flow that owns this deferred scenario (H6)
|
|
41
|
-
/** Ordered steps with their resolved bucket (And/But inherit) — flow handoff analysis (#569). */
|
|
42
|
-
steps?: Array<{ bucket: 'given' | 'when' | 'then'; text: string }>;
|
|
43
41
|
}
|
|
44
42
|
|
|
45
43
|
/** Format-tolerant: is this token an ID (project's scheme), not a prose word?
|
|
@@ -93,29 +91,6 @@ export function parseViewpointOverview(filePath: string): ViewpointEntry[] {
|
|
|
93
91
|
}
|
|
94
92
|
}
|
|
95
93
|
|
|
96
|
-
// 1b) Flow-style declarations (#569). Flow viewpoint files commonly declare per-item
|
|
97
|
-
// ids at the END of a bullet ("… → **FL-HP-001**") under phase section headers
|
|
98
|
-
// ("## FL-HP — Happy Path"). Neither matched the table/group passes, so every flow
|
|
99
|
-
// audit collapsed to taxonomy=0% / traceability n-a — scenarios correctly tagged
|
|
100
|
-
// FL-HP-001 were reported as unmapped. Both forms are additive here.
|
|
101
|
-
for (const raw of lines) {
|
|
102
|
-
const line = raw.trim();
|
|
103
|
-
const section = line.match(/^##\s+([A-Z]{2,}(?:-[A-Z0-9]{2,})*)\s+[—–-]\s*(.*)$/);
|
|
104
|
-
if (section && isViewpointId(section[1] + '-0')) {
|
|
105
|
-
const id = section[1].toUpperCase();
|
|
106
|
-
if (!entries.has(id)) entries.set(id, { id, priority: 'Unknown', reason: section[2] ?? '' });
|
|
107
|
-
}
|
|
108
|
-
if (/^[-*+]\s/.test(line)) {
|
|
109
|
-
const arrow = line.match(/(?:→|->)\s*\*{0,2}([A-Z][A-Z0-9]*(?:-[A-Z0-9]+)*-\d+[a-zA-Z]?)\*{0,2}\s*$/);
|
|
110
|
-
if (arrow) {
|
|
111
|
-
const id = arrow[1].toUpperCase();
|
|
112
|
-
if (!entries.has(id)) {
|
|
113
|
-
entries.set(id, { id, priority: 'Unknown', reason: line.replace(/\s*(?:→|->).*$/, '').replace(/^[-*+]\s+/, '') });
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
|
|
119
94
|
// 2) Viewpoint Grouping: ### Required / ### Recommended / ### Optional → bullet list
|
|
120
95
|
let group: ViewpointEntry['group'] | undefined;
|
|
121
96
|
for (const raw of lines) {
|
|
@@ -169,9 +144,7 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
169
144
|
// Category is everything between `VP-` and the final `-<sequence>` — INCLUDING hyphens, so
|
|
170
145
|
// compound categories (VP-LIST-DISPLAY-01, VP-ADD-TO-CART-03, VP-PRODUCT-DISCOVERY-02) parse,
|
|
171
146
|
// not just single-word ones. A single-word category (VP-CART-001) still works. (H1)
|
|
172
|
-
|
|
173
|
-
// flow scenario had NO category and the whole suite bucketed `other` (taxonomy=0%, #569).
|
|
174
|
-
const codeMatch = sc.name.match(/\b(?:VP|FL)-([A-Z]+(?:-[A-Z]+)*)-\d+/i);
|
|
147
|
+
const codeMatch = sc.name.match(/\bVP-([A-Z]+(?:-[A-Z]+)*)-\d+/i);
|
|
175
148
|
const vpCode = codeMatch ? codeMatch[0].toUpperCase() : undefined;
|
|
176
149
|
const category = codeMatch ? codeMatch[1].toUpperCase() : undefined;
|
|
177
150
|
// Project-scheme ID: the leading token of the title (VP0-001 / MS-HP-001 / VP-LIST-001).
|
|
@@ -185,14 +158,12 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
185
158
|
const skeletonParts: string[] = [];
|
|
186
159
|
const textParts: string[] = [sc.name];
|
|
187
160
|
const stepTextParts: string[] = [];
|
|
188
|
-
const orderedSteps: Array<{ bucket: 'given' | 'when' | 'then'; text: string }> = [];
|
|
189
161
|
|
|
190
162
|
for (const step of sc.steps as ParsedStep[]) {
|
|
191
163
|
const kw = step.keyword.trim();
|
|
192
164
|
if (kw === 'Given' || kw === 'When' || kw === 'Then') last = kw;
|
|
193
165
|
textParts.push(step.text);
|
|
194
166
|
stepTextParts.push(step.text);
|
|
195
|
-
orderedSteps.push({ bucket: (kw === 'And' || kw === 'But' ? last : kw).toLowerCase() as 'given' | 'when' | 'then', text: step.text });
|
|
196
167
|
// normalized skeleton: keep [refs] (distinct targets = distinct tests),
|
|
197
168
|
// but neutralize {{vars}} and quoted values so EP/data families collapse.
|
|
198
169
|
const skel = step.text
|
|
@@ -228,7 +199,6 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
228
199
|
stepSkeleton: skeletonParts.join(' | '),
|
|
229
200
|
haystack: textParts.join(' ').toLowerCase(),
|
|
230
201
|
stepsText: stepTextParts.join(' ').toLowerCase(),
|
|
231
|
-
steps: orderedSteps,
|
|
232
202
|
vpId,
|
|
233
203
|
casesDataset,
|
|
234
204
|
queryRefs: queryRefs.size ? [...queryRefs] : undefined,
|
package/src/harness/sensors.ts
CHANGED
|
@@ -33,22 +33,10 @@ const BUCKET_ORDER: Array<[string, string[]]> = [
|
|
|
33
33
|
];
|
|
34
34
|
const BUCKETS: Record<string, string[]> = Object.fromEntries(BUCKET_ORDER);
|
|
35
35
|
|
|
36
|
-
// Flow journey-phase categories (FL-HP-001, FL-ER-002 …). Matched on exact SEGMENTS,
|
|
37
|
-
// never by containment — 'SHOP'.includes('HP') is true, which is exactly the kind of
|
|
38
|
-
// false hit substring matching would produce for two-letter phase tokens. (#569)
|
|
39
|
-
const PHASE_BUCKETS: Record<string, string> = {
|
|
40
|
-
HP: 'business-core', // happy path = the business goal itself
|
|
41
|
-
ER: 'validation-security', // error recovery (validation must not trap the journey)
|
|
42
|
-
EH: 'validation-security', // guards & leakage (direct access, back, refresh)
|
|
43
|
-
};
|
|
44
|
-
|
|
45
36
|
/** Classify a VP category into a balance bucket by keyword containment + precedence (H1). */
|
|
46
37
|
export function bucketForCategory(category: string | undefined): string {
|
|
47
38
|
const cat = (category || '').toUpperCase();
|
|
48
39
|
if (!cat) return 'other';
|
|
49
|
-
for (const seg of cat.split('-')) {
|
|
50
|
-
if (PHASE_BUCKETS[seg]) return PHASE_BUCKETS[seg];
|
|
51
|
-
}
|
|
52
40
|
for (const [bucket, kws] of BUCKET_ORDER) {
|
|
53
41
|
if (kws.some((k) => cat.includes(k))) return bucket;
|
|
54
42
|
}
|
|
@@ -363,7 +351,7 @@ export function flowRegressionDepth(scenarios: ScenarioInfo[]): FlowDepthResult
|
|
|
363
351
|
// 1. Count/quantity proof — a row count or item quantity, not just presence of a row.
|
|
364
352
|
const countProof = any(/\b(quantity|qty|two (?:rows|lines|cart)|row count|count column|number of items|one[_ ]row|two[_ ]rows|qty[_ ])/i);
|
|
365
353
|
// 2. Teardown — removes the item and verifies the empty/zero state (the inverse operation).
|
|
366
|
-
const teardown = any(/\b(remove|delete|clear)
|
|
354
|
+
const teardown = any(/\b(remove|delete|clear)\b/i) && any(/\b(empty|no items|zero|removed|0 items)\b/i);
|
|
367
355
|
// 3. Multi-source — the cart is fed from >1 source (the main list AND a recommended/related rail).
|
|
368
356
|
const multiSource = any(/\b(recommended|related|you may also|suggest)\b/i) && addsToCart;
|
|
369
357
|
|
|
@@ -66,43 +66,21 @@ export function parseSpecClauses(specPath: string): { frs: FrClause[]; valRows:
|
|
|
66
66
|
if (!fs.existsSync(specPath)) return { frs: [], valRows: [] };
|
|
67
67
|
const lines = readTextFile(specPath).split('\n');
|
|
68
68
|
|
|
69
|
-
// Requirement ids follow the PROJECT's scheme, not ours (#572): `**FR-1**:` is one
|
|
70
|
-
// convention among many — a real spec declared ~30 MUST clauses as `` `REQ-SRCH-001`: ``
|
|
71
|
-
// and the FR-locked pattern returned zero, so the MUST-coverage gate never ran and the
|
|
72
|
-
// specFR axis was excluded "for lack of evidence" that was sitting right there. Same
|
|
73
|
-
// silent-failure class as the CRLF parsers, same id-scheme-tolerance lesson as delivery.
|
|
74
|
-
//
|
|
75
|
-
// A declaration is: line-leading (optionally bulleted / bold / backticked) `<ID>:` where
|
|
76
|
-
// the id ends in a number. Table rows are EXCLUDED — traceability tables cite requirement
|
|
77
|
-
// ids without declaring them — and so are prefixes that are never requirements
|
|
78
|
-
// (test cases, viewpoints, known-defect records, data-factory checks, flows, delivery items).
|
|
79
|
-
const NON_REQUIREMENT_PREFIX = /^(TC|VP|KD|CHK|FL|DI)-/i;
|
|
80
69
|
const frs: FrClause[] = [];
|
|
81
|
-
const seen = new Set<string>();
|
|
82
70
|
for (const line of lines) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
if (!m || NON_REQUIREMENT_PREFIX.test(m[1])) continue;
|
|
86
|
-
const id = m[1].toUpperCase();
|
|
87
|
-
if (seen.has(id)) continue; // first declaration wins
|
|
88
|
-
seen.add(id);
|
|
89
|
-
frs.push({ id, text: m[2].replace(/\*\*/g, '').trim(), modality: modalityOf(m[2]) });
|
|
71
|
+
const m = line.match(/\*\*FR-(\d+)\*\*\s*:\s*(.+)$/);
|
|
72
|
+
if (m) frs.push({ id: `FR-${m[1]}`, text: m[2].replace(/\*\*/g, '').trim(), modality: modalityOf(m[2]) });
|
|
90
73
|
}
|
|
91
74
|
|
|
92
75
|
// Validation Rules table: a row carries a Constraint, a Trigger cell, and (often) a code.
|
|
93
|
-
// A "Trigger" column alone is NOT enough to claim the table (#578): a screen-STATES table
|
|
94
|
-
// ("State ID | Trigger | URL/heading oracle | …") uses Trigger for the user action that
|
|
95
|
-
// enters the state, and reading it as validation rows invented three gate-relevant
|
|
96
|
-
// TRIGGER-UNCOVERED gaps on a real spec. The table must also name a Constraint/Rule/
|
|
97
|
-
// Validation column — the thing a validation row is ABOUT.
|
|
98
76
|
const valRows: ValRow[] = [];
|
|
99
77
|
let cTrigger = -1, cConstraint = -1, cCode = -1, inTable = false;
|
|
100
78
|
for (const raw of lines) {
|
|
101
79
|
const line = raw.trim();
|
|
102
|
-
if (line.startsWith('|') && /\btrigger\b/i.test(line) &&
|
|
80
|
+
if (line.startsWith('|') && /\btrigger\b/i.test(line) && cTrigger < 0) {
|
|
103
81
|
const cells = line.split('|').map((c) => c.trim());
|
|
104
82
|
cTrigger = cells.findIndex((c) => /^trigger$/i.test(c));
|
|
105
|
-
cConstraint = cells.findIndex((c) => /constraint
|
|
83
|
+
cConstraint = cells.findIndex((c) => /constraint/i.test(c));
|
|
106
84
|
cCode = cells.findIndex((c) => /code/i.test(c));
|
|
107
85
|
inTable = cTrigger >= 0;
|
|
108
86
|
continue;
|