@sun-asterisk/sungen 3.2.20 → 3.2.21-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +8 -0
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +9 -0
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/trace.d.ts.map +1 -1
- package/dist/cli/commands/trace.js +9 -0
- package/dist/cli/commands/trace.js.map +1 -1
- package/dist/exporters/api-testcase-formatter.d.ts +8 -0
- package/dist/exporters/api-testcase-formatter.d.ts.map +1 -1
- package/dist/exporters/api-testcase-formatter.js +44 -22
- package/dist/exporters/api-testcase-formatter.js.map +1 -1
- package/dist/exporters/matrix/build.d.ts +4 -2
- package/dist/exporters/matrix/build.d.ts.map +1 -1
- package/dist/exporters/matrix/build.js +25 -2
- package/dist/exporters/matrix/build.js.map +1 -1
- package/dist/exporters/matrix/export.d.ts +6 -0
- package/dist/exporters/matrix/export.d.ts.map +1 -1
- package/dist/exporters/matrix/export.js +20 -1
- package/dist/exporters/matrix/export.js.map +1 -1
- package/dist/exporters/matrix/gates.js +7 -1
- package/dist/exporters/matrix/gates.js.map +1 -1
- package/dist/exporters/matrix/render-xlsx.d.ts.map +1 -1
- package/dist/exporters/matrix/render-xlsx.js +15 -0
- package/dist/exporters/matrix/render-xlsx.js.map +1 -1
- package/dist/exporters/matrix/types.d.ts +7 -0
- package/dist/exporters/matrix/types.d.ts.map +1 -1
- package/dist/exporters/matrix/types.js.map +1 -1
- package/dist/exporters/playwright-report-parser.d.ts.map +1 -1
- package/dist/exporters/playwright-report-parser.js +1 -0
- package/dist/exporters/playwright-report-parser.js.map +1 -1
- package/dist/exporters/types.d.ts +2 -0
- package/dist/exporters/types.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.d.ts +6 -1
- package/dist/generators/test-generator/diagnostics.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.js +5 -0
- package/dist/generators/test-generator/diagnostics.js.map +1 -1
- package/dist/generators/test-generator/patterns/index.d.ts.map +1 -1
- package/dist/generators/test-generator/patterns/index.js +27 -19
- package/dist/generators/test-generator/patterns/index.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.d.ts.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +22 -0
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/harness/audit.d.ts +2 -0
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +79 -9
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-contract.d.ts +71 -0
- package/dist/harness/flow-contract.d.ts.map +1 -0
- package/dist/harness/flow-contract.js +235 -0
- package/dist/harness/flow-contract.js.map +1 -0
- package/dist/harness/flow-plan.d.ts +3 -0
- package/dist/harness/flow-plan.d.ts.map +1 -1
- package/dist/harness/flow-plan.js +6 -2
- package/dist/harness/flow-plan.js.map +1 -1
- package/dist/harness/parse.d.ts +5 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +29 -1
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/perf.d.ts +40 -0
- package/dist/harness/perf.d.ts.map +1 -0
- package/dist/harness/perf.js +136 -0
- package/dist/harness/perf.js.map +1 -0
- package/dist/harness/script-check.d.ts.map +1 -1
- package/dist/harness/script-check.js +17 -4
- package/dist/harness/script-check.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +13 -1
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/spec-coverage.d.ts.map +1 -1
- package/dist/harness/spec-coverage.js +29 -5
- package/dist/harness/spec-coverage.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/add-flow.md +41 -3
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +10 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +35 -16
- package/dist/orchestrator/templates/qa-context.md +14 -1
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +8 -0
- package/src/cli/commands/delivery.ts +8 -0
- package/src/cli/commands/trace.ts +9 -0
- package/src/exporters/api-testcase-formatter.ts +46 -20
- package/src/exporters/matrix/build.ts +30 -4
- package/src/exporters/matrix/export.ts +28 -1
- package/src/exporters/matrix/gates.ts +7 -1
- package/src/exporters/matrix/render-xlsx.ts +15 -0
- package/src/exporters/matrix/types.ts +7 -0
- package/src/exporters/playwright-report-parser.ts +2 -0
- package/src/exporters/types.ts +2 -0
- package/src/generators/test-generator/diagnostics.ts +6 -1
- package/src/generators/test-generator/patterns/index.ts +30 -24
- package/src/generators/test-generator/step-mapper.ts +22 -0
- package/src/harness/audit.ts +82 -10
- package/src/harness/flow-contract.ts +229 -0
- package/src/harness/flow-plan.ts +10 -3
- package/src/harness/parse.ts +31 -1
- package/src/harness/perf.ts +112 -0
- package/src/harness/script-check.ts +18 -4
- package/src/harness/sensors.ts +13 -1
- package/src/harness/spec-coverage.ts +26 -4
- package/src/orchestrator/templates/ai-src/commands/add-flow.md +41 -3
- package/src/orchestrator/templates/ai-src/commands/create-test.md +10 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +35 -16
- package/src/orchestrator/templates/qa-context.md +14 -1
|
@@ -73,34 +73,40 @@ export class PatternRegistry {
|
|
|
73
73
|
* Prefers resolver (returns template data) over generator (renders directly).
|
|
74
74
|
*/
|
|
75
75
|
generateStep(step: ParsedStep, context: PatternContext): MappedStep | null {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
76
|
+
// Matchers see only the STEP; some decisions need context (a resolver reading the
|
|
77
|
+
// test-data value, e.g. row-count's numeric check). A resolver may therefore return
|
|
78
|
+
// null to say "not mine after all" — the search continues with the next matching
|
|
79
|
+
// pattern in priority order instead of crashing or silently dropping the step.
|
|
80
|
+
const matchStep: ParsedStep = { ...step, text: maskRefsForMatching(step.text) };
|
|
81
|
+
for (const pattern of this.patterns) {
|
|
82
|
+
if (!this.matchesPattern(matchStep, pattern.matcher)) continue;
|
|
83
|
+
|
|
84
|
+
// Prefer resolver (framework-agnostic) over generator (legacy)
|
|
85
|
+
if (pattern.resolver) {
|
|
86
|
+
const resolved = pattern.resolver(step, context);
|
|
87
|
+
if (!resolved) continue; // resolver declined — fall through
|
|
88
|
+
|
|
89
|
+
// Auto-inject parent scoping if step has parentRef
|
|
90
|
+
// Skip for table-* patterns — they resolve the table name internally from step text
|
|
91
|
+
if (step.parentRef && step.parentType && !pattern.name.startsWith('table-')) {
|
|
92
|
+
resolved.data.parentLocator = PatternRegistry.resolveParentLocator(
|
|
93
|
+
step.parentRef, step.parentType, context
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const code = context.templateEngine.renderStep(resolved.templateName, resolved.data);
|
|
98
|
+
return {
|
|
99
|
+
code,
|
|
100
|
+
comment: resolved.comment,
|
|
101
|
+
};
|
|
91
102
|
}
|
|
92
103
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
comment: resolved.comment,
|
|
97
|
-
};
|
|
98
|
-
}
|
|
104
|
+
if (pattern.generator) {
|
|
105
|
+
return pattern.generator(step, context);
|
|
106
|
+
}
|
|
99
107
|
|
|
100
|
-
|
|
101
|
-
return pattern.generator(step, context);
|
|
108
|
+
return null;
|
|
102
109
|
}
|
|
103
|
-
|
|
104
110
|
return null;
|
|
105
111
|
}
|
|
106
112
|
|
|
@@ -175,6 +175,28 @@ export class StepMapper {
|
|
|
175
175
|
this.lastPrimaryKeyword = keyword;
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
+
// SG-W015 — `see [X] card with {{n}}` where n is a bare number compiles to a TEXT
|
|
179
|
+
// assertion (toHaveText('3')), not a count. An author reaching for "3 cards are shown"
|
|
180
|
+
// gets an assertion that can only pass if the card's TEXT is the digit — a wrong test
|
|
181
|
+
// with no error. Count semantics live on row/table ("see [T] table with {{n}}",
|
|
182
|
+
// "see [Ref] row with {{n}}") and lists ("should have count"); on the container types
|
|
183
|
+
// below, a bare-number text expectation is almost never what the author meant. A text
|
|
184
|
+
// element asserting a numeric readout ("[Row Quantity] text with 4") stays silent.
|
|
185
|
+
const COUNT_MISREAD_TYPES = new Set(['card', 'item', 'items', 'cards', 'option', 'options']);
|
|
186
|
+
if (this.diagnostics && step.dataRef && step.selectorRef &&
|
|
187
|
+
COUNT_MISREAD_TYPES.has((step.elementType || '').toLowerCase()) &&
|
|
188
|
+
/\bsees?\s+\[/.test(step.text) && /\bwith\b/.test(step.text)) {
|
|
189
|
+
// peekBaseValue reads the literal from test-data even in runtime-data mode,
|
|
190
|
+
// where resolveData returns an opaque marker (never numeric-looking).
|
|
191
|
+
let resolvedValue: string | undefined;
|
|
192
|
+
try { resolvedValue = this.dataResolver.peekBaseValue(step.dataRef, this.featureName); } catch { /* unresolved at compile time → cannot judge */ }
|
|
193
|
+
if (resolvedValue !== undefined && /^\d+$/.test(resolvedValue.trim())) {
|
|
194
|
+
this.diagnostics.add('SG-W015',
|
|
195
|
+
`\`see [${step.selectorRef}] ${step.elementType} with {{${step.dataRef}}}\` compiles to a TEXT assertion toHaveText('${resolvedValue.trim()}'), NOT a count of ${step.elementType}s. For a count use \`see [Ref] row with {{n}}\` / \`see [T] table with {{n}}\` (or a list "should have count"); if '${resolvedValue.trim()}' really is the ${step.elementType}'s text, assert it on a text/label element instead`,
|
|
196
|
+
{ feature: this.featureName, step: step.text });
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
178
200
|
// SG-W007 — a two-step-form browser-alert handler (`click [OK] alert`) written AFTER
|
|
179
201
|
// an action step registers its page.once('dialog') listener too late: a dialog
|
|
180
202
|
// triggered by the earlier action auto-dismisses with no error (issue #472).
|
package/src/harness/audit.ts
CHANGED
|
@@ -10,10 +10,11 @@ import * as path from 'path';
|
|
|
10
10
|
import * as fs from 'fs';
|
|
11
11
|
import { loadUnitScenarios, readUnitFeatureText, parseViewpointOverview, ScenarioInfo, ViewpointEntry } from './parse';
|
|
12
12
|
import { checkViewpointBaseline, ViewpointBaseline } from './viewpoint-baseline';
|
|
13
|
+
import { flowQuality, statefulDepthFor, FlowQualityResult } from './flow-contract';
|
|
13
14
|
import { featureFilesFor } from './unit-paths';
|
|
14
15
|
import {
|
|
15
16
|
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, depthThresholdFor, coverageBalance, duplicateClusters, traceability, claimProof, taxonomyLint,
|
|
16
|
-
automatableManual, flowCoveredThemes, flowRegressionDepth, oracleStrength,
|
|
17
|
+
automatableManual, flowCoveredThemes, flowRegressionDepth, oracleStrength, declaredPageType,
|
|
17
18
|
GateResult, DepthResult, BalanceResult, DuplicateResult, TraceResult, ClaimProofResult, TaxonomyResult, Catalog, AutomatableManualResult, FlowDepthResult, OracleStrengthResult,
|
|
18
19
|
} from './sensors';
|
|
19
20
|
import { loadFlowScenarios } from './flow-check';
|
|
@@ -48,6 +49,7 @@ export interface AuditReport {
|
|
|
48
49
|
oracle: OracleStrengthResult; // H4 — facet-oracle strength (weak name-substring vs title/detail/API/DB)
|
|
49
50
|
ledger: LedgerResult; // atomic viewpoint-item coverage (per-bullet status)
|
|
50
51
|
viewpointBaseline: ViewpointBaseline; // is the yardstick still the accepted one? (#557)
|
|
52
|
+
flowQuality?: FlowQualityResult; // #569 — flow contract verification (flows only)
|
|
51
53
|
calibration: { // #8 — multi-axis score so a high overall can't hide a weak axis
|
|
52
54
|
axes: Record<string, number>;
|
|
53
55
|
weakest: { axis: string; value: number };
|
|
@@ -154,15 +156,28 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
154
156
|
// back to the in-core functions if the capability provides none.
|
|
155
157
|
// A capability gate may need project context (the API gate resolves endpoint methods from the
|
|
156
158
|
// catalog) + the focus depth threshold (so it scores depth with the SAME bar as the UI gate).
|
|
157
|
-
const capGate = scoringCap?.gateProvider as
|
|
158
|
-
((i: { scenarios: ScenarioInfo[]; viewpoints: ViewpointEntry[]; catalog: Catalog; focus: typeof intent.focus; cwd: string; screenName: string; threshold: number; businessCriticalMethods?: string[] }) => { gate: GateResult; depth: DepthResult }) | undefined;
|
|
159
|
-
const provided = capGate?.({ scenarios, viewpoints, catalog, focus: intent.focus, cwd: projectRootFromScreenDir(screenDir), screenName: catalogScreenName, threshold: depthThresholdFor(intent.focus), businessCriticalMethods: intent.businessCriticalMethods });
|
|
160
159
|
// Is the declaration this suite is measured against still the one a human accepted?
|
|
161
160
|
const viewpointBaseline = checkViewpointBaseline(
|
|
162
161
|
projectRootFromScreenDir(screenDir), catalogScreenName, viewpointPath);
|
|
163
162
|
const viewpointText = fs.existsSync(viewpointPath) ? readTextFile(viewpointPath) : '';
|
|
164
|
-
|
|
165
|
-
|
|
163
|
+
// #569 — a flow WITH a contract is measured against ITS declaration, not a screen's.
|
|
164
|
+
// Keyword page-type detection force-fit flows into screen checklists (a registration
|
|
165
|
+
// flow judged as a `form` screen scored coverage 0%), so for contract-holding flows the
|
|
166
|
+
// page-type machinery only applies when the project DECLARED a page type explicitly.
|
|
167
|
+
// The stripped catalog feeds the CAPABILITY gate too — driver-ui provides the gate on
|
|
168
|
+
// real projects, and handing it the unstripped catalog re-introduced the force-fit.
|
|
169
|
+
const isUiFlowUnit = /^flows\//.test(catalogScreenName);
|
|
170
|
+
const flowQ: FlowQualityResult = isUiFlowUnit
|
|
171
|
+
? flowQuality(screenDir, scenarios)
|
|
172
|
+
: { hasContract: false, errors: [], outcomeProven: false, outcomeManualOnly: false, offGoal: [], offGoalRatio: 0, offGoalCategories: [], phases: [], phaseRatio: 1, handoffs: { total: 0, asserted: 0, ratio: 1 } };
|
|
173
|
+
const gateCatalog = (flowQ.hasContract && !declaredPageType(viewpointText, catalog))
|
|
174
|
+
? { ...catalog, page_types: {} } as Catalog
|
|
175
|
+
: catalog;
|
|
176
|
+
const capGate = scoringCap?.gateProvider as
|
|
177
|
+
((i: { scenarios: ScenarioInfo[]; viewpoints: ViewpointEntry[]; catalog: Catalog; focus: typeof intent.focus; cwd: string; screenName: string; threshold: number; businessCriticalMethods?: string[] }) => { gate: GateResult; depth: DepthResult }) | undefined;
|
|
178
|
+
const provided = capGate?.({ scenarios, viewpoints, catalog: gateCatalog, focus: intent.focus, cwd: projectRootFromScreenDir(screenDir), screenName: catalogScreenName, threshold: depthThresholdFor(intent.focus), businessCriticalMethods: intent.businessCriticalMethods });
|
|
179
|
+
const gate = provided?.gate ?? viewpointGate(scenarios, viewpoints, gateCatalog, platform === 'mobile', viewpointText);
|
|
180
|
+
const depth = provided?.depth ?? assertionDepth(scenarios, dataThemesFor(gateCatalog, gate.pageType), intent.focus);
|
|
166
181
|
|
|
167
182
|
// TQ-4 — deferral-aware coverage credit: an inherently cross-screen theme (cart / detail /
|
|
168
183
|
// filter correctness) belongs in a FLOW, not on the screen. When a flow deeply covers a screen
|
|
@@ -200,8 +215,15 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
200
215
|
// collection, the regression dimensions (count/quantity proof · teardown · multi-source) cap the
|
|
201
216
|
// businessDepth headroom: it can reach 1.0 only when all three are exercised, so a present-but-
|
|
202
217
|
// shallow flow can't claim a perfect score (floor 0.5 — assertion depth still dominates).
|
|
203
|
-
const isUiFlow =
|
|
204
|
-
|
|
218
|
+
const isUiFlow = isUiFlowUnit;
|
|
219
|
+
let flowDepth = isUiFlow ? flowRegressionDepth(scenarios) : { stateful: false, countProof: false, teardown: false, multiSource: false, ratio: 1, missing: [] } as FlowDepthResult;
|
|
220
|
+
// Contract-declared statefulness generalizes the cart-hardcoded vocabulary: a flow that
|
|
221
|
+
// mutates ANY named collection (order, application, submission …) gets the same three
|
|
222
|
+
// regression dims. The legacy cart detection stays for contract-less flows (snapshots).
|
|
223
|
+
if (isUiFlow && !flowDepth.stateful && flowQ.contract?.stateful) {
|
|
224
|
+
const d = statefulDepthFor(flowQ.contract.stateful, scenarios);
|
|
225
|
+
flowDepth = { stateful: true, countProof: d.countProof, teardown: d.teardown, multiSource: d.multiSource, ratio: d.ratio, missing: d.missing };
|
|
226
|
+
}
|
|
205
227
|
const FLOW_DEPTH_FLOOR = 0.5;
|
|
206
228
|
// H4 — oracle strength: a weak facet oracle (name-substring "proves" category/brand membership)
|
|
207
229
|
// caps businessDepth the same way (floor 0.5). A suite with no facet claim, or a strong oracle, is
|
|
@@ -211,10 +233,14 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
211
233
|
|
|
212
234
|
// Sub-scores
|
|
213
235
|
const coverage = gate.coverageRatio;
|
|
236
|
+
// #569 — handoff integrity: a flow's depth is also whether each cross-screen transition
|
|
237
|
+
// is followed by an assertion on the carried state (same floor discipline as the other caps).
|
|
238
|
+
const HANDOFF_FLOOR = 0.5;
|
|
214
239
|
const businessDepth = Math.min(
|
|
215
240
|
depth.bcDepthRatio,
|
|
216
241
|
flowDepth.stateful ? FLOW_DEPTH_FLOOR + (1 - FLOW_DEPTH_FLOOR) * flowDepth.ratio : 1,
|
|
217
242
|
oracle.weak.length ? ORACLE_FLOOR + (1 - ORACLE_FLOOR) * oracle.ratio : 1,
|
|
243
|
+
flowQ.hasContract && flowQ.handoffs.total > 0 ? HANDOFF_FLOOR + (1 - HANDOFF_FLOOR) * flowQ.handoffs.ratio : 1,
|
|
218
244
|
);
|
|
219
245
|
// When the taxonomy drifted (most scenarios unclassified), the balance axis is unreliable — cap it
|
|
220
246
|
// at 0.5 instead of awarding a vacuous 1.0 so a stale taxonomy fails loudly, not silently (H1).
|
|
@@ -246,8 +272,15 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
246
272
|
// what a create-test run did, silently dropping the performance viewpoint on
|
|
247
273
|
// the way. While the declaration is unconfirmed, neither axis is evidence.
|
|
248
274
|
const viewpointMoved = viewpointBaseline.status === 'changed';
|
|
275
|
+
// 4. A flow with a contract is measured on ITS coverage: journey phases (HP/ER/EH
|
|
276
|
+
// automated) — not a screen's page-type themes. Same weight, same criticality,
|
|
277
|
+
// so a flow can finally EARN the coverage axis instead of always losing it to
|
|
278
|
+
// PAGE-TYPE-UNDETERMINED (or worse, being judged against a form checklist).
|
|
279
|
+
const flowScored = flowQ.hasContract;
|
|
249
280
|
const axisDefs: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean }> = [
|
|
250
|
-
|
|
281
|
+
flowScored
|
|
282
|
+
? { key: 'flowCoverage', value: flowQ.phaseRatio, weight: 0.22, applicable: true, critical: true }
|
|
283
|
+
: { key: 'coverage', value: coverage, weight: 0.22, applicable: !!gate.pageType && gate.themesTotal > 0, critical: true },
|
|
251
284
|
{ key: 'specFR', value: specRatio, weight: 0.15, applicable: spec.hasSpec && spec.frTotal > 0, critical: true },
|
|
252
285
|
{ key: 'atomicLedger', value: ledger.ratio, weight: 0.13, applicable: ledger.hasViewpoint && ledger.total > 0 && !viewpointMoved, critical: true },
|
|
253
286
|
{ key: 'businessDepth', value: businessDepth, weight: 0.20, applicable: true, critical: true },
|
|
@@ -297,9 +330,40 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
297
330
|
].filter(Boolean).join(', ');
|
|
298
331
|
findings.push(`VIEWPOINT-BASELINE-CHANGED: test-viewpoint.md no longer declares what it did when this unit was last accepted — ${moved || 'the declared ids were reordered or replaced'}. atomicLedger + traceability measure the suite AGAINST this file, so they are excluded from the score until the change is confirmed: a generator that rewrites the declaration scores both 100% by construction, and a viewpoint dropped from the file stops being missing from anything. Review the diff (a removed id means that coverage is now unclaimed), then run \`sungen audit --screen ${screenName} --accept-viewpoint\`.`);
|
|
299
332
|
}
|
|
300
|
-
|
|
333
|
+
// On a contract flow the page type is INTENTIONALLY not applied — the contract is the checklist.
|
|
334
|
+
if (gate.pageTypeSource === 'undetermined' && !flowScored) {
|
|
301
335
|
findings.push(`PAGE-TYPE-UNDETERMINED: no page type fits this screen with enough confidence (best ${gate.pageTypeEvidence?.hits ?? 0} keyword hit(s)), so NO critical themes were demanded and the coverage axis is excluded from the score — declare it in test-viewpoint.md (\`page-type: <id>\`) to have the theme checklist applied.`);
|
|
302
336
|
}
|
|
337
|
+
// ---- #569: flow-quality findings -----------------------------------------
|
|
338
|
+
if (isUiFlow && !flowQ.hasContract) {
|
|
339
|
+
const why = flowQ.errors.length ? ` (${flowQ.errors.join('; ')})` : '';
|
|
340
|
+
findings.push(`FLOW-CONTRACT-MISSING: this flow declares no contract${why} — without \`requirements/flow-contract.yaml\` (goal · trigger · outcome.screen · phases) the flow cannot be scored on ITS coverage (journey phases) and falls back to screen machinery. Author it with the boundary checklist: one business goal, a clear trigger, ONE observable outcome valuable to the actor, name = "Verb + outcome"; split anything that forms an independently valuable flow.`);
|
|
341
|
+
}
|
|
342
|
+
if (flowQ.hasContract) {
|
|
343
|
+
if (!flowQ.outcomeProven) {
|
|
344
|
+
findings.push(flowQ.outcomeManualOnly
|
|
345
|
+
? `FLOW-OUTCOME-UNPROVEN: the goal "${flowQ.contract!.goal}" is covered only by @manual scenarios — the flow's own outcome (\`${flowQ.contract!.outcome.screen}\`) has no AUTOMATED data assertion, so a regression run cannot prove the journey still works. Automate the happy-path proof first; it is the reason this flow exists.`
|
|
346
|
+
: `FLOW-OUTCOME-UNPROVEN: no scenario asserts data on the outcome screen \`${flowQ.contract!.outcome.screen}\` — the flow never proves its own goal ("${flowQ.contract!.goal}"). Add the happy-path scenario that ends in the declared final assertion.`);
|
|
347
|
+
}
|
|
348
|
+
if (flowQ.offGoalRatio > 0.35) {
|
|
349
|
+
findings.push(`FLOW-SCOPE-CREEP: ${flowQ.offGoal.length}/${scenarios.length} scenarios never touch the outcome screen \`${flowQ.contract!.outcome.screen}\` and are not guards/error-recovery (categories: ${flowQ.offGoalCategories.join(', ')}) — evidence of a SECOND business goal in this flow. Split them into their own flow (one flow = one goal, one observable outcome).`);
|
|
350
|
+
}
|
|
351
|
+
for (const ph of flowQ.phases.filter((p) => !p.covered || !p.automated)) {
|
|
352
|
+
findings.push(`FLOW-PHASE-${ph.phase}-MISSING: journey phase ${ph.phase} (${ph.phase === 'HP' ? 'happy path proving the outcome' : ph.phase === 'ER' ? 'error recovery — validation must not trap the journey' : 'guards — direct access / back / refresh'}) is ${ph.covered ? 'covered only by @manual' : 'not covered'} → it does not count toward flowCoverage until automated.`);
|
|
353
|
+
}
|
|
354
|
+
if (flowQ.handoffs.total > 0 && flowQ.handoffs.ratio < 0.7) {
|
|
355
|
+
findings.push(`FLOW-HANDOFF-SHALLOW: only ${flowQ.handoffs.asserted}/${flowQ.handoffs.total} cross-screen transitions are followed by an assertion in the new screen — carried state (the thing a flow exists to verify) is unproven at the handoff. businessDepth is capped until asserted.`);
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
if (isUiFlow) {
|
|
359
|
+
// Flows defer more than screens, so an UNCODED @manual is costlier here: the capability
|
|
360
|
+
// planner cannot route it to the driver that would automate it (password-reset: 11/22
|
|
361
|
+
// manual, most of them M5-inbox — invisible while tagged bare).
|
|
362
|
+
const bareManual = (featureText.match(/@manual(?![:\w])/g) || []).length;
|
|
363
|
+
if (bareManual > 0) {
|
|
364
|
+
findings.push(`MANUAL-CODE-MISSING: ${bareManual} @manual tag(s) carry no reason code — tag @manual:M1..M9 (M5 = external dependency e.g. inbox, M3 = mock/network, M7 = environment) so the capability planner can route what a driver would automate.`);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
303
367
|
for (const c of flowCredits) {
|
|
304
368
|
findings.push(`COVERED-VIA-FLOW: critical theme "${c.theme}" is not covered on this screen but is deeply covered by flow "${c.flow}" — cross-screen depth correctly owned by the flow, so it is credited (not a screen gap). Verify with \`sungen flow-check\`.`);
|
|
305
369
|
}
|
|
@@ -376,6 +440,13 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
376
440
|
const shown = spec.inferredOnly.slice(0, 8).join(', ');
|
|
377
441
|
findings.push(`SPEC-TRACE-IMPLICIT: ${spec.inferredOnly.length} requirement(s) are covered by keyword inference only (${shown}${spec.inferredOnly.length > 8 ? ', …' : ''}) — add \`@spec:<id>\` to the scenario that proves each so the trace survives edits and delivery can report it.`);
|
|
378
442
|
}
|
|
443
|
+
// Safety net for the parser (#572): the feature CITES requirement ids the spec parser
|
|
444
|
+
// could not find a declaration for. Silence here would repeat the original failure —
|
|
445
|
+
// "nothing found" quietly excluding the specFR axis while the evidence sits in the file.
|
|
446
|
+
if (spec.hasSpec && spec.frTotal === 0 && /@spec:/i.test(featureText)) {
|
|
447
|
+
const cited = [...new Set([...featureText.matchAll(/@spec:([A-Za-z0-9.-]+)/g)].map((m) => m[1].toUpperCase()))];
|
|
448
|
+
findings.push(`SPEC-IDS-UNPARSED: the feature cites ${cited.length} requirement id(s) via @spec: (${cited.slice(0, 5).join(', ')}${cited.length > 5 ? ', …' : ''}) but requirements/spec.md yielded ZERO parseable requirement declarations — the spec likely declares ids in a scheme the parser cannot read. Declare each requirement on its own line as \`<ID>: <clause>\` (bullet/bold/backtick all fine; the id must end in a number) so the MUST-coverage gate and the specFR axis can run.`);
|
|
449
|
+
}
|
|
379
450
|
if (taxonomyMismatch) {
|
|
380
451
|
findings.push(`VP-TAXONOMY-MISMATCH: only ${(trace.mappedRatio * 100).toFixed(0)}% of scenarios use the viewpoint IDs declared in test-viewpoint.md — scenarios invented a generic VP-<CAT> scheme. Re-tag to the project's viewpoint IDs so the coverage matrix is accurate.`);
|
|
381
452
|
}
|
|
@@ -485,6 +556,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
485
556
|
scenarioCount: scenarios.length,
|
|
486
557
|
gate, depth, claim, taxonomy, balance, duplicates, trace, spec,
|
|
487
558
|
taxonomyMismatch, downstream, manualOracle: manualOracleResult, automatableManual: autoManual, flowDepth, oracle, ledger, viewpointBaseline, calibration,
|
|
559
|
+
...(isUiFlow ? { flowQuality: flowQ } : {}),
|
|
488
560
|
score: {
|
|
489
561
|
overall: Math.round(overall * 10) / 10,
|
|
490
562
|
coverage: Math.round(coverage * 100) / 100,
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flow Contract — the declared boundary of a flow, and the sensors that verify
|
|
3
|
+
* the suite against it. (#569, docs/spec/sungen-flow-quality-spec.md)
|
|
4
|
+
*
|
|
5
|
+
* A flow is the SMALLEST complete business action chain: one clear trigger ending
|
|
6
|
+
* in ONE observable, valuable outcome. Nothing enforced that: `add-flow` asked only
|
|
7
|
+
* "which screens, in order?", so a real example mixed three business goals (cart,
|
|
8
|
+
* filter, product-detail) in one "flow", and the harness scored flows with screen
|
|
9
|
+
* machinery — a registration flow was judged against the `form` page-type checklist
|
|
10
|
+
* (coverage 0%), and every flow audit showed taxonomy=0% because flow phase ids
|
|
11
|
+
* (FL-HP-001) don't even parse as categories.
|
|
12
|
+
*
|
|
13
|
+
* The contract is a declaration the QA owns (AI proposes at add-flow; a filled
|
|
14
|
+
* contract is an INPUT to generation, never an output — same rule as
|
|
15
|
+
* test-viewpoint.md). These sensors are deterministic checks against it:
|
|
16
|
+
*
|
|
17
|
+
* - outcome proof — the flow proves its own goal with an automated data assertion
|
|
18
|
+
* - scope creep — scenarios that never touch the outcome are a second goal
|
|
19
|
+
* - phase coverage — HP / ER / EH journey phases, the flow's coverage axis
|
|
20
|
+
* - handoff — a cross-screen transition is followed by an assertion
|
|
21
|
+
* - stateful depth — generalizes the cart-hardcoded regression dims to any
|
|
22
|
+
* declared collection (order, application, submission …)
|
|
23
|
+
*/
|
|
24
|
+
import * as fs from 'fs';
|
|
25
|
+
import * as path from 'path';
|
|
26
|
+
import { parse as parseYaml } from 'yaml';
|
|
27
|
+
import { ScenarioInfo } from './parse';
|
|
28
|
+
import { readTextFile } from './read-text';
|
|
29
|
+
|
|
30
|
+
export interface FlowContract {
|
|
31
|
+
goal: string;
|
|
32
|
+
actor?: string;
|
|
33
|
+
trigger?: string;
|
|
34
|
+
precondition?: string;
|
|
35
|
+
outcome: { screen: string; assertion?: string };
|
|
36
|
+
value?: string;
|
|
37
|
+
/** Journey phases this flow declares. Default [HP, ER, EH]; UI is allowed but
|
|
38
|
+
* never demanded (presentation is the balance axis's business, not coverage's). */
|
|
39
|
+
phases: string[];
|
|
40
|
+
/** The mutated collection (cart, order, application …) — enables regression dims. */
|
|
41
|
+
stateful?: string;
|
|
42
|
+
budgets?: Record<string, number>;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface FlowQualityResult {
|
|
46
|
+
hasContract: boolean;
|
|
47
|
+
contract?: FlowContract;
|
|
48
|
+
/** Parse/shape errors — a broken contract is reported, never silently ignored. */
|
|
49
|
+
errors: string[];
|
|
50
|
+
outcomeProven: boolean;
|
|
51
|
+
/** Manual-only proof: the goal is covered but not by automation. */
|
|
52
|
+
outcomeManualOnly: boolean;
|
|
53
|
+
/** Scenario names that never touch the outcome screen (guard/error phases excluded). */
|
|
54
|
+
offGoal: string[];
|
|
55
|
+
offGoalRatio: number;
|
|
56
|
+
/** Off-goal categories, for the split suggestion ("VP-FILTER-* looks like its own flow"). */
|
|
57
|
+
offGoalCategories: string[];
|
|
58
|
+
phases: { phase: string; covered: boolean; automated: boolean }[];
|
|
59
|
+
/** Covered-and-automated phases / declared phases (UI excluded) — the flow coverage axis. */
|
|
60
|
+
phaseRatio: number;
|
|
61
|
+
/** Cross-namespace transitions followed by an assertion / all transitions. */
|
|
62
|
+
handoffs: { total: number; asserted: number; ratio: number };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const DEFAULT_PHASES = ['HP', 'ER', 'EH'];
|
|
66
|
+
|
|
67
|
+
export function flowContractPath(unitDir: string): string {
|
|
68
|
+
return path.join(unitDir, 'requirements', 'flow-contract.yaml');
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Load + validate. Returns null when absent; a present-but-broken file returns errors. */
|
|
72
|
+
export function loadFlowContract(unitDir: string): { contract: FlowContract | null; errors: string[] } {
|
|
73
|
+
const p = flowContractPath(unitDir);
|
|
74
|
+
if (!fs.existsSync(p)) return { contract: null, errors: [] };
|
|
75
|
+
let raw: Record<string, unknown>;
|
|
76
|
+
try {
|
|
77
|
+
raw = parseYaml(readTextFile(p)) as Record<string, unknown>;
|
|
78
|
+
} catch (e) {
|
|
79
|
+
return { contract: null, errors: [`flow-contract.yaml does not parse: ${(e as Error).message}`] };
|
|
80
|
+
}
|
|
81
|
+
if (!raw || typeof raw !== 'object') return { contract: null, errors: ['flow-contract.yaml is empty'] };
|
|
82
|
+
const errors: string[] = [];
|
|
83
|
+
if (!raw.goal || typeof raw.goal !== 'string') errors.push('missing `goal:` (Verb + outcome, e.g. "Place an order for a product added from home")');
|
|
84
|
+
const outcome = raw.outcome as { screen?: unknown; assertion?: unknown } | undefined;
|
|
85
|
+
if (!outcome || typeof outcome.screen !== 'string' || !outcome.screen.trim()) {
|
|
86
|
+
errors.push('missing `outcome.screen:` — the screen namespace that carries the final proof');
|
|
87
|
+
}
|
|
88
|
+
if (errors.length > 0) return { contract: null, errors };
|
|
89
|
+
const phases = Array.isArray(raw.phases) && raw.phases.length > 0
|
|
90
|
+
? (raw.phases as unknown[]).map((x) => String(x).toUpperCase())
|
|
91
|
+
: DEFAULT_PHASES;
|
|
92
|
+
return {
|
|
93
|
+
contract: {
|
|
94
|
+
goal: String(raw.goal),
|
|
95
|
+
actor: raw.actor !== undefined ? String(raw.actor) : undefined,
|
|
96
|
+
trigger: raw.trigger !== undefined ? String(raw.trigger) : undefined,
|
|
97
|
+
precondition: raw.precondition !== undefined ? String(raw.precondition) : undefined,
|
|
98
|
+
outcome: { screen: String(outcome!.screen).toLowerCase(), assertion: outcome!.assertion !== undefined ? String(outcome!.assertion) : undefined },
|
|
99
|
+
value: raw.value !== undefined ? String(raw.value) : undefined,
|
|
100
|
+
phases,
|
|
101
|
+
stateful: raw.stateful !== undefined ? String(raw.stateful).toLowerCase() : undefined,
|
|
102
|
+
budgets: (raw.budgets && typeof raw.budgets === 'object') ? raw.budgets as Record<string, number> : undefined,
|
|
103
|
+
},
|
|
104
|
+
errors: [],
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** `[screen:element]` namespaces referenced by a scenario's steps, in step order. */
|
|
109
|
+
function namespacesInOrder(s: ScenarioInfo): string[] {
|
|
110
|
+
const out: string[] = [];
|
|
111
|
+
for (const m of s.stepsText.matchAll(/\[([a-z0-9_.-]+):/g)) out.push(m[1]);
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function touchesOutcome(s: ScenarioInfo, outcomeScreen: string): boolean {
|
|
116
|
+
return namespacesInOrder(s).includes(outcomeScreen);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Phase of a scenario: its declared phase token (FL-HP-001 / VP-FLOW-ER-02 / MS-EH-005)
|
|
120
|
+
* when present, else vocabulary detection. */
|
|
121
|
+
export function phaseOf(s: ScenarioInfo, declared: string[]): string | null {
|
|
122
|
+
const id = (s.vpId ?? '').toUpperCase();
|
|
123
|
+
for (const ph of declared) {
|
|
124
|
+
if (new RegExp(`(^|-)${ph}(-|$)`).test(id)) return ph;
|
|
125
|
+
}
|
|
126
|
+
const hay = s.haystack;
|
|
127
|
+
if (declared.includes('EH') && /\b(direct access|without (a |the )?(submit|login)|browser back|refresh|expired|tamper|unauthoriz|redirect(ed)? (back )?to|guard)\b/.test(hay)) return 'EH';
|
|
128
|
+
if (declared.includes('ER') && /\b(invalid|error|required|validation|malformed|blocked|then correct|recover)\b/.test(hay)) return 'ER';
|
|
129
|
+
if (declared.includes('HP') && s.hasDataAssertion) return 'HP';
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Verify the suite against the contract. Deterministic; a flow without a contract
|
|
135
|
+
* returns hasContract:false and neutral values (the audit reports the checklist).
|
|
136
|
+
*/
|
|
137
|
+
export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQualityResult {
|
|
138
|
+
const { contract, errors } = loadFlowContract(unitDir);
|
|
139
|
+
const neutral: FlowQualityResult = {
|
|
140
|
+
hasContract: false, errors, outcomeProven: false, outcomeManualOnly: false,
|
|
141
|
+
offGoal: [], offGoalRatio: 0, offGoalCategories: [],
|
|
142
|
+
phases: [], phaseRatio: 1, handoffs: { total: 0, asserted: 0, ratio: 1 },
|
|
143
|
+
};
|
|
144
|
+
if (!contract) return neutral;
|
|
145
|
+
|
|
146
|
+
const outcomeScreen = contract.outcome.screen;
|
|
147
|
+
|
|
148
|
+
// --- Outcome proof: the flow proves its own goal, by automation -------------
|
|
149
|
+
const proofs = scenarios.filter((s) => touchesOutcome(s, outcomeScreen) && s.hasDataAssertion);
|
|
150
|
+
const outcomeProven = proofs.some((s) => !s.manual);
|
|
151
|
+
const outcomeManualOnly = !outcomeProven && proofs.length > 0;
|
|
152
|
+
|
|
153
|
+
// --- Scope creep: a scenario that never touches the outcome and is not a ----
|
|
154
|
+
// guard/error phase is evidence of a SECOND business goal in this flow.
|
|
155
|
+
const declaredPhases = contract.phases;
|
|
156
|
+
const offGoalScenarios = scenarios.filter((s) => {
|
|
157
|
+
if (touchesOutcome(s, outcomeScreen)) return false;
|
|
158
|
+
const ph = phaseOf(s, declaredPhases);
|
|
159
|
+
return ph !== 'EH' && ph !== 'ER'; // guards/error-recovery legitimately stop early
|
|
160
|
+
});
|
|
161
|
+
const offGoalRatio = scenarios.length ? offGoalScenarios.length / scenarios.length : 0;
|
|
162
|
+
const offGoalCategories = Array.from(new Set(
|
|
163
|
+
offGoalScenarios.map((s) => s.category ?? s.vpId?.replace(/-\d+.*$/, '') ?? '?')));
|
|
164
|
+
|
|
165
|
+
// --- Phase coverage: the flow's coverage axis (UI never demanded) -----------
|
|
166
|
+
const demanded = declaredPhases.filter((p) => p !== 'UI');
|
|
167
|
+
const phases = demanded.map((phase) => {
|
|
168
|
+
const inPhase = scenarios.filter((s) => phaseOf(s, declaredPhases) === phase);
|
|
169
|
+
// HP must additionally prove the outcome — a data assertion elsewhere is not the goal.
|
|
170
|
+
const relevant = phase === 'HP' ? inPhase.filter((s) => touchesOutcome(s, outcomeScreen)) : inPhase;
|
|
171
|
+
return {
|
|
172
|
+
phase,
|
|
173
|
+
covered: relevant.length > 0,
|
|
174
|
+
automated: relevant.some((s) => !s.manual),
|
|
175
|
+
};
|
|
176
|
+
});
|
|
177
|
+
const phaseRatio = demanded.length
|
|
178
|
+
? phases.filter((p) => p.covered && p.automated).length / demanded.length
|
|
179
|
+
: 1;
|
|
180
|
+
|
|
181
|
+
// --- Handoff integrity: no blind tail after a cross-namespace transition. ---
|
|
182
|
+
// A transition counts as asserted when ANY assertion follows it — in the entered
|
|
183
|
+
// namespace or later. Demanding the assertion in the entered namespace itself
|
|
184
|
+
// flagged two legitimate shapes: a guard that asserts the REDIRECT target
|
|
185
|
+
// ("go to [Checkout] → see [Home] page"), and a passthrough click en route
|
|
186
|
+
// ("click [Cart:Checkout]" asserting on the next screen). What the sensor
|
|
187
|
+
// actually guards against is a flow that clicks through screens and ends blind.
|
|
188
|
+
let total = 0; let asserted = 0;
|
|
189
|
+
for (const s of scenarios) {
|
|
190
|
+
if (s.manual) continue;
|
|
191
|
+
const steps = s.steps ?? [];
|
|
192
|
+
let current: string | null = null;
|
|
193
|
+
for (let i = 0; i < steps.length; i++) {
|
|
194
|
+
const ns = (steps[i].text.match(/\[([A-Za-z0-9_.-]+):/) || [])[1]?.toLowerCase() ?? null;
|
|
195
|
+
if (!ns) continue;
|
|
196
|
+
if (current !== null && ns !== current) {
|
|
197
|
+
total++;
|
|
198
|
+
if (steps.slice(i).some((st) => st.bucket === 'then')) asserted++;
|
|
199
|
+
}
|
|
200
|
+
current = ns;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
const handoffs = { total, asserted, ratio: total ? asserted / total : 1 };
|
|
204
|
+
|
|
205
|
+
return {
|
|
206
|
+
hasContract: true, contract, errors: [],
|
|
207
|
+
outcomeProven, outcomeManualOnly,
|
|
208
|
+
offGoal: offGoalScenarios.map((s) => s.name.slice(0, 80)),
|
|
209
|
+
offGoalRatio, offGoalCategories,
|
|
210
|
+
phases, phaseRatio, handoffs,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Generalized stateful regression depth: the contract names the mutated collection,
|
|
216
|
+
* so the three dims (count-proof · teardown · multi-source) stop being cart-only.
|
|
217
|
+
*/
|
|
218
|
+
export function statefulDepthFor(collection: string, scenarios: ScenarioInfo[]): { countProof: boolean; teardown: boolean; multiSource: boolean; missing: string[]; ratio: number } {
|
|
219
|
+
const hay = scenarios.map((s) => s.haystack);
|
|
220
|
+
const any = (re: RegExp) => hay.some((h) => re.test(h));
|
|
221
|
+
const noun = collection.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
222
|
+
const countProof = any(new RegExp(`\\b(quantity|qty|row count|count|number of|two (rows|lines|items)|\\d+ (rows|lines|items))\\b`)) ;
|
|
223
|
+
const teardown = any(/\b(remove|delete|clear|cancel|withdraw)(?:s|d|ed|ing|n)?\b/) && any(new RegExp(`\\b(empty|emptied|no items|zero|removed|cleared|cancelled|withdrawn|0 items)\\b|empty[- ]${noun}`));
|
|
224
|
+
const adds = hay.filter((h) => new RegExp(`\\b(add|submit|create|place).{0,40}${noun}|${noun}.{0,40}\\b(add|submit|create|place)`).test(h));
|
|
225
|
+
const multiSource = any(/\b(recommended|related|you may also|another source|both sources|second (list|source))\b/) && adds.length > 0;
|
|
226
|
+
const dims: Array<[string, boolean]> = [['count-proof', countProof], ['teardown', teardown], ['multi-source', multiSource]];
|
|
227
|
+
const missing = dims.filter(([, v]) => !v).map(([k]) => k);
|
|
228
|
+
return { countProof, teardown, multiSource, missing, ratio: (dims.length - missing.length) / dims.length };
|
|
229
|
+
}
|
package/src/harness/flow-plan.ts
CHANGED
|
@@ -71,6 +71,9 @@ export interface FlowPlan {
|
|
|
71
71
|
legs: LegPlan[];
|
|
72
72
|
byReason: Record<string, number>;
|
|
73
73
|
capabilityManual: number;
|
|
74
|
+
/** @manual whose reason is "cross-screen → automate via flow" (class XS) — inside a flow this
|
|
75
|
+
* usually means the scenario should simply BE automated here. (#569) */
|
|
76
|
+
crossScreenManual: number;
|
|
74
77
|
judgmentManual: number;
|
|
75
78
|
contracts: Contract[];
|
|
76
79
|
readiness: 'ready' | 'not-ready';
|
|
@@ -87,14 +90,18 @@ export function buildFlowPlan(cwd: string, flow: string): FlowPlan {
|
|
|
87
90
|
// Legs = distinct screen namespaces.
|
|
88
91
|
const legMap = new Map<string, { scenarios: Set<string>; refs: Set<string>; automated: boolean }>();
|
|
89
92
|
const byReason: Record<string, number> = {};
|
|
90
|
-
let capabilityManual = 0, judgmentManual = 0;
|
|
93
|
+
let capabilityManual = 0, judgmentManual = 0, crossScreenManual = 0;
|
|
91
94
|
|
|
92
95
|
for (const sc of scenarios) {
|
|
93
96
|
if (sc.manual) {
|
|
94
97
|
const { code } = inferReasonCode(sc.tags, sc.reason);
|
|
95
98
|
byReason[code] = (byReason[code] || 0) + 1;
|
|
96
99
|
const cls = MANUAL_REASONS[code]?.cls;
|
|
97
|
-
|
|
100
|
+
// XS ("cross-screen → automate via flow") is a THIRD class; it used to be silently
|
|
101
|
+
// dropped from both counters, understating the plan's manual load. (#569)
|
|
102
|
+
if (cls === 'capability') capabilityManual++;
|
|
103
|
+
else if (cls === 'keep') judgmentManual++;
|
|
104
|
+
else if (cls === 'flow') crossScreenManual++;
|
|
98
105
|
}
|
|
99
106
|
for (const r of sc.refs) {
|
|
100
107
|
const leg = r.screen.toLowerCase();
|
|
@@ -133,5 +140,5 @@ export function buildFlowPlan(cwd: string, flow: string): FlowPlan {
|
|
|
133
140
|
}
|
|
134
141
|
if (readiness === 'ready') plan.unshift('Selectors present for every automated leg — ready to compile + run.');
|
|
135
142
|
|
|
136
|
-
return { flow, total: scenarios.length, legs, byReason, capabilityManual, judgmentManual, contracts, readiness, missingLegs, plan };
|
|
143
|
+
return { flow, total: scenarios.length, legs, byReason, capabilityManual, judgmentManual, crossScreenManual, contracts, readiness, missingLegs, plan };
|
|
137
144
|
}
|
package/src/harness/parse.ts
CHANGED
|
@@ -38,6 +38,8 @@ export interface ScenarioInfo {
|
|
|
38
38
|
requiresCaps?: string[]; // @requires:<cap> — automation-ready but needs an opt-in driver (TQ-11)
|
|
39
39
|
deferredToFlow?: boolean; // @deferred:flow — owned by a flow, not automated on this screen (H6)
|
|
40
40
|
ownedByFlow?: string; // @owned-by:<flow> — the flow that owns this deferred scenario (H6)
|
|
41
|
+
/** Ordered steps with their resolved bucket (And/But inherit) — flow handoff analysis (#569). */
|
|
42
|
+
steps?: Array<{ bucket: 'given' | 'when' | 'then'; text: string }>;
|
|
41
43
|
}
|
|
42
44
|
|
|
43
45
|
/** Format-tolerant: is this token an ID (project's scheme), not a prose word?
|
|
@@ -91,6 +93,29 @@ export function parseViewpointOverview(filePath: string): ViewpointEntry[] {
|
|
|
91
93
|
}
|
|
92
94
|
}
|
|
93
95
|
|
|
96
|
+
// 1b) Flow-style declarations (#569). Flow viewpoint files commonly declare per-item
|
|
97
|
+
// ids at the END of a bullet ("… → **FL-HP-001**") under phase section headers
|
|
98
|
+
// ("## FL-HP — Happy Path"). Neither matched the table/group passes, so every flow
|
|
99
|
+
// audit collapsed to taxonomy=0% / traceability n-a — scenarios correctly tagged
|
|
100
|
+
// FL-HP-001 were reported as unmapped. Both forms are additive here.
|
|
101
|
+
for (const raw of lines) {
|
|
102
|
+
const line = raw.trim();
|
|
103
|
+
const section = line.match(/^##\s+([A-Z]{2,}(?:-[A-Z0-9]{2,})*)\s+[—–-]\s*(.*)$/);
|
|
104
|
+
if (section && isViewpointId(section[1] + '-0')) {
|
|
105
|
+
const id = section[1].toUpperCase();
|
|
106
|
+
if (!entries.has(id)) entries.set(id, { id, priority: 'Unknown', reason: section[2] ?? '' });
|
|
107
|
+
}
|
|
108
|
+
if (/^[-*+]\s/.test(line)) {
|
|
109
|
+
const arrow = line.match(/(?:→|->)\s*\*{0,2}([A-Z][A-Z0-9]*(?:-[A-Z0-9]+)*-\d+[a-zA-Z]?)\*{0,2}\s*$/);
|
|
110
|
+
if (arrow) {
|
|
111
|
+
const id = arrow[1].toUpperCase();
|
|
112
|
+
if (!entries.has(id)) {
|
|
113
|
+
entries.set(id, { id, priority: 'Unknown', reason: line.replace(/\s*(?:→|->).*$/, '').replace(/^[-*+]\s+/, '') });
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
94
119
|
// 2) Viewpoint Grouping: ### Required / ### Recommended / ### Optional → bullet list
|
|
95
120
|
let group: ViewpointEntry['group'] | undefined;
|
|
96
121
|
for (const raw of lines) {
|
|
@@ -144,7 +169,9 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
144
169
|
// Category is everything between `VP-` and the final `-<sequence>` — INCLUDING hyphens, so
|
|
145
170
|
// compound categories (VP-LIST-DISPLAY-01, VP-ADD-TO-CART-03, VP-PRODUCT-DISCOVERY-02) parse,
|
|
146
171
|
// not just single-word ones. A single-word category (VP-CART-001) still works. (H1)
|
|
147
|
-
|
|
172
|
+
// Flows use journey-phase ids (FL-HP-001 / FL-ER-002) — the VP- anchor rejected them, so every
|
|
173
|
+
// flow scenario had NO category and the whole suite bucketed `other` (taxonomy=0%, #569).
|
|
174
|
+
const codeMatch = sc.name.match(/\b(?:VP|FL)-([A-Z]+(?:-[A-Z]+)*)-\d+/i);
|
|
148
175
|
const vpCode = codeMatch ? codeMatch[0].toUpperCase() : undefined;
|
|
149
176
|
const category = codeMatch ? codeMatch[1].toUpperCase() : undefined;
|
|
150
177
|
// Project-scheme ID: the leading token of the title (VP0-001 / MS-HP-001 / VP-LIST-001).
|
|
@@ -158,12 +185,14 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
158
185
|
const skeletonParts: string[] = [];
|
|
159
186
|
const textParts: string[] = [sc.name];
|
|
160
187
|
const stepTextParts: string[] = [];
|
|
188
|
+
const orderedSteps: Array<{ bucket: 'given' | 'when' | 'then'; text: string }> = [];
|
|
161
189
|
|
|
162
190
|
for (const step of sc.steps as ParsedStep[]) {
|
|
163
191
|
const kw = step.keyword.trim();
|
|
164
192
|
if (kw === 'Given' || kw === 'When' || kw === 'Then') last = kw;
|
|
165
193
|
textParts.push(step.text);
|
|
166
194
|
stepTextParts.push(step.text);
|
|
195
|
+
orderedSteps.push({ bucket: (kw === 'And' || kw === 'But' ? last : kw).toLowerCase() as 'given' | 'when' | 'then', text: step.text });
|
|
167
196
|
// normalized skeleton: keep [refs] (distinct targets = distinct tests),
|
|
168
197
|
// but neutralize {{vars}} and quoted values so EP/data families collapse.
|
|
169
198
|
const skel = step.text
|
|
@@ -199,6 +228,7 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
199
228
|
stepSkeleton: skeletonParts.join(' | '),
|
|
200
229
|
haystack: textParts.join(' ').toLowerCase(),
|
|
201
230
|
stepsText: stepTextParts.join(' ').toLowerCase(),
|
|
231
|
+
steps: orderedSteps,
|
|
202
232
|
vpId,
|
|
203
233
|
casesDataset,
|
|
204
234
|
queryRefs: queryRefs.size ? [...queryRefs] : undefined,
|