@sun-asterisk/sungen 3.2.20-beta.1 → 3.2.20-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +8 -0
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +7 -0
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/exporters/matrix/export.d.ts.map +1 -1
- package/dist/exporters/matrix/export.js +11 -0
- package/dist/exporters/matrix/export.js.map +1 -1
- package/dist/exporters/matrix/render-xlsx.d.ts.map +1 -1
- package/dist/exporters/matrix/render-xlsx.js +15 -0
- package/dist/exporters/matrix/render-xlsx.js.map +1 -1
- package/dist/exporters/matrix/types.d.ts +2 -0
- package/dist/exporters/matrix/types.d.ts.map +1 -1
- package/dist/exporters/matrix/types.js.map +1 -1
- package/dist/exporters/playwright-report-parser.d.ts.map +1 -1
- package/dist/exporters/playwright-report-parser.js +1 -0
- package/dist/exporters/playwright-report-parser.js.map +1 -1
- package/dist/exporters/types.d.ts +2 -0
- package/dist/exporters/types.d.ts.map +1 -1
- package/dist/harness/audit.d.ts +2 -0
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +79 -9
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-contract.d.ts +71 -0
- package/dist/harness/flow-contract.d.ts.map +1 -0
- package/dist/harness/flow-contract.js +235 -0
- package/dist/harness/flow-contract.js.map +1 -0
- package/dist/harness/flow-plan.d.ts +3 -0
- package/dist/harness/flow-plan.d.ts.map +1 -1
- package/dist/harness/flow-plan.js +6 -2
- package/dist/harness/flow-plan.js.map +1 -1
- package/dist/harness/parse.d.ts +5 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +29 -1
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/perf.d.ts +40 -0
- package/dist/harness/perf.d.ts.map +1 -0
- package/dist/harness/perf.js +136 -0
- package/dist/harness/perf.js.map +1 -0
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +13 -1
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/spec-coverage.d.ts.map +1 -1
- package/dist/harness/spec-coverage.js +22 -3
- package/dist/harness/spec-coverage.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/add-flow.md +41 -3
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +35 -16
- package/dist/orchestrator/templates/qa-context.md +14 -1
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +8 -0
- package/src/cli/commands/delivery.ts +6 -0
- package/src/exporters/matrix/export.ts +11 -0
- package/src/exporters/matrix/render-xlsx.ts +15 -0
- package/src/exporters/matrix/types.ts +2 -0
- package/src/exporters/playwright-report-parser.ts +2 -0
- package/src/exporters/types.ts +2 -0
- package/src/harness/audit.ts +82 -10
- package/src/harness/flow-contract.ts +229 -0
- package/src/harness/flow-plan.ts +10 -3
- package/src/harness/parse.ts +31 -1
- package/src/harness/perf.ts +112 -0
- package/src/harness/sensors.ts +13 -1
- package/src/harness/spec-coverage.ts +19 -2
- package/src/orchestrator/templates/ai-src/commands/add-flow.md +41 -3
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +35 -16
- package/src/orchestrator/templates/qa-context.md +14 -1
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flow Contract — the declared boundary of a flow, and the sensors that verify
|
|
3
|
+
* the suite against it. (#569, docs/spec/sungen-flow-quality-spec.md)
|
|
4
|
+
*
|
|
5
|
+
* A flow is the SMALLEST complete business action chain: one clear trigger ending
|
|
6
|
+
* in ONE observable, valuable outcome. Nothing enforced that: `add-flow` asked only
|
|
7
|
+
* "which screens, in order?", so a real example mixed three business goals (cart,
|
|
8
|
+
* filter, product-detail) in one "flow", and the harness scored flows with screen
|
|
9
|
+
* machinery — a registration flow was judged against the `form` page-type checklist
|
|
10
|
+
* (coverage 0%), and every flow audit showed taxonomy=0% because flow phase ids
|
|
11
|
+
* (FL-HP-001) don't even parse as categories.
|
|
12
|
+
*
|
|
13
|
+
* The contract is a declaration the QA owns (AI proposes at add-flow; a filled
|
|
14
|
+
* contract is an INPUT to generation, never an output — same rule as
|
|
15
|
+
* test-viewpoint.md). These sensors are deterministic checks against it:
|
|
16
|
+
*
|
|
17
|
+
* - outcome proof — the flow proves its own goal with an automated data assertion
|
|
18
|
+
* - scope creep — scenarios that never touch the outcome are a second goal
|
|
19
|
+
* - phase coverage — HP / ER / EH journey phases, the flow's coverage axis
|
|
20
|
+
* - handoff — a cross-screen transition is followed by an assertion
|
|
21
|
+
* - stateful depth — generalizes the cart-hardcoded regression dims to any
|
|
22
|
+
* declared collection (order, application, submission …)
|
|
23
|
+
*/
|
|
24
|
+
import * as fs from 'fs';
|
|
25
|
+
import * as path from 'path';
|
|
26
|
+
import { parse as parseYaml } from 'yaml';
|
|
27
|
+
import { ScenarioInfo } from './parse';
|
|
28
|
+
import { readTextFile } from './read-text';
|
|
29
|
+
|
|
30
|
+
export interface FlowContract {
|
|
31
|
+
goal: string;
|
|
32
|
+
actor?: string;
|
|
33
|
+
trigger?: string;
|
|
34
|
+
precondition?: string;
|
|
35
|
+
outcome: { screen: string; assertion?: string };
|
|
36
|
+
value?: string;
|
|
37
|
+
/** Journey phases this flow declares. Default [HP, ER, EH]; UI is allowed but
|
|
38
|
+
* never demanded (presentation is the balance axis's business, not coverage's). */
|
|
39
|
+
phases: string[];
|
|
40
|
+
/** The mutated collection (cart, order, application …) — enables regression dims. */
|
|
41
|
+
stateful?: string;
|
|
42
|
+
budgets?: Record<string, number>;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface FlowQualityResult {
|
|
46
|
+
hasContract: boolean;
|
|
47
|
+
contract?: FlowContract;
|
|
48
|
+
/** Parse/shape errors — a broken contract is reported, never silently ignored. */
|
|
49
|
+
errors: string[];
|
|
50
|
+
outcomeProven: boolean;
|
|
51
|
+
/** Manual-only proof: the goal is covered but not by automation. */
|
|
52
|
+
outcomeManualOnly: boolean;
|
|
53
|
+
/** Scenario names that never touch the outcome screen (guard/error phases excluded). */
|
|
54
|
+
offGoal: string[];
|
|
55
|
+
offGoalRatio: number;
|
|
56
|
+
/** Off-goal categories, for the split suggestion ("VP-FILTER-* looks like its own flow"). */
|
|
57
|
+
offGoalCategories: string[];
|
|
58
|
+
phases: { phase: string; covered: boolean; automated: boolean }[];
|
|
59
|
+
/** Covered-and-automated phases / declared phases (UI excluded) — the flow coverage axis. */
|
|
60
|
+
phaseRatio: number;
|
|
61
|
+
/** Cross-namespace transitions followed by an assertion / all transitions. */
|
|
62
|
+
handoffs: { total: number; asserted: number; ratio: number };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const DEFAULT_PHASES = ['HP', 'ER', 'EH'];
|
|
66
|
+
|
|
67
|
+
export function flowContractPath(unitDir: string): string {
|
|
68
|
+
return path.join(unitDir, 'requirements', 'flow-contract.yaml');
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Load + validate. Returns null when absent; a present-but-broken file returns errors. */
|
|
72
|
+
export function loadFlowContract(unitDir: string): { contract: FlowContract | null; errors: string[] } {
|
|
73
|
+
const p = flowContractPath(unitDir);
|
|
74
|
+
if (!fs.existsSync(p)) return { contract: null, errors: [] };
|
|
75
|
+
let raw: Record<string, unknown>;
|
|
76
|
+
try {
|
|
77
|
+
raw = parseYaml(readTextFile(p)) as Record<string, unknown>;
|
|
78
|
+
} catch (e) {
|
|
79
|
+
return { contract: null, errors: [`flow-contract.yaml does not parse: ${(e as Error).message}`] };
|
|
80
|
+
}
|
|
81
|
+
if (!raw || typeof raw !== 'object') return { contract: null, errors: ['flow-contract.yaml is empty'] };
|
|
82
|
+
const errors: string[] = [];
|
|
83
|
+
if (!raw.goal || typeof raw.goal !== 'string') errors.push('missing `goal:` (Verb + outcome, e.g. "Place an order for a product added from home")');
|
|
84
|
+
const outcome = raw.outcome as { screen?: unknown; assertion?: unknown } | undefined;
|
|
85
|
+
if (!outcome || typeof outcome.screen !== 'string' || !outcome.screen.trim()) {
|
|
86
|
+
errors.push('missing `outcome.screen:` — the screen namespace that carries the final proof');
|
|
87
|
+
}
|
|
88
|
+
if (errors.length > 0) return { contract: null, errors };
|
|
89
|
+
const phases = Array.isArray(raw.phases) && raw.phases.length > 0
|
|
90
|
+
? (raw.phases as unknown[]).map((x) => String(x).toUpperCase())
|
|
91
|
+
: DEFAULT_PHASES;
|
|
92
|
+
return {
|
|
93
|
+
contract: {
|
|
94
|
+
goal: String(raw.goal),
|
|
95
|
+
actor: raw.actor !== undefined ? String(raw.actor) : undefined,
|
|
96
|
+
trigger: raw.trigger !== undefined ? String(raw.trigger) : undefined,
|
|
97
|
+
precondition: raw.precondition !== undefined ? String(raw.precondition) : undefined,
|
|
98
|
+
outcome: { screen: String(outcome!.screen).toLowerCase(), assertion: outcome!.assertion !== undefined ? String(outcome!.assertion) : undefined },
|
|
99
|
+
value: raw.value !== undefined ? String(raw.value) : undefined,
|
|
100
|
+
phases,
|
|
101
|
+
stateful: raw.stateful !== undefined ? String(raw.stateful).toLowerCase() : undefined,
|
|
102
|
+
budgets: (raw.budgets && typeof raw.budgets === 'object') ? raw.budgets as Record<string, number> : undefined,
|
|
103
|
+
},
|
|
104
|
+
errors: [],
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** `[screen:element]` namespaces referenced by a scenario's steps, in step order. */
|
|
109
|
+
function namespacesInOrder(s: ScenarioInfo): string[] {
|
|
110
|
+
const out: string[] = [];
|
|
111
|
+
for (const m of s.stepsText.matchAll(/\[([a-z0-9_.-]+):/g)) out.push(m[1]);
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function touchesOutcome(s: ScenarioInfo, outcomeScreen: string): boolean {
|
|
116
|
+
return namespacesInOrder(s).includes(outcomeScreen);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Phase of a scenario: its declared phase token (FL-HP-001 / VP-FLOW-ER-02 / MS-EH-005)
|
|
120
|
+
* when present, else vocabulary detection. */
|
|
121
|
+
export function phaseOf(s: ScenarioInfo, declared: string[]): string | null {
|
|
122
|
+
const id = (s.vpId ?? '').toUpperCase();
|
|
123
|
+
for (const ph of declared) {
|
|
124
|
+
if (new RegExp(`(^|-)${ph}(-|$)`).test(id)) return ph;
|
|
125
|
+
}
|
|
126
|
+
const hay = s.haystack;
|
|
127
|
+
if (declared.includes('EH') && /\b(direct access|without (a |the )?(submit|login)|browser back|refresh|expired|tamper|unauthoriz|redirect(ed)? (back )?to|guard)\b/.test(hay)) return 'EH';
|
|
128
|
+
if (declared.includes('ER') && /\b(invalid|error|required|validation|malformed|blocked|then correct|recover)\b/.test(hay)) return 'ER';
|
|
129
|
+
if (declared.includes('HP') && s.hasDataAssertion) return 'HP';
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Verify the suite against the contract. Deterministic; a flow without a contract
|
|
135
|
+
* returns hasContract:false and neutral values (the audit reports the checklist).
|
|
136
|
+
*/
|
|
137
|
+
export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQualityResult {
|
|
138
|
+
const { contract, errors } = loadFlowContract(unitDir);
|
|
139
|
+
const neutral: FlowQualityResult = {
|
|
140
|
+
hasContract: false, errors, outcomeProven: false, outcomeManualOnly: false,
|
|
141
|
+
offGoal: [], offGoalRatio: 0, offGoalCategories: [],
|
|
142
|
+
phases: [], phaseRatio: 1, handoffs: { total: 0, asserted: 0, ratio: 1 },
|
|
143
|
+
};
|
|
144
|
+
if (!contract) return neutral;
|
|
145
|
+
|
|
146
|
+
const outcomeScreen = contract.outcome.screen;
|
|
147
|
+
|
|
148
|
+
// --- Outcome proof: the flow proves its own goal, by automation -------------
|
|
149
|
+
const proofs = scenarios.filter((s) => touchesOutcome(s, outcomeScreen) && s.hasDataAssertion);
|
|
150
|
+
const outcomeProven = proofs.some((s) => !s.manual);
|
|
151
|
+
const outcomeManualOnly = !outcomeProven && proofs.length > 0;
|
|
152
|
+
|
|
153
|
+
// --- Scope creep: a scenario that never touches the outcome and is not a ----
|
|
154
|
+
// guard/error phase is evidence of a SECOND business goal in this flow.
|
|
155
|
+
const declaredPhases = contract.phases;
|
|
156
|
+
const offGoalScenarios = scenarios.filter((s) => {
|
|
157
|
+
if (touchesOutcome(s, outcomeScreen)) return false;
|
|
158
|
+
const ph = phaseOf(s, declaredPhases);
|
|
159
|
+
return ph !== 'EH' && ph !== 'ER'; // guards/error-recovery legitimately stop early
|
|
160
|
+
});
|
|
161
|
+
const offGoalRatio = scenarios.length ? offGoalScenarios.length / scenarios.length : 0;
|
|
162
|
+
const offGoalCategories = Array.from(new Set(
|
|
163
|
+
offGoalScenarios.map((s) => s.category ?? s.vpId?.replace(/-\d+.*$/, '') ?? '?')));
|
|
164
|
+
|
|
165
|
+
// --- Phase coverage: the flow's coverage axis (UI never demanded) -----------
|
|
166
|
+
const demanded = declaredPhases.filter((p) => p !== 'UI');
|
|
167
|
+
const phases = demanded.map((phase) => {
|
|
168
|
+
const inPhase = scenarios.filter((s) => phaseOf(s, declaredPhases) === phase);
|
|
169
|
+
// HP must additionally prove the outcome — a data assertion elsewhere is not the goal.
|
|
170
|
+
const relevant = phase === 'HP' ? inPhase.filter((s) => touchesOutcome(s, outcomeScreen)) : inPhase;
|
|
171
|
+
return {
|
|
172
|
+
phase,
|
|
173
|
+
covered: relevant.length > 0,
|
|
174
|
+
automated: relevant.some((s) => !s.manual),
|
|
175
|
+
};
|
|
176
|
+
});
|
|
177
|
+
const phaseRatio = demanded.length
|
|
178
|
+
? phases.filter((p) => p.covered && p.automated).length / demanded.length
|
|
179
|
+
: 1;
|
|
180
|
+
|
|
181
|
+
// --- Handoff integrity: no blind tail after a cross-namespace transition. ---
|
|
182
|
+
// A transition counts as asserted when ANY assertion follows it — in the entered
|
|
183
|
+
// namespace or later. Demanding the assertion in the entered namespace itself
|
|
184
|
+
// flagged two legitimate shapes: a guard that asserts the REDIRECT target
|
|
185
|
+
// ("go to [Checkout] → see [Home] page"), and a passthrough click en route
|
|
186
|
+
// ("click [Cart:Checkout]" asserting on the next screen). What the sensor
|
|
187
|
+
// actually guards against is a flow that clicks through screens and ends blind.
|
|
188
|
+
let total = 0; let asserted = 0;
|
|
189
|
+
for (const s of scenarios) {
|
|
190
|
+
if (s.manual) continue;
|
|
191
|
+
const steps = s.steps ?? [];
|
|
192
|
+
let current: string | null = null;
|
|
193
|
+
for (let i = 0; i < steps.length; i++) {
|
|
194
|
+
const ns = (steps[i].text.match(/\[([A-Za-z0-9_.-]+):/) || [])[1]?.toLowerCase() ?? null;
|
|
195
|
+
if (!ns) continue;
|
|
196
|
+
if (current !== null && ns !== current) {
|
|
197
|
+
total++;
|
|
198
|
+
if (steps.slice(i).some((st) => st.bucket === 'then')) asserted++;
|
|
199
|
+
}
|
|
200
|
+
current = ns;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
const handoffs = { total, asserted, ratio: total ? asserted / total : 1 };
|
|
204
|
+
|
|
205
|
+
return {
|
|
206
|
+
hasContract: true, contract, errors: [],
|
|
207
|
+
outcomeProven, outcomeManualOnly,
|
|
208
|
+
offGoal: offGoalScenarios.map((s) => s.name.slice(0, 80)),
|
|
209
|
+
offGoalRatio, offGoalCategories,
|
|
210
|
+
phases, phaseRatio, handoffs,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Generalized stateful regression depth: the contract names the mutated collection,
|
|
216
|
+
* so the three dims (count-proof · teardown · multi-source) stop being cart-only.
|
|
217
|
+
*/
|
|
218
|
+
export function statefulDepthFor(collection: string, scenarios: ScenarioInfo[]): { countProof: boolean; teardown: boolean; multiSource: boolean; missing: string[]; ratio: number } {
|
|
219
|
+
const hay = scenarios.map((s) => s.haystack);
|
|
220
|
+
const any = (re: RegExp) => hay.some((h) => re.test(h));
|
|
221
|
+
const noun = collection.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
222
|
+
const countProof = any(new RegExp(`\\b(quantity|qty|row count|count|number of|two (rows|lines|items)|\\d+ (rows|lines|items))\\b`)) ;
|
|
223
|
+
const teardown = any(/\b(remove|delete|clear|cancel|withdraw)(?:s|d|ed|ing|n)?\b/) && any(new RegExp(`\\b(empty|emptied|no items|zero|removed|cleared|cancelled|withdrawn|0 items)\\b|empty[- ]${noun}`));
|
|
224
|
+
const adds = hay.filter((h) => new RegExp(`\\b(add|submit|create|place).{0,40}${noun}|${noun}.{0,40}\\b(add|submit|create|place)`).test(h));
|
|
225
|
+
const multiSource = any(/\b(recommended|related|you may also|another source|both sources|second (list|source))\b/) && adds.length > 0;
|
|
226
|
+
const dims: Array<[string, boolean]> = [['count-proof', countProof], ['teardown', teardown], ['multi-source', multiSource]];
|
|
227
|
+
const missing = dims.filter(([, v]) => !v).map(([k]) => k);
|
|
228
|
+
return { countProof, teardown, multiSource, missing, ratio: (dims.length - missing.length) / dims.length };
|
|
229
|
+
}
|
package/src/harness/flow-plan.ts
CHANGED
|
@@ -71,6 +71,9 @@ export interface FlowPlan {
|
|
|
71
71
|
legs: LegPlan[];
|
|
72
72
|
byReason: Record<string, number>;
|
|
73
73
|
capabilityManual: number;
|
|
74
|
+
/** @manual whose reason is "cross-screen → automate via flow" (class XS) — inside a flow this
|
|
75
|
+
* usually means the scenario should simply BE automated here. (#569) */
|
|
76
|
+
crossScreenManual: number;
|
|
74
77
|
judgmentManual: number;
|
|
75
78
|
contracts: Contract[];
|
|
76
79
|
readiness: 'ready' | 'not-ready';
|
|
@@ -87,14 +90,18 @@ export function buildFlowPlan(cwd: string, flow: string): FlowPlan {
|
|
|
87
90
|
// Legs = distinct screen namespaces.
|
|
88
91
|
const legMap = new Map<string, { scenarios: Set<string>; refs: Set<string>; automated: boolean }>();
|
|
89
92
|
const byReason: Record<string, number> = {};
|
|
90
|
-
let capabilityManual = 0, judgmentManual = 0;
|
|
93
|
+
let capabilityManual = 0, judgmentManual = 0, crossScreenManual = 0;
|
|
91
94
|
|
|
92
95
|
for (const sc of scenarios) {
|
|
93
96
|
if (sc.manual) {
|
|
94
97
|
const { code } = inferReasonCode(sc.tags, sc.reason);
|
|
95
98
|
byReason[code] = (byReason[code] || 0) + 1;
|
|
96
99
|
const cls = MANUAL_REASONS[code]?.cls;
|
|
97
|
-
|
|
100
|
+
// XS ("cross-screen → automate via flow") is a THIRD class; it used to be silently
|
|
101
|
+
// dropped from both counters, understating the plan's manual load. (#569)
|
|
102
|
+
if (cls === 'capability') capabilityManual++;
|
|
103
|
+
else if (cls === 'keep') judgmentManual++;
|
|
104
|
+
else if (cls === 'flow') crossScreenManual++;
|
|
98
105
|
}
|
|
99
106
|
for (const r of sc.refs) {
|
|
100
107
|
const leg = r.screen.toLowerCase();
|
|
@@ -133,5 +140,5 @@ export function buildFlowPlan(cwd: string, flow: string): FlowPlan {
|
|
|
133
140
|
}
|
|
134
141
|
if (readiness === 'ready') plan.unshift('Selectors present for every automated leg — ready to compile + run.');
|
|
135
142
|
|
|
136
|
-
return { flow, total: scenarios.length, legs, byReason, capabilityManual, judgmentManual, contracts, readiness, missingLegs, plan };
|
|
143
|
+
return { flow, total: scenarios.length, legs, byReason, capabilityManual, judgmentManual, crossScreenManual, contracts, readiness, missingLegs, plan };
|
|
137
144
|
}
|
package/src/harness/parse.ts
CHANGED
|
@@ -38,6 +38,8 @@ export interface ScenarioInfo {
|
|
|
38
38
|
requiresCaps?: string[]; // @requires:<cap> — automation-ready but needs an opt-in driver (TQ-11)
|
|
39
39
|
deferredToFlow?: boolean; // @deferred:flow — owned by a flow, not automated on this screen (H6)
|
|
40
40
|
ownedByFlow?: string; // @owned-by:<flow> — the flow that owns this deferred scenario (H6)
|
|
41
|
+
/** Ordered steps with their resolved bucket (And/But inherit) — flow handoff analysis (#569). */
|
|
42
|
+
steps?: Array<{ bucket: 'given' | 'when' | 'then'; text: string }>;
|
|
41
43
|
}
|
|
42
44
|
|
|
43
45
|
/** Format-tolerant: is this token an ID (project's scheme), not a prose word?
|
|
@@ -91,6 +93,29 @@ export function parseViewpointOverview(filePath: string): ViewpointEntry[] {
|
|
|
91
93
|
}
|
|
92
94
|
}
|
|
93
95
|
|
|
96
|
+
// 1b) Flow-style declarations (#569). Flow viewpoint files commonly declare per-item
|
|
97
|
+
// ids at the END of a bullet ("… → **FL-HP-001**") under phase section headers
|
|
98
|
+
// ("## FL-HP — Happy Path"). Neither matched the table/group passes, so every flow
|
|
99
|
+
// audit collapsed to taxonomy=0% / traceability n-a — scenarios correctly tagged
|
|
100
|
+
// FL-HP-001 were reported as unmapped. Both forms are additive here.
|
|
101
|
+
for (const raw of lines) {
|
|
102
|
+
const line = raw.trim();
|
|
103
|
+
const section = line.match(/^##\s+([A-Z]{2,}(?:-[A-Z0-9]{2,})*)\s+[—–-]\s*(.*)$/);
|
|
104
|
+
if (section && isViewpointId(section[1] + '-0')) {
|
|
105
|
+
const id = section[1].toUpperCase();
|
|
106
|
+
if (!entries.has(id)) entries.set(id, { id, priority: 'Unknown', reason: section[2] ?? '' });
|
|
107
|
+
}
|
|
108
|
+
if (/^[-*+]\s/.test(line)) {
|
|
109
|
+
const arrow = line.match(/(?:→|->)\s*\*{0,2}([A-Z][A-Z0-9]*(?:-[A-Z0-9]+)*-\d+[a-zA-Z]?)\*{0,2}\s*$/);
|
|
110
|
+
if (arrow) {
|
|
111
|
+
const id = arrow[1].toUpperCase();
|
|
112
|
+
if (!entries.has(id)) {
|
|
113
|
+
entries.set(id, { id, priority: 'Unknown', reason: line.replace(/\s*(?:→|->).*$/, '').replace(/^[-*+]\s+/, '') });
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
94
119
|
// 2) Viewpoint Grouping: ### Required / ### Recommended / ### Optional → bullet list
|
|
95
120
|
let group: ViewpointEntry['group'] | undefined;
|
|
96
121
|
for (const raw of lines) {
|
|
@@ -144,7 +169,9 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
144
169
|
// Category is everything between `VP-` and the final `-<sequence>` — INCLUDING hyphens, so
|
|
145
170
|
// compound categories (VP-LIST-DISPLAY-01, VP-ADD-TO-CART-03, VP-PRODUCT-DISCOVERY-02) parse,
|
|
146
171
|
// not just single-word ones. A single-word category (VP-CART-001) still works. (H1)
|
|
147
|
-
|
|
172
|
+
// Flows use journey-phase ids (FL-HP-001 / FL-ER-002) — the VP- anchor rejected them, so every
|
|
173
|
+
// flow scenario had NO category and the whole suite bucketed `other` (taxonomy=0%, #569).
|
|
174
|
+
const codeMatch = sc.name.match(/\b(?:VP|FL)-([A-Z]+(?:-[A-Z]+)*)-\d+/i);
|
|
148
175
|
const vpCode = codeMatch ? codeMatch[0].toUpperCase() : undefined;
|
|
149
176
|
const category = codeMatch ? codeMatch[1].toUpperCase() : undefined;
|
|
150
177
|
// Project-scheme ID: the leading token of the title (VP0-001 / MS-HP-001 / VP-LIST-001).
|
|
@@ -158,12 +185,14 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
158
185
|
const skeletonParts: string[] = [];
|
|
159
186
|
const textParts: string[] = [sc.name];
|
|
160
187
|
const stepTextParts: string[] = [];
|
|
188
|
+
const orderedSteps: Array<{ bucket: 'given' | 'when' | 'then'; text: string }> = [];
|
|
161
189
|
|
|
162
190
|
for (const step of sc.steps as ParsedStep[]) {
|
|
163
191
|
const kw = step.keyword.trim();
|
|
164
192
|
if (kw === 'Given' || kw === 'When' || kw === 'Then') last = kw;
|
|
165
193
|
textParts.push(step.text);
|
|
166
194
|
stepTextParts.push(step.text);
|
|
195
|
+
orderedSteps.push({ bucket: (kw === 'And' || kw === 'But' ? last : kw).toLowerCase() as 'given' | 'when' | 'then', text: step.text });
|
|
167
196
|
// normalized skeleton: keep [refs] (distinct targets = distinct tests),
|
|
168
197
|
// but neutralize {{vars}} and quoted values so EP/data families collapse.
|
|
169
198
|
const skel = step.text
|
|
@@ -199,6 +228,7 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
199
228
|
stepSkeleton: skeletonParts.join(' | '),
|
|
200
229
|
haystack: textParts.join(' ').toLowerCase(),
|
|
201
230
|
stepsText: stepTextParts.join(' ').toLowerCase(),
|
|
231
|
+
steps: orderedSteps,
|
|
202
232
|
vpId,
|
|
203
233
|
casesDataset,
|
|
204
234
|
queryRefs: queryRefs.size ? [...queryRefs] : undefined,
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Performance budgets — config + percentile math + the per-unit verdict. (#569)
|
|
3
|
+
*
|
|
4
|
+
* A flow's regression value includes "still fast enough": after a lib/framework
|
|
5
|
+
* upgrade the main journeys must not only pass but hold their response-time
|
|
6
|
+
* budget. Sungen had no perf concept at all — the Playwright JSON parser even
|
|
7
|
+
* dropped the `duration` field Playwright already emits on every result.
|
|
8
|
+
*
|
|
9
|
+
* Scope discipline:
|
|
10
|
+
* - This is config + measurement + report over runs sungen already makes.
|
|
11
|
+
* Real load tests stay @manual:M8 → a dedicated tool.
|
|
12
|
+
* - The AUDIT never reads it: the quality score is documented as a pure
|
|
13
|
+
* function of the design artifacts ("reads no test-results, live page, or
|
|
14
|
+
* clock"). Perf reports where runs are already read — `sungen delivery`
|
|
15
|
+
* and the dashboard. Advisory: a blown budget never fails the design gate.
|
|
16
|
+
*
|
|
17
|
+
* Config: qa/perf.yaml
|
|
18
|
+
* percentile: p75 # default p75 — "≥75% of runs meet the budget"
|
|
19
|
+
* defaults:
|
|
20
|
+
* scenario_ms: 30000 # whole-scenario wall clock (Playwright duration)
|
|
21
|
+
* page_load_ms: 3000 # Phase B — needs per-transition runtime timing
|
|
22
|
+
* transition_ms: 2000 # Phase B
|
|
23
|
+
* units:
|
|
24
|
+
* place-order: { scenario_ms: 20000 }
|
|
25
|
+
*/
|
|
26
|
+
import * as fs from 'fs';
|
|
27
|
+
import * as path from 'path';
|
|
28
|
+
import { parse as parseYaml } from 'yaml';
|
|
29
|
+
import { readTextFile } from './read-text';
|
|
30
|
+
|
|
31
|
+
export interface PerfConfig {
|
|
32
|
+
/** 0..100 — e.g. 75 for p75. */
|
|
33
|
+
percentile: number;
|
|
34
|
+
defaults: Record<string, number>;
|
|
35
|
+
units: Record<string, Record<string, number>>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface PerfVerdict {
|
|
39
|
+
unit: string;
|
|
40
|
+
metric: string; // 'scenario_ms' today; page_load_ms/transition_ms in Phase B
|
|
41
|
+
percentile: number; // 75
|
|
42
|
+
budgetMs: number;
|
|
43
|
+
measuredMs: number; // the pXX of the observed durations
|
|
44
|
+
samples: number;
|
|
45
|
+
pass: boolean;
|
|
46
|
+
/** Titles of the slowest offenders (only when failing), for the report. */
|
|
47
|
+
slowest: Array<{ title: string; ms: number }>;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function perfConfigPath(projectRoot: string): string {
|
|
51
|
+
return path.join(projectRoot, 'qa', 'perf.yaml');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Absent file → null (perf reporting is opt-in; nothing changes until configured). */
|
|
55
|
+
export function loadPerfConfig(projectRoot: string): PerfConfig | null {
|
|
56
|
+
const p = perfConfigPath(projectRoot);
|
|
57
|
+
if (!fs.existsSync(p)) return null;
|
|
58
|
+
let raw: Record<string, unknown>;
|
|
59
|
+
try { raw = parseYaml(readTextFile(p)) as Record<string, unknown>; } catch { return null; }
|
|
60
|
+
if (!raw || typeof raw !== 'object') return null;
|
|
61
|
+
const pctRaw = String(raw.percentile ?? 'p75').toLowerCase().replace(/^p/, '');
|
|
62
|
+
const percentile = Math.min(100, Math.max(1, Number(pctRaw) || 75));
|
|
63
|
+
const num = (o: unknown): Record<string, number> => {
|
|
64
|
+
const out: Record<string, number> = {};
|
|
65
|
+
if (o && typeof o === 'object') {
|
|
66
|
+
for (const [k, v] of Object.entries(o as Record<string, unknown>)) {
|
|
67
|
+
const n = Number(v);
|
|
68
|
+
if (Number.isFinite(n) && n > 0) out[k] = n;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return out;
|
|
72
|
+
};
|
|
73
|
+
const units: Record<string, Record<string, number>> = {};
|
|
74
|
+
if (raw.units && typeof raw.units === 'object') {
|
|
75
|
+
for (const [u, o] of Object.entries(raw.units as Record<string, unknown>)) units[u] = num(o);
|
|
76
|
+
}
|
|
77
|
+
return { percentile, defaults: num(raw.defaults), units };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Nearest-rank percentile (ceil), the standard "≥pXX of samples meet the budget"
|
|
82
|
+
* reading: p75 of [a…] is the value at ceil(0.75·n) in the sorted list. One
|
|
83
|
+
* sample → that sample. Deterministic, no interpolation.
|
|
84
|
+
*/
|
|
85
|
+
export function percentileOf(p: number, values: number[]): number {
|
|
86
|
+
if (values.length === 0) return 0;
|
|
87
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
88
|
+
const rank = Math.min(sorted.length, Math.max(1, Math.ceil((p / 100) * sorted.length)));
|
|
89
|
+
return sorted[rank - 1];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Budget for a metric on a unit: per-unit override, else defaults, else none. */
|
|
93
|
+
export function budgetFor(config: PerfConfig, unit: string, metric: string): number | undefined {
|
|
94
|
+
return config.units[unit]?.[metric] ?? config.defaults[metric];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* The scenario_ms verdict for one unit's run. `durations` = per-test wall-clock ms
|
|
99
|
+
* (a @cases scenario contributes one sample per row-test — each is a real run).
|
|
100
|
+
*/
|
|
101
|
+
export function perfVerdict(
|
|
102
|
+
config: PerfConfig,
|
|
103
|
+
unit: string,
|
|
104
|
+
samples: Array<{ title: string; ms: number }>,
|
|
105
|
+
): PerfVerdict | null {
|
|
106
|
+
const budgetMs = budgetFor(config, unit, 'scenario_ms');
|
|
107
|
+
if (budgetMs === undefined || samples.length === 0) return null;
|
|
108
|
+
const measuredMs = percentileOf(config.percentile, samples.map((s) => s.ms));
|
|
109
|
+
const pass = measuredMs <= budgetMs;
|
|
110
|
+
const slowest = pass ? [] : [...samples].sort((a, b) => b.ms - a.ms).slice(0, 3);
|
|
111
|
+
return { unit, metric: 'scenario_ms', percentile: config.percentile, budgetMs, measuredMs, samples: samples.length, pass, slowest };
|
|
112
|
+
}
|
package/src/harness/sensors.ts
CHANGED
|
@@ -33,10 +33,22 @@ const BUCKET_ORDER: Array<[string, string[]]> = [
|
|
|
33
33
|
];
|
|
34
34
|
const BUCKETS: Record<string, string[]> = Object.fromEntries(BUCKET_ORDER);
|
|
35
35
|
|
|
36
|
+
// Flow journey-phase categories (FL-HP-001, FL-ER-002 …). Matched on exact SEGMENTS,
|
|
37
|
+
// never by containment — 'SHOP'.includes('HP') is true, which is exactly the kind of
|
|
38
|
+
// false hit substring matching would produce for two-letter phase tokens. (#569)
|
|
39
|
+
const PHASE_BUCKETS: Record<string, string> = {
|
|
40
|
+
HP: 'business-core', // happy path = the business goal itself
|
|
41
|
+
ER: 'validation-security', // error recovery (validation must not trap the journey)
|
|
42
|
+
EH: 'validation-security', // guards & leakage (direct access, back, refresh)
|
|
43
|
+
};
|
|
44
|
+
|
|
36
45
|
/** Classify a VP category into a balance bucket by keyword containment + precedence (H1). */
|
|
37
46
|
export function bucketForCategory(category: string | undefined): string {
|
|
38
47
|
const cat = (category || '').toUpperCase();
|
|
39
48
|
if (!cat) return 'other';
|
|
49
|
+
for (const seg of cat.split('-')) {
|
|
50
|
+
if (PHASE_BUCKETS[seg]) return PHASE_BUCKETS[seg];
|
|
51
|
+
}
|
|
40
52
|
for (const [bucket, kws] of BUCKET_ORDER) {
|
|
41
53
|
if (kws.some((k) => cat.includes(k))) return bucket;
|
|
42
54
|
}
|
|
@@ -351,7 +363,7 @@ export function flowRegressionDepth(scenarios: ScenarioInfo[]): FlowDepthResult
|
|
|
351
363
|
// 1. Count/quantity proof — a row count or item quantity, not just presence of a row.
|
|
352
364
|
const countProof = any(/\b(quantity|qty|two (?:rows|lines|cart)|row count|count column|number of items|one[_ ]row|two[_ ]rows|qty[_ ])/i);
|
|
353
365
|
// 2. Teardown — removes the item and verifies the empty/zero state (the inverse operation).
|
|
354
|
-
const teardown = any(/\b(remove|delete|clear)
|
|
366
|
+
const teardown = any(/\b(remove|delete|clear)(?:s|d|ed|ing)?\b/i) && any(/\b(empty|emptied|no items|zero|removed|cleared|0 items)\b/i);
|
|
355
367
|
// 3. Multi-source — the cart is fed from >1 source (the main list AND a recommended/related rail).
|
|
356
368
|
const multiSource = any(/\b(recommended|related|you may also|suggest)\b/i) && addsToCart;
|
|
357
369
|
|
|
@@ -66,10 +66,27 @@ export function parseSpecClauses(specPath: string): { frs: FrClause[]; valRows:
|
|
|
66
66
|
if (!fs.existsSync(specPath)) return { frs: [], valRows: [] };
|
|
67
67
|
const lines = readTextFile(specPath).split('\n');
|
|
68
68
|
|
|
69
|
+
// Requirement ids follow the PROJECT's scheme, not ours (#572): `**FR-1**:` is one
|
|
70
|
+
// convention among many — a real spec declared ~30 MUST clauses as `` `REQ-SRCH-001`: ``
|
|
71
|
+
// and the FR-locked pattern returned zero, so the MUST-coverage gate never ran and the
|
|
72
|
+
// specFR axis was excluded "for lack of evidence" that was sitting right there. Same
|
|
73
|
+
// silent-failure class as the CRLF parsers, same id-scheme-tolerance lesson as delivery.
|
|
74
|
+
//
|
|
75
|
+
// A declaration is: line-leading (optionally bulleted / bold / backticked) `<ID>:` where
|
|
76
|
+
// the id ends in a number. Table rows are EXCLUDED — traceability tables cite requirement
|
|
77
|
+
// ids without declaring them — and so are prefixes that are never requirements
|
|
78
|
+
// (test cases, viewpoints, known-defect records, data-factory checks, flows, delivery items).
|
|
79
|
+
const NON_REQUIREMENT_PREFIX = /^(TC|VP|KD|CHK|FL|DI)-/i;
|
|
69
80
|
const frs: FrClause[] = [];
|
|
81
|
+
const seen = new Set<string>();
|
|
70
82
|
for (const line of lines) {
|
|
71
|
-
|
|
72
|
-
|
|
83
|
+
if (/^\s*\|/.test(line)) continue; // table row = citation, not declaration
|
|
84
|
+
const m = line.match(/^\s*(?:[-*+]\s+)?[*_`]*([A-Z][A-Z0-9]*(?:-[A-Z0-9]+)*-\d+[a-zA-Z]?)[*_`]*\s*:\s*(.+)$/);
|
|
85
|
+
if (!m || NON_REQUIREMENT_PREFIX.test(m[1])) continue;
|
|
86
|
+
const id = m[1].toUpperCase();
|
|
87
|
+
if (seen.has(id)) continue; // first declaration wins
|
|
88
|
+
seen.add(id);
|
|
89
|
+
frs.push({ id, text: m[2].replace(/\*\*/g, '').trim(), modality: modalityOf(m[2]) });
|
|
73
90
|
}
|
|
74
91
|
|
|
75
92
|
// Validation Rules table: a row carries a Constraint, a Trigger cell, and (often) a code.
|
|
@@ -86,15 +86,52 @@ qa/flows/${input:flow}/
|
|
|
86
86
|
└── ui/ # Screenshots, mockups
|
|
87
87
|
```
|
|
88
88
|
|
|
89
|
-
### 1a.
|
|
89
|
+
### 1a. Define the flow's BOUNDARY, then its screens
|
|
90
90
|
|
|
91
|
-
|
|
91
|
+
A flow is the **smallest complete business action chain**: one clear trigger ending in ONE
|
|
92
|
+
observable, valuable outcome. Before asking for screens, walk this checklist with the user —
|
|
93
|
+
if 1, 3 or 8 fails, propose SPLITTING into separate flows:
|
|
94
|
+
|
|
95
|
+
1. Exactly **one business goal**? (cart correctness + category filtering = two flows)
|
|
96
|
+
2. A clear **trigger** and precondition?
|
|
97
|
+
3. **One observable final outcome**? (a final assertion you can write in one sentence)
|
|
98
|
+
4. Is that outcome **valuable to the actor**? (an order placed, a password reset — not "a page rendered")
|
|
99
|
+
5. Is **every step necessary** for that outcome?
|
|
100
|
+
6. Are all steps at the **same business abstraction**?
|
|
101
|
+
7. Are optional/error branches **phases of this goal** (ER/EH), not new goals?
|
|
102
|
+
8. Does **no segment** form an independently valuable flow on its own?
|
|
103
|
+
9. Can you write **a single clear final assertion**?
|
|
104
|
+
10. Can you name it "**Verb + outcome**"? (`place-order`, `reset-password` — not `cart-and-filter`)
|
|
105
|
+
|
|
106
|
+
Then ask: "Which screens does this flow visit, in order? (e.g., login → dashboard → award-form → confirmation)"
|
|
92
107
|
|
|
93
108
|
Record the screen list — you will need it for:
|
|
94
109
|
- Filling `spec.md` (Step 3)
|
|
95
110
|
- Suggesting `[Screen:Element]` namespace prefixes
|
|
96
111
|
- Capturing visuals per screen (Step 2)
|
|
97
112
|
|
|
113
|
+
### 1b. Author the Flow Contract (`requirements/flow-contract.yaml`)
|
|
114
|
+
|
|
115
|
+
Write the answers down as the flow's contract — `sungen audit` scores the flow **against it**
|
|
116
|
+
(the `flowCoverage` axis: HP/ER/EH journey phases; `FLOW-OUTCOME-UNPROVEN` when no automated
|
|
117
|
+
scenario asserts data on the outcome screen; `FLOW-SCOPE-CREEP` when scenarios never touch it):
|
|
118
|
+
|
|
119
|
+
```yaml
|
|
120
|
+
goal: "Place an order for a product added from home" # Verb + outcome
|
|
121
|
+
actor: user
|
|
122
|
+
trigger: "Add a product to the cart from the home featured list"
|
|
123
|
+
precondition: "A registered account; an empty cart"
|
|
124
|
+
outcome:
|
|
125
|
+
screen: checkout # the [Screen:...] namespace carrying the final proof
|
|
126
|
+
assertion: "The confirmation shows the order number and the paid total"
|
|
127
|
+
value: "The customer has paid; the shop has a new order"
|
|
128
|
+
phases: [HP, ER, EH] # journey phases (default); add UI only if the flow owns UI states
|
|
129
|
+
stateful: cart # the mutated collection, if any — enables regression-depth dims
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
**A filled contract is an INPUT to generation — never an output.** Like `test-viewpoint.md`,
|
|
133
|
+
generation must not rewrite it to match what was generated; disagree → propose the diff and ask.
|
|
134
|
+
|
|
98
135
|
### 2. Capture visual source
|
|
99
136
|
|
|
100
137
|
**Mobile path** (`platform: mobile`):
|
|
@@ -187,7 +224,8 @@ If user picks `/sungen:create-test`, **you MUST use the Skill tool** to invoke i
|
|
|
187
224
|
- Test data namespaced by phase: `login.email`, `submission.nominee`
|
|
188
225
|
- `@flow` tag required at feature level
|
|
189
226
|
- `Background:` should only contain the starting navigation — the URL path (web) or the `--reach` nav recipe (mobile)
|
|
190
|
-
- Each scenario = one phase of the journey
|
|
227
|
+
- Each scenario = one phase of the journey; ids are `FL-<PHASE>-NNN` (`HP`/`ER`/`EH`, optional `UI`)
|
|
228
|
+
- One flow = ONE business goal with ONE observable outcome (`requirements/flow-contract.yaml`) — a segment with its own value is its own flow
|
|
191
229
|
{{#cap parallel-subagents}}
|
|
192
230
|
- Mobile flows are tagged `@platform:mobile` and run via `/sungen:run-test <flow>` (WebdriverIO, not Playwright)
|
|
193
231
|
{{/cap}}
|