@sun-asterisk/sungen 3.2.26 → 3.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/next.js +2 -2
- package/dist/cli/commands/next.js.map +1 -1
- package/dist/generators/test-generator/adapters/appium/templates/steps/assertions/page-assertion.hbs +6 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/page-assertion.hbs +2 -1
- package/dist/generators/test-generator/adapters/playwright/templates/steps/partials/url-assertion.hbs +6 -2
- package/dist/generators/test-generator/code-generator.js +1 -1
- package/dist/generators/test-generator/code-generator.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +4 -4
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/generators/test-generator/utils/selector-resolver.d.ts +1 -0
- package/dist/generators/test-generator/utils/selector-resolver.d.ts.map +1 -1
- package/dist/generators/test-generator/utils/selector-resolver.js +6 -1
- package/dist/generators/test-generator/utils/selector-resolver.js.map +1 -1
- package/dist/harness/audit.js +4 -4
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-contract.d.ts +0 -14
- package/dist/harness/flow-contract.d.ts.map +1 -1
- package/dist/harness/flow-contract.js +44 -9
- package/dist/harness/flow-contract.js.map +1 -1
- package/dist/harness/next-step.d.ts +8 -2
- package/dist/harness/next-step.d.ts.map +1 -1
- package/dist/harness/next-step.js +3 -2
- package/dist/harness/next-step.js.map +1 -1
- package/dist/harness/sensors.d.ts +1 -0
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +34 -1
- package/dist/harness/sensors.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-gherkin-syntax/SKILL.md +21 -3
- package/dist/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +18 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-selector-keys/SKILL.md +4 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +12 -2
- package/dist/orchestrator/templates/ai-src/skills/sungen-viewpoint/group-d-display.md +1 -0
- package/dist/orchestrator/templates/specs-url-assert.d.ts +31 -0
- package/dist/orchestrator/templates/specs-url-assert.d.ts.map +1 -1
- package/dist/orchestrator/templates/specs-url-assert.js +43 -0
- package/dist/orchestrator/templates/specs-url-assert.js.map +1 -1
- package/dist/orchestrator/templates/specs-url-assert.ts +54 -0
- package/dist/utils/ref-key.d.ts +30 -0
- package/dist/utils/ref-key.d.ts.map +1 -0
- package/dist/utils/ref-key.js +48 -0
- package/dist/utils/ref-key.js.map +1 -0
- package/package.json +3 -3
- package/src/cli/commands/next.ts +2 -2
- package/src/generators/test-generator/adapters/appium/templates/steps/assertions/page-assertion.hbs +6 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/page-assertion.hbs +2 -1
- package/src/generators/test-generator/adapters/playwright/templates/steps/partials/url-assertion.hbs +6 -2
- package/src/generators/test-generator/code-generator.ts +1 -1
- package/src/generators/test-generator/step-mapper.ts +2 -2
- package/src/generators/test-generator/utils/selector-resolver.ts +12 -1
- package/src/harness/audit.ts +4 -4
- package/src/harness/flow-contract.ts +41 -8
- package/src/harness/next-step.ts +11 -4
- package/src/harness/sensors.ts +34 -3
- package/src/orchestrator/templates/ai-src/commands/create-test.md +1 -1
- package/src/orchestrator/templates/ai-src/skills/sungen-gherkin-syntax/SKILL.md +21 -3
- package/src/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +18 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-selector-keys/SKILL.md +4 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +12 -2
- package/src/orchestrator/templates/ai-src/skills/sungen-viewpoint/group-d-display.md +1 -0
- package/src/orchestrator/templates/specs-url-assert.ts +54 -0
- package/src/utils/ref-key.ts +43 -0
package/src/cli/commands/next.ts
CHANGED
|
@@ -40,8 +40,8 @@ function render(s: UnitState): void {
|
|
|
40
40
|
if (s.audit) L(` audit: ${s.audit.overall}/10 [${s.audit.gateStatus}] · ${s.audit.findings.length} finding(s)`);
|
|
41
41
|
L('');
|
|
42
42
|
for (const st of s.steps) {
|
|
43
|
-
const mark = st.kind === 'blocked' ? '⛔' : st.kind === 'optional' ? '○' : '→';
|
|
44
|
-
L(` ${mark} ${st.command}`);
|
|
43
|
+
const mark = st.kind === 'blocked' ? '⛔' : st.kind === 'optional' ? '○' : st.kind === 'decide' ? '⚖' : '→';
|
|
44
|
+
L(` ${mark} ${st.command}${st.kind === 'decide' ? ' (a QA decision — ask, do not run)' : ''}`);
|
|
45
45
|
L(` ${st.because}`);
|
|
46
46
|
}
|
|
47
47
|
L('');
|
package/src/generators/test-generator/adapters/appium/templates/steps/assertions/page-assertion.hbs
CHANGED
|
@@ -1 +1,7 @@
|
|
|
1
|
+
{{#if anchor~}}
|
|
2
|
+
{{!-- A native app has no URL; the page's declared `anchor:` (a screen marker) IS the page oracle
|
|
3
|
+
here — the same rendered-content proof the web side adds after its URL check (#662). --}}
|
|
4
|
+
await __assertVisible({{#with anchor}}{{> appium-selector-expr}}{{/with}});
|
|
5
|
+
{{~else~}}
|
|
1
6
|
// page/URL assertion is not applicable on mobile (no URL) — [{{value}}]
|
|
7
|
+
{{~/if}}
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
{{> url-assertion}}
|
|
1
|
+
{{> url-assertion}}{{#if anchor}}
|
|
2
|
+
await expect({{#with anchor}}{{> locator}}{{/with}}).toBeVisible();{{/if}}
|
package/src/generators/test-generator/adapters/playwright/templates/steps/partials/url-assertion.hbs
CHANGED
|
@@ -11,5 +11,9 @@
|
|
|
11
11
|
Only `pathRegex` is the generator's business: it alone knows the page selector's value and
|
|
12
12
|
its `:id` wildcards, and it is pre-escaped by pathToRegexSource. `\/?$` normalises a
|
|
13
13
|
trailing slash so `/vi/search/` does not read as a different page. The declared value stays
|
|
14
|
-
a plain string literal so the runtime-data marker pass can rewrite it to testData.get(…).
|
|
15
|
-
|
|
14
|
+
a plain string literal so the runtime-data marker pass can rewrite it to testData.get(…).
|
|
15
|
+
|
|
16
|
+
`expectPage`, not a bare `toHaveURL`: a polling URL assertion passes on the first transient
|
|
17
|
+
match, before a route guard has bounced the user away (#662). The helper arrives, lets the
|
|
18
|
+
page settle, holds, and re-asserts — read specs/url-assert.ts. --}}
|
|
19
|
+
await expectPage(page, { pathname: /^{{pathRegex}}\/?$/{{#if query}}, dataRef: '{{escapeQuotes dataRef}}', declared: '{{escapeQuotes query}}'{{/if}} });
|
|
@@ -346,7 +346,7 @@ export class CodeGenerator {
|
|
|
346
346
|
// helper into every UI spec — including the many that touch no URL at all. Hence the body is
|
|
347
347
|
// rendered BEFORE the imports. Importing exactly the names used keeps the import honest; the
|
|
348
348
|
// helper file itself is synced only when something needs it (an api-only unit carries none).
|
|
349
|
-
const urlAssertImports = ['urlMatches', 'joinPath'].filter((n) => testCode.includes(`${n}(`)).join(', ');
|
|
349
|
+
const urlAssertImports = ['urlMatches', 'joinPath', 'expectPage'].filter((n) => testCode.includes(`${n}(`)).join(', ');
|
|
350
350
|
if (urlAssertImports) this.syncGeneratedHelper(outputDir, 'url-assert.ts', 'specs-url-assert.ts');
|
|
351
351
|
|
|
352
352
|
// Same body-first rule for the runtime regex-escape helper: only a spec whose
|
|
@@ -490,7 +490,7 @@ export class StepMapper {
|
|
|
490
490
|
*/
|
|
491
491
|
private checkPageAssertionFallback(step: ParsedStep, mapped: MappedStep): void {
|
|
492
492
|
if (!this.diagnostics || !step.selectorRef || step.elementType !== 'page') return;
|
|
493
|
-
if (
|
|
493
|
+
if (!/\b(?:urlMatches|expectPage)\(/.test(mapped.code || '')) return; // both URL-assertion shapes (#662)
|
|
494
494
|
let resolvedAsPage = false;
|
|
495
495
|
try {
|
|
496
496
|
const resolved = this.selectorResolver.resolveSelector(
|
|
@@ -518,7 +518,7 @@ export class StepMapper {
|
|
|
518
518
|
*/
|
|
519
519
|
private checkDeclaredUrlHasNoQuery(step: ParsedStep, mapped: MappedStep): void {
|
|
520
520
|
if (!this.diagnostics || !step.dataRef || step.elementType !== 'page') return;
|
|
521
|
-
if (
|
|
521
|
+
if (!/\b(?:urlMatches|expectPage)\(/.test(mapped.code || '')) return; // both URL-assertion shapes (#662)
|
|
522
522
|
// The base value, never resolveData(): in runtime mode that returns the marker, which carries
|
|
523
523
|
// no query by construction — the diagnostic would then fire on every single step.
|
|
524
524
|
const value = this.dataResolver.peekBaseValue(step.dataRef, this.featureName);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as fs from 'fs';
|
|
2
|
+
import { normalizeRefLabel } from '../../../utils/ref-key';
|
|
2
3
|
import * as path from 'path';
|
|
3
4
|
import yaml from 'yaml';
|
|
4
5
|
import { readYaml, readYamlIfExists } from '../../../utils/yaml-io';
|
|
@@ -22,6 +23,12 @@ interface SelectorEntry {
|
|
|
22
23
|
scope?: string; // Parent landmark aria-label to scope within (e.g., 'desktop navigation')
|
|
23
24
|
match?: 'exact' | 'partial'; // For getByText matching (default: partial)
|
|
24
25
|
|
|
26
|
+
// === Page anchor (#662) ===
|
|
27
|
+
// For a `type: page` entry: the element that proves the page has RENDERED — a heading, a
|
|
28
|
+
// landmark, a title. `see [X] page` asserts the URL and then this element, so a route guard
|
|
29
|
+
// that bounces the user after the URL has already matched turns the step red instead of green.
|
|
30
|
+
anchor?: SelectorEntry;
|
|
31
|
+
|
|
25
32
|
// === Mobile per-platform variant (issue #392) ===
|
|
26
33
|
// `android:`/`ios:` sub-selectors for one logical element whose locator differs per OS (composite
|
|
27
34
|
// content-desc, partial/dynamic text, native predicate). Each is a full SelectorEntry.
|
|
@@ -126,6 +133,7 @@ export interface ResolvedSelector {
|
|
|
126
133
|
attribute?: string; // Attribute to check
|
|
127
134
|
pattern?: string; // Regex for attribute value
|
|
128
135
|
expanded?: { class?: string; attribute?: string; state?: 'none' }; // Expand/collapse state signal
|
|
136
|
+
anchor?: ResolvedSelector; // page entries only — the rendered-content proof behind `see [X] page` (#662)
|
|
129
137
|
}
|
|
130
138
|
|
|
131
139
|
/**
|
|
@@ -189,7 +197,7 @@ export class SelectorResolver {
|
|
|
189
197
|
* "書類一覧" → "書類一覧"
|
|
190
198
|
*/
|
|
191
199
|
static generateKey(label: string): string {
|
|
192
|
-
return label
|
|
200
|
+
return normalizeRefLabel(label); // the one definition — the harness reads the same rule (#660)
|
|
193
201
|
}
|
|
194
202
|
|
|
195
203
|
/**
|
|
@@ -579,6 +587,9 @@ export class SelectorResolver {
|
|
|
579
587
|
if (entry.attribute) v2Fields.attribute = entry.attribute;
|
|
580
588
|
if (entry.pattern) v2Fields.pattern = entry.pattern;
|
|
581
589
|
if (entry.expanded) v2Fields.expanded = entry.expanded;
|
|
590
|
+
// The anchor is a selector in its own right; it must not inherit the PAGE's label as its
|
|
591
|
+
// accessible name, so it resolves against its own name (or nothing).
|
|
592
|
+
if (entry.anchor) v2Fields.anchor = this.resolveFromEntry(entry.anchor, entry.anchor.name ?? '');
|
|
582
593
|
|
|
583
594
|
// Helper to attach v2 fields and inputMethod
|
|
584
595
|
const withExtras = (resolved: ResolvedSelector): ResolvedSelector => {
|
package/src/harness/audit.ts
CHANGED
|
@@ -427,7 +427,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
427
427
|
viewpointBaseline.removed?.length ? `removed [${viewpointBaseline.removed.join(', ')}]` : '',
|
|
428
428
|
viewpointBaseline.added?.length ? `added [${viewpointBaseline.added.join(', ')}]` : '',
|
|
429
429
|
].filter(Boolean).join(', ');
|
|
430
|
-
findings.push(`VIEWPOINT-BASELINE-CHANGED: test-viewpoint.md no longer declares what it did when this unit was last accepted — ${moved || 'the declared ids were reordered or replaced'}. atomicLedger + traceability measure the suite AGAINST this file, so they are excluded from the score until the change is confirmed: a generator that rewrites the declaration scores both 100% by construction, and a viewpoint dropped from the file stops being missing from anything. Review the diff (a removed id means that coverage is now unclaimed), then run \`sungen audit --screen ${screenName} --accept-viewpoint
|
|
430
|
+
findings.push(`VIEWPOINT-BASELINE-CHANGED: test-viewpoint.md no longer declares what it did when this unit was last accepted — ${moved || 'the declared ids were reordered or replaced'}. atomicLedger + traceability measure the suite AGAINST this file, so they are excluded from the score until the change is confirmed: a generator that rewrites the declaration scores both 100% by construction, and a viewpoint dropped from the file stops being missing from anything. Review the diff (a removed id means that coverage is now unclaimed), then run \`sungen audit --screen ${screenName} --accept-viewpoint\` — a QA decision: an agent presents it and does not run it, because a yardstick accepted by the process it measures reads 100% by construction.`);
|
|
431
431
|
}
|
|
432
432
|
// On a contract flow the page type is INTENTIONALLY not applied — the contract is the checklist.
|
|
433
433
|
if (gate.pageTypeSource === 'undetermined' && !flowScored) {
|
|
@@ -445,7 +445,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
445
445
|
: `FLOW-OUTCOME-UNPROVEN: no scenario asserts data on the outcome screen \`${flowQ.contract!.outcome.screen}\` — the flow never proves its own goal ("${flowQ.contract!.goal}"). Add the happy-path scenario that ends in the declared final assertion.`);
|
|
446
446
|
}
|
|
447
447
|
if (flowQ.offGoalRatio > 0.35) {
|
|
448
|
-
findings.push(`FLOW-SCOPE-CREEP: ${flowQ.offGoal.length}/${scenarios.length} scenarios never touch the outcome screen \`${flowQ.contract!.outcome.screen}\` and are not guards/error-recovery (categories: ${flowQ.offGoalCategories.join(', ')}) — evidence of a SECOND business goal in this flow. Split them into their own flow (one flow = one goal, one observable outcome).`);
|
|
448
|
+
findings.push(`FLOW-SCOPE-CREEP: ${flowQ.offGoal.length}/${scenarios.length} scenarios never touch the outcome screen \`${flowQ.contract!.outcome.screen}\` and are not guards/error-recovery (categories: ${flowQ.offGoalCategories.join(', ')}) — evidence of a SECOND business goal in this flow. Split them into their own flow (one flow = one goal, one observable outcome) — or, when the spec KEEPS a variant deliberately light (an entry variant that only proves it reaches the action bar), declare that flow in \`flows:\` with its own \`outcome:\` and it is judged against that, not against the use case's. Never widen a scenario to satisfy this finding: that trades a reviewer's "over-testing" for the gate's approval, and the gate is the one that is wrong.`);
|
|
449
449
|
}
|
|
450
450
|
for (const ph of flowQ.phases.filter((p) => !p.covered || !p.automated)) {
|
|
451
451
|
findings.push(`FLOW-PHASE-${ph.phase}-MISSING: journey phase ${ph.phase} (${ph.phase === 'HP' ? 'happy path proving the outcome' : ph.phase === 'ER' ? 'error recovery — validation must not trap the journey' : 'guards — direct access / back / refresh'}) is ${ph.covered ? 'covered only by @manual' : 'not covered'} → it does not count toward flowCoverage until automated.`);
|
|
@@ -591,7 +591,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
591
591
|
findings.push(`FLOW-DEPTH: this stateful flow (${what}) exercises ${dims - flowDepth.missing.length}/${dims} applicable regression dimensions — missing [${flowDepth.missing.join(', ')}] → ${flowDepth.missing.map((m) => how[m]).join('; ')}. (businessDepth is capped until covered.)`);
|
|
592
592
|
}
|
|
593
593
|
for (const w of oracle.weak) {
|
|
594
|
-
findings.push(
|
|
594
|
+
findings.push(`${w.code ?? 'ORACLE-WEAK'}: "${w.name}" — ${w.hint}`);
|
|
595
595
|
}
|
|
596
596
|
for (const u of claim.unproven) {
|
|
597
597
|
const tag = u.severity === 'fail' ? 'CLAIM-UNPROVEN' : 'CLAIM-WEAK';
|
|
@@ -606,7 +606,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
606
606
|
findings.push(`BALANCE: ${balance.note} Stop expanding secondary viewpoints until business-core gaps are filled.`);
|
|
607
607
|
}
|
|
608
608
|
if (adoptedPostHoc) {
|
|
609
|
-
findings.push(`VIEWPOINT-ADOPTED-POST-HOC: this is the first audit of \`requirements/test-viewpoint.md\` and the suite ALREADY has ${scenarios.length} scenarios, so the yardstick is being introduced against tests that already exist. A declaration written after the suite can only measure what its author already thought of — and if it was derived FROM the suite, \`atomicLedger\` and \`traceability\` read 100% by construction while anything dropped along the way stops being missing from anything. Both axes are held as unverified until you review the claims (add what you care about, delete what you do not) and confirm: \`sungen audit --screen ${screenName} --accept-viewpoint
|
|
609
|
+
findings.push(`VIEWPOINT-ADOPTED-POST-HOC: this is the first audit of \`requirements/test-viewpoint.md\` and the suite ALREADY has ${scenarios.length} scenarios, so the yardstick is being introduced against tests that already exist. A declaration written after the suite can only measure what its author already thought of — and if it was derived FROM the suite, \`atomicLedger\` and \`traceability\` read 100% by construction while anything dropped along the way stops being missing from anything. Both axes are held as unverified until you review the claims (add what you care about, delete what you do not) and confirm: \`sungen audit --screen ${screenName} --accept-viewpoint\` — a QA decision: an agent presents it and does not run it.`);
|
|
610
610
|
}
|
|
611
611
|
if (!hasYardstick) {
|
|
612
612
|
const lost = [
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
* declared collection (order, application, submission …)
|
|
23
23
|
*/
|
|
24
24
|
import * as fs from 'fs';
|
|
25
|
+
import { refNamespaces, screenKey } from '../utils/ref-key';
|
|
25
26
|
import * as path from 'path';
|
|
26
27
|
import { parse as parseYaml } from 'yaml';
|
|
27
28
|
import { ScenarioInfo } from './parse';
|
|
@@ -179,7 +180,7 @@ export function loadFlowContract(unitDir: string): { contract: FlowContract | nu
|
|
|
179
180
|
actor: raw.actor !== undefined ? String(raw.actor) : undefined,
|
|
180
181
|
trigger: raw.trigger !== undefined ? String(raw.trigger) : undefined,
|
|
181
182
|
precondition: raw.precondition !== undefined ? String(raw.precondition) : undefined,
|
|
182
|
-
outcome: { screen: String(outcome!.screen)
|
|
183
|
+
outcome: { screen: screenKey(String(outcome!.screen)), assertion: outcome!.assertion !== undefined ? String(outcome!.assertion) : undefined },
|
|
183
184
|
value: raw.value !== undefined ? String(raw.value) : undefined,
|
|
184
185
|
successGuarantee: raw.successGuarantee !== undefined ? String(raw.successGuarantee) : (raw.success_guarantee !== undefined ? String(raw.success_guarantee) : undefined),
|
|
185
186
|
minimalGuarantee: raw.minimalGuarantee !== undefined ? String(raw.minimalGuarantee) : (raw.minimal_guarantee !== undefined ? String(raw.minimal_guarantee) : undefined),
|
|
@@ -210,7 +211,7 @@ export function loadFlowContract(unitDir: string): { contract: FlowContract | nu
|
|
|
210
211
|
.map((e) => ({
|
|
211
212
|
name: String(e.name),
|
|
212
213
|
owner: e.owner !== undefined ? String(e.owner) : undefined,
|
|
213
|
-
screens: Array.isArray(e.screens) ? (e.screens as unknown[]).map((x) => String(x)
|
|
214
|
+
screens: Array.isArray(e.screens) ? (e.screens as unknown[]).map((x) => screenKey(String(x))) : [],
|
|
214
215
|
}))
|
|
215
216
|
: undefined,
|
|
216
217
|
golden: raw.golden === true,
|
|
@@ -223,13 +224,12 @@ export function loadFlowContract(unitDir: string): { contract: FlowContract | nu
|
|
|
223
224
|
|
|
224
225
|
/** `[screen:element]` namespaces referenced by a scenario's steps, in step order. */
|
|
225
226
|
function namespacesInOrder(s: ScenarioInfo): string[] {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
return out;
|
|
227
|
+
// Through the shared vocabulary — a private regex here once dropped every multi-word screen (#660).
|
|
228
|
+
return refNamespaces(s.stepsText);
|
|
229
229
|
}
|
|
230
230
|
|
|
231
231
|
function touchesOutcome(s: ScenarioInfo, outcomeScreen: string): boolean {
|
|
232
|
-
return namespacesInOrder(s).includes(outcomeScreen);
|
|
232
|
+
return namespacesInOrder(s).includes(screenKey(outcomeScreen));
|
|
233
233
|
}
|
|
234
234
|
|
|
235
235
|
/**
|
|
@@ -244,7 +244,8 @@ function reachesOutcome(s: ScenarioInfo, outcomeScreen: string): boolean {
|
|
|
244
244
|
// anywhere in the scenario would suppress every positive assertion in it.
|
|
245
245
|
const steps = (s.steps ?? []).map((st) => st.text.toLowerCase());
|
|
246
246
|
if (steps.length === 0) return touchesOutcome(s, outcomeScreen);
|
|
247
|
-
|
|
247
|
+
const key = screenKey(outcomeScreen);
|
|
248
|
+
return steps.some((t) => refNamespaces(t, { bare: true }).includes(key)
|
|
248
249
|
&& !/\b(is hidden|is not visible|does not exist|is absent|no longer)\b/.test(t));
|
|
249
250
|
}
|
|
250
251
|
|
|
@@ -366,15 +367,39 @@ export function flowInventory(contract: FlowContract, scenarios: ScenarioInfo[])
|
|
|
366
367
|
* Both inflate the flow count while adding no branch coverage, which is exactly what makes a
|
|
367
368
|
* suite look complete to the harness and thin to a reviewer.
|
|
368
369
|
*/
|
|
370
|
+
/** An interaction, whichever keyword the author put in front of it — `Then User click` exists. */
|
|
371
|
+
const INTERACTION_VERB = /\buser\s+(?:click|clicks|tap|taps|fill|fills|type|types|select|selects|press|presses|check|checks|uncheck|unchecks|upload|uploads|submit|submits|enter|enters|choose|chooses|toggle|toggles|drag|drags|swipe|swipes|scroll|scrolls|hover|hovers|clear|clears)\b/i;
|
|
372
|
+
|
|
369
373
|
export function misfiledPhases(
|
|
370
374
|
contract: FlowContract, scenarios: ScenarioInfo[], basicPhase: string,
|
|
371
375
|
): Array<{ scenario: string; id: string; why: string }> {
|
|
372
376
|
const out: Array<{ scenario: string; id: string; why: string }> = [];
|
|
373
377
|
const outcome = contract.outcome.screen;
|
|
378
|
+
// A branch point is a STRUCTURAL fact before it is a vocabulary one. The check used to demand a
|
|
379
|
+
// branch WORD (back / cancel / retry…), so a precondition guard — `Given` already authenticated,
|
|
380
|
+
// or no originating session → `Then` redirected — had no way to satisfy it: such a scenario has
|
|
381
|
+
// no interaction step at all, by construction, and its branch is in the `Given`, not in a verb.
|
|
382
|
+
// Reported under AF and EF alike on a real project (#660). Two structural signals now count:
|
|
383
|
+
// no `When` at all (a guard by construction), or a `Given` the basic flow never establishes
|
|
384
|
+
// (a precondition branch). Vocabulary remains as the fallback for scenarios without step data.
|
|
385
|
+
const basicGivens = new Set(scenarios
|
|
386
|
+
.filter((sc) => phaseToken(flowIdOf(sc, contract.phases) ?? '') === phaseToken(basicPhase))
|
|
387
|
+
.flatMap((sc) => (sc.steps ?? []).filter((st) => st.bucket === 'given').map((st) => st.text.toLowerCase().trim())));
|
|
388
|
+
const branchesStructurally = (sc: ScenarioInfo): boolean => {
|
|
389
|
+
const steps = sc.steps ?? [];
|
|
390
|
+
if (steps.length === 0) return false;
|
|
391
|
+
// No interaction AND the journey does not complete: a guard by construction (the redirect or
|
|
392
|
+
// the blocked screen IS the branch). No interaction but the outcome IS reached is a different
|
|
393
|
+
// animal — a postcondition on the basic path — and stays subject to the checks below.
|
|
394
|
+
const interacts = steps.some((st) => st.bucket === 'when' || INTERACTION_VERB.test(st.text));
|
|
395
|
+
if (!interacts && !reachesOutcome(sc, outcome)) return true;
|
|
396
|
+
return steps.some((st) => st.bucket === 'given' && !basicGivens.has(st.text.toLowerCase().trim()));
|
|
397
|
+
};
|
|
374
398
|
for (const s of scenarios) {
|
|
375
399
|
const id = flowIdOf(s, contract.phases);
|
|
376
400
|
if (!id || phaseToken(id) === phaseToken(basicPhase)) continue;
|
|
377
401
|
const ph = phaseToken(id);
|
|
402
|
+
if (branchesStructurally(s)) continue;
|
|
378
403
|
if (ph === 'EF' && reachesOutcome(s, outcome)) {
|
|
379
404
|
// An error-then-recover flow legitimately ends at the outcome — it is the recovery that
|
|
380
405
|
// is being proven, and a guard flow is blocked rather than "failed". Only a scenario with
|
|
@@ -415,8 +440,16 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
415
440
|
// guard/error phase is evidence of a SECOND business goal in this flow.
|
|
416
441
|
const declaredPhases = contract.phases;
|
|
417
442
|
const basic = declaredPhases.filter((p) => p !== 'UI')[0];
|
|
443
|
+
// A flow the inventory declares WITH ITS OWN outcome is measured against that outcome, never
|
|
444
|
+
// against the use case's. A spec that keeps an entry variant deliberately light ("prove the
|
|
445
|
+
// entry reaches the action bar; do not repeat the full assertion") could only satisfy the old
|
|
446
|
+
// check by widening the scenario — which the human reviewer then called over-testing. The gate
|
|
447
|
+
// and the reviewer were optimising different things; the declaration is where they agree (#660).
|
|
448
|
+
const ownOutcome = new Set((contract.flows ?? []).filter((f) => f.outcome).map((f) => f.id));
|
|
418
449
|
const offGoalScenarios = scenarios.filter((s) => {
|
|
419
450
|
if (touchesOutcome(s, outcomeScreen)) return false;
|
|
451
|
+
const fid = flowIdOf(s, declaredPhases);
|
|
452
|
+
if (fid && ownOutcome.has(fid)) return false;
|
|
420
453
|
const ph = phaseOf(s, declaredPhases, contract.phaseDetails);
|
|
421
454
|
// Any NON-basic declared phase (guards, error recovery, alternate branches) legitimately
|
|
422
455
|
// stops before the outcome — that is what a branch IS. Only an unclassified scenario that
|
|
@@ -470,7 +503,7 @@ export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQua
|
|
|
470
503
|
const steps = s.steps ?? [];
|
|
471
504
|
let current: string | null = null;
|
|
472
505
|
for (let i = 0; i < steps.length; i++) {
|
|
473
|
-
const ns = (steps[i].text
|
|
506
|
+
const ns = refNamespaces(steps[i].text)[0] ?? null;
|
|
474
507
|
if (!ns) continue;
|
|
475
508
|
if (current !== null && ns !== current) {
|
|
476
509
|
total++;
|
package/src/harness/next-step.ts
CHANGED
|
@@ -25,8 +25,14 @@ export interface NextStep {
|
|
|
25
25
|
command: string;
|
|
26
26
|
/** Why it is next — stated so an agent can weigh it, not just obey it. */
|
|
27
27
|
because: string;
|
|
28
|
-
/**
|
|
29
|
-
|
|
28
|
+
/**
|
|
29
|
+
* `blocked` steps are not runnable yet; they name what to resolve first. `decide` steps are a
|
|
30
|
+
* HUMAN'S to run: an agent presents them and stops. Accepting the viewpoint baseline is the
|
|
31
|
+
* case — offered as a plain `do`, an agent that had just rewritten `test-viewpoint.md` from the
|
|
32
|
+
* spec ran it, and `atomicLedger`/`traceability` read 100% against a checklist it wrote itself
|
|
33
|
+
* in the same session (#660).
|
|
34
|
+
*/
|
|
35
|
+
kind: 'do' | 'blocked' | 'optional' | 'decide';
|
|
30
36
|
}
|
|
31
37
|
|
|
32
38
|
export interface UnitState {
|
|
@@ -160,6 +166,7 @@ export function readUnitState(
|
|
|
160
166
|
const REPAIRABLE = [
|
|
161
167
|
'FLOW-CONTRACT-MISSING', 'FLOW-INVENTORY-MISSING', 'FLOW-UNCOVERED', 'FLOW-UNDECLARED',
|
|
162
168
|
'FLOW-PHASE-MISFILED', 'FLOW-GUARANTEE-MISSING', 'FLOW-OUTCOME-UNPROVEN', 'CONTINUITY-ONE-SIDED',
|
|
169
|
+
'ORACLE-URL-ONLY',
|
|
163
170
|
'SPEC-RESTATED-UNVERIFIED', 'VIEWPOINT-GESTURE-SUBSTITUTED', 'SPEC-UNCOVERED', 'TRIGGER-UNCOVERED',
|
|
164
171
|
'VIEWPOINT-ITEM-MISSING', 'MANUAL-CODE-MISSING', 'MANUAL-AUTOMATABLE', 'DEPTH-DEFERRED',
|
|
165
172
|
];
|
|
@@ -228,9 +235,9 @@ export function deriveSteps(s: UnitState): NextStep[] {
|
|
|
228
235
|
}
|
|
229
236
|
if (s.audit.viewpointBaselineStatus === 'changed') {
|
|
230
237
|
out.push({
|
|
231
|
-
kind: '
|
|
238
|
+
kind: 'decide',
|
|
232
239
|
command: `sungen audit --screen ${s.unit} --accept-viewpoint`,
|
|
233
|
-
because: 'test-viewpoint.md changed since the accepted baseline, so the ledger and traceability axes are not evidence until
|
|
240
|
+
because: 'test-viewpoint.md changed since the accepted baseline, so the ledger and traceability axes are not evidence until a QA confirms the new declaration. A decision, not a step: an agent presents it and does not run it — accepting the yardstick you just wrote makes both axes read 100% by construction',
|
|
234
241
|
});
|
|
235
242
|
}
|
|
236
243
|
const repair = s.audit.findings
|
package/src/harness/sensors.ts
CHANGED
|
@@ -386,7 +386,7 @@ export function flowRegressionDepth(scenarios: ScenarioInfo[]): FlowDepthResult
|
|
|
386
386
|
// ---------- Sensor: Oracle strength (H4) ----------
|
|
387
387
|
|
|
388
388
|
export interface OracleStrengthResult {
|
|
389
|
-
weak: { name: string; hint: string }[]; //
|
|
389
|
+
weak: { name: string; hint: string; code?: string }[]; // facet-by-name-substring, or URL-only (code ORACLE-URL-ONLY)
|
|
390
390
|
facetClaims: number; // scenarios that touch a category/brand facet (denominator)
|
|
391
391
|
ratio: number; // 1 - weak/facetClaims (1 when none) — caps businessDepth
|
|
392
392
|
}
|
|
@@ -395,11 +395,34 @@ export interface OracleStrengthResult {
|
|
|
395
395
|
// a category/brand term does NOT prove the item BELONGS to that facet (a "Dress" item need not contain
|
|
396
396
|
// "Dress" in its name). The strong oracle is the results-page title/header, a detail-page facet field,
|
|
397
397
|
// an API/DB query, or an explicit @manual:M2 deferral.
|
|
398
|
+
/** A `Then` that only asserts where the browser IS: `see [X] page`, `is on [X] page`, `should see route`. */
|
|
399
|
+
const URL_ONLY_THEN = /\]\s*page\b|\bis on \[|\bshould (?:see route|remain on)\b/i;
|
|
400
|
+
/** The scenarios where a transient URL lies most expensively: permission, auth, guards, redirects. */
|
|
401
|
+
const GUARD_LIKE = /\b(?:auth\w*|role|permission|access|guard|redirect\w*|forbidden|403|unauthori[sz]\w*|logged[- ]?(?:in|out)|session)\b|権限|quyền|đăng nhập/i;
|
|
402
|
+
|
|
398
403
|
const WEAK_FACET_ORACLE = /\bsee all\b\s*\[[^\]]*\b(name|title|label)\b[^\]]*\][^{[]*\bcontains?\b[^{]*\{\{[^}]*\b(categ|brand|facet|filter|term)/i;
|
|
399
404
|
const FACET_REF = /\{\{[^}]*\b(categ|brand|facet|filter)\b[^}]*\}\}|\b(category|brand)\b/i;
|
|
400
405
|
|
|
401
406
|
export function oracleStrength(scenarios: ScenarioInfo[]): OracleStrengthResult {
|
|
402
|
-
const weak: { name: string; hint: string }[] = [];
|
|
407
|
+
const weak: { name: string; hint: string; code?: string }[] = [];
|
|
408
|
+
// A scenario whose EVERY `Then` is a URL assertion has a URL-only oracle. `toHaveURL` passes on
|
|
409
|
+
// the first transient match, and a client-side router pushes the destination before the guard or
|
|
410
|
+
// the API has answered — so "role X can open screen Y" stayed green while the server returned 403
|
|
411
|
+
// and the app bounced to its error page (#662). The runtime now settles and re-asserts, which
|
|
412
|
+
// catches the bounce; this names the scenarios that still prove nothing about the page having
|
|
413
|
+
// RENDERED, and the one-line-per-screen fix.
|
|
414
|
+
for (const s of scenarios) {
|
|
415
|
+
if (s.manual) continue;
|
|
416
|
+
const thens = (s.steps ?? []).filter((st) => st.bucket === 'then');
|
|
417
|
+
if (thens.length === 0) continue;
|
|
418
|
+
if (!thens.every((st) => URL_ONLY_THEN.test(st.text))) continue;
|
|
419
|
+
const guardLike = GUARD_LIKE.test(s.haystack);
|
|
420
|
+
weak.push({
|
|
421
|
+
name: s.name.slice(0, 80),
|
|
422
|
+
code: 'ORACLE-URL-ONLY',
|
|
423
|
+
hint: `every \`Then\` is a URL assertion${guardLike ? ' on a permission / guard scenario' : ''} — it proves the router pushed a path, not that the page rendered for this user${guardLike ? ', and this is exactly the scenario a late 403 redirect turns falsely green' : ''}. Give the page selector an \`anchor:\` (a heading/landmark only the rendered page has) — one line per screen, and every \`see [X] page\` on it becomes a content oracle — or add a content assertion after it.`,
|
|
424
|
+
});
|
|
425
|
+
}
|
|
403
426
|
for (const s of scenarios) {
|
|
404
427
|
if (s.manual) continue; // a @manual facet check is a deliberate deferral, not a weak automated oracle
|
|
405
428
|
if (WEAK_FACET_ORACLE.test(s.stepsText)) {
|
|
@@ -659,7 +682,15 @@ const CLAIM_RULES: ClaimRule[] = [
|
|
|
659
682
|
// category behind "browser back does not re-submit", "does not re-charge the card",
|
|
660
683
|
// "double-click does not create two orders" — not a per-feature keyword.
|
|
661
684
|
claim: 'no-side-effect/no-duplicate',
|
|
662
|
-
|
|
685
|
+
// The negation must attach to an EFFECT — something created, sent, stored, charged, fired
|
|
686
|
+
// twice — not to the INPUT being refused. "duplicate address is rejected" is an observable
|
|
687
|
+
// rejection, proven by the error assertion in front of the user; "no duplicate record is
|
|
688
|
+
// created" is an absence nothing on the page can show. The rule used to fire on any action
|
|
689
|
+
// word beside any negation word, so ordinary rejection titles were told to go find a DB count,
|
|
690
|
+
// and renaming around the trigger words became the workflow (#660). Two shapes now qualify:
|
|
691
|
+
// (a) a negation followed by an effect token, (b) a repeat token (twice / again / double- or
|
|
692
|
+
// re-submit / second …) beside a negation, "only once" or a disabled control.
|
|
693
|
+
title: /(?:\b(?:no|not|n['’]t|never|without|cannot|không|chưa)\b[^.;]{0,40}?\b(?:creat(?:e|es|ed|ion)|record(?:s|ed)?|rows?|entr(?:y|ies)|sen[dt]|resen[dt]|charg(?:e|es|ed)|order(?:s|ed)?|payment|paid|insert(?:s|ed)?|submit(?:s|ted)?|submission|stor(?:e|es|ed)|sav(?:e|es|ed)|persist(?:s|ed)?|fir(?:e|es|ed)|issu(?:e|es|ed)|post(?:s|ed)?|email(?:s|ed)?|request(?:s|ed)?|\botp\b|twice|again|second|double|duplicate[sd]?|re-?submit(?:s|ted)?|tạo|gửi|lưu)\b)|(?=.*\b(?:double[- ]?(?:submit|click|tap|activation)|re-?submi(?:t|ts|tted|ssion)|twice|again|second (?:submit|submission|request|click|tap|order|charge|email|record))\b)(?=.*\b(?:no|not|n['’]t|never|without|cannot|prevent(?:s|ed)?|block(?:s|ed)?|avoid(?:s|ed)?|only once|exactly one|single|idempotent|disabled|không|chưa)\b)/i,
|
|
663
694
|
// `is disabled` counts: when the spec's own mechanism against a repeat is "the control
|
|
664
695
|
// is disabled immediately" (FR-014-style), asserting the disabled state IS the contrast —
|
|
665
696
|
// the second activation cannot occur. Without it the canonical double-submit proof shape
|
|
@@ -96,7 +96,7 @@ dropped along the way stops being missing from anything. `sungen audit` now repo
|
|
|
96
96
|
`VIEWPOINT-ADOPTED-POST-HOC` on the first sighting of a viewpoint alongside an existing suite and
|
|
97
97
|
holds `atomicLedger` + `traceability` as unverified until a human accepts it — so the honest move
|
|
98
98
|
is to write the claims from `spec.md` and the contract, mark plainly that they are a draft for QA
|
|
99
|
-
review, and let the QA revise before `--accept-viewpoint`.
|
|
99
|
+
review, and let the QA revise before `--accept-viewpoint`. **You never run `--accept-viewpoint` yourself** — `sungen next` lists it as a `⚖ decide` step for exactly this reason: an agent accepting the yardstick it wrote in the same session makes `atomicLedger` and `traceability` read 100% against its own checklist, and the Exception Flow it forgot stops being missing from anything. Present the acceptance to the QA (`AskUserQuestion`) with what the file declares, and stop.
|
|
100
100
|
|
|
101
101
|
**When you give a claim an id, a scenario must carry that id.** The id is the contract: a claim
|
|
102
102
|
whose id no scenario carries is reported MISSING, and word overlap will not rescue it. Several
|
|
@@ -121,10 +121,28 @@ two content-filtered queries can hit different rows if the table re-renders in b
|
|
|
121
121
|
> assert the page you expect (`Then User see [Other] page`); to say "this went away", assert a marker
|
|
122
122
|
> element (`Then User see [Some Element] is hidden`).
|
|
123
123
|
|
|
124
|
+
**Pattern 8 is a URL oracle, and a URL is not a rendered page.** `see [T] page` proves the router
|
|
125
|
+
pushed a path — it does NOT prove the page rendered for this user. A client-side router pushes the
|
|
126
|
+
destination BEFORE the route guard or the permission API has answered, so a user with no permission
|
|
127
|
+
is "on" `/items/detail` for a few hundred milliseconds before being bounced to the 403 page. Since
|
|
128
|
+
3.2.28 the step compiles to `expectPage(page, {…})` — arrive, let the page settle, HOLD the URL, and
|
|
129
|
+
re-assert — so a late redirect fails and names where the user ended up. But the page having
|
|
130
|
+
rendered is still unproven: **for permission / auth-guard / "role X can open screen Y" scenarios,
|
|
131
|
+
give the page selector an `anchor:`** (a heading or landmark only the rendered page has) — every
|
|
132
|
+
`see [T] page` on that screen then asserts the anchor too — or follow the page step with a content
|
|
133
|
+
assertion. A scenario whose every `Then` is a page assertion is reported as `ORACLE-URL-ONLY`.
|
|
134
|
+
|
|
135
|
+
```yaml
|
|
136
|
+
detail:
|
|
137
|
+
type: 'page'
|
|
138
|
+
value: '/items/detail'
|
|
139
|
+
anchor: { type: 'role', value: 'heading', name: 'Item Detail' } # the rendered-content proof
|
|
140
|
+
```
|
|
141
|
+
|
|
124
142
|
**Pattern 8 — the page assertion judges pathname AND query together.** Both `see [T] page` and
|
|
125
|
-
`is on [T] page` (Then-side) compile to ONE web-first
|
|
126
|
-
|
|
127
|
-
normalised) plus the query:
|
|
143
|
+
`is on [T] page` (Then-side) compile to ONE web-first check — `expectPage(page, {…})`, built on
|
|
144
|
+
`toHaveURL(urlMatches({…}))` and retrying like any other assertion — comparing the **exact** pathname
|
|
145
|
+
(anchored, trailing slash normalised) plus the query:
|
|
128
146
|
|
|
129
147
|
| step | asserts |
|
|
130
148
|
|---|---|
|
|
@@ -34,6 +34,7 @@ user-invocable: false
|
|
|
34
34
|
| **DEPTH** | business-critical scenarios assert only visibility/navigation | Replace `Then User see [X] page/section` with **observable data assertions**: `Then User see [X] with {{value}}`, `Then User see [T] table match data:`. Capture real expected values into `test-data.yaml`. |
|
|
35
35
|
| **BALANCE** | secondary viewpoints (UI/validation/security) outweigh business-core | **Stop expanding** secondary viewpoints; generate the missing business-core scenarios first. Do not add more subscription/UI variants while core is thin. |
|
|
36
36
|
| **TRACE** | scenarios use ad-hoc `VP-<CAT>-NNN` codes not linked to the viewpoint-overview | Make each scenario map to a viewpoint-overview id (align category codes, or add a mapping comment). |
|
|
37
|
+
| **ORACLE-URL-ONLY** | every `Then` in the scenario is a page/URL assertion — it proves a path was pushed, not that the page rendered; on a permission/guard scenario a late 403 redirect turns it falsely green | Add `anchor:` to the page selector (one line per screen; every `see [X] page` on it becomes a content oracle) or a content assertion after the page step. Never delete the page step to silence it. |
|
|
37
38
|
| **UNIVERSAL** | a universal theme (error/empty-state, accessibility) is absent | Low priority — add if in scope; otherwise note as out-of-scope with reason. |
|
|
38
39
|
|
|
39
40
|
## P5 steps for deep cross-screen / list coverage
|
|
@@ -68,6 +69,23 @@ Use these when repairing GATE/DEPTH findings for the hard viewpoints (cart/detai
|
|
|
68
69
|
4. **EP/data families are OK.** A `duplicates` cluster with `sameDataLikely=false` is an intentional equivalence-partition family (e.g. many invalid-email cases) — keep it; only collapse `sameDataLikely=true` exact duplicates.
|
|
69
70
|
5. **Advisory findings — surface, don't gate.** `MANUAL-REASON-MISMATCH` → fix the scenario's `@manual:Mx` code (so the planner recommends the right driver) during repair. `CAPABILITY-SUGGESTION` → **present it to the user as a next-step option** (e.g. "N @manual could be automated — `sungen capability add api db`?"), **recommend-only — never auto-install**. `VERIFICATION-OUT-OF-SCOPE (api|db)` → the unit uses `@api`/`@query`/`@requires:api|db` verification the project's recorded **test type** never opted into (E2E/UI-only per `capabilities.yaml`, and the viewpoint doesn't ask for it). During repair: **rewrite the oracle to be UI-observable** (or downgrade the deep check to an `(optional deep check: …)` note); only keep the API/DB verification if the project genuinely tests that layer — then record it (`verification: [ui, api]` / `sungen capability add`). Keep API/DB-in-E2E to the ≤20% band. Neither of these fails the gate.
|
|
70
71
|
|
|
72
|
+
6. **A gate that contradicts the spec's intent is a question, not a repair.** When the only way to
|
|
73
|
+
satisfy a finding is to change what the spec deliberately asked for — widening an entry variant the
|
|
74
|
+
spec keeps light so `FLOW-SCOPE-CREEP` stops firing, renaming a title around a trigger word so a
|
|
75
|
+
claim rule stops firing, adding an assertion the spec never demands — **stop and put the conflict
|
|
76
|
+
to the user** (`AskUserQuestion`: the finding, the spec passage it collides with, the two ways out).
|
|
77
|
+
A score reached by editing toward the gate is not a measurement of the suite; it is a measurement
|
|
78
|
+
of how well the gate was gamed, and the human reviewer will read it as over-testing or a false
|
|
79
|
+
claim. Where the harness provides a declaration route — `flows:` with a per-flow `outcome:`, a
|
|
80
|
+
`status: deferred` with a reason, `@manual` naming the gesture — that route is the fix, never the
|
|
81
|
+
scenario.
|
|
82
|
+
7. **`--accept-viewpoint` is a QA decision. Never run it.** `atomicLedger` and `traceability` are
|
|
83
|
+
evidence only while `test-viewpoint.md` is independent of what was generated. `sungen next` lists
|
|
84
|
+
the acceptance as a `⚖ decide` step: present it (the diff of declared ids, what was added, what was
|
|
85
|
+
dropped) and let the QA run the command. An agent that accepts the yardstick it just wrote makes
|
|
86
|
+
both axes read 100% by construction — and the spec's Exception Flow it forgot stops being missing
|
|
87
|
+
from anything.
|
|
88
|
+
|
|
71
89
|
## Discovery / fallback tree (when input is limited)
|
|
72
90
|
|
|
73
91
|
```
|
|
@@ -63,6 +63,10 @@ login:
|
|
|
63
63
|
awards:
|
|
64
64
|
type: 'page'
|
|
65
65
|
value: '/awards'
|
|
66
|
+
anchor: # optional — the element that proves the page RENDERED (#662).
|
|
67
|
+
type: 'role' # `see [Awards] page` asserts the URL, then this; without it the step is
|
|
68
|
+
value: 'heading' # a URL-only oracle a late 403 redirect can satisfy. One line per screen.
|
|
69
|
+
name: 'Awards'
|
|
66
70
|
|
|
67
71
|
"awards:submit":
|
|
68
72
|
type: 'role'
|
|
@@ -340,6 +340,9 @@ Security: [S1 – admin only]
|
|
|
340
340
|
`sungen audit` enforces these. Generate compliant output up front:
|
|
341
341
|
|
|
342
342
|
1. **Taxonomy-match** (`VP-TAXONOMY-MISMATCH`, gate-FAIL) — when `test-viewpoint.md` declares its own viewpoint IDs (e.g. `VP0`, `VP1`, … `VP12`, `MS-HP-001`, `MS-EH-001`), **reuse those IDs verbatim as the scenario codes**. Do NOT invent a generic `VP-UI / VP-LOGIC / VP-VAL` scheme — that breaks the coverage matrix. Only fall back to `VP-<CATEGORY>-<NNN>` when the viewpoint file declares no IDs.
|
|
343
|
+
- **`--accept-viewpoint` is the QA's command, not yours.** `sungen next` shows it as a `⚖ decide`
|
|
344
|
+
step. Present the declared-id diff and stop; a yardstick accepted by the process it measures
|
|
345
|
+
reads 100% by construction.
|
|
343
346
|
- **Match the scenarios to the file — never the file to the scenarios.** A filled `test-viewpoint.md` is an input; do not rewrite its declarations to fit what you generated. That is not compliance, it is moving the yardstick: `traceability` + `atomicLedger` then read 100% by construction and a dropped viewpoint stops being missing from anything. Disagree with the taxonomy → propose the diff and ask. `sungen audit` reports `VIEWPOINT-BASELINE-CHANGED` and excludes both axes until a human accepts the change (`sungen audit --screen <name> --accept-viewpoint`).
|
|
344
347
|
2. **Spec-coverage triggers** (`TRIGGER-UNCOVERED`, gate-FAIL) — the Validation-Rules table lists a **trigger** per constraint (e.g. `blur, submit`). Generate one scenario **per (constraint × trigger)** — a `format` rule validating *on blur AND on submit* needs BOTH a blur scenario (`press Tab`) and a submit scenario (`click [Submit]` / `press Enter`). Never collapse the trigger × input matrix to one representative case.
|
|
345
348
|
3. **Claim-Proof** (`CLAIM-UNPROVEN`) — a title claiming `all`/`only`/`every`/`single`/`correct`/`same`/`changes`/`hidden`/`cleared`/`restored`/`independent`/`sanitized`/`announces` MUST have the matching assertion (`see all …`, count, `remember`+compare, `is hidden`, return-and-assert-empty, etc.). If the title promises it, the steps must prove it.
|
|
@@ -372,7 +375,7 @@ Security: [S1 – admin only]
|
|
|
372
375
|
| Validation rule | 1 exact-message TC per rule | `@high` |
|
|
373
376
|
| Business rule | 1 behavioral TC per rule | `@high` |
|
|
374
377
|
| **Secondary behavior / tiebreaker** | **1 TC per tiebreaker or fallback rule in `Secondary behaviors`** | **`@high`** |
|
|
375
|
-
| Auth / OAuth / permissions | 1 VP-SEC TC | `@high` |
|
|
378
|
+
| Auth / OAuth / permissions | 1 VP-SEC TC — **with a rendered-content oracle**: `see [X] page` alone is a URL check a late 403 redirect passes; give the page selector an `anchor:` or add a content assertion (`ORACLE-URL-ONLY`) | `@high` |
|
|
376
379
|
| Free-text input | 1 XSS TC **and** 1 SQL injection TC (separate) | `@high` |
|
|
377
380
|
| **Free-text LIKE / partial-match field** | **1 field-level SQL TC + 1 API-level SQL `@manual` TC** | **`@high`** |
|
|
378
381
|
| Lifecycle states | 1 key state transition TC | `@high` |
|
|
@@ -821,6 +824,13 @@ is a gap to REPORT, not to quietly absorb.
|
|
|
821
824
|
that never touches `outcome.screen` and is not a guard (`EH`) or error-recovery (`ER`) belongs in a
|
|
822
825
|
DIFFERENT flow — propose the split instead of writing it here (`FLOW-SCOPE-CREEP` will flag it).
|
|
823
826
|
Auth persistence across transitions is part of `EH` unless the project declares it its own phase.
|
|
827
|
+
**The exception is declared, not widened:** when the spec KEEPS a variant deliberately light — an
|
|
828
|
+
entry variant that only proves the alternative entry reaches the action bar, without repeating the
|
|
829
|
+
full assertion the basic flow already carries — declare that flow in `flows:` with its own
|
|
830
|
+
`outcome:` and the audit judges it against that outcome. Never widen the scenario to satisfy the
|
|
831
|
+
gate; a reviewer reads the result as over-testing, and the finding was the thing that was wrong.
|
|
832
|
+
When a gate and the spec's stated intent collide and no declaration route exists, stop and put the
|
|
833
|
+
conflict to the user — do not edit toward the score.
|
|
824
834
|
|
|
825
835
|
**Manual in flows**: always `@manual:Mx` with the reason code — bare `@manual` is flagged
|
|
826
836
|
(`MANUAL-CODE-MISSING`) because the capability planner cannot route it. Typical flow deferrals:
|
|
@@ -847,7 +857,7 @@ Feature: Award Submission Flow
|
|
|
847
857
|
|
|
848
858
|
@high
|
|
849
859
|
Scenario: FL-EH-001 Direct access to the award form without login redirects to login
|
|
850
|
-
|
|
860
|
+
Given User is on [Awards] page
|
|
851
861
|
Then User see [Login] page
|
|
852
862
|
```
|
|
853
863
|
|
|
@@ -147,6 +147,7 @@ See `SKILL.md` for the 4 Viewpoints, Shared Checks, and Security Tag Rules.
|
|
|
147
147
|
|
|
148
148
|
- A menu item for a restricted page is shown but the user has no permission → click → a 403 page; or the item is hidden from the menu (verify the DOM has no link and the API returns no data)
|
|
149
149
|
- Direct URL access to a restricted page while not logged in → redirect to Login, the URL is preserved so post-login redirects correctly
|
|
150
|
+
- **The oracle for "role X can open screen Y" is rendered content, not the URL.** A client-side router shows the destination URL before the permission check answers; assert a heading/landmark of the screen (or give its page selector an `anchor:`) so a late 403 bounce fails the case
|
|
150
151
|
|
|
151
152
|
---
|
|
152
153
|
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
* URL, and needs no regex escaping of runtime values, which is what forced the split.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
+
import { expect, type Page } from '@playwright/test';
|
|
22
|
+
|
|
21
23
|
export interface UrlExpectation {
|
|
22
24
|
/**
|
|
23
25
|
* Anchored pathname pattern, compiled by the generator from the page selector's `value`
|
|
@@ -132,3 +134,55 @@ export function urlMatches({ pathname, declared, dataRef }: UrlExpectation): (u:
|
|
|
132
134
|
return queryOk && hashOk;
|
|
133
135
|
};
|
|
134
136
|
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* `toHaveURL` is a polling assertion: it passes on the FIRST sample that matches. A client-side
|
|
140
|
+
* router pushes the destination URL before the route guard or the API has answered, so a user
|
|
141
|
+
* with no permission is "on" `/items/detail` for a few hundred milliseconds — long enough for
|
|
142
|
+
* the assertion to sample it, pass, and end the test green — and is then bounced to `/forbidden`
|
|
143
|
+
* with no assertion left to see it. A real project shipped a permission matrix on that oracle;
|
|
144
|
+
* the 403s were found by hand (#662).
|
|
145
|
+
*
|
|
146
|
+
* `expectPage` is the page oracle `see [X] page` compiles to now. Arriving is not enough: the URL
|
|
147
|
+
* has to be STILL matching once the page has settled.
|
|
148
|
+
* 1. arrive — the same predicate `toHaveURL` always used;
|
|
149
|
+
* 2. settle — wait for network to go quiet (bounded: an app that polls forever must not hang
|
|
150
|
+
* the test, so a timeout here is not a failure);
|
|
151
|
+
* 3. hold — sample the URL over a settle window; the moment it stops matching, fail and
|
|
152
|
+
* NAME where it went, because that destination is the finding;
|
|
153
|
+
* 4. re-assert — the arrival predicate, once more, on the settled page.
|
|
154
|
+
*
|
|
155
|
+
* `SUNGEN_URL_SETTLE_MS` (default 1000) is the hold window; `SUNGEN_URL_NETWORKIDLE_MS` (default
|
|
156
|
+
* 3000) bounds step 2. Both are runtime knobs, so a slow environment is a config change, not a
|
|
157
|
+
* regenerate.
|
|
158
|
+
*/
|
|
159
|
+
export async function expectPage(
|
|
160
|
+
page: { url(): string; waitForLoadState(state: 'networkidle', opts?: { timeout?: number }): Promise<void> },
|
|
161
|
+
expectation: UrlExpectation,
|
|
162
|
+
opts: { settleMs?: number; networkIdleMs?: number; assertUrl?: (matches: (u: URL) => boolean) => Promise<void> } = {},
|
|
163
|
+
): Promise<void> {
|
|
164
|
+
const settleMs = opts.settleMs ?? Number(process.env.SUNGEN_URL_SETTLE_MS ?? 1000);
|
|
165
|
+
const networkIdleMs = opts.networkIdleMs ?? Number(process.env.SUNGEN_URL_NETWORKIDLE_MS ?? 3000);
|
|
166
|
+
const matches = urlMatches(expectation);
|
|
167
|
+
const assertUrl = opts.assertUrl ?? (async (m: (u: URL) => boolean) => { await expect(page as unknown as Page).toHaveURL(m); });
|
|
168
|
+
|
|
169
|
+
await assertUrl(matches);
|
|
170
|
+
const arrivedAt = page.url();
|
|
171
|
+
|
|
172
|
+
await page.waitForLoadState('networkidle', { timeout: networkIdleMs }).catch(() => undefined);
|
|
173
|
+
|
|
174
|
+
const started = Date.now();
|
|
175
|
+
while (Date.now() - started < settleMs) {
|
|
176
|
+
const now = page.url();
|
|
177
|
+
if (!matches(new URL(now))) {
|
|
178
|
+
throw new Error(
|
|
179
|
+
`page assertion: reached ${arrivedAt} and then left for ${now} after ${Date.now() - started}ms — `
|
|
180
|
+
+ 'a route guard, an error boundary or a redirect moved the user away after the URL had already '
|
|
181
|
+
+ 'matched. The page the test asserted is not the page the user ended on.',
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
await assertUrl(matches);
|
|
188
|
+
}
|