@sun-asterisk/sungen 3.2.27 → 3.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/generators/test-generator/adapters/appium/templates/steps/assertions/page-assertion.hbs +6 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/page-assertion.hbs +2 -1
- package/dist/generators/test-generator/adapters/playwright/templates/steps/partials/url-assertion.hbs +6 -2
- package/dist/generators/test-generator/code-generator.js +1 -1
- package/dist/generators/test-generator/code-generator.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +4 -4
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/generators/test-generator/utils/selector-resolver.d.ts +1 -0
- package/dist/generators/test-generator/utils/selector-resolver.d.ts.map +1 -1
- package/dist/generators/test-generator/utils/selector-resolver.js +4 -0
- package/dist/generators/test-generator/utils/selector-resolver.js.map +1 -1
- package/dist/harness/audit.js +1 -1
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/next-step.d.ts.map +1 -1
- package/dist/harness/next-step.js +1 -0
- package/dist/harness/next-step.js.map +1 -1
- package/dist/harness/sensors.d.ts +1 -0
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +25 -0
- package/dist/harness/sensors.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-gherkin-syntax/SKILL.md +21 -3
- package/dist/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +1 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-selector-keys/SKILL.md +4 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-viewpoint/group-d-display.md +1 -0
- package/dist/orchestrator/templates/specs-url-assert.d.ts +31 -0
- package/dist/orchestrator/templates/specs-url-assert.d.ts.map +1 -1
- package/dist/orchestrator/templates/specs-url-assert.js +43 -0
- package/dist/orchestrator/templates/specs-url-assert.js.map +1 -1
- package/dist/orchestrator/templates/specs-url-assert.ts +54 -0
- package/package.json +3 -3
- package/src/generators/test-generator/adapters/appium/templates/steps/assertions/page-assertion.hbs +6 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/page-assertion.hbs +2 -1
- package/src/generators/test-generator/adapters/playwright/templates/steps/partials/url-assertion.hbs +6 -2
- package/src/generators/test-generator/code-generator.ts +1 -1
- package/src/generators/test-generator/step-mapper.ts +2 -2
- package/src/generators/test-generator/utils/selector-resolver.ts +10 -0
- package/src/harness/audit.ts +1 -1
- package/src/harness/next-step.ts +1 -0
- package/src/harness/sensors.ts +25 -2
- package/src/orchestrator/templates/ai-src/skills/sungen-gherkin-syntax/SKILL.md +21 -3
- package/src/orchestrator/templates/ai-src/skills/sungen-harness-audit/SKILL.md +1 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-selector-keys/SKILL.md +4 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +1 -1
- package/src/orchestrator/templates/ai-src/skills/sungen-viewpoint/group-d-display.md +1 -0
- package/src/orchestrator/templates/specs-url-assert.ts +54 -0
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
22
|
exports.joinPath = joinPath;
|
|
23
23
|
exports.urlMatches = urlMatches;
|
|
24
|
+
exports.expectPage = expectPage;
|
|
25
|
+
const test_1 = require("@playwright/test");
|
|
24
26
|
/**
|
|
25
27
|
* Join a page's URL with the `with {{v}}` value for a NAVIGATION step
|
|
26
28
|
* (`Given User is on [X] page with {{v}}`).
|
|
@@ -118,4 +120,45 @@ function urlMatches({ pathname, declared, dataRef }) {
|
|
|
118
120
|
return queryOk && hashOk;
|
|
119
121
|
};
|
|
120
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* `toHaveURL` is a polling assertion: it passes on the FIRST sample that matches. A client-side
|
|
125
|
+
* router pushes the destination URL before the route guard or the API has answered, so a user
|
|
126
|
+
* with no permission is "on" `/items/detail` for a few hundred milliseconds — long enough for
|
|
127
|
+
* the assertion to sample it, pass, and end the test green — and is then bounced to `/forbidden`
|
|
128
|
+
* with no assertion left to see it. A real project shipped a permission matrix on that oracle;
|
|
129
|
+
* the 403s were found by hand (#662).
|
|
130
|
+
*
|
|
131
|
+
* `expectPage` is the page oracle `see [X] page` compiles to now. Arriving is not enough: the URL
|
|
132
|
+
* has to be STILL matching once the page has settled.
|
|
133
|
+
* 1. arrive — the same predicate `toHaveURL` always used;
|
|
134
|
+
* 2. settle — wait for network to go quiet (bounded: an app that polls forever must not hang
|
|
135
|
+
* the test, so a timeout here is not a failure);
|
|
136
|
+
* 3. hold — sample the URL over a settle window; the moment it stops matching, fail and
|
|
137
|
+
* NAME where it went, because that destination is the finding;
|
|
138
|
+
* 4. re-assert — the arrival predicate, once more, on the settled page.
|
|
139
|
+
*
|
|
140
|
+
* `SUNGEN_URL_SETTLE_MS` (default 1000) is the hold window; `SUNGEN_URL_NETWORKIDLE_MS` (default
|
|
141
|
+
* 3000) bounds step 2. Both are runtime knobs, so a slow environment is a config change, not a
|
|
142
|
+
* regenerate.
|
|
143
|
+
*/
|
|
144
|
+
async function expectPage(page, expectation, opts = {}) {
|
|
145
|
+
const settleMs = opts.settleMs ?? Number(process.env.SUNGEN_URL_SETTLE_MS ?? 1000);
|
|
146
|
+
const networkIdleMs = opts.networkIdleMs ?? Number(process.env.SUNGEN_URL_NETWORKIDLE_MS ?? 3000);
|
|
147
|
+
const matches = urlMatches(expectation);
|
|
148
|
+
const assertUrl = opts.assertUrl ?? (async (m) => { await (0, test_1.expect)(page).toHaveURL(m); });
|
|
149
|
+
await assertUrl(matches);
|
|
150
|
+
const arrivedAt = page.url();
|
|
151
|
+
await page.waitForLoadState('networkidle', { timeout: networkIdleMs }).catch(() => undefined);
|
|
152
|
+
const started = Date.now();
|
|
153
|
+
while (Date.now() - started < settleMs) {
|
|
154
|
+
const now = page.url();
|
|
155
|
+
if (!matches(new URL(now))) {
|
|
156
|
+
throw new Error(`page assertion: reached ${arrivedAt} and then left for ${now} after ${Date.now() - started}ms — `
|
|
157
|
+
+ 'a route guard, an error boundary or a redirect moved the user away after the URL had already '
|
|
158
|
+
+ 'matched. The page the test asserted is not the page the user ended on.');
|
|
159
|
+
}
|
|
160
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
161
|
+
}
|
|
162
|
+
await assertUrl(matches);
|
|
163
|
+
}
|
|
121
164
|
//# sourceMappingURL=specs-url-assert.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"specs-url-assert.js","sourceRoot":"","sources":["../../../src/orchestrator/templates/specs-url-assert.ts"],"names":[],"mappings":";AAAA,oBAAoB;AACpB;;;;;;;;;;;;;;;;;GAiBG;;
|
|
1
|
+
{"version":3,"file":"specs-url-assert.js","sourceRoot":"","sources":["../../../src/orchestrator/templates/specs-url-assert.ts"],"names":[],"mappings":";AAAA,oBAAoB;AACpB;;;;;;;;;;;;;;;;;GAiBG;;AAwCH,4BAYC;AA0BD,gCAuCC;AAuBD,gCA6BC;AAvKD,2CAAqD;AAmBrD;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAgB,QAAQ,CAAC,IAAY,EAAE,IAAY,EAAE,QAAwB;IAC3E,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,EAAE;QAAE,OAAO,IAAI,GAAG,IAAI,CAAC;IACvF,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,2FAA2F;IAC3F,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,GAAG,IAAI,GAAG,KAAK,CAAC;IAC/E,yEAAyE;IACzE,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9C,0FAA0F;IAC1F,qEAAqE;IACrE,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,GAAG,KAAK,CAAC;IAC/C,4DAA4D;IAC5D,OAAO,IAAI,GAAG,IAAI,GAAG,KAAK,CAAC;AAC7B,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,QAAgB;IACnC,IAAI,CAAC;QACH,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,QAAQ,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAgB,UAAU,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAkB;IACxE,OAAO,CAAC,CAAM,EAAW,EAAE;QACzB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAExF,8FAA8F;QAC9F,sFAAsF;QACtF,+FAA+F;QAC/F,2FAA2F;QAC3F,mEAAmE;QACnE,IAAI,CAAC,OAAO;YAAE,OAAO,CAAC,CAAC,MAAM,KAAK,EAAE,CAAC;QAErC,8FAA8F;QAC9F,4FAA4F;QAC5F,6FAA6F;QAC7F,2FAA2F;QAC3F,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,iCAAiC,OAAO,kCAAkC,CAAC,CAAC;QAC9F,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,iCAAiC,OAAO,gCAAgC,CAAC,CAAC;QAC5F,CAAC;QAED,6FAA6F;QAC7F,6FAA6F;QAC7F,wFAAwF;QACxF,iFAAiF;QACjF,uFAAuF;QACvF,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,uBAAuB,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAE1E,4FAA4F;QAC5F,4FAA4F;QAC5F,0FAA0F;QAC1F,yDAAyD;QACzD,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5F,uDAAuD;QACvD,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC;QAC5C,OAAO,OAAO,IAAI,MAAM,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACI,KAAK,UAAU,UAAU,CAC9B,IAA2G,EAC3G,WAA2B,EAC3B,OAAmH,EAAE;IAErH,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,IAAI,CAAC,CAAC;IACnF,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,IAAI,CAAC,CAAC;IAClG,MAAM,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,EAAE,CAAsB,EAAE,EAAE,GAAG,MAAM,IAAA,aAAM,EAAC,IAAuB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhI,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,GAAG,QAAQ,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,2BAA2B,SAAS,sBAAsB,GAAG,UAAU,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,OAAO;kBAChG,+FAA+F;kBAC/F,wEAAwE,CAC3E,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;AAC3B,CAAC"}
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
* URL, and needs no regex escaping of runtime values, which is what forced the split.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
+
import { expect, type Page } from '@playwright/test';
|
|
22
|
+
|
|
21
23
|
export interface UrlExpectation {
|
|
22
24
|
/**
|
|
23
25
|
* Anchored pathname pattern, compiled by the generator from the page selector's `value`
|
|
@@ -132,3 +134,55 @@ export function urlMatches({ pathname, declared, dataRef }: UrlExpectation): (u:
|
|
|
132
134
|
return queryOk && hashOk;
|
|
133
135
|
};
|
|
134
136
|
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* `toHaveURL` is a polling assertion: it passes on the FIRST sample that matches. A client-side
|
|
140
|
+
* router pushes the destination URL before the route guard or the API has answered, so a user
|
|
141
|
+
* with no permission is "on" `/items/detail` for a few hundred milliseconds — long enough for
|
|
142
|
+
* the assertion to sample it, pass, and end the test green — and is then bounced to `/forbidden`
|
|
143
|
+
* with no assertion left to see it. A real project shipped a permission matrix on that oracle;
|
|
144
|
+
* the 403s were found by hand (#662).
|
|
145
|
+
*
|
|
146
|
+
* `expectPage` is the page oracle `see [X] page` compiles to now. Arriving is not enough: the URL
|
|
147
|
+
* has to be STILL matching once the page has settled.
|
|
148
|
+
* 1. arrive — the same predicate `toHaveURL` always used;
|
|
149
|
+
* 2. settle — wait for network to go quiet (bounded: an app that polls forever must not hang
|
|
150
|
+
* the test, so a timeout here is not a failure);
|
|
151
|
+
* 3. hold — sample the URL over a settle window; the moment it stops matching, fail and
|
|
152
|
+
* NAME where it went, because that destination is the finding;
|
|
153
|
+
* 4. re-assert — the arrival predicate, once more, on the settled page.
|
|
154
|
+
*
|
|
155
|
+
* `SUNGEN_URL_SETTLE_MS` (default 1000) is the hold window; `SUNGEN_URL_NETWORKIDLE_MS` (default
|
|
156
|
+
* 3000) bounds step 2. Both are runtime knobs, so a slow environment is a config change, not a
|
|
157
|
+
* regenerate.
|
|
158
|
+
*/
|
|
159
|
+
export async function expectPage(
|
|
160
|
+
page: { url(): string; waitForLoadState(state: 'networkidle', opts?: { timeout?: number }): Promise<void> },
|
|
161
|
+
expectation: UrlExpectation,
|
|
162
|
+
opts: { settleMs?: number; networkIdleMs?: number; assertUrl?: (matches: (u: URL) => boolean) => Promise<void> } = {},
|
|
163
|
+
): Promise<void> {
|
|
164
|
+
const settleMs = opts.settleMs ?? Number(process.env.SUNGEN_URL_SETTLE_MS ?? 1000);
|
|
165
|
+
const networkIdleMs = opts.networkIdleMs ?? Number(process.env.SUNGEN_URL_NETWORKIDLE_MS ?? 3000);
|
|
166
|
+
const matches = urlMatches(expectation);
|
|
167
|
+
const assertUrl = opts.assertUrl ?? (async (m: (u: URL) => boolean) => { await expect(page as unknown as Page).toHaveURL(m); });
|
|
168
|
+
|
|
169
|
+
await assertUrl(matches);
|
|
170
|
+
const arrivedAt = page.url();
|
|
171
|
+
|
|
172
|
+
await page.waitForLoadState('networkidle', { timeout: networkIdleMs }).catch(() => undefined);
|
|
173
|
+
|
|
174
|
+
const started = Date.now();
|
|
175
|
+
while (Date.now() - started < settleMs) {
|
|
176
|
+
const now = page.url();
|
|
177
|
+
if (!matches(new URL(now))) {
|
|
178
|
+
throw new Error(
|
|
179
|
+
`page assertion: reached ${arrivedAt} and then left for ${now} after ${Date.now() - started}ms — `
|
|
180
|
+
+ 'a route guard, an error boundary or a redirect moved the user away after the URL had already '
|
|
181
|
+
+ 'matched. The page the test asserted is not the page the user ended on.',
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
await assertUrl(matches);
|
|
188
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sun-asterisk/sungen",
|
|
3
|
-
"version": "3.2.
|
|
3
|
+
"version": "3.2.28",
|
|
4
4
|
"description": "Deterministic E2E Test Compiler - Gherkin + Selectors → Playwright tests",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
@@ -40,8 +40,8 @@
|
|
|
40
40
|
"@babel/types": "^7.28.5",
|
|
41
41
|
"@cucumber/gherkin": "^37.0.0",
|
|
42
42
|
"@cucumber/messages": "^31.0.0",
|
|
43
|
-
"@sungen/driver-data-factory": "3.2.
|
|
44
|
-
"@sungen/driver-ui": "3.2.
|
|
43
|
+
"@sungen/driver-data-factory": "3.2.28",
|
|
44
|
+
"@sungen/driver-ui": "3.2.28",
|
|
45
45
|
"chalk": "^5.6.2",
|
|
46
46
|
"commander": "^14.0.2",
|
|
47
47
|
"dotenv": "^17.2.3",
|
package/src/generators/test-generator/adapters/appium/templates/steps/assertions/page-assertion.hbs
CHANGED
|
@@ -1 +1,7 @@
|
|
|
1
|
+
{{#if anchor~}}
|
|
2
|
+
{{!-- A native app has no URL; the page's declared `anchor:` (a screen marker) IS the page oracle
|
|
3
|
+
here — the same rendered-content proof the web side adds after its URL check (#662). --}}
|
|
4
|
+
await __assertVisible({{#with anchor}}{{> appium-selector-expr}}{{/with}});
|
|
5
|
+
{{~else~}}
|
|
1
6
|
// page/URL assertion is not applicable on mobile (no URL) — [{{value}}]
|
|
7
|
+
{{~/if}}
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
{{> url-assertion}}
|
|
1
|
+
{{> url-assertion}}{{#if anchor}}
|
|
2
|
+
await expect({{#with anchor}}{{> locator}}{{/with}}).toBeVisible();{{/if}}
|
package/src/generators/test-generator/adapters/playwright/templates/steps/partials/url-assertion.hbs
CHANGED
|
@@ -11,5 +11,9 @@
|
|
|
11
11
|
Only `pathRegex` is the generator's business: it alone knows the page selector's value and
|
|
12
12
|
its `:id` wildcards, and it is pre-escaped by pathToRegexSource. `\/?$` normalises a
|
|
13
13
|
trailing slash so `/vi/search/` does not read as a different page. The declared value stays
|
|
14
|
-
a plain string literal so the runtime-data marker pass can rewrite it to testData.get(…).
|
|
15
|
-
|
|
14
|
+
a plain string literal so the runtime-data marker pass can rewrite it to testData.get(…).
|
|
15
|
+
|
|
16
|
+
`expectPage`, not a bare `toHaveURL`: a polling URL assertion passes on the first transient
|
|
17
|
+
match, before a route guard has bounced the user away (#662). The helper arrives, lets the
|
|
18
|
+
page settle, holds, and re-asserts — read specs/url-assert.ts. --}}
|
|
19
|
+
await expectPage(page, { pathname: /^{{pathRegex}}\/?$/{{#if query}}, dataRef: '{{escapeQuotes dataRef}}', declared: '{{escapeQuotes query}}'{{/if}} });
|
|
@@ -346,7 +346,7 @@ export class CodeGenerator {
|
|
|
346
346
|
// helper into every UI spec — including the many that touch no URL at all. Hence the body is
|
|
347
347
|
// rendered BEFORE the imports. Importing exactly the names used keeps the import honest; the
|
|
348
348
|
// helper file itself is synced only when something needs it (an api-only unit carries none).
|
|
349
|
-
const urlAssertImports = ['urlMatches', 'joinPath'].filter((n) => testCode.includes(`${n}(`)).join(', ');
|
|
349
|
+
const urlAssertImports = ['urlMatches', 'joinPath', 'expectPage'].filter((n) => testCode.includes(`${n}(`)).join(', ');
|
|
350
350
|
if (urlAssertImports) this.syncGeneratedHelper(outputDir, 'url-assert.ts', 'specs-url-assert.ts');
|
|
351
351
|
|
|
352
352
|
// Same body-first rule for the runtime regex-escape helper: only a spec whose
|
|
@@ -490,7 +490,7 @@ export class StepMapper {
|
|
|
490
490
|
*/
|
|
491
491
|
private checkPageAssertionFallback(step: ParsedStep, mapped: MappedStep): void {
|
|
492
492
|
if (!this.diagnostics || !step.selectorRef || step.elementType !== 'page') return;
|
|
493
|
-
if (
|
|
493
|
+
if (!/\b(?:urlMatches|expectPage)\(/.test(mapped.code || '')) return; // both URL-assertion shapes (#662)
|
|
494
494
|
let resolvedAsPage = false;
|
|
495
495
|
try {
|
|
496
496
|
const resolved = this.selectorResolver.resolveSelector(
|
|
@@ -518,7 +518,7 @@ export class StepMapper {
|
|
|
518
518
|
*/
|
|
519
519
|
private checkDeclaredUrlHasNoQuery(step: ParsedStep, mapped: MappedStep): void {
|
|
520
520
|
if (!this.diagnostics || !step.dataRef || step.elementType !== 'page') return;
|
|
521
|
-
if (
|
|
521
|
+
if (!/\b(?:urlMatches|expectPage)\(/.test(mapped.code || '')) return; // both URL-assertion shapes (#662)
|
|
522
522
|
// The base value, never resolveData(): in runtime mode that returns the marker, which carries
|
|
523
523
|
// no query by construction — the diagnostic would then fire on every single step.
|
|
524
524
|
const value = this.dataResolver.peekBaseValue(step.dataRef, this.featureName);
|
|
@@ -23,6 +23,12 @@ interface SelectorEntry {
|
|
|
23
23
|
scope?: string; // Parent landmark aria-label to scope within (e.g., 'desktop navigation')
|
|
24
24
|
match?: 'exact' | 'partial'; // For getByText matching (default: partial)
|
|
25
25
|
|
|
26
|
+
// === Page anchor (#662) ===
|
|
27
|
+
// For a `type: page` entry: the element that proves the page has RENDERED — a heading, a
|
|
28
|
+
// landmark, a title. `see [X] page` asserts the URL and then this element, so a route guard
|
|
29
|
+
// that bounces the user after the URL has already matched turns the step red instead of green.
|
|
30
|
+
anchor?: SelectorEntry;
|
|
31
|
+
|
|
26
32
|
// === Mobile per-platform variant (issue #392) ===
|
|
27
33
|
// `android:`/`ios:` sub-selectors for one logical element whose locator differs per OS (composite
|
|
28
34
|
// content-desc, partial/dynamic text, native predicate). Each is a full SelectorEntry.
|
|
@@ -127,6 +133,7 @@ export interface ResolvedSelector {
|
|
|
127
133
|
attribute?: string; // Attribute to check
|
|
128
134
|
pattern?: string; // Regex for attribute value
|
|
129
135
|
expanded?: { class?: string; attribute?: string; state?: 'none' }; // Expand/collapse state signal
|
|
136
|
+
anchor?: ResolvedSelector; // page entries only — the rendered-content proof behind `see [X] page` (#662)
|
|
130
137
|
}
|
|
131
138
|
|
|
132
139
|
/**
|
|
@@ -580,6 +587,9 @@ export class SelectorResolver {
|
|
|
580
587
|
if (entry.attribute) v2Fields.attribute = entry.attribute;
|
|
581
588
|
if (entry.pattern) v2Fields.pattern = entry.pattern;
|
|
582
589
|
if (entry.expanded) v2Fields.expanded = entry.expanded;
|
|
590
|
+
// The anchor is a selector in its own right; it must not inherit the PAGE's label as its
|
|
591
|
+
// accessible name, so it resolves against its own name (or nothing).
|
|
592
|
+
if (entry.anchor) v2Fields.anchor = this.resolveFromEntry(entry.anchor, entry.anchor.name ?? '');
|
|
583
593
|
|
|
584
594
|
// Helper to attach v2 fields and inputMethod
|
|
585
595
|
const withExtras = (resolved: ResolvedSelector): ResolvedSelector => {
|
package/src/harness/audit.ts
CHANGED
|
@@ -591,7 +591,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
591
591
|
findings.push(`FLOW-DEPTH: this stateful flow (${what}) exercises ${dims - flowDepth.missing.length}/${dims} applicable regression dimensions — missing [${flowDepth.missing.join(', ')}] → ${flowDepth.missing.map((m) => how[m]).join('; ')}. (businessDepth is capped until covered.)`);
|
|
592
592
|
}
|
|
593
593
|
for (const w of oracle.weak) {
|
|
594
|
-
findings.push(
|
|
594
|
+
findings.push(`${w.code ?? 'ORACLE-WEAK'}: "${w.name}" — ${w.hint}`);
|
|
595
595
|
}
|
|
596
596
|
for (const u of claim.unproven) {
|
|
597
597
|
const tag = u.severity === 'fail' ? 'CLAIM-UNPROVEN' : 'CLAIM-WEAK';
|
package/src/harness/next-step.ts
CHANGED
|
@@ -166,6 +166,7 @@ export function readUnitState(
|
|
|
166
166
|
const REPAIRABLE = [
|
|
167
167
|
'FLOW-CONTRACT-MISSING', 'FLOW-INVENTORY-MISSING', 'FLOW-UNCOVERED', 'FLOW-UNDECLARED',
|
|
168
168
|
'FLOW-PHASE-MISFILED', 'FLOW-GUARANTEE-MISSING', 'FLOW-OUTCOME-UNPROVEN', 'CONTINUITY-ONE-SIDED',
|
|
169
|
+
'ORACLE-URL-ONLY',
|
|
169
170
|
'SPEC-RESTATED-UNVERIFIED', 'VIEWPOINT-GESTURE-SUBSTITUTED', 'SPEC-UNCOVERED', 'TRIGGER-UNCOVERED',
|
|
170
171
|
'VIEWPOINT-ITEM-MISSING', 'MANUAL-CODE-MISSING', 'MANUAL-AUTOMATABLE', 'DEPTH-DEFERRED',
|
|
171
172
|
];
|
package/src/harness/sensors.ts
CHANGED
|
@@ -386,7 +386,7 @@ export function flowRegressionDepth(scenarios: ScenarioInfo[]): FlowDepthResult
|
|
|
386
386
|
// ---------- Sensor: Oracle strength (H4) ----------
|
|
387
387
|
|
|
388
388
|
export interface OracleStrengthResult {
|
|
389
|
-
weak: { name: string; hint: string }[]; //
|
|
389
|
+
weak: { name: string; hint: string; code?: string }[]; // facet-by-name-substring, or URL-only (code ORACLE-URL-ONLY)
|
|
390
390
|
facetClaims: number; // scenarios that touch a category/brand facet (denominator)
|
|
391
391
|
ratio: number; // 1 - weak/facetClaims (1 when none) — caps businessDepth
|
|
392
392
|
}
|
|
@@ -395,11 +395,34 @@ export interface OracleStrengthResult {
|
|
|
395
395
|
// a category/brand term does NOT prove the item BELONGS to that facet (a "Dress" item need not contain
|
|
396
396
|
// "Dress" in its name). The strong oracle is the results-page title/header, a detail-page facet field,
|
|
397
397
|
// an API/DB query, or an explicit @manual:M2 deferral.
|
|
398
|
+
/** A `Then` that only asserts where the browser IS: `see [X] page`, `is on [X] page`, `should see route`. */
|
|
399
|
+
const URL_ONLY_THEN = /\]\s*page\b|\bis on \[|\bshould (?:see route|remain on)\b/i;
|
|
400
|
+
/** The scenarios where a transient URL lies most expensively: permission, auth, guards, redirects. */
|
|
401
|
+
const GUARD_LIKE = /\b(?:auth\w*|role|permission|access|guard|redirect\w*|forbidden|403|unauthori[sz]\w*|logged[- ]?(?:in|out)|session)\b|権限|quyền|đăng nhập/i;
|
|
402
|
+
|
|
398
403
|
const WEAK_FACET_ORACLE = /\bsee all\b\s*\[[^\]]*\b(name|title|label)\b[^\]]*\][^{[]*\bcontains?\b[^{]*\{\{[^}]*\b(categ|brand|facet|filter|term)/i;
|
|
399
404
|
const FACET_REF = /\{\{[^}]*\b(categ|brand|facet|filter)\b[^}]*\}\}|\b(category|brand)\b/i;
|
|
400
405
|
|
|
401
406
|
export function oracleStrength(scenarios: ScenarioInfo[]): OracleStrengthResult {
|
|
402
|
-
const weak: { name: string; hint: string }[] = [];
|
|
407
|
+
const weak: { name: string; hint: string; code?: string }[] = [];
|
|
408
|
+
// A scenario whose EVERY `Then` is a URL assertion has a URL-only oracle. `toHaveURL` passes on
|
|
409
|
+
// the first transient match, and a client-side router pushes the destination before the guard or
|
|
410
|
+
// the API has answered — so "role X can open screen Y" stayed green while the server returned 403
|
|
411
|
+
// and the app bounced to its error page (#662). The runtime now settles and re-asserts, which
|
|
412
|
+
// catches the bounce; this names the scenarios that still prove nothing about the page having
|
|
413
|
+
// RENDERED, and the one-line-per-screen fix.
|
|
414
|
+
for (const s of scenarios) {
|
|
415
|
+
if (s.manual) continue;
|
|
416
|
+
const thens = (s.steps ?? []).filter((st) => st.bucket === 'then');
|
|
417
|
+
if (thens.length === 0) continue;
|
|
418
|
+
if (!thens.every((st) => URL_ONLY_THEN.test(st.text))) continue;
|
|
419
|
+
const guardLike = GUARD_LIKE.test(s.haystack);
|
|
420
|
+
weak.push({
|
|
421
|
+
name: s.name.slice(0, 80),
|
|
422
|
+
code: 'ORACLE-URL-ONLY',
|
|
423
|
+
hint: `every \`Then\` is a URL assertion${guardLike ? ' on a permission / guard scenario' : ''} — it proves the router pushed a path, not that the page rendered for this user${guardLike ? ', and this is exactly the scenario a late 403 redirect turns falsely green' : ''}. Give the page selector an \`anchor:\` (a heading/landmark only the rendered page has) — one line per screen, and every \`see [X] page\` on it becomes a content oracle — or add a content assertion after it.`,
|
|
424
|
+
});
|
|
425
|
+
}
|
|
403
426
|
for (const s of scenarios) {
|
|
404
427
|
if (s.manual) continue; // a @manual facet check is a deliberate deferral, not a weak automated oracle
|
|
405
428
|
if (WEAK_FACET_ORACLE.test(s.stepsText)) {
|
|
@@ -121,10 +121,28 @@ two content-filtered queries can hit different rows if the table re-renders in b
|
|
|
121
121
|
> assert the page you expect (`Then User see [Other] page`); to say "this went away", assert a marker
|
|
122
122
|
> element (`Then User see [Some Element] is hidden`).
|
|
123
123
|
|
|
124
|
+
**Pattern 8 is a URL oracle, and a URL is not a rendered page.** `see [T] page` proves the router
|
|
125
|
+
pushed a path — it does NOT prove the page rendered for this user. A client-side router pushes the
|
|
126
|
+
destination BEFORE the route guard or the permission API has answered, so a user with no permission
|
|
127
|
+
is "on" `/items/detail` for a few hundred milliseconds before being bounced to the 403 page. Since
|
|
128
|
+
3.2.28 the step compiles to `expectPage(page, {…})` — arrive, let the page settle, HOLD the URL, and
|
|
129
|
+
re-assert — so a late redirect fails and names where the user ended up. But the page having
|
|
130
|
+
rendered is still unproven: **for permission / auth-guard / "role X can open screen Y" scenarios,
|
|
131
|
+
give the page selector an `anchor:`** (a heading or landmark only the rendered page has) — every
|
|
132
|
+
`see [T] page` on that screen then asserts the anchor too — or follow the page step with a content
|
|
133
|
+
assertion. A scenario whose every `Then` is a page assertion is reported as `ORACLE-URL-ONLY`.
|
|
134
|
+
|
|
135
|
+
```yaml
|
|
136
|
+
detail:
|
|
137
|
+
type: 'page'
|
|
138
|
+
value: '/items/detail'
|
|
139
|
+
anchor: { type: 'role', value: 'heading', name: 'Item Detail' } # the rendered-content proof
|
|
140
|
+
```
|
|
141
|
+
|
|
124
142
|
**Pattern 8 — the page assertion judges pathname AND query together.** Both `see [T] page` and
|
|
125
|
-
`is on [T] page` (Then-side) compile to ONE web-first
|
|
126
|
-
|
|
127
|
-
normalised) plus the query:
|
|
143
|
+
`is on [T] page` (Then-side) compile to ONE web-first check — `expectPage(page, {…})`, built on
|
|
144
|
+
`toHaveURL(urlMatches({…}))` and retrying like any other assertion — comparing the **exact** pathname
|
|
145
|
+
(anchored, trailing slash normalised) plus the query:
|
|
128
146
|
|
|
129
147
|
| step | asserts |
|
|
130
148
|
|---|---|
|
|
@@ -34,6 +34,7 @@ user-invocable: false
|
|
|
34
34
|
| **DEPTH** | business-critical scenarios assert only visibility/navigation | Replace `Then User see [X] page/section` with **observable data assertions**: `Then User see [X] with {{value}}`, `Then User see [T] table match data:`. Capture real expected values into `test-data.yaml`. |
|
|
35
35
|
| **BALANCE** | secondary viewpoints (UI/validation/security) outweigh business-core | **Stop expanding** secondary viewpoints; generate the missing business-core scenarios first. Do not add more subscription/UI variants while core is thin. |
|
|
36
36
|
| **TRACE** | scenarios use ad-hoc `VP-<CAT>-NNN` codes not linked to the viewpoint-overview | Make each scenario map to a viewpoint-overview id (align category codes, or add a mapping comment). |
|
|
37
|
+
| **ORACLE-URL-ONLY** | every `Then` in the scenario is a page/URL assertion — it proves a path was pushed, not that the page rendered; on a permission/guard scenario a late 403 redirect turns it falsely green | Add `anchor:` to the page selector (one line per screen; every `see [X] page` on it becomes a content oracle) or a content assertion after the page step. Never delete the page step to silence it. |
|
|
37
38
|
| **UNIVERSAL** | a universal theme (error/empty-state, accessibility) is absent | Low priority — add if in scope; otherwise note as out-of-scope with reason. |
|
|
38
39
|
|
|
39
40
|
## P5 steps for deep cross-screen / list coverage
|
|
@@ -63,6 +63,10 @@ login:
|
|
|
63
63
|
awards:
|
|
64
64
|
type: 'page'
|
|
65
65
|
value: '/awards'
|
|
66
|
+
anchor: # optional — the element that proves the page RENDERED (#662).
|
|
67
|
+
type: 'role' # `see [Awards] page` asserts the URL, then this; without it the step is
|
|
68
|
+
value: 'heading' # a URL-only oracle a late 403 redirect can satisfy. One line per screen.
|
|
69
|
+
name: 'Awards'
|
|
66
70
|
|
|
67
71
|
"awards:submit":
|
|
68
72
|
type: 'role'
|
|
@@ -375,7 +375,7 @@ Security: [S1 – admin only]
|
|
|
375
375
|
| Validation rule | 1 exact-message TC per rule | `@high` |
|
|
376
376
|
| Business rule | 1 behavioral TC per rule | `@high` |
|
|
377
377
|
| **Secondary behavior / tiebreaker** | **1 TC per tiebreaker or fallback rule in `Secondary behaviors`** | **`@high`** |
|
|
378
|
-
| Auth / OAuth / permissions | 1 VP-SEC TC | `@high` |
|
|
378
|
+
| Auth / OAuth / permissions | 1 VP-SEC TC — **with a rendered-content oracle**: `see [X] page` alone is a URL check a late 403 redirect passes; give the page selector an `anchor:` or add a content assertion (`ORACLE-URL-ONLY`) | `@high` |
|
|
379
379
|
| Free-text input | 1 XSS TC **and** 1 SQL injection TC (separate) | `@high` |
|
|
380
380
|
| **Free-text LIKE / partial-match field** | **1 field-level SQL TC + 1 API-level SQL `@manual` TC** | **`@high`** |
|
|
381
381
|
| Lifecycle states | 1 key state transition TC | `@high` |
|
|
@@ -147,6 +147,7 @@ See `SKILL.md` for the 4 Viewpoints, Shared Checks, and Security Tag Rules.
|
|
|
147
147
|
|
|
148
148
|
- A menu item for a restricted page is shown but the user has no permission → click → a 403 page; or the item is hidden from the menu (verify the DOM has no link and the API returns no data)
|
|
149
149
|
- Direct URL access to a restricted page while not logged in → redirect to Login, the URL is preserved so post-login redirects correctly
|
|
150
|
+
- **The oracle for "role X can open screen Y" is rendered content, not the URL.** A client-side router shows the destination URL before the permission check answers; assert a heading/landmark of the screen (or give its page selector an `anchor:`) so a late 403 bounce fails the case
|
|
150
151
|
|
|
151
152
|
---
|
|
152
153
|
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
* URL, and needs no regex escaping of runtime values, which is what forced the split.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
+
import { expect, type Page } from '@playwright/test';
|
|
22
|
+
|
|
21
23
|
export interface UrlExpectation {
|
|
22
24
|
/**
|
|
23
25
|
* Anchored pathname pattern, compiled by the generator from the page selector's `value`
|
|
@@ -132,3 +134,55 @@ export function urlMatches({ pathname, declared, dataRef }: UrlExpectation): (u:
|
|
|
132
134
|
return queryOk && hashOk;
|
|
133
135
|
};
|
|
134
136
|
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* `toHaveURL` is a polling assertion: it passes on the FIRST sample that matches. A client-side
|
|
140
|
+
* router pushes the destination URL before the route guard or the API has answered, so a user
|
|
141
|
+
* with no permission is "on" `/items/detail` for a few hundred milliseconds — long enough for
|
|
142
|
+
* the assertion to sample it, pass, and end the test green — and is then bounced to `/forbidden`
|
|
143
|
+
* with no assertion left to see it. A real project shipped a permission matrix on that oracle;
|
|
144
|
+
* the 403s were found by hand (#662).
|
|
145
|
+
*
|
|
146
|
+
* `expectPage` is the page oracle `see [X] page` compiles to now. Arriving is not enough: the URL
|
|
147
|
+
* has to be STILL matching once the page has settled.
|
|
148
|
+
* 1. arrive — the same predicate `toHaveURL` always used;
|
|
149
|
+
* 2. settle — wait for network to go quiet (bounded: an app that polls forever must not hang
|
|
150
|
+
* the test, so a timeout here is not a failure);
|
|
151
|
+
* 3. hold — sample the URL over a settle window; the moment it stops matching, fail and
|
|
152
|
+
* NAME where it went, because that destination is the finding;
|
|
153
|
+
* 4. re-assert — the arrival predicate, once more, on the settled page.
|
|
154
|
+
*
|
|
155
|
+
* `SUNGEN_URL_SETTLE_MS` (default 1000) is the hold window; `SUNGEN_URL_NETWORKIDLE_MS` (default
|
|
156
|
+
* 3000) bounds step 2. Both are runtime knobs, so a slow environment is a config change, not a
|
|
157
|
+
* regenerate.
|
|
158
|
+
*/
|
|
159
|
+
export async function expectPage(
|
|
160
|
+
page: { url(): string; waitForLoadState(state: 'networkidle', opts?: { timeout?: number }): Promise<void> },
|
|
161
|
+
expectation: UrlExpectation,
|
|
162
|
+
opts: { settleMs?: number; networkIdleMs?: number; assertUrl?: (matches: (u: URL) => boolean) => Promise<void> } = {},
|
|
163
|
+
): Promise<void> {
|
|
164
|
+
const settleMs = opts.settleMs ?? Number(process.env.SUNGEN_URL_SETTLE_MS ?? 1000);
|
|
165
|
+
const networkIdleMs = opts.networkIdleMs ?? Number(process.env.SUNGEN_URL_NETWORKIDLE_MS ?? 3000);
|
|
166
|
+
const matches = urlMatches(expectation);
|
|
167
|
+
const assertUrl = opts.assertUrl ?? (async (m: (u: URL) => boolean) => { await expect(page as unknown as Page).toHaveURL(m); });
|
|
168
|
+
|
|
169
|
+
await assertUrl(matches);
|
|
170
|
+
const arrivedAt = page.url();
|
|
171
|
+
|
|
172
|
+
await page.waitForLoadState('networkidle', { timeout: networkIdleMs }).catch(() => undefined);
|
|
173
|
+
|
|
174
|
+
const started = Date.now();
|
|
175
|
+
while (Date.now() - started < settleMs) {
|
|
176
|
+
const now = page.url();
|
|
177
|
+
if (!matches(new URL(now))) {
|
|
178
|
+
throw new Error(
|
|
179
|
+
`page assertion: reached ${arrivedAt} and then left for ${now} after ${Date.now() - started}ms — `
|
|
180
|
+
+ 'a route guard, an error boundary or a redirect moved the user away after the URL had already '
|
|
181
|
+
+ 'matched. The page the test asserted is not the page the user ended on.',
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
await assertUrl(matches);
|
|
188
|
+
}
|