@sun-asterisk/sungen 3.2.18 → 3.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capabilities/discover.d.ts.map +1 -1
- package/dist/capabilities/discover.js +4 -1
- package/dist/capabilities/discover.js.map +1 -1
- package/dist/cli/commands/add.d.ts.map +1 -1
- package/dist/cli/commands/add.js +4 -2
- package/dist/cli/commands/add.js.map +1 -1
- package/dist/cli/commands/dashboard.d.ts.map +1 -1
- package/dist/cli/commands/dashboard.js +10 -1
- package/dist/cli/commands/dashboard.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +19 -1
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/generate.d.ts.map +1 -1
- package/dist/cli/commands/generate.js +10 -1
- package/dist/cli/commands/generate.js.map +1 -1
- package/dist/dashboard/html-renderer.d.ts.map +1 -1
- package/dist/dashboard/html-renderer.js +7 -2
- package/dist/dashboard/html-renderer.js.map +1 -1
- package/dist/dashboard/snapshot-builder.d.ts.map +1 -1
- package/dist/dashboard/snapshot-builder.js +3 -0
- package/dist/dashboard/snapshot-builder.js.map +1 -1
- package/dist/dashboard/types.d.ts +4 -0
- package/dist/dashboard/types.d.ts.map +1 -1
- package/dist/exporters/api-environment.d.ts.map +1 -1
- package/dist/exporters/api-environment.js +4 -24
- package/dist/exporters/api-environment.js.map +1 -1
- package/dist/exporters/api-testcase-formatter.d.ts +6 -0
- package/dist/exporters/api-testcase-formatter.d.ts.map +1 -1
- package/dist/exporters/api-testcase-formatter.js +11 -4
- package/dist/exporters/api-testcase-formatter.js.map +1 -1
- package/dist/exporters/csv-exporter.d.ts.map +1 -1
- package/dist/exporters/csv-exporter.js +40 -9
- package/dist/exporters/csv-exporter.js.map +1 -1
- package/dist/exporters/env-qa.d.ts +13 -0
- package/dist/exporters/env-qa.d.ts.map +1 -0
- package/dist/exporters/env-qa.js +82 -0
- package/dist/exporters/env-qa.js.map +1 -0
- package/dist/exporters/feature-parser.d.ts.map +1 -1
- package/dist/exporters/feature-parser.js +16 -33
- package/dist/exporters/feature-parser.js.map +1 -1
- package/dist/exporters/json-exporter.d.ts.map +1 -1
- package/dist/exporters/json-exporter.js +57 -20
- package/dist/exporters/json-exporter.js.map +1 -1
- package/dist/exporters/matrix/build.d.ts.map +1 -1
- package/dist/exporters/matrix/build.js +15 -6
- package/dist/exporters/matrix/build.js.map +1 -1
- package/dist/exporters/matrix/gates.d.ts.map +1 -1
- package/dist/exporters/matrix/gates.js +3 -2
- package/dist/exporters/matrix/gates.js.map +1 -1
- package/dist/exporters/matrix/step-buckets.d.ts +13 -0
- package/dist/exporters/matrix/step-buckets.d.ts.map +1 -0
- package/dist/exporters/matrix/step-buckets.js +7 -0
- package/dist/exporters/matrix/step-buckets.js.map +1 -0
- package/dist/exporters/qa-env-secret-denylist.d.ts +43 -0
- package/dist/exporters/qa-env-secret-denylist.d.ts.map +1 -0
- package/dist/exporters/qa-env-secret-denylist.js +143 -0
- package/dist/exporters/qa-env-secret-denylist.js.map +1 -0
- package/dist/exporters/scenario-merger.d.ts +20 -13
- package/dist/exporters/scenario-merger.d.ts.map +1 -1
- package/dist/exporters/scenario-merger.js +55 -30
- package/dist/exporters/scenario-merger.js.map +1 -1
- package/dist/exporters/step-formatter.d.ts +42 -33
- package/dist/exporters/step-formatter.d.ts.map +1 -1
- package/dist/exporters/step-formatter.js +68 -68
- package/dist/exporters/step-formatter.js.map +1 -1
- package/dist/exporters/test-data-resolver.d.ts +15 -0
- package/dist/exporters/test-data-resolver.d.ts.map +1 -1
- package/dist/exporters/test-data-resolver.js +34 -1
- package/dist/exporters/test-data-resolver.js.map +1 -1
- package/dist/exporters/types.d.ts +9 -8
- package/dist/exporters/types.d.ts.map +1 -1
- package/dist/generators/test-generator/adapters/adapter-interface.d.ts +2 -0
- package/dist/generators/test-generator/adapters/adapter-interface.d.ts.map +1 -1
- package/dist/generators/test-generator/adapters/appium/appium-adapter.d.ts +2 -0
- package/dist/generators/test-generator/adapters/appium/appium-adapter.d.ts.map +1 -1
- package/dist/generators/test-generator/adapters/appium/appium-adapter.js.map +1 -1
- package/dist/generators/test-generator/adapters/appium/templates/steps/actions/keyboard-global-action.hbs +3 -1
- package/dist/generators/test-generator/adapters/playwright/playwright-adapter.d.ts +2 -0
- package/dist/generators/test-generator/adapters/playwright/playwright-adapter.d.ts.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/playwright-adapter.js.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/templates/imports.hbs +6 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/actions/keyboard-global-action.hbs +3 -1
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/page-assertion.hbs +1 -1
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/storage-key-assertion.hbs +4 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/assertions/tab-order-assertion.hbs +30 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/navigation/navigation.hbs +4 -0
- package/dist/generators/test-generator/adapters/playwright/templates/steps/navigation/route-assertion.hbs +1 -4
- package/dist/generators/test-generator/adapters/playwright/templates/steps/partials/dialog-root.hbs +1 -1
- package/dist/generators/test-generator/adapters/playwright/templates/steps/partials/url-assertion.hbs +15 -0
- package/dist/generators/test-generator/code-generator.d.ts.map +1 -1
- package/dist/generators/test-generator/code-generator.js +78 -7
- package/dist/generators/test-generator/code-generator.js.map +1 -1
- package/dist/generators/test-generator/diagnostics.d.ts +16 -1
- package/dist/generators/test-generator/diagnostics.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.js +15 -0
- package/dist/generators/test-generator/diagnostics.js.map +1 -1
- package/dist/generators/test-generator/recipe-expander.d.ts +1 -1
- package/dist/generators/test-generator/recipe-expander.d.ts.map +1 -1
- package/dist/generators/test-generator/recipe-expander.js +1 -1
- package/dist/generators/test-generator/recipe-expander.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.d.ts +41 -0
- package/dist/generators/test-generator/step-mapper.d.ts.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +161 -17
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/generators/test-generator/template-engine.d.ts +2 -0
- package/dist/generators/test-generator/template-engine.d.ts.map +1 -1
- package/dist/generators/test-generator/template-engine.js +1 -1
- package/dist/generators/test-generator/template-engine.js.map +1 -1
- package/dist/generators/test-generator/utils/data-resolver.d.ts +18 -1
- package/dist/generators/test-generator/utils/data-resolver.d.ts.map +1 -1
- package/dist/generators/test-generator/utils/data-resolver.js +28 -4
- package/dist/generators/test-generator/utils/data-resolver.js.map +1 -1
- package/dist/generators/test-generator/utils/runtime-data-transformer.d.ts +2 -2
- package/dist/generators/test-generator/utils/runtime-data-transformer.d.ts.map +1 -1
- package/dist/generators/test-generator/utils/runtime-data-transformer.js +56 -11
- package/dist/generators/test-generator/utils/runtime-data-transformer.js.map +1 -1
- package/dist/generators/test-generator/utils/selector-resolver.d.ts +18 -2
- package/dist/generators/test-generator/utils/selector-resolver.d.ts.map +1 -1
- package/dist/generators/test-generator/utils/selector-resolver.js +29 -6
- package/dist/generators/test-generator/utils/selector-resolver.js.map +1 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +8 -7
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/capability-plan.d.ts.map +1 -1
- package/dist/harness/capability-plan.js +2 -3
- package/dist/harness/capability-plan.js.map +1 -1
- package/dist/harness/catalog/drivers.yaml +1 -1
- package/dist/harness/challenge.d.ts.map +1 -1
- package/dist/harness/challenge.js +1 -55
- package/dist/harness/challenge.js.map +1 -1
- package/dist/harness/data-driven-lint.d.ts.map +1 -1
- package/dist/harness/data-driven-lint.js +14 -9
- package/dist/harness/data-driven-lint.js.map +1 -1
- package/dist/harness/depth-lint.d.ts.map +1 -1
- package/dist/harness/depth-lint.js +1 -3
- package/dist/harness/depth-lint.js.map +1 -1
- package/dist/harness/flow-check.d.ts.map +1 -1
- package/dist/harness/flow-check.js +9 -7
- package/dist/harness/flow-check.js.map +1 -1
- package/dist/harness/flow-plan.d.ts.map +1 -1
- package/dist/harness/flow-plan.js +4 -3
- package/dist/harness/flow-plan.js.map +1 -1
- package/dist/harness/manifest.js +2 -2
- package/dist/harness/manifest.js.map +1 -1
- package/dist/harness/parse.d.ts +12 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +28 -1
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/script-check.d.ts.map +1 -1
- package/dist/harness/script-check.js +61 -31
- package/dist/harness/script-check.js.map +1 -1
- package/dist/harness/secret-scan.d.ts +3 -1
- package/dist/harness/secret-scan.d.ts.map +1 -1
- package/dist/harness/secret-scan.js +27 -11
- package/dist/harness/secret-scan.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +15 -4
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/trace.js +1 -1
- package/dist/harness/trace.js.map +1 -1
- package/dist/harness/unit-paths.d.ts +16 -0
- package/dist/harness/unit-paths.d.ts.map +1 -1
- package/dist/harness/unit-paths.js +39 -2
- package/dist/harness/unit-paths.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/orchestrator/screen-manager.d.ts.map +1 -1
- package/dist/orchestrator/screen-manager.js +8 -2
- package/dist/orchestrator/screen-manager.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/dashboard.md +2 -1
- package/dist/orchestrator/templates/ai-src/commands/run-test.md +1 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +8 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-error-mapping/SKILL.md +4 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-gherkin-syntax/SKILL.md +79 -2
- package/dist/orchestrator/templates/ai-src/skills/sungen-selector-fix/SKILL.md +7 -1
- package/dist/orchestrator/templates/ai-src/skills/sungen-selector-keys/SKILL.md +12 -3
- package/dist/orchestrator/templates/playwright.config.ts +33 -2
- package/dist/orchestrator/templates/specs-mock.d.ts +66 -0
- package/dist/orchestrator/templates/specs-mock.d.ts.map +1 -0
- package/dist/orchestrator/templates/specs-mock.js +137 -0
- package/dist/orchestrator/templates/specs-mock.js.map +1 -0
- package/dist/orchestrator/templates/specs-mock.ts +179 -0
- package/dist/orchestrator/templates/specs-url-assert.d.ts +67 -0
- package/dist/orchestrator/templates/specs-url-assert.d.ts.map +1 -0
- package/dist/orchestrator/templates/specs-url-assert.js +121 -0
- package/dist/orchestrator/templates/specs-url-assert.js.map +1 -0
- package/dist/orchestrator/templates/specs-url-assert.ts +134 -0
- package/dist/orchestrator/test-data-guide.d.ts.map +1 -1
- package/dist/orchestrator/test-data-guide.js +4 -0
- package/dist/orchestrator/test-data-guide.js.map +1 -1
- package/package.json +3 -3
- package/src/capabilities/discover.ts +4 -1
- package/src/cli/commands/add.ts +4 -2
- package/src/cli/commands/dashboard.ts +10 -1
- package/src/cli/commands/delivery.ts +19 -1
- package/src/cli/commands/generate.ts +9 -1
- package/src/dashboard/html-renderer.ts +8 -2
- package/src/dashboard/snapshot-builder.ts +3 -0
- package/src/dashboard/types.ts +4 -0
- package/src/exporters/api-environment.ts +3 -20
- package/src/exporters/api-testcase-formatter.ts +8 -2
- package/src/exporters/csv-exporter.ts +47 -8
- package/src/exporters/env-qa.ts +41 -0
- package/src/exporters/feature-parser.ts +17 -38
- package/src/exporters/json-exporter.ts +71 -24
- package/src/exporters/matrix/build.ts +15 -6
- package/src/exporters/matrix/gates.ts +3 -2
- package/src/exporters/matrix/step-buckets.ts +15 -0
- package/src/exporters/qa-env-secret-denylist.ts +118 -0
- package/src/exporters/scenario-merger.ts +80 -42
- package/src/exporters/step-formatter.ts +67 -66
- package/src/exporters/test-data-resolver.ts +34 -1
- package/src/exporters/types.ts +9 -8
- package/src/generators/test-generator/adapters/adapter-interface.ts +1 -1
- package/src/generators/test-generator/adapters/appium/appium-adapter.ts +1 -1
- package/src/generators/test-generator/adapters/appium/templates/steps/actions/keyboard-global-action.hbs +3 -1
- package/src/generators/test-generator/adapters/playwright/playwright-adapter.ts +1 -1
- package/src/generators/test-generator/adapters/playwright/templates/imports.hbs +6 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/actions/keyboard-global-action.hbs +3 -1
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/page-assertion.hbs +1 -1
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/storage-key-assertion.hbs +4 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/assertions/tab-order-assertion.hbs +30 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/navigation/navigation.hbs +4 -0
- package/src/generators/test-generator/adapters/playwright/templates/steps/navigation/route-assertion.hbs +1 -4
- package/src/generators/test-generator/adapters/playwright/templates/steps/partials/dialog-root.hbs +1 -1
- package/src/generators/test-generator/adapters/playwright/templates/steps/partials/url-assertion.hbs +15 -0
- package/src/generators/test-generator/code-generator.ts +91 -8
- package/src/generators/test-generator/diagnostics.ts +16 -1
- package/src/generators/test-generator/recipe-expander.ts +2 -2
- package/src/generators/test-generator/step-mapper.ts +160 -15
- package/src/generators/test-generator/template-engine.ts +2 -2
- package/src/generators/test-generator/utils/data-resolver.ts +27 -5
- package/src/generators/test-generator/utils/runtime-data-transformer.ts +57 -11
- package/src/generators/test-generator/utils/selector-resolver.ts +29 -5
- package/src/harness/audit.ts +10 -9
- package/src/harness/capability-plan.ts +2 -2
- package/src/harness/catalog/drivers.yaml +1 -1
- package/src/harness/challenge.ts +2 -3
- package/src/harness/data-driven-lint.ts +13 -9
- package/src/harness/depth-lint.ts +2 -4
- package/src/harness/flow-check.ts +9 -7
- package/src/harness/flow-plan.ts +4 -3
- package/src/harness/manifest.ts +3 -3
- package/src/harness/parse.ts +28 -1
- package/src/harness/script-check.ts +61 -33
- package/src/harness/secret-scan.ts +23 -8
- package/src/harness/sensors.ts +15 -4
- package/src/harness/trace.ts +2 -2
- package/src/harness/unit-paths.ts +39 -2
- package/src/index.ts +1 -0
- package/src/orchestrator/screen-manager.ts +7 -2
- package/src/orchestrator/templates/ai-src/commands/dashboard.md +2 -1
- package/src/orchestrator/templates/ai-src/commands/run-test.md +1 -1
- package/src/orchestrator/templates/ai-src/skills/sungen-delivery/SKILL.md +8 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-error-mapping/SKILL.md +4 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-gherkin-syntax/SKILL.md +79 -2
- package/src/orchestrator/templates/ai-src/skills/sungen-selector-fix/SKILL.md +7 -1
- package/src/orchestrator/templates/ai-src/skills/sungen-selector-keys/SKILL.md +12 -3
- package/src/orchestrator/templates/playwright.config.ts +33 -2
- package/src/orchestrator/templates/specs-mock.ts +179 -0
- package/src/orchestrator/templates/specs-url-assert.ts +134 -0
- package/src/orchestrator/test-data-guide.ts +4 -0
|
@@ -56,6 +56,23 @@ interface SelectorEntry {
|
|
|
56
56
|
};
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
/**
|
|
60
|
+
* Roles whose accessible name comes from their own CONTENT, not from a label — so a name the
|
|
61
|
+
* compiler invented from the `[Reference]` text can never match a real element.
|
|
62
|
+
*
|
|
63
|
+
* The `name:` key is defaulted to the Gherkin reference when an entry omits it, which is right for
|
|
64
|
+
* label-named roles (button/link/textbox) and wrong for these. Left unchecked it compiles to
|
|
65
|
+
* `getByRole('option', { name: 'Shop Item' })` — zero elements — and `toBeHidden()` PASSES on an
|
|
66
|
+
* empty locator, so the scenario goes green without ever testing anything.
|
|
67
|
+
*
|
|
68
|
+
* The rule lives HERE, at the point the name is invented, rather than in each consumer: it used to
|
|
69
|
+
* be enforced in exactly one of the branches that read `resolved.name`, and every other branch
|
|
70
|
+
* (`is hidden`, `is disabled`) silently kept the bogus filter.
|
|
71
|
+
*
|
|
72
|
+
* An EXPLICIT `name:` in selectors.yaml is always honoured — the user knows their DOM.
|
|
73
|
+
*/
|
|
74
|
+
export const LIST_MEMBER_ROLES = new Set(['listitem', 'option', 'cell']);
|
|
75
|
+
|
|
59
76
|
// New selector file structure: flat key-value pairs
|
|
60
77
|
type SelectorFile = Record<string, SelectorEntry>;
|
|
61
78
|
|
|
@@ -275,7 +292,9 @@ export class SelectorResolver {
|
|
|
275
292
|
'list': () => ({ strategy: 'role', role: 'list', name: label, value: 'list' }),
|
|
276
293
|
'listitem': () => ({ strategy: 'role', role: 'listitem', value: 'listitem' }),
|
|
277
294
|
'searchbox': () => ({ strategy: 'role', role: 'searchbox', name: label, value: 'searchbox' }),
|
|
278
|
-
|
|
295
|
+
// no `name` — a list-member role names itself from its content (see LIST_MEMBER_ROLES);
|
|
296
|
+
// `listitem` already omitted it, `option`/`cell` used to disagree.
|
|
297
|
+
'option': () => ({ strategy: 'role', role: 'option', value: 'option' }),
|
|
279
298
|
'slider': () => ({ strategy: 'role', role: 'slider', name: label, value: 'slider' }),
|
|
280
299
|
'switch': () => ({ strategy: 'role', role: 'switch', name: label, value: 'switch' }),
|
|
281
300
|
'tab': () => ({ strategy: 'role', role: 'tab', name: label, value: 'tab' }),
|
|
@@ -286,7 +305,7 @@ export class SelectorResolver {
|
|
|
286
305
|
'progressbar': () => ({ strategy: 'role', role: 'progressbar', name: label, value: 'progressbar' }),
|
|
287
306
|
'region': () => ({ strategy: 'role', role: 'region', name: label, value: 'region' }),
|
|
288
307
|
'article': () => ({ strategy: 'role', role: 'article', name: label, value: 'article' }),
|
|
289
|
-
'cell': () => ({ strategy: 'role', role: 'cell',
|
|
308
|
+
'cell': () => ({ strategy: 'role', role: 'cell', value: 'cell' }),
|
|
290
309
|
'status': () => ({ strategy: 'role', role: 'status', name: label, value: 'status' }),
|
|
291
310
|
'navigation': () => ({ strategy: 'role', role: 'navigation', name: label, value: 'navigation' }),
|
|
292
311
|
};
|
|
@@ -491,8 +510,8 @@ export class SelectorResolver {
|
|
|
491
510
|
* Replace {{variable}} references in selector values with runtime data markers.
|
|
492
511
|
* This enables i18n: selectors can reference locale-dependent text from test-data.
|
|
493
512
|
*
|
|
494
|
-
* "{{lbl_submit}}" → "
|
|
495
|
-
* "Hello {{name}}" → "Hello
|
|
513
|
+
* "{{lbl_submit}}" → "__SUNGEN_TD$lbl_submit$__"
|
|
514
|
+
* "Hello {{name}}" → "Hello __SUNGEN_TD$name$__"
|
|
496
515
|
* "No variables" → "No variables" (unchanged)
|
|
497
516
|
*/
|
|
498
517
|
private static resolveTemplateVars(value: string): string {
|
|
@@ -517,7 +536,12 @@ export class SelectorResolver {
|
|
|
517
536
|
// Explicit `name: null` in YAML is a deliberate opt-out (e.g. a role selector that must NOT
|
|
518
537
|
// filter by accessible name); it must not be conflated with "unset". The role template omits
|
|
519
538
|
// `{ name }` when this resolves to a falsy value.
|
|
520
|
-
|
|
539
|
+
// A list-member role gets NO defaulted name — see LIST_MEMBER_ROLES. `type` is the entry's
|
|
540
|
+
// declared kind and `rawValue` the role, so the check reads the entry, not a resolved shape.
|
|
541
|
+
const defaultsNameFromLabel = !(type === 'role' && LIST_MEMBER_ROLES.has(String(rawValue)));
|
|
542
|
+
const rawName = entry.name === undefined
|
|
543
|
+
? (defaultsNameFromLabel ? originalLabel : '')
|
|
544
|
+
: (entry.name ?? '');
|
|
521
545
|
const value = SelectorResolver.resolveTemplateVars(rawValue);
|
|
522
546
|
const name = SelectorResolver.resolveTemplateVars(rawName);
|
|
523
547
|
const nth = entry.nth !== undefined && entry.nth !== null ? entry.nth : undefined;
|
package/src/harness/audit.ts
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import * as path from 'path';
|
|
10
10
|
import * as fs from 'fs';
|
|
11
|
-
import {
|
|
11
|
+
import { loadUnitScenarios, readUnitFeatureText, parseViewpointOverview, ScenarioInfo, ViewpointEntry } from './parse';
|
|
12
12
|
import { checkViewpointBaseline, ViewpointBaseline } from './viewpoint-baseline';
|
|
13
|
-
import {
|
|
13
|
+
import { featureFilesFor } from './unit-paths';
|
|
14
14
|
import {
|
|
15
15
|
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, depthThresholdFor, coverageBalance, duplicateClusters, traceability, claimProof, taxonomyLint,
|
|
16
16
|
automatableManual, flowCoveredThemes, flowRegressionDepth, oracleStrength,
|
|
@@ -117,9 +117,10 @@ export function seniorBandedOverall(
|
|
|
117
117
|
}
|
|
118
118
|
|
|
119
119
|
export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
|
|
120
|
+
// A unit is a DIRECTORY of features — `add --feature` puts sub-features beside the canonical
|
|
121
|
+
// one, and they all count. `featureFiles[0]` is that canonical `<basename>.feature` (an api
|
|
122
|
+
// flow `flows/<flow>` lives at `<dir>/features/<flow>.feature`, not `features/flows/…`).
|
|
123
|
+
const featureFiles = featureFilesFor(screenDir, screenName);
|
|
123
124
|
const viewpointPath = path.join(screenDir, 'requirements', 'test-viewpoint.md');
|
|
124
125
|
// Catalog-resolution id (for the @api/@query gate sensors): the unit's path relative to qa/ —
|
|
125
126
|
// `flows/<flow>`, `api/<area>`, `api/flows/<flow>`, else the bare screen. A bare screen matches
|
|
@@ -127,9 +128,9 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
127
128
|
const catalogScreenName = catalogIdFromScreenDir(screenDir);
|
|
128
129
|
|
|
129
130
|
const specPath = path.join(screenDir, 'requirements', 'spec.md');
|
|
130
|
-
const featureText =
|
|
131
|
+
const featureText = readUnitFeatureText(screenDir, screenName);
|
|
131
132
|
|
|
132
|
-
const scenarios: ScenarioInfo[] =
|
|
133
|
+
const scenarios: ScenarioInfo[] = loadUnitScenarios(screenDir, screenName);
|
|
133
134
|
const viewpoints: ViewpointEntry[] = parseViewpointOverview(viewpointPath);
|
|
134
135
|
// AO-1 — capability-routed scoring: the viewpoint catalog + score-bearing gate are owned by the
|
|
135
136
|
// unit's capability, resolved from the unit id (`api/<area>` → `api`; screen/flow → the default
|
|
@@ -390,14 +391,14 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
390
391
|
}
|
|
391
392
|
// TQ-9 — manual reason-code mismatch: an explicit @manual:Mx that disagrees with the reason
|
|
392
393
|
// text makes the capability planner recommend the wrong driver. Surface it so the code is fixed.
|
|
393
|
-
for (const mm of manualReasonMismatches
|
|
394
|
+
for (const mm of featureFiles.flatMap(manualReasonMismatches).slice(0, 8)) {
|
|
394
395
|
const label = MANUAL_REASONS[mm.inferred]?.label ?? (mm.inferred === 'XS' ? 'Cross-screen (→ flow)' : mm.inferred);
|
|
395
396
|
findings.push(`MANUAL-REASON-MISMATCH: "${mm.scenario}" is tagged @manual:${mm.explicit} but its reason reads as ${mm.inferred} (${label}) → fix the code so \`sungen capability plan\` recommends the right driver to automate it.`);
|
|
396
397
|
}
|
|
397
398
|
// TQ-10 — surface the Capability Planner recommendation (recommend-only; never installs). Silenced
|
|
398
399
|
// by `capability_suggestions: off` in qa/context.md. Reuses the planner (trustworthy after TQ-9).
|
|
399
400
|
if (intent.capabilitySuggestions) {
|
|
400
|
-
const plan = buildPlan(screenDir,
|
|
401
|
+
const plan = buildPlan(screenDir, screenName);
|
|
401
402
|
if (plan.recommendations.length) {
|
|
402
403
|
const recs = plan.recommendations.map((r) => `\`sungen capability add ${r.driver}\` (automates ${r.count})`).join(' · ');
|
|
403
404
|
findings.push(`CAPABILITY-SUGGESTION: ${plan.capabilityManual} @manual scenario(s) are capability-manual (a driver could automate them) — ${recs}. Recommend-only: nothing is installed automatically; the ${plan.judgmentManual} judgment-manual (M6/M8/M9) correctly stay manual.`);
|
|
@@ -10,6 +10,7 @@ import * as fs from 'fs';
|
|
|
10
10
|
import * as path from 'path';
|
|
11
11
|
import { loadDriverCatalog } from './capability';
|
|
12
12
|
import { readTextFile } from './read-text';
|
|
13
|
+
import { featureFilesFor } from './unit-paths';
|
|
13
14
|
|
|
14
15
|
export type ReasonClass = 'capability' | 'keep' | 'flow';
|
|
15
16
|
export interface ReasonDef { code: string; label: string; cls: ReasonClass; drivers: string[] }
|
|
@@ -193,8 +194,7 @@ export interface CapabilityPlan {
|
|
|
193
194
|
}
|
|
194
195
|
|
|
195
196
|
export function buildPlan(screenDir: string, screenName: string): CapabilityPlan {
|
|
196
|
-
const
|
|
197
|
-
const scenarios = parseScenarios(featurePath);
|
|
197
|
+
const scenarios = featureFilesFor(screenDir, screenName).flatMap(parseScenarios);
|
|
198
198
|
const catalog = loadDriverCatalog();
|
|
199
199
|
|
|
200
200
|
const modes: Record<string, number> = {};
|
package/src/harness/challenge.ts
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* The AI agent layer adds the semantic + novelty judgement on top of this spine.
|
|
19
19
|
*/
|
|
20
20
|
import * as path from 'path';
|
|
21
|
-
import {
|
|
21
|
+
import { loadUnitScenarios, ScenarioInfo } from './parse';
|
|
22
22
|
import { runAudit } from './audit';
|
|
23
23
|
|
|
24
24
|
export interface ChallengeFinding {
|
|
@@ -63,8 +63,7 @@ const NOVELTY_LENSES = [
|
|
|
63
63
|
];
|
|
64
64
|
|
|
65
65
|
export function buildChallenge(screenDir: string, screenName: string): ChallengeReport {
|
|
66
|
-
const
|
|
67
|
-
const scenarios: ScenarioInfo[] = loadScenarios(featurePath);
|
|
66
|
+
const scenarios: ScenarioInfo[] = loadUnitScenarios(screenDir, screenName);
|
|
68
67
|
const audit = runAudit(screenDir, screenName);
|
|
69
68
|
|
|
70
69
|
// 1. Coverage critic — over-covered buckets (secondary >> business-core) + shallow themes.
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import * as fs from 'fs';
|
|
10
10
|
import * as path from 'path';
|
|
11
|
+
import { featureFilesFor } from './unit-paths';
|
|
11
12
|
import { parse as parseYaml } from 'yaml';
|
|
12
13
|
import { GherkinParser, ParsedScenario } from '../generators/gherkin-parser';
|
|
13
14
|
import { resolveQuery, lintCatalog } from './query-catalog';
|
|
@@ -44,15 +45,18 @@ export function lintDataDriven(screenDir: string, cwd: string = process.cwd()):
|
|
|
44
45
|
const base = path.basename(screenDir);
|
|
45
46
|
const isFlow = path.basename(path.dirname(screenDir)) === 'flows';
|
|
46
47
|
const screenName = isFlow ? `flows/${base}` : base;
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
48
|
+
// Every `.feature` of the unit — `add --feature` sub-features lint too. A file that
|
|
49
|
+
// fails to parse contributes nothing rather than voiding the whole unit's lint.
|
|
50
|
+
const scenarios: ParsedScenario[] = featureFilesFor(screenDir, base)
|
|
51
|
+
.filter((p) => fs.existsSync(p))
|
|
52
|
+
.flatMap((p) => {
|
|
53
|
+
try {
|
|
54
|
+
return new GherkinParser().parseFeatureFile(p).scenarios || [];
|
|
55
|
+
} catch {
|
|
56
|
+
return [];
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
if (scenarios.length === 0) return [];
|
|
56
60
|
|
|
57
61
|
const tdPath = path.join(screenDir, 'test-data', `${base}.yaml`);
|
|
58
62
|
const td: Record<string, any> = tdPath && fs.existsSync(tdPath) ? parseYaml(fs.readFileSync(tdPath, 'utf8')) || {} : {};
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import * as fs from 'fs';
|
|
15
15
|
import * as path from 'path';
|
|
16
|
-
import {
|
|
16
|
+
import { loadUnitScenarios, parseViewpointOverview, ScenarioInfo, ViewpointEntry } from './parse';
|
|
17
17
|
import { loadCatalog, viewpointGate, assertionDepth, dataThemesFor, CatalogTheme } from './sensors';
|
|
18
18
|
|
|
19
19
|
export type DepthAction = 'deepen' | 'defer';
|
|
@@ -50,11 +50,9 @@ function matchTheme(s: ScenarioInfo, dataThemes: CatalogTheme[]): CatalogTheme |
|
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
export function runDepthLint(screenDir: string, screenName: string, focus = 'functional'): DepthLintReport {
|
|
53
|
-
const last = screenName.split('/').pop() || screenName;
|
|
54
|
-
const featurePath = path.join(screenDir, 'features', `${last}.feature`);
|
|
55
53
|
const viewpointPath = path.join(screenDir, 'requirements', 'test-viewpoint.md');
|
|
56
54
|
|
|
57
|
-
const scenarios: ScenarioInfo[] =
|
|
55
|
+
const scenarios: ScenarioInfo[] = loadUnitScenarios(screenDir, screenName);
|
|
58
56
|
const viewpoints: ViewpointEntry[] = parseViewpointOverview(viewpointPath);
|
|
59
57
|
const catalog = loadCatalog();
|
|
60
58
|
const gate = viewpointGate(scenarios, viewpoints, catalog);
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import * as fs from 'fs';
|
|
13
13
|
import * as path from 'path';
|
|
14
|
+
import { featureFilesFor } from './unit-paths';
|
|
14
15
|
import { loadScenarios } from './parse';
|
|
15
16
|
import { parseScenarios } from './capability-plan';
|
|
16
17
|
import { readTextFile } from './read-text';
|
|
@@ -29,8 +30,9 @@ export interface FlowCheckReport {
|
|
|
29
30
|
function listDirs(p: string): string[] {
|
|
30
31
|
return fs.existsSync(p) ? fs.readdirSync(p, { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name) : [];
|
|
31
32
|
}
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
/** Every `.feature` of the unit — a unit is a directory, so a sub-feature counts too. */
|
|
34
|
+
function featurePaths(base: string, kind: 'screens' | 'flows', name: string): string[] {
|
|
35
|
+
return featureFilesFor(path.join(base, 'qa', kind, name), name);
|
|
34
36
|
}
|
|
35
37
|
function featureTags(fp: string): string[] {
|
|
36
38
|
if (!fs.existsSync(fp)) return [];
|
|
@@ -57,7 +59,7 @@ export interface FlowScenario { flow: string; name: string; haystack: string; de
|
|
|
57
59
|
export function loadFlowScenarios(cwd: string): FlowScenario[] {
|
|
58
60
|
const out: FlowScenario[] = [];
|
|
59
61
|
for (const f of listDirs(path.join(cwd, 'qa', 'flows'))) {
|
|
60
|
-
for (const s of
|
|
62
|
+
for (const s of featurePaths(cwd, 'flows', f).flatMap(loadScenarios)) {
|
|
61
63
|
out.push({ flow: f, name: s.name, haystack: s.haystack, deep: s.hasDataAssertion });
|
|
62
64
|
}
|
|
63
65
|
}
|
|
@@ -74,7 +76,7 @@ export function buildFlowCheck(cwd: string, onlyFlow?: string): FlowCheckReport
|
|
|
74
76
|
// A. Deferral integrity (screens).
|
|
75
77
|
const deferrals: Deferral[] = [];
|
|
76
78
|
for (const sc of screens) {
|
|
77
|
-
for (const s of
|
|
79
|
+
for (const s of featurePaths(cwd, 'screens', sc).flatMap(parseScenarios)) {
|
|
78
80
|
// A deferral is the first-class `@deferred:flow` tag (H6) OR the legacy `@manual` + a
|
|
79
81
|
// "deferred to a flow" comment (back-compat). Either marks a cross-screen case owned by a flow.
|
|
80
82
|
const isDeferral = s.deferredToFlow || (s.manual && /deferred to a flow/i.test(s.reason));
|
|
@@ -98,9 +100,9 @@ export function buildFlowCheck(cwd: string, onlyFlow?: string): FlowCheckReport
|
|
|
98
100
|
// B. Run-test contract (per flow).
|
|
99
101
|
const contracts: Contract[] = [];
|
|
100
102
|
for (const f of flows) {
|
|
101
|
-
const
|
|
102
|
-
const parallel = featureTags
|
|
103
|
-
const scs = loadScenarios
|
|
103
|
+
const fps = featurePaths(cwd, 'flows', f);
|
|
104
|
+
const parallel = fps.flatMap(featureTags).some((t) => /^@parallel$/i.test(t));
|
|
105
|
+
const scs = fps.flatMap(loadScenarios);
|
|
104
106
|
let cartStateCount = 0;
|
|
105
107
|
for (const s of scs) {
|
|
106
108
|
const h = s.haystack;
|
package/src/harness/flow-plan.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import * as fs from 'fs';
|
|
11
11
|
import * as path from 'path';
|
|
12
|
+
import { featureFilesFor } from './unit-paths';
|
|
12
13
|
import { parse as parseYaml } from 'yaml';
|
|
13
14
|
import { MANUAL_REASONS, inferReasonCode } from './capability-plan';
|
|
14
15
|
import { buildFlowCheck, Contract } from './flow-check';
|
|
@@ -78,9 +79,9 @@ export interface FlowPlan {
|
|
|
78
79
|
}
|
|
79
80
|
|
|
80
81
|
export function buildFlowPlan(cwd: string, flow: string): FlowPlan {
|
|
81
|
-
const
|
|
82
|
-
const selectorsPath = path.join(
|
|
83
|
-
const scenarios =
|
|
82
|
+
const flowDir = path.join(cwd, 'qa', 'flows', flow);
|
|
83
|
+
const selectorsPath = path.join(flowDir, 'selectors', `${flow}.yaml`);
|
|
84
|
+
const scenarios = featureFilesFor(flowDir, flow).flatMap(parseFlowScenarios);
|
|
84
85
|
const keys = selectorKeys(selectorsPath);
|
|
85
86
|
|
|
86
87
|
// Legs = distinct screen namespaces.
|
package/src/harness/manifest.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import * as fs from 'fs';
|
|
12
12
|
import * as path from 'path';
|
|
13
13
|
import { createHash } from 'crypto';
|
|
14
|
-
import {
|
|
14
|
+
import { featureFilesFor, reportSlug } from './unit-paths';
|
|
15
15
|
import { readTextFile } from './read-text';
|
|
16
16
|
|
|
17
17
|
export interface SpecSection { name: string; hash: string }
|
|
@@ -99,13 +99,13 @@ function parseFeatureSections(featurePath: string): { scenario: string; vpCode?:
|
|
|
99
99
|
|
|
100
100
|
export function buildManifest(screenDir: string, screenName: string): Manifest {
|
|
101
101
|
const specPath = path.join(screenDir, 'requirements', 'spec.md');
|
|
102
|
-
const
|
|
102
|
+
const featureFiles = featureFilesFor(screenDir, screenName);
|
|
103
103
|
const specSections = parseSpecSections(specPath);
|
|
104
104
|
const specMap: Record<string, string> = {};
|
|
105
105
|
for (const s of specSections) specMap[s.name] = s.hash;
|
|
106
106
|
|
|
107
107
|
const specNames = Object.keys(specMap);
|
|
108
|
-
const featScenarios = parseFeatureSections
|
|
108
|
+
const featScenarios = featureFiles.flatMap(parseFeatureSections);
|
|
109
109
|
const entries: ManifestEntry[] = featScenarios.map((f) => {
|
|
110
110
|
const matched = matchSection(f.section, specNames);
|
|
111
111
|
return {
|
package/src/harness/parse.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import * as fs from 'fs';
|
|
9
9
|
import { GherkinParser, ParsedScenario, ParsedStep } from '../generators/gherkin-parser';
|
|
10
10
|
import { readTextFile } from './read-text';
|
|
11
|
+
import { featureFilesFor } from './unit-paths';
|
|
11
12
|
|
|
12
13
|
export type Priority = 'high' | 'normal' | 'low' | 'unknown';
|
|
13
14
|
|
|
@@ -175,7 +176,12 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
175
176
|
|
|
176
177
|
if (last === 'Then') {
|
|
177
178
|
thenCount++;
|
|
178
|
-
|
|
179
|
+
// Storage assertions count as data-bearing: 4 of the 5 shapes carry no {{ref}}
|
|
180
|
+
// (`exists`, `does not exist`, `is not empty`, `key matching "…"`), yet they
|
|
181
|
+
// assert persisted state — without this they'd be scored `shallow`, penalizing
|
|
182
|
+
// exactly the checks the review encourages. `tab order` is a STRUCTURAL
|
|
183
|
+
// assertion (a data table of focus stops) with the same no-{{ref}} property.
|
|
184
|
+
if (/\{\{|contains|match data|toHaveText|in (local|session) storage|tab order/i.test(step.text)) hasData = true;
|
|
179
185
|
}
|
|
180
186
|
}
|
|
181
187
|
|
|
@@ -211,3 +217,24 @@ export function loadScenarios(featurePath: string): ScenarioInfo[] {
|
|
|
211
217
|
.filter((s) => !s.stepsName && !s.hookType) // skip @steps/@hook blocks
|
|
212
218
|
.map(classifyScenario);
|
|
213
219
|
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Scenarios of a WHOLE unit — every `.feature` in its features/ directory, not just the
|
|
223
|
+
* canonical one. Sub-features created by `add --feature` used to be invisible to every
|
|
224
|
+
* sensor; the score they produced was a subset presented as the whole.
|
|
225
|
+
*/
|
|
226
|
+
export function loadUnitScenarios(unitDir: string, unit: string): ScenarioInfo[] {
|
|
227
|
+
return featureFilesFor(unitDir, unit).flatMap(loadScenarios);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Raw Gherkin of a WHOLE unit, concatenated in `featureFilesFor` order — the input for the
|
|
232
|
+
* text-level sensors (isolation risk, serial cascade, manual oracle, …). Missing files
|
|
233
|
+
* contribute nothing, so a unit with no features yields '' exactly as before.
|
|
234
|
+
*/
|
|
235
|
+
export function readUnitFeatureText(unitDir: string, unit: string): string {
|
|
236
|
+
return featureFilesFor(unitDir, unit)
|
|
237
|
+
.filter((p) => fs.existsSync(p))
|
|
238
|
+
.map((p) => readTextFile(p))
|
|
239
|
+
.join('\n');
|
|
240
|
+
}
|
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
import * as fs from 'fs';
|
|
16
16
|
import * as path from 'path';
|
|
17
17
|
import * as os from 'os';
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
18
|
+
import { loadUnitScenarios, ScenarioInfo } from './parse';
|
|
19
|
+
import { featureFilesFor } from './unit-paths';
|
|
20
20
|
import { readTextFile } from './read-text';
|
|
21
21
|
|
|
22
22
|
export interface ScriptCheckResult {
|
|
@@ -57,7 +57,15 @@ export function extractTestBlocks(specSrc: string): { title: string; body: strin
|
|
|
57
57
|
return blocks;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
// A line counts as executable code when it interacts with the page/assertions/data,
|
|
61
|
+
// assigns, OR is a standalone call expression (`mock.setPassthrough(page);`,
|
|
62
|
+
// `driver.enable(...)`). The call branch is what keeps capability helpers from
|
|
63
|
+
// reading as hollow: a sync void call after its step comment matched none of the
|
|
64
|
+
// original alternatives (no await/expect/testData, `page` only as a bare argument,
|
|
65
|
+
// no `=`), so @mock:passthrough scenarios were flagged BYPASS (field report) — on
|
|
66
|
+
// sungen's own mock-route golden fixture, no less. Comment lines are step markers
|
|
67
|
+
// (never inside a scanned region), so `identifier(` cannot false-match non-code.
|
|
68
|
+
const CODE_LINE = /\b(await|expect|testData|page\.)\b|=\s*[^=]|[\w$]\s*\(/;
|
|
61
69
|
|
|
62
70
|
/**
|
|
63
71
|
* Faithfulness / anti-bypass: a non-@manual test that performs actions but proves
|
|
@@ -120,7 +128,12 @@ function specSubdir(dir: string, name: string, kind: UnitKind): string {
|
|
|
120
128
|
return kind === 'flow' ? path.join(dir, 'flows', name) : kind === 'api' ? path.join(dir, 'api', name) : path.join(dir, name);
|
|
121
129
|
}
|
|
122
130
|
|
|
123
|
-
|
|
131
|
+
/**
|
|
132
|
+
* EVERY compiled spec of the unit. A unit compiles to one `.spec.ts` per `.feature`, so
|
|
133
|
+
* taking only the first hit checked a multi-feature screen against a fraction of its own
|
|
134
|
+
* compiled output — every sub-feature scenario then read as MISSING in spec.
|
|
135
|
+
*/
|
|
136
|
+
function findSpecs(dir: string, name: string, kind: UnitKind): string[] {
|
|
124
137
|
// Screens compile to <dir>/<name>/<feature>.spec.ts
|
|
125
138
|
// Flows compile to <dir>/flows/<name>/<feature>.spec.ts
|
|
126
139
|
// Api compile to <dir>/api/<name>/<feature>.spec.ts
|
|
@@ -137,37 +150,45 @@ function findSpec(dir: string, name: string, kind: UnitKind): string | null {
|
|
|
137
150
|
}
|
|
138
151
|
};
|
|
139
152
|
const scoped = specSubdir(dir, name, kind);
|
|
140
|
-
if (!fs.existsSync(scoped)) return
|
|
153
|
+
if (!fs.existsSync(scoped)) return []; // no spec for this target (e.g. not compiled yet)
|
|
141
154
|
walk(scoped);
|
|
142
|
-
return hits
|
|
155
|
+
return hits.sort((a, b) => a.localeCompare(b, 'en'));
|
|
143
156
|
}
|
|
144
157
|
|
|
145
158
|
export async function runScriptCheck(screenDir: string, screenName: string, kind: UnitKind): Promise<ScriptCheckResult> {
|
|
146
|
-
const
|
|
147
|
-
const scenarios =
|
|
159
|
+
const featureFiles = featureFilesFor(screenDir, screenName);
|
|
160
|
+
const scenarios = loadUnitScenarios(screenDir, screenName);
|
|
148
161
|
const automated = scenarios.filter((s) => !s.manual);
|
|
149
162
|
const manual = scenarios.filter((s) => s.manual);
|
|
150
163
|
|
|
151
|
-
const
|
|
164
|
+
const committedSpecs = findSpecs(path.join(process.cwd(), 'specs', 'generated'), screenName, kind);
|
|
165
|
+
// Representative path for the report; the checks below read ALL of them.
|
|
166
|
+
const committedSpec = committedSpecs[0] ?? null;
|
|
152
167
|
|
|
153
168
|
const findings: string[] = [];
|
|
154
169
|
let specTitles: string[] = [];
|
|
155
170
|
let specSrc = '';
|
|
156
171
|
if (committedSpec) {
|
|
157
|
-
specSrc = readTextFile(
|
|
158
|
-
specTitles = extractTestTitles(
|
|
172
|
+
specSrc = committedSpecs.map((p) => readTextFile(p)).join('\n');
|
|
173
|
+
specTitles = committedSpecs.flatMap((p) => extractTestTitles(readTextFile(p)));
|
|
159
174
|
} else {
|
|
160
175
|
findings.push('No generated spec found under specs/generated/ — run `sungen generate` / `/sungen:run-test` first.');
|
|
161
176
|
}
|
|
162
177
|
|
|
163
178
|
// A. Structural 1:1
|
|
164
|
-
// A @cases scenario emits ONE source test() inside a per-row loop
|
|
165
|
-
//
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
179
|
+
// A @cases scenario emits ONE source test() inside a per-row loop. Its title
|
|
180
|
+
// appears in the spec as `'<name> — ' + __row.__label` (CONCAT — what
|
|
181
|
+
// scenario.hbs emits today; the extracted literal is `<name> —`) or as the
|
|
182
|
+
// legacy template form `<name> — ${__row.__label}`. This check used to expect
|
|
183
|
+
// ONLY the template form, so on the current codegen every @cases scenario was
|
|
184
|
+
// reported MISSING (+ its test EXTRA) despite compiling and running fine.
|
|
185
|
+
// Canonicalize the extracted title to the bare scenario name so both shapes
|
|
186
|
+
// (and plain titles) compare on the same footing.
|
|
187
|
+
const canonSpecTitle = (t: string) => t.replace(/\s*—(\s*\$\{__row\.__label\})?$/, '');
|
|
188
|
+
const specTitleSet = new Set(specTitles.map(canonSpecTitle));
|
|
189
|
+
const expectedSet = new Set(automated.map((s) => s.name));
|
|
190
|
+
const missingInSpec = automated.filter((s) => !specTitleSet.has(s.name)).map((s) => s.name);
|
|
191
|
+
const extraInSpec = specTitles.filter((t) => !expectedSet.has(canonSpecTitle(t)));
|
|
171
192
|
const countMatch = committedSpec ? automated.length === specTitles.length : false;
|
|
172
193
|
if (committedSpec && !countMatch) {
|
|
173
194
|
findings.push(`Count mismatch: ${automated.length} automated scenarios vs ${specTitles.length} test() blocks.`);
|
|
@@ -192,25 +213,32 @@ export async function runScriptCheck(screenDir: string, screenName: string, kind
|
|
|
192
213
|
const gen = kind === 'api'
|
|
193
214
|
? new CodeGenerator({ framework, runtimeData: true })
|
|
194
215
|
: new CodeGenerator({ framework, screenName, runtimeData: true, flowMode: kind === 'flow' });
|
|
195
|
-
await gen.generateAllTests(qaSourceDir, tmp,
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
216
|
+
await gen.generateAllTests(qaSourceDir, tmp, featureFiles);
|
|
217
|
+
// Pair each committed spec with its regenerated twin by filename — one `.feature`
|
|
218
|
+
// compiles to one `.spec.ts`, so a multi-feature unit has several of each.
|
|
219
|
+
const freshByName = new Map(findSpecs(tmp, screenName, kind).map((p) => [path.basename(p), p]));
|
|
220
|
+
const multi = committedSpecs.length > 1;
|
|
221
|
+
for (const committed of committedSpecs) {
|
|
222
|
+
const fresh = freshByName.get(path.basename(committed));
|
|
223
|
+
if (!fresh) continue;
|
|
224
|
+
const a = normalize(readTextFile(committed));
|
|
199
225
|
const b = normalize(readTextFile(fresh));
|
|
200
|
-
if (a
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
226
|
+
if (a === b) continue;
|
|
227
|
+
drift = 'drift';
|
|
228
|
+
// collect a few differing lines
|
|
229
|
+
const al = a.split('\n'), bl = b.split('\n');
|
|
230
|
+
const max = Math.max(al.length, bl.length);
|
|
231
|
+
const where = multi ? `${path.basename(committed)} ` : '';
|
|
232
|
+
for (let i = 0, shown = 0; i < max && shown < 6 && driftHunks.length < 6; i++) {
|
|
233
|
+
if (al[i] !== bl[i]) {
|
|
234
|
+
driftHunks.push(` ${where}L${i + 1}\n committed: ${(al[i] ?? '∅').trim().slice(0, 100)}\n expected : ${(bl[i] ?? '∅').trim().slice(0, 100)}`);
|
|
235
|
+
shown++;
|
|
210
236
|
}
|
|
211
|
-
findings.push('DRIFT: committed spec differs from a fresh regenerate → spec was hand-edited or the .feature changed without `sungen generate`. The test code no longer reflects the Gherkin.');
|
|
212
237
|
}
|
|
213
238
|
}
|
|
239
|
+
if (drift === 'drift') {
|
|
240
|
+
findings.push('DRIFT: committed spec differs from a fresh regenerate → spec was hand-edited or the .feature changed without `sungen generate`. The test code no longer reflects the Gherkin.');
|
|
241
|
+
}
|
|
214
242
|
fs.rmSync(tmp, { recursive: true, force: true });
|
|
215
243
|
} catch (e) {
|
|
216
244
|
findings.push(`Drift check skipped (regenerate failed): ${e instanceof Error ? e.message : e}`);
|
|
@@ -27,7 +27,18 @@ function scanText(text: string, file: string): SecretHit[] {
|
|
|
27
27
|
}
|
|
28
28
|
const m = raw.match(SECRET_KEY);
|
|
29
29
|
if (m) {
|
|
30
|
-
|
|
30
|
+
// Take only the FIRST scalar after the key. The old `(.+)$` capture swallowed
|
|
31
|
+
// the rest of the line, so in an inline-map @cases row
|
|
32
|
+
// - { …, password: "${QA_X}", expected_status: 201 }
|
|
33
|
+
// the "value" included `, expected_status: 201 }` — whose digits + length
|
|
34
|
+
// pushed EVERY such row over the threshold regardless of the actual value
|
|
35
|
+
// (field report: ${QA_*} refs and a deliberate wrong_password all warned).
|
|
36
|
+
const tail = m[2].trim();
|
|
37
|
+
const quoted = tail.match(/^(["'])((?:\\.|(?!\1).)*)\1/);
|
|
38
|
+
const val = (quoted ? quoted[2] : tail.split(/[,}\s]/, 1)[0]).trim();
|
|
39
|
+
// A ${VAR} env ref IS the recommended practice (.env.qa overlays) — never
|
|
40
|
+
// warn on it, or the gate teaches users the opposite of the convention.
|
|
41
|
+
if (/^\$\{[A-Za-z_][A-Za-z0-9_]*\}$/.test(val)) return;
|
|
31
42
|
// Long, high-entropy-ish, not an obvious placeholder/test value.
|
|
32
43
|
if (val.length >= 20 && !PLACEHOLDERish.test(raw) && /[A-Za-z]/.test(val) && /[0-9]/.test(val)) {
|
|
33
44
|
hits.push({ file, line, reason: `secret-named key "${m[1]}" with a long literal value` });
|
|
@@ -37,15 +48,19 @@ function scanText(text: string, file: string): SecretHit[] {
|
|
|
37
48
|
return hits;
|
|
38
49
|
}
|
|
39
50
|
|
|
40
|
-
/** Scan a
|
|
51
|
+
/** Scan a unit dir's committed yaml sinks for likely real secrets: test-data/*.yaml,
|
|
52
|
+
* mock/*.yaml (mock response bodies are a credential magnet — pasted DevTools
|
|
53
|
+
* responses carry real tokens), and api/*.yaml (endpoint catalogs carry headers). */
|
|
41
54
|
export function scanTestDataSecrets(baseDir: string): SecretHit[] {
|
|
42
|
-
const tdDir = path.join(baseDir, 'test-data');
|
|
43
|
-
if (!fs.existsSync(tdDir)) return [];
|
|
44
55
|
const hits: SecretHit[] = [];
|
|
45
|
-
for (const
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
56
|
+
for (const sub of ['test-data', 'mock', 'api']) {
|
|
57
|
+
const dir = path.join(baseDir, sub);
|
|
58
|
+
if (!fs.existsSync(dir)) continue;
|
|
59
|
+
for (const f of fs.readdirSync(dir)) {
|
|
60
|
+
if (!/\.ya?ml$/i.test(f)) continue;
|
|
61
|
+
const p = path.join(dir, f);
|
|
62
|
+
try { hits.push(...scanText(fs.readFileSync(p, 'utf-8'), path.join(sub, f))); } catch { /* ignore */ }
|
|
63
|
+
}
|
|
49
64
|
}
|
|
50
65
|
return hits;
|
|
51
66
|
}
|
package/src/harness/sensors.ts
CHANGED
|
@@ -279,8 +279,12 @@ export interface AutomatableManualResult {
|
|
|
279
279
|
|
|
280
280
|
// Genuine-judgment markers (M6/M8/M9 territory): visual/responsive/a11y/mock/network/
|
|
281
281
|
// external/empty-state — these legitimately stay @manual (or need a future driver).
|
|
282
|
+
// `focus order` is no longer a judgment marker — `Then User see tab order:` automates
|
|
283
|
+
// it deterministically. `keyboard` is NARROWED (keyboard-only navigation / focus traps
|
|
284
|
+
// still need a human or AT), not removed: a scenario merely mentioning the keyboard is
|
|
285
|
+
// automatable, a keyboard-ONLY audit is not.
|
|
282
286
|
const JUDGMENT_MARKER =
|
|
283
|
-
/\b(visual|responsive|layout|breakpoint|mobile|tablet|viewport|accessib|a11y|keyboard|screen reader
|
|
287
|
+
/\b(visual|responsive|layout|breakpoint|mobile|tablet|viewport|accessib|a11y|keyboard[- ]only|keyboard trap|screen reader|\baria\b|empty[- ]?(state|product|list|category|cart)|no[- ]?result|missing (image|product|data)|placeholder|fallback|slow|failing|offline|network|loading|spinner|external|new tab|video tutorial|email|mailbox|download|payment gateway|exploratory|not worth)\b/;
|
|
284
288
|
|
|
285
289
|
/**
|
|
286
290
|
* Automatable-@manual (TQ-2) — a `@manual` scenario whose steps are fully DSL-expressible
|
|
@@ -637,7 +641,10 @@ const CLAIM_RULES: ClaimRule[] = [
|
|
|
637
641
|
{
|
|
638
642
|
claim: 'independent/separate',
|
|
639
643
|
title: /\b(independent|separate|isolat(ed|es)|per[- ]tab|two tabs|each tab)\b/,
|
|
640
|
-
|
|
644
|
+
// A storage `does not exist` check on the OTHER tenant's key proves isolation
|
|
645
|
+
// ("the other shop's token is not exposed"). Only the absence shape counts — a
|
|
646
|
+
// positive `exists` check looks the same whether or not the claim holds.
|
|
647
|
+
proof: /\bcontext\b|tab a|tab b|second (tab|context)|does not exist/,
|
|
641
648
|
need: 'a multi-context proof (tab A vs tab B)',
|
|
642
649
|
hint: 'independence across tabs/contexts is rarely DSL-expressible — mark @manual with a clear setup/action/oracle.',
|
|
643
650
|
severity: 'warn',
|
|
@@ -653,7 +660,7 @@ const CLAIM_RULES: ClaimRule[] = [
|
|
|
653
660
|
{
|
|
654
661
|
claim: 'announces/aria',
|
|
655
662
|
title: /\b(announce[sd]?|aria|screen[- ]reader|programmatically associated)\b/,
|
|
656
|
-
proof: /aria|role|@manual|describedby|is focused/,
|
|
663
|
+
proof: /aria|role|@manual|describedby|is focused|tab order/,
|
|
657
664
|
need: 'an aria/role assertion (or @manual with a screen-reader oracle)',
|
|
658
665
|
hint: 'ARIA announcement is usually not DSL-expressible — assert aria attributes if possible, else @manual with an NVDA/VoiceOver oracle.',
|
|
659
666
|
severity: 'warn',
|
|
@@ -668,7 +675,11 @@ const CLAIM_RULES: ClaimRule[] = [
|
|
|
668
675
|
// authoritative recall layer for the residue this can't judge structurally.
|
|
669
676
|
claim: 'negative-claim/absence',
|
|
670
677
|
title: /(\bno\b|\bnot\b|n['’]t\b|\bnever\b|\bwithout\b|\bcannot\b|prevent|block|avoid|reject|disabl|\bdeny\b|denies|\bkhông\b|\bchưa\b)/i,
|
|
671
|
-
|
|
678
|
+
// `does not exist` covers the storage-absence shape — the broad "any storage
|
|
679
|
+
// assertion" alternative would let a positive `exists` silence a negative claim.
|
|
680
|
+
// `tab order` proves "focus never lands on X"-style claims (the table enumerates
|
|
681
|
+
// exactly where focus goes).
|
|
682
|
+
proof: /is hidden|are hidden|is empty|no longer|not complete|disabl|invalid|rejected|\berror\b|\bcount\b|row with \{\{|table with|\bremember\b|\bexactly\b|tohavecount|does not exist|tab order/i,
|
|
672
683
|
need: 'a proof of the ABSENCE — a contrast/empty/hidden/error/count assertion, or @manual with an oracle',
|
|
673
684
|
hint: 'a negative claim ("no / not / không …") is not proven by a positive `see [X]` that looks the same whether or not the claim holds. Assert the contrast (state hidden/empty, error shown, count unchanged), or mark @manual.',
|
|
674
685
|
severity: 'warn',
|