@sun-asterisk/sungen 3.1.2-beta.99 → 3.2.0-beta.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capabilities/context.d.ts +17 -0
- package/dist/capabilities/context.d.ts.map +1 -1
- package/dist/capabilities/discover.d.ts.map +1 -1
- package/dist/capabilities/discover.js +70 -9
- package/dist/capabilities/discover.js.map +1 -1
- package/dist/capabilities/registry.d.ts +3 -1
- package/dist/capabilities/registry.d.ts.map +1 -1
- package/dist/capabilities/registry.js.map +1 -1
- package/dist/capabilities/sensor.d.ts +3 -0
- package/dist/capabilities/sensor.d.ts.map +1 -1
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +17 -11
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/capability.d.ts.map +1 -1
- package/dist/cli/commands/capability.js +57 -5
- package/dist/cli/commands/capability.js.map +1 -1
- package/dist/cli/commands/context.d.ts +9 -0
- package/dist/cli/commands/context.d.ts.map +1 -0
- package/dist/cli/commands/context.js +91 -0
- package/dist/cli/commands/context.js.map +1 -0
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +42 -30
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/generate.d.ts.map +1 -1
- package/dist/cli/commands/generate.js +28 -5
- package/dist/cli/commands/generate.js.map +1 -1
- package/dist/cli/commands/ledger.d.ts.map +1 -1
- package/dist/cli/commands/ledger.js +15 -5
- package/dist/cli/commands/ledger.js.map +1 -1
- package/dist/cli/commands/manifest.d.ts.map +1 -1
- package/dist/cli/commands/manifest.js +10 -9
- package/dist/cli/commands/manifest.js.map +1 -1
- package/dist/cli/commands/repair.d.ts +8 -0
- package/dist/cli/commands/repair.d.ts.map +1 -0
- package/dist/cli/commands/repair.js +97 -0
- package/dist/cli/commands/repair.js.map +1 -0
- package/dist/cli/commands/script-check.d.ts.map +1 -1
- package/dist/cli/commands/script-check.js +13 -9
- package/dist/cli/commands/script-check.js.map +1 -1
- package/dist/cli/commands/trace.d.ts.map +1 -1
- package/dist/cli/commands/trace.js +7 -4
- package/dist/cli/commands/trace.js.map +1 -1
- package/dist/cli/index.js +4 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/generators/test-generator/code-generator.d.ts +7 -0
- package/dist/generators/test-generator/code-generator.d.ts.map +1 -1
- package/dist/generators/test-generator/code-generator.js +112 -42
- package/dist/generators/test-generator/code-generator.js.map +1 -1
- package/dist/harness/annotation-overrides.d.ts +3 -1
- package/dist/harness/annotation-overrides.d.ts.map +1 -1
- package/dist/harness/annotation-overrides.js +3 -1
- package/dist/harness/annotation-overrides.js.map +1 -1
- package/dist/harness/audit.d.ts +9 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +114 -12
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/capability-plan.d.ts +14 -0
- package/dist/harness/capability-plan.d.ts.map +1 -1
- package/dist/harness/capability-plan.js +63 -1
- package/dist/harness/capability-plan.js.map +1 -1
- package/dist/harness/data-driven-lint.d.ts.map +1 -1
- package/dist/harness/data-driven-lint.js +23 -0
- package/dist/harness/data-driven-lint.js.map +1 -1
- package/dist/harness/flow-check.d.ts +9 -0
- package/dist/harness/flow-check.d.ts.map +1 -1
- package/dist/harness/flow-check.js +13 -6
- package/dist/harness/flow-check.js.map +1 -1
- package/dist/harness/intent.d.ts +6 -0
- package/dist/harness/intent.d.ts.map +1 -1
- package/dist/harness/intent.js +20 -4
- package/dist/harness/intent.js.map +1 -1
- package/dist/harness/ledger.d.ts.map +1 -1
- package/dist/harness/ledger.js +3 -2
- package/dist/harness/ledger.js.map +1 -1
- package/dist/harness/manifest.d.ts.map +1 -1
- package/dist/harness/manifest.js +3 -2
- package/dist/harness/manifest.js.map +1 -1
- package/dist/harness/parse.d.ts +1 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +3 -0
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/quality-gates.js +1 -1
- package/dist/harness/quality-gates.js.map +1 -1
- package/dist/harness/query-catalog.d.ts.map +1 -1
- package/dist/harness/query-catalog.js +0 -0
- package/dist/harness/query-catalog.js.map +1 -1
- package/dist/harness/repair.d.ts +20 -0
- package/dist/harness/repair.d.ts.map +1 -0
- package/dist/harness/repair.js +111 -0
- package/dist/harness/repair.js.map +1 -0
- package/dist/harness/script-check.d.ts +3 -1
- package/dist/harness/script-check.d.ts.map +1 -1
- package/dist/harness/script-check.js +22 -8
- package/dist/harness/script-check.js.map +1 -1
- package/dist/harness/sensors.d.ts +40 -0
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +54 -2
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/trace.d.ts.map +1 -1
- package/dist/harness/trace.js +4 -3
- package/dist/harness/trace.js.map +1 -1
- package/dist/harness/unit-paths.d.ts +3 -0
- package/dist/harness/unit-paths.d.ts.map +1 -0
- package/dist/harness/unit-paths.js +52 -0
- package/dist/harness/unit-paths.js.map +1 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/orchestrator/ai-rules-updater.d.ts.map +1 -1
- package/dist/orchestrator/ai-rules-updater.js +2 -0
- package/dist/orchestrator/ai-rules-updater.js.map +1 -1
- package/dist/orchestrator/context-discovery.d.ts +12 -0
- package/dist/orchestrator/context-discovery.d.ts.map +1 -0
- package/dist/orchestrator/context-discovery.js +46 -0
- package/dist/orchestrator/context-discovery.js.map +1 -0
- package/dist/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +7 -1
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-create-test.md +6 -2
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-run-test.md +18 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-api-design.md +62 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +2 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +16 -2
- package/dist/orchestrator/templates/ai-instructions/claude-skill-viewpoint.md +14 -0
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-create-test.md +6 -2
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-run-test.md +11 -1
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-api-design.md +62 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +2 -1
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +16 -2
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-viewpoint.md +14 -0
- package/dist/orchestrator/templates/specs-api.d.ts +38 -2
- package/dist/orchestrator/templates/specs-api.d.ts.map +1 -1
- package/dist/orchestrator/templates/specs-api.js +65 -22
- package/dist/orchestrator/templates/specs-api.js.map +1 -1
- package/dist/orchestrator/templates/specs-api.ts +71 -18
- package/dist/orchestrator/templates/specs-db.d.ts +3 -0
- package/dist/orchestrator/templates/specs-db.d.ts.map +1 -1
- package/dist/orchestrator/templates/specs-db.js +78 -1
- package/dist/orchestrator/templates/specs-db.js.map +1 -1
- package/dist/orchestrator/templates/specs-db.ts +78 -1
- package/dist/orchestrator/templates/specs-test-data.ts +2 -1
- package/package.json +2 -2
- package/src/capabilities/context.ts +19 -0
- package/src/capabilities/discover.ts +27 -7
- package/src/capabilities/registry.ts +3 -1
- package/src/capabilities/sensor.ts +1 -1
- package/src/cli/commands/audit.ts +15 -9
- package/src/cli/commands/capability.ts +53 -5
- package/src/cli/commands/context.ts +52 -0
- package/src/cli/commands/delivery.ts +40 -31
- package/src/cli/commands/generate.ts +30 -5
- package/src/cli/commands/ledger.ts +13 -5
- package/src/cli/commands/manifest.ts +9 -7
- package/src/cli/commands/repair.ts +57 -0
- package/src/cli/commands/script-check.ts +12 -8
- package/src/cli/commands/trace.ts +7 -4
- package/src/cli/index.ts +4 -0
- package/src/generators/test-generator/code-generator.ts +115 -40
- package/src/harness/annotation-overrides.ts +3 -1
- package/src/harness/audit.ts +115 -15
- package/src/harness/capability-plan.ts +51 -1
- package/src/harness/data-driven-lint.ts +20 -0
- package/src/harness/flow-check.ts +15 -6
- package/src/harness/intent.ts +25 -4
- package/src/harness/ledger.ts +3 -2
- package/src/harness/manifest.ts +3 -2
- package/src/harness/parse.ts +4 -0
- package/src/harness/quality-gates.ts +1 -1
- package/src/harness/query-catalog.ts +0 -0
- package/src/harness/repair.ts +75 -0
- package/src/harness/script-check.ts +25 -8
- package/src/harness/sensors.ts +71 -2
- package/src/harness/trace.ts +4 -3
- package/src/harness/unit-paths.ts +14 -0
- package/src/index.ts +4 -2
- package/src/orchestrator/ai-rules-updater.ts +2 -0
- package/src/orchestrator/context-discovery.ts +50 -0
- package/src/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +7 -1
- package/src/orchestrator/templates/ai-instructions/claude-cmd-create-test.md +6 -2
- package/src/orchestrator/templates/ai-instructions/claude-cmd-run-test.md +18 -1
- package/src/orchestrator/templates/ai-instructions/claude-skill-api-design.md +62 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +2 -1
- package/src/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +16 -2
- package/src/orchestrator/templates/ai-instructions/claude-skill-viewpoint.md +14 -0
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-create-test.md +6 -2
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-run-test.md +11 -1
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-api-design.md +62 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +2 -1
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +16 -2
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-viewpoint.md +14 -0
- package/src/orchestrator/templates/specs-api.ts +71 -18
- package/src/orchestrator/templates/specs-db.ts +78 -1
- package/src/orchestrator/templates/specs-test-data.ts +2 -1
|
@@ -2,28 +2,32 @@ import { Command } from 'commander';
|
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
import * as fs from 'fs';
|
|
4
4
|
import { runScriptCheck } from '../../harness/script-check';
|
|
5
|
+
import { reportSlug } from '../../harness/unit-paths';
|
|
5
6
|
|
|
6
7
|
export function registerScriptCheckCommand(program: Command): void {
|
|
7
8
|
program
|
|
8
9
|
.command('script-check')
|
|
9
10
|
.description('Verify the generated Playwright spec is a faithful 1:1 of the Gherkin feature (no hand-edit / stale drift)')
|
|
10
11
|
.option('-s, --screen <name>', 'Screen or flow name')
|
|
12
|
+
.option('--api <name>', 'API-first area or api flow (qa/api/<name>)')
|
|
13
|
+
.option('--area <name>', 'Alias of --api — an API-first area (qa/api/<name>)')
|
|
11
14
|
.option('--json', 'Output raw JSON')
|
|
12
15
|
.action(async (options) => {
|
|
13
16
|
try {
|
|
14
|
-
const name = options.screen;
|
|
15
|
-
if (!name) throw new Error('Provide --screen <name>');
|
|
17
|
+
const name = options.screen || options.api || options.area;
|
|
18
|
+
if (!name) throw new Error('Provide --screen <name> (or --api <area>)');
|
|
16
19
|
const screen = path.join(process.cwd(), 'qa', 'screens', name);
|
|
17
20
|
const flow = path.join(process.cwd(), 'qa', 'flows', name);
|
|
18
|
-
const
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
+
const api = path.join(process.cwd(), 'qa', 'api', name);
|
|
22
|
+
const kind = fs.existsSync(screen) ? 'screen' : fs.existsSync(flow) ? 'flow' : fs.existsSync(api) ? 'api' : null;
|
|
23
|
+
const dir = kind === 'screen' ? screen : kind === 'flow' ? flow : kind === 'api' ? api : null;
|
|
24
|
+
if (!dir || !kind) throw new Error(`Not found: qa/screens|flows|api/${name}`);
|
|
21
25
|
|
|
22
|
-
const r = await runScriptCheck(dir, name,
|
|
26
|
+
const r = await runScriptCheck(dir, name, kind);
|
|
23
27
|
|
|
24
28
|
const outDir = path.join(process.cwd(), '.sungen', 'reports');
|
|
25
29
|
fs.mkdirSync(outDir, { recursive: true });
|
|
26
|
-
fs.writeFileSync(path.join(outDir, `${name}-script-check.json`), JSON.stringify(r, null, 2), 'utf-8');
|
|
30
|
+
fs.writeFileSync(path.join(outDir, `${reportSlug(name)}-script-check.json`), JSON.stringify(r, null, 2), 'utf-8');
|
|
27
31
|
|
|
28
32
|
if (options.json) { console.log(JSON.stringify(r, null, 2)); process.exit(r.status === 'OK' ? 0 : 2); }
|
|
29
33
|
|
|
@@ -39,7 +43,7 @@ export function registerScriptCheckCommand(program: Command): void {
|
|
|
39
43
|
if (r.findings.length) { L(' findings:'); for (const f of r.findings) L(` • ${f}`); }
|
|
40
44
|
else L(' ✓ The test code faithfully reflects the Gherkin (1:1).');
|
|
41
45
|
L('');
|
|
42
|
-
if (r.drift === 'drift') L(
|
|
46
|
+
if (r.drift === 'drift') L(` → Fix: re-run \`sungen generate --${kind === 'api' ? 'api' : kind === 'flow' ? 'flow' : 'screen'} ${name}\` (or /sungen:run-test) so the spec matches the feature. Never hand-edit generated specs.`);
|
|
43
47
|
L('');
|
|
44
48
|
process.exit(r.status === 'OK' ? 0 : 2);
|
|
45
49
|
} catch (error) {
|
|
@@ -8,16 +8,19 @@ export function registerTraceCommand(program: Command): void {
|
|
|
8
8
|
.command('trace')
|
|
9
9
|
.description('Visualise the executed test-design process (workflow/skill steps, repair loops), find bottlenecks, and show where to focus human review')
|
|
10
10
|
.option('-s, --screen <name>', 'Screen or flow name')
|
|
11
|
+
.option('--api <name>', 'API-first area or api flow (qa/api/<name>)')
|
|
12
|
+
.option('--area <name>', 'Alias of --api — an API-first area (qa/api/<name>)')
|
|
11
13
|
.option('--json', 'Output raw JSON')
|
|
12
14
|
.option('--mermaid', 'Print only the Mermaid flowchart')
|
|
13
15
|
.action((options) => {
|
|
14
16
|
try {
|
|
15
|
-
const name = options.screen;
|
|
16
|
-
if (!name) throw new Error('Provide --screen <name>');
|
|
17
|
+
const name = options.screen || options.api || options.area;
|
|
18
|
+
if (!name) throw new Error('Provide --screen <name> (or --api <area>)');
|
|
17
19
|
const screen = path.join(process.cwd(), 'qa', 'screens', name);
|
|
18
20
|
const flow = path.join(process.cwd(), 'qa', 'flows', name);
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
+
const api = path.join(process.cwd(), 'qa', 'api', name);
|
|
22
|
+
const dir = fs.existsSync(screen) ? screen : fs.existsSync(flow) ? flow : fs.existsSync(api) ? api : null;
|
|
23
|
+
if (!dir) throw new Error(`Not found: qa/screens|flows|api/${name}`);
|
|
21
24
|
|
|
22
25
|
const r = buildTrace(dir, name);
|
|
23
26
|
if (options.json) { console.log(JSON.stringify(r, null, 2)); return; }
|
package/src/cli/index.ts
CHANGED
|
@@ -26,6 +26,8 @@ import { registerChallengeCommand } from './commands/challenge';
|
|
|
26
26
|
import { registerBlindspotCommand } from './commands/blindspot';
|
|
27
27
|
import { registerCapabilityCommand } from './commands/capability';
|
|
28
28
|
import { registerFlowCheckCommand } from './commands/flow-check';
|
|
29
|
+
import { registerContextCommand } from './commands/context';
|
|
30
|
+
import { registerRepairCommand } from './commands/repair';
|
|
29
31
|
import { capabilityRegistry } from '../capabilities/registry';
|
|
30
32
|
import { discoverAndRegisterCapabilities } from '../capabilities/discover';
|
|
31
33
|
|
|
@@ -64,6 +66,8 @@ async function main() {
|
|
|
64
66
|
registerBlindspotCommand(program);
|
|
65
67
|
registerCapabilityCommand(program);
|
|
66
68
|
registerFlowCheckCommand(program);
|
|
69
|
+
registerContextCommand(program);
|
|
70
|
+
registerRepairCommand(program);
|
|
67
71
|
registerIngestCommand(program);
|
|
68
72
|
registerEvalCommand(program);
|
|
69
73
|
|
|
@@ -6,6 +6,7 @@ import { TestGeneratorAdapter, adapterRegistry } from './adapters';
|
|
|
6
6
|
import { transformToRuntimeData } from './utils/runtime-data-transformer';
|
|
7
7
|
import { capabilityRegistry } from '../../capabilities/registry';
|
|
8
8
|
import { discoverAndRegisterCapabilities } from '../../capabilities/discover';
|
|
9
|
+
import { readCapabilities } from '../../harness/capability';
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* Filter base scenario steps for @extend: only keep Given→When steps.
|
|
@@ -88,6 +89,40 @@ function extractPassThroughTags(scenarioTags: string[], featureTags: string[]):
|
|
|
88
89
|
return unique.map(t => `'${t}'`).join(', ');
|
|
89
90
|
}
|
|
90
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Derive a feature's unit from its source path: the catalog-resolution id (relative to qa/), the
|
|
94
|
+
* output subdir, and whether it is a UI flow. Recognizes the api-first project model:
|
|
95
|
+
* qa/screens/<screen>/features/… → id `<screen>` (subdir `<screen>`)
|
|
96
|
+
* qa/flows/<flow>/features/… → id `flows/<flow>` (subdir `flows/<flow>`, isFlow)
|
|
97
|
+
* qa/api/<area>/features/… → id `api/<area>` (subdir `api/<area>`)
|
|
98
|
+
* qa/api/flows/<flow>/features/… → id `api/flows/<flow>` (subdir `api/flows/<flow>`)
|
|
99
|
+
* api-first units are NOT UI flows (no cross-screen namespacing) — they resolve their own api/db catalogs.
|
|
100
|
+
*/
|
|
101
|
+
function deriveUnitFromFeaturePath(sourceFile: string): { unitId: string; outputSubdir: string; isFlow: boolean } {
|
|
102
|
+
const parts = path.dirname(sourceFile).split(path.sep);
|
|
103
|
+
const qa = parts.lastIndexOf('qa');
|
|
104
|
+
if (qa >= 0 && parts[qa + 1] === 'api') {
|
|
105
|
+
if (parts[qa + 2] === 'flows' && parts[qa + 3]) {
|
|
106
|
+
const id = `api/flows/${parts[qa + 3]}`;
|
|
107
|
+
return { unitId: id, outputSubdir: id, isFlow: false };
|
|
108
|
+
}
|
|
109
|
+
if (parts[qa + 2]) {
|
|
110
|
+
const id = `api/${parts[qa + 2]}`;
|
|
111
|
+
return { unitId: id, outputSubdir: id, isFlow: false };
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
const flowsIndex = parts.indexOf('flows');
|
|
115
|
+
const screensIndex = parts.indexOf('screens');
|
|
116
|
+
if (flowsIndex >= 0 && flowsIndex < parts.length - 1) {
|
|
117
|
+
const f = parts[flowsIndex + 1];
|
|
118
|
+
return { unitId: `flows/${f}`, outputSubdir: `flows/${f}`, isFlow: true };
|
|
119
|
+
}
|
|
120
|
+
if (screensIndex >= 0 && screensIndex < parts.length - 1) {
|
|
121
|
+
return { unitId: parts[screensIndex + 1], outputSubdir: parts[screensIndex + 1], isFlow: false };
|
|
122
|
+
}
|
|
123
|
+
return { unitId: '', outputSubdir: '', isFlow: false };
|
|
124
|
+
}
|
|
125
|
+
|
|
91
126
|
/**
|
|
92
127
|
* Check for @screenshot:on-failure tag
|
|
93
128
|
*/
|
|
@@ -141,6 +176,19 @@ function isManual(tags: string[]): boolean {
|
|
|
141
176
|
return tags.some(tag => tag === '@manual');
|
|
142
177
|
}
|
|
143
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Capabilities a scenario declares via `@requires:<cap>` (TQ-11) — automation-ready work that
|
|
181
|
+
* needs an opt-in driver (e.g. `@requires:db`, `@requires:api`). When the cap is enabled the
|
|
182
|
+
* scenario compiles to a real test; when absent it compiles to a clean `test.skip` stub (no
|
|
183
|
+
* driver imports) so the run never breaks and the case is visible as "pending capability".
|
|
184
|
+
*/
|
|
185
|
+
function requiresCaps(tags: string[]): string[] {
|
|
186
|
+
return tags
|
|
187
|
+
.filter(t => /^@requires:/i.test(t))
|
|
188
|
+
.map(t => t.slice('@requires:'.length).trim().toLowerCase())
|
|
189
|
+
.filter(Boolean);
|
|
190
|
+
}
|
|
191
|
+
|
|
144
192
|
/**
|
|
145
193
|
* Check for multiple auth tags and log warning
|
|
146
194
|
*/
|
|
@@ -210,29 +258,16 @@ export class CodeGenerator {
|
|
|
210
258
|
fileName = this.featureNameToFileName(feature.name);
|
|
211
259
|
}
|
|
212
260
|
|
|
213
|
-
//
|
|
214
|
-
|
|
215
|
-
// qa/flows/{flowName}/features/{featureName}.feature -> flows/flowName
|
|
216
|
-
let outputSubdir = '';
|
|
217
|
-
if (feature.sourceFile) {
|
|
218
|
-
const sourceDir = path.dirname(feature.sourceFile);
|
|
219
|
-
const parts = sourceDir.split(path.sep);
|
|
220
|
-
const flowsIndex = parts.indexOf('flows');
|
|
221
|
-
const screensIndex = parts.indexOf('screens');
|
|
222
|
-
if (flowsIndex >= 0 && flowsIndex < parts.length - 2) {
|
|
223
|
-
outputSubdir = path.join('flows', parts[flowsIndex + 1]);
|
|
224
|
-
} else if (screensIndex >= 0 && screensIndex < parts.length - 2) {
|
|
225
|
-
outputSubdir = parts[screensIndex + 1];
|
|
226
|
-
}
|
|
227
|
-
}
|
|
261
|
+
// Output subdirectory from the source path (screens / flows / api-first areas+flows).
|
|
262
|
+
const outputSubdir = feature.sourceFile ? deriveUnitFromFeaturePath(feature.sourceFile).outputSubdir : '';
|
|
228
263
|
|
|
229
264
|
// Build output path with subdirectory
|
|
230
265
|
const filePath = outputSubdir
|
|
231
266
|
? path.join(outputDir, outputSubdir, fileName)
|
|
232
267
|
: path.join(outputDir, fileName);
|
|
233
268
|
|
|
234
|
-
// Compute relative path from output file back to specs/generated/
|
|
235
|
-
const depth = outputSubdir ? outputSubdir.split(
|
|
269
|
+
// Compute relative path from output file back to specs/generated/ (subdir uses '/').
|
|
270
|
+
const depth = outputSubdir ? outputSubdir.split('/').length : 0;
|
|
236
271
|
const basePath = depth > 0 ? Array(depth).fill('..').join('/') : '..';
|
|
237
272
|
|
|
238
273
|
// Serial + @cleanup tags → need cleanupPage import from base
|
|
@@ -336,12 +371,22 @@ export class CodeGenerator {
|
|
|
336
371
|
*/
|
|
337
372
|
private capabilityPreconditions(scenario: ParsedScenario): Array<{ comment?: string; code: string; boundVars?: string[] }> {
|
|
338
373
|
discoverAndRegisterCapabilities();
|
|
374
|
+
const tags = scenario.tags || [];
|
|
339
375
|
const out: Array<{ comment?: string; code: string; boundVars?: string[] }> = [];
|
|
340
376
|
for (const cap of capabilityRegistry.all()) {
|
|
341
377
|
if (!cap.preconditionCodegen) continue;
|
|
342
|
-
out.push(...cap.preconditionCodegen({ tags
|
|
343
|
-
}
|
|
344
|
-
|
|
378
|
+
out.push(...cap.preconditionCodegen({ tags, screenName: this.queryScreenName, cwd: process.cwd() }));
|
|
379
|
+
}
|
|
380
|
+
// Order by each precondition's annotation position on the scenario (not capability-registry order),
|
|
381
|
+
// so a cross-capability sequence runs as authored — e.g. `@api:pay @concurrent:2 @query:charge_count`
|
|
382
|
+
// mutates THEN reads the DB (the idempotency cross-check). Stable: ties keep their original order.
|
|
383
|
+
const pos = (p: { boundVars?: string[] }): number => {
|
|
384
|
+
const name = p.boundVars?.[0];
|
|
385
|
+
if (!name) return tags.length;
|
|
386
|
+
const i = tags.findIndex((t) => new RegExp(`^@\\w+:${name}\\b`).test(t));
|
|
387
|
+
return i < 0 ? tags.length : i;
|
|
388
|
+
};
|
|
389
|
+
return out.map((p, i) => ({ p, i })).sort((a, b) => pos(a.p) - pos(b.p) || a.i - b.i).map((x) => x.p);
|
|
345
390
|
}
|
|
346
391
|
|
|
347
392
|
/**
|
|
@@ -382,27 +427,23 @@ export class CodeGenerator {
|
|
|
382
427
|
featureName = this.featureNameToFileName(feature.name).replace('.spec.ts', '');
|
|
383
428
|
}
|
|
384
429
|
|
|
385
|
-
// Derive
|
|
386
|
-
// qa/
|
|
387
|
-
//
|
|
430
|
+
// Derive the unit from the source path when not explicitly set. `unitId` is the catalog/test-data
|
|
431
|
+
// resolution id relative to qa/ (`<screen>` · `flows/<flow>` · `api/<area>` · `api/flows/<flow>`);
|
|
432
|
+
// `effectiveScreenName` is the bare name for UI screen-context (selector/data namespacing).
|
|
388
433
|
let effectiveScreenName = this.screenName;
|
|
389
434
|
let isFlowFeature = !!this.options.flowMode;
|
|
435
|
+
let unitId = isFlowFeature ? `flows/${effectiveScreenName || ''}` : (effectiveScreenName || '');
|
|
390
436
|
if (!this.screenName && feature.sourceFile) {
|
|
391
|
-
const
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
effectiveScreenName
|
|
397
|
-
isFlowFeature = true;
|
|
398
|
-
} else if (screensIndex >= 0 && screensIndex < parts.length - 2) {
|
|
399
|
-
effectiveScreenName = parts[screensIndex + 1];
|
|
400
|
-
isFlowFeature = false;
|
|
437
|
+
const u = deriveUnitFromFeaturePath(feature.sourceFile);
|
|
438
|
+
if (u.unitId) {
|
|
439
|
+
unitId = u.unitId;
|
|
440
|
+
isFlowFeature = u.isFlow;
|
|
441
|
+
effectiveScreenName = u.unitId.split('/').pop() || u.unitId; // bare name (no api/ or flows/ prefix)
|
|
442
|
+
this.stepMapper.setScreenContext(effectiveScreenName);
|
|
401
443
|
}
|
|
402
|
-
this.stepMapper.setScreenContext(effectiveScreenName);
|
|
403
444
|
}
|
|
404
|
-
// Catalog-resolution
|
|
405
|
-
this.queryScreenName =
|
|
445
|
+
// Catalog + test-data resolution id (flows → `flows/<flow>`, api-first → `api/<area>` / `api/flows/<flow>`).
|
|
446
|
+
this.queryScreenName = unitId;
|
|
406
447
|
|
|
407
448
|
// Reset flow mode per feature to prevent state leak in --all mode
|
|
408
449
|
this.stepMapper.setFlowMode(isFlowFeature);
|
|
@@ -466,6 +507,10 @@ export class CodeGenerator {
|
|
|
466
507
|
// Generate all scenarios with feature tags for inheritance
|
|
467
508
|
// Skip scenarios tagged with @manual
|
|
468
509
|
// Track auth role per scenario for grouping
|
|
510
|
+
// TQ-11 — enabled capabilities (qa/capabilities.yaml): a @requires:<cap> scenario whose cap is
|
|
511
|
+
// absent compiles to a skip stub instead of a real test (read once per feature).
|
|
512
|
+
const enabledCaps = new Set<string>(readCapabilities(process.cwd()).enabled.map(d => d.toLowerCase()));
|
|
513
|
+
|
|
469
514
|
const renderedScenarios: Array<{ code: string; authRole?: string }> = [];
|
|
470
515
|
for (const scenario of feature.scenarios) {
|
|
471
516
|
if (isManual(scenario.tags)) {
|
|
@@ -480,6 +525,16 @@ export class CodeGenerator {
|
|
|
480
525
|
continue;
|
|
481
526
|
}
|
|
482
527
|
|
|
528
|
+
// TQ-11 — @requires:<cap> with the cap NOT enabled → emit a clean skip stub (no driver
|
|
529
|
+
// imports, so the spec still loads), visible as skipped-with-reason. With the cap enabled it
|
|
530
|
+
// falls through and compiles as a real test.
|
|
531
|
+
const reqAbsent = requiresCaps(scenario.tags).filter(c => !enabledCaps.has(c));
|
|
532
|
+
if (reqAbsent.length) {
|
|
533
|
+
if (this.options.verbose) console.log(` ⏸ Pending capability (${reqAbsent.join(', ')}): ${scenario.name}`);
|
|
534
|
+
renderedScenarios.push({ code: this.generateRequiresSkipStub(scenario.name, reqAbsent), authRole: undefined });
|
|
535
|
+
continue;
|
|
536
|
+
}
|
|
537
|
+
|
|
483
538
|
// Resolve auth tags for @extend scenarios (same logic as generateScenario)
|
|
484
539
|
let authFeatureTags = feature.tags || [];
|
|
485
540
|
if (scenario.extendsName) {
|
|
@@ -553,7 +608,7 @@ export class CodeGenerator {
|
|
|
553
608
|
cleanupConfig,
|
|
554
609
|
screenshotOnFailure,
|
|
555
610
|
runtimeData: this.options.runtimeData,
|
|
556
|
-
screenName:
|
|
611
|
+
screenName: this.queryScreenName, // catalog/test-data id (screen · flows/<flow> · api/<area>)
|
|
557
612
|
featureFileName: featureName,
|
|
558
613
|
isParallel,
|
|
559
614
|
flowMode: isFlowFeature,
|
|
@@ -615,6 +670,21 @@ export class CodeGenerator {
|
|
|
615
670
|
return renderMap[hookType]();
|
|
616
671
|
}
|
|
617
672
|
|
|
673
|
+
/**
|
|
674
|
+
* TQ-11 — a `@requires:<cap>` scenario whose capability is NOT enabled. Emits a real `test()`
|
|
675
|
+
* that skips itself with an actionable reason, and references NO driver runtime (so the spec
|
|
676
|
+
* loads even though the driver isn't installed). Once the cap is added + regenerated, the
|
|
677
|
+
* scenario compiles to its full automated body instead.
|
|
678
|
+
*/
|
|
679
|
+
private generateRequiresSkipStub(name: string, caps: string[]): string {
|
|
680
|
+
const reason = `requires ${caps.join(' + ')} — run \`sungen capability add ${caps.join(' ')}\` to automate this`;
|
|
681
|
+
return [
|
|
682
|
+
` test(${JSON.stringify(name)}, async () => {`,
|
|
683
|
+
` test.skip(true, ${JSON.stringify(reason)});`,
|
|
684
|
+
` });`,
|
|
685
|
+
].join('\n');
|
|
686
|
+
}
|
|
687
|
+
|
|
618
688
|
private async generateScenario(
|
|
619
689
|
scenario: ParsedScenario,
|
|
620
690
|
hasBackground: boolean,
|
|
@@ -705,8 +775,8 @@ export class CodeGenerator {
|
|
|
705
775
|
}
|
|
706
776
|
}
|
|
707
777
|
|
|
708
|
-
// Capability preconditions (db `@query:<name>` → bind {{name}}; computed
|
|
709
|
-
// scenario's own steps — prepend them, indenting the capability
|
|
778
|
+
// Capability preconditions (db `@query:<name>` / api `@api:<name>` → bind {{name}}; computed
|
|
779
|
+
// above) run BEFORE the scenario's own steps — prepend them, indenting the capability statements.
|
|
710
780
|
if (preconditions.length) {
|
|
711
781
|
steps.unshift(...preconditions.map((p) => ({ comment: p.comment, code: this.indentCode(p.code, 4) })));
|
|
712
782
|
}
|
|
@@ -729,7 +799,12 @@ export class CodeGenerator {
|
|
|
729
799
|
// global `testData` transform that runs next on the rest of the file leaves it alone.
|
|
730
800
|
// The loop header's `testData.cases()/withRow()` are literal code (no markers) → untouched.
|
|
731
801
|
if (casesDataset && this.options.runtimeData) {
|
|
732
|
-
|
|
802
|
+
// AP-3: capability preconditions (`@api`/`@query`) and value assertions (`expect {{x}} is {{y}}`)
|
|
803
|
+
// emit LITERAL `testData.get/bind/set(…)` (not markers), so the marker transform above misses
|
|
804
|
+
// them. Rewrite those to the per-row `rowData` view — each row then fires its own `@api` call
|
|
805
|
+
// with that row's input and asserts that row's expected status/body (the success/failure matrix).
|
|
806
|
+
// `testData.cases()/withRow()` (the global loader, no `.get/.bind/.set`) is intentionally left alone.
|
|
807
|
+
return transformToRuntimeData(rendered, 'rowData').replace(/\btestData\.(get|bind|set)\(/g, 'rowData.$1(');
|
|
733
808
|
}
|
|
734
809
|
return rendered;
|
|
735
810
|
}
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Parses `name(a={{x}},b="lit",c=3)` overrides into a map of JS expressions, e.g.
|
|
5
5
|
* `{ a: "testData.get('x')", b: "\"lit\"", c: "3" }`. Used by the DB and API capability drivers'
|
|
6
|
-
* precondition codegen; lives in core so both drivers
|
|
6
|
+
* precondition codegen; lives in core so both drivers can share it. Gherkin tags carry no whitespace,
|
|
7
|
+
* so values are single tokens — flows thread a prior response via a whole-value ref, e.g.
|
|
8
|
+
* `@api:get_profile(token={{login.body.token}})`, with the auth scheme declared in the catalog header.
|
|
7
9
|
*/
|
|
8
10
|
export function parseQueryOverrides(raw?: string): Record<string, string> {
|
|
9
11
|
const out: Record<string, string> = {};
|
package/src/harness/audit.ts
CHANGED
|
@@ -9,10 +9,15 @@
|
|
|
9
9
|
import * as path from 'path';
|
|
10
10
|
import * as fs from 'fs';
|
|
11
11
|
import { loadScenarios, parseViewpointOverview, ScenarioInfo, ViewpointEntry } from './parse';
|
|
12
|
+
import { featureBasename } from './unit-paths';
|
|
12
13
|
import {
|
|
13
|
-
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, coverageBalance, duplicateClusters, traceability, claimProof, taxonomyLint,
|
|
14
|
-
|
|
14
|
+
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, depthThresholdFor, coverageBalance, duplicateClusters, traceability, claimProof, taxonomyLint,
|
|
15
|
+
automatableManual, flowCoveredThemes,
|
|
16
|
+
GateResult, DepthResult, BalanceResult, DuplicateResult, TraceResult, ClaimProofResult, TaxonomyResult, Catalog, AutomatableManualResult,
|
|
15
17
|
} from './sensors';
|
|
18
|
+
import { loadFlowScenarios } from './flow-check';
|
|
19
|
+
import { manualReasonMismatches, MANUAL_REASONS, buildPlan } from './capability-plan';
|
|
20
|
+
import { readCapabilities } from './capability';
|
|
16
21
|
import { readIntent, projectRootFromScreenDir, IntentProfile } from './intent';
|
|
17
22
|
import { getProvenance, Provenance } from './provenance';
|
|
18
23
|
import { specCoverage, SpecCoverageResult, parseSpecClauses } from './spec-coverage';
|
|
@@ -36,6 +41,7 @@ export interface AuditReport {
|
|
|
36
41
|
taxonomyMismatch: boolean; // scenarios use IDs not in the project's test-viewpoint.md
|
|
37
42
|
downstream: DownstreamResult; // downstream screens referenced but under-covered
|
|
38
43
|
manualOracle: ManualOracleResult; // @manual scenarios lacking setup/action/oracle
|
|
44
|
+
automatableManual: AutomatableManualResult; // @manual that is actually automatable (deferred, not judgment) — TQ-2
|
|
39
45
|
ledger: LedgerResult; // atomic viewpoint-item coverage (per-bullet status)
|
|
40
46
|
calibration: { // #8 — multi-axis score so a high overall can't hide a weak axis
|
|
41
47
|
axes: Record<string, number>;
|
|
@@ -57,32 +63,84 @@ export interface AuditReport {
|
|
|
57
63
|
spec: SpecCoverageResult; // G2 — spec-clause coverage (FR + validation-trigger matrix)
|
|
58
64
|
}
|
|
59
65
|
|
|
66
|
+
/** The catalog-resolution id for a unit dir (relative to qa/): screen · flows/<flow> · api/<area> · api/flows/<flow>. */
|
|
67
|
+
function catalogIdFromScreenDir(screenDir: string): string {
|
|
68
|
+
const parts = screenDir.split(path.sep);
|
|
69
|
+
const qa = parts.lastIndexOf('qa');
|
|
70
|
+
if (qa >= 0) {
|
|
71
|
+
if (parts[qa + 1] === 'api' && parts[qa + 2] === 'flows' && parts[qa + 3]) return `api/flows/${parts[qa + 3]}`;
|
|
72
|
+
if (parts[qa + 1] === 'api' && parts[qa + 2]) return `api/${parts[qa + 2]}`;
|
|
73
|
+
if (parts[qa + 1] === 'flows' && parts[qa + 2]) return `flows/${parts[qa + 2]}`;
|
|
74
|
+
}
|
|
75
|
+
return path.basename(screenDir);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The capability that owns SCORING for a unit (AO-1): the first path segment of the unit id when it
|
|
80
|
+
* is a registered capability (`api/<area>` → `api`), else the default (UI) capability. Generic — a
|
|
81
|
+
* future `mobile/<x>` or `perf/<x>` unit routes to that capability with no core change. `flows/<flow>`
|
|
82
|
+
* has no `flows` capability → default (UI), which is correct (flows are a UI concept).
|
|
83
|
+
*/
|
|
84
|
+
export function scoringCapabilityFor(catalogScreenName: string, defaultCap: string | undefined): string | undefined {
|
|
85
|
+
const seg = catalogScreenName.split('/')[0];
|
|
86
|
+
return seg && capabilityRegistry.get(seg) ? seg : defaultCap;
|
|
87
|
+
}
|
|
88
|
+
|
|
60
89
|
export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
61
|
-
|
|
90
|
+
// The feature filename is the unit's LAST segment — an api flow (`flows/<flow>`) lives at
|
|
91
|
+
// `<dir>/features/<flow>.feature`, not `features/flows/<flow>.feature` (which found 0 scenarios).
|
|
92
|
+
const featurePath = path.join(screenDir, 'features', `${featureBasename(screenName)}.feature`);
|
|
62
93
|
const viewpointPath = path.join(screenDir, 'requirements', 'test-viewpoint.md');
|
|
94
|
+
// Catalog-resolution id (for the @api/@query gate sensors): the unit's path relative to qa/ —
|
|
95
|
+
// `flows/<flow>`, `api/<area>`, `api/flows/<flow>`, else the bare screen. A bare screen matches
|
|
96
|
+
// the old behaviour (so the audit-sample snapshot is unchanged); flows/api now resolve correctly.
|
|
97
|
+
const catalogScreenName = catalogIdFromScreenDir(screenDir);
|
|
63
98
|
|
|
64
99
|
const specPath = path.join(screenDir, 'requirements', 'spec.md');
|
|
65
100
|
const featureText = fs.existsSync(featurePath) ? fs.readFileSync(featurePath, 'utf-8') : '';
|
|
66
101
|
|
|
67
102
|
const scenarios: ScenarioInfo[] = loadScenarios(featurePath);
|
|
68
103
|
const viewpoints: ViewpointEntry[] = parseViewpointOverview(viewpointPath);
|
|
69
|
-
//
|
|
70
|
-
//
|
|
104
|
+
// AO-1 — capability-routed scoring: the viewpoint catalog + score-bearing gate are owned by the
|
|
105
|
+
// unit's capability, resolved from the unit id (`api/<area>` → `api`; screen/flow → the default
|
|
106
|
+
// UI capability). A capability that provides no catalog/gate falls back to the in-core UI
|
|
107
|
+
// functions, so UI units — and api units until AO-2 adds the api providers — are byte-identical.
|
|
71
108
|
discoverAndRegisterCapabilities();
|
|
72
109
|
const defaultCap = capabilityRegistry.defaultCapabilityId();
|
|
73
|
-
const
|
|
110
|
+
const scoringCapId = scoringCapabilityFor(catalogScreenName, defaultCap);
|
|
111
|
+
const scoringCap = scoringCapId ? capabilityRegistry.get(scoringCapId) : undefined;
|
|
112
|
+
const catalog = (scoringCap?.viewpoints?.() as Catalog | undefined) || loadCatalog();
|
|
74
113
|
const spec = specCoverage(specPath, scenarios, featureText);
|
|
75
114
|
|
|
76
115
|
// P3 — intent profile from qa/context.md drives the depth threshold (focus).
|
|
77
116
|
const intent = readIntent(projectRootFromScreenDir(screenDir));
|
|
78
|
-
// The viewpoint coverage
|
|
79
|
-
//
|
|
80
|
-
//
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
const
|
|
117
|
+
// The score-bearing gate (viewpoint coverage + assertion depth) is the scoring capability's
|
|
118
|
+
// `gateProvider`; same functions underneath → byte-identical gate/depth → identical score. Falls
|
|
119
|
+
// back to the in-core functions if the capability provides none.
|
|
120
|
+
// A capability gate may need project context (the API gate resolves endpoint methods from the
|
|
121
|
+
// catalog) + the focus depth threshold (so it scores depth with the SAME bar as the UI gate).
|
|
122
|
+
const capGate = scoringCap?.gateProvider as
|
|
123
|
+
((i: { scenarios: ScenarioInfo[]; viewpoints: ViewpointEntry[]; catalog: Catalog; focus: typeof intent.focus; cwd: string; screenName: string; threshold: number; businessCriticalMethods?: string[] }) => { gate: GateResult; depth: DepthResult }) | undefined;
|
|
124
|
+
const provided = capGate?.({ scenarios, viewpoints, catalog, focus: intent.focus, cwd: projectRootFromScreenDir(screenDir), screenName: catalogScreenName, threshold: depthThresholdFor(intent.focus), businessCriticalMethods: intent.businessCriticalMethods });
|
|
84
125
|
const gate = provided?.gate ?? viewpointGate(scenarios, viewpoints, catalog);
|
|
85
126
|
const depth = provided?.depth ?? assertionDepth(scenarios, dataThemesFor(catalog, gate.pageType), intent.focus);
|
|
127
|
+
|
|
128
|
+
// TQ-4 — deferral-aware coverage credit: an inherently cross-screen theme (cart / detail /
|
|
129
|
+
// filter correctness) belongs in a FLOW, not on the screen. When a flow deeply covers a screen
|
|
130
|
+
// gate gap, credit it to the flow instead of double-counting it as a screen gap. Screens only
|
|
131
|
+
// (a flow/api unit is not credited by sibling flows); mutates the gate before coverage is scored.
|
|
132
|
+
const flowCredits: { theme: string; flow: string }[] = [];
|
|
133
|
+
const isScreenUnit = !/^(flows|api)\//.test(catalogScreenName);
|
|
134
|
+
if (isScreenUnit && gate.gaps.length) {
|
|
135
|
+
const flowScenarios = loadFlowScenarios(projectRootFromScreenDir(screenDir));
|
|
136
|
+
if (flowScenarios.length) {
|
|
137
|
+
for (const c of flowCoveredThemes(gate.gaps, flowScenarios)) {
|
|
138
|
+
const i = gate.gaps.findIndex((g) => g.theme === c.theme);
|
|
139
|
+
if (i >= 0) { gate.gaps.splice(i, 1); gate.themesCovered++; flowCredits.push(c); }
|
|
140
|
+
}
|
|
141
|
+
gate.coverageRatio = gate.themesTotal ? gate.themesCovered / gate.themesTotal : 1;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
86
144
|
const claim = claimProof(scenarios, intent.focus);
|
|
87
145
|
const taxonomy = taxonomyLint(scenarios);
|
|
88
146
|
const balance = coverageBalance(scenarios);
|
|
@@ -93,6 +151,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
93
151
|
// #2 downstream-scope + #4 manual-oracle
|
|
94
152
|
const downstream = downstreamScope(readText(specPath), scenarios);
|
|
95
153
|
const manualOracleResult = manualOracle(featureText);
|
|
154
|
+
const autoManual = automatableManual(scenarios); // TQ-2 — @manual that is really automatable
|
|
96
155
|
const ledger = viewpointLedger(viewpointPath, scenarios, featureText);
|
|
97
156
|
const negSideEffect = negativeSideEffect(scenarios);
|
|
98
157
|
const ownership = crossArtifactOwnership(screenDir, scenarios);
|
|
@@ -110,6 +169,9 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
110
169
|
const overall = (0.4 * coverage + 0.3 * businessDepth + 0.15 * balanceScore + 0.15 * traceScore) * 10;
|
|
111
170
|
|
|
112
171
|
const findings: string[] = [];
|
|
172
|
+
for (const c of flowCredits) {
|
|
173
|
+
findings.push(`COVERED-VIA-FLOW: critical theme "${c.theme}" is not covered on this screen but is deeply covered by flow "${c.flow}" — cross-screen depth correctly owned by the flow, so it is credited (not a screen gap). Verify with \`sungen flow-check\`.`);
|
|
174
|
+
}
|
|
113
175
|
for (const g of gate.gaps) {
|
|
114
176
|
if (g.status === 'shallow') {
|
|
115
177
|
findings.push(`GATE: critical theme "${g.theme}" is covered only by SHALLOW scenarios (no data assertion) → deepen with \`... with {{value}}\` / \`table ... with {{value}}\` (count @manual cross-screen too).`);
|
|
@@ -154,6 +216,40 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
154
216
|
for (const m of manualOracleResult.insufficient.slice(0, 8)) {
|
|
155
217
|
findings.push(`MANUAL-STEPS-INSUFFICIENT: "${m}" — a @manual scenario needs setup · action · observable expected · oracle/tool (not just a one-line note).`);
|
|
156
218
|
}
|
|
219
|
+
// TQ-2 — automatable @manual: deferred (usually cross-screen) but fully DSL-expressible.
|
|
220
|
+
for (const m of autoManual.scenarios.slice(0, 8)) {
|
|
221
|
+
findings.push(`MANUAL-AUTOMATABLE: "${m.name}" is @manual but its steps are fully automatable (a data assertion, no visual/mock/a11y judgment) → generate it as an AUTOMATED flow scenario (or tag @manual:Mx with a real capability reason). Don't leave a non-running @manual duplicate of a flow scenario.`);
|
|
222
|
+
}
|
|
223
|
+
// TQ-9 — manual reason-code mismatch: an explicit @manual:Mx that disagrees with the reason
|
|
224
|
+
// text makes the capability planner recommend the wrong driver. Surface it so the code is fixed.
|
|
225
|
+
for (const mm of manualReasonMismatches(featurePath).slice(0, 8)) {
|
|
226
|
+
const label = MANUAL_REASONS[mm.inferred]?.label ?? (mm.inferred === 'XS' ? 'Cross-screen (→ flow)' : mm.inferred);
|
|
227
|
+
findings.push(`MANUAL-REASON-MISMATCH: "${mm.scenario}" is tagged @manual:${mm.explicit} but its reason reads as ${mm.inferred} (${label}) → fix the code so \`sungen capability plan\` recommends the right driver to automate it.`);
|
|
228
|
+
}
|
|
229
|
+
// TQ-10 — surface the Capability Planner recommendation (recommend-only; never installs). Silenced
|
|
230
|
+
// by `capability_suggestions: off` in qa/context.md. Reuses the planner (trustworthy after TQ-9).
|
|
231
|
+
if (intent.capabilitySuggestions) {
|
|
232
|
+
const plan = buildPlan(screenDir, featureBasename(screenName));
|
|
233
|
+
if (plan.recommendations.length) {
|
|
234
|
+
const recs = plan.recommendations.map((r) => `\`sungen capability add ${r.driver}\` (automates ${r.count})`).join(' · ');
|
|
235
|
+
findings.push(`CAPABILITY-SUGGESTION: ${plan.capabilityManual} @manual scenario(s) are capability-manual (a driver could automate them) — ${recs}. Recommend-only: nothing is installed automatically; the ${plan.judgmentManual} judgment-manual (M6/M8/M9) correctly stay manual.`);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
// TQ-11b — automation-ready (pending capability): @requires:<cap> scenarios whose cap isn't enabled.
|
|
239
|
+
// They are NOT manual (real steps, compiled the moment the cap is added) — surface them distinctly.
|
|
240
|
+
{
|
|
241
|
+
const enabledCaps = new Set(readCapabilities(projectRootFromScreenDir(screenDir)).enabled.map((d) => d.toLowerCase()));
|
|
242
|
+
const pending = scenarios.filter((s) => (s.requiresCaps ?? []).some((c) => !enabledCaps.has(c)));
|
|
243
|
+
if (pending.length) {
|
|
244
|
+
const caps = [...new Set(pending.flatMap((s) => (s.requiresCaps ?? []).filter((c) => !enabledCaps.has(c))))];
|
|
245
|
+
findings.push(`AUTOMATION-READY-PENDING: ${pending.length} scenario(s) are automation-ready but need a capability — \`sungen capability add ${caps.join(' ')}\` to run them. They are skipped (not manual, not a gap) until the driver is enabled.`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
// TQ-3 — businessDepth de-inflation: a high ratio on a tiny denominator because business-critical
|
|
249
|
+
// scenarios were deferred to @manual is misleading. Surface the deferral so 1.0 isn't read as "done".
|
|
250
|
+
if (depth.deferredBusinessCritical > 0 && depth.deferredBusinessCritical >= depth.businessCriticalTotal) {
|
|
251
|
+
findings.push(`DEPTH-DEFERRED: businessDepth ${businessDepth.toFixed(2)} is computed over only ${depth.businessCriticalTotal} on-screen scenario(s); ${depth.deferredBusinessCritical} business-critical scenario(s) are deferred to @manual (excluded from the ratio). Automate them in a flow and verify with \`sungen flow-check\` — this ratio is NOT "all business depth covered".`);
|
|
252
|
+
}
|
|
157
253
|
if (ledger.hasViewpoint && ledger.missing.length) {
|
|
158
254
|
const sample = ledger.missing.slice(0, 6).map((m) => m.id || `"${m.text}"`).join(', ');
|
|
159
255
|
findings.push(`VIEWPOINT-ITEM-MISSING: ${ledger.missing.length}/${ledger.total} atomic viewpoint items have no covering scenario (${(ledger.ratio * 100).toFixed(0)}% covered) — e.g. ${sample}. Cover each item or mark it deferred/spec-gap.`);
|
|
@@ -180,7 +276,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
180
276
|
const routedGateIds = contextRouter.route({ target: { kind: 'screen', id: screenName }, artifact: 'feature', tags: featureTags }).gateSensorIds;
|
|
181
277
|
const gateSensorFindings = capabilityRegistry.sensors('gate')
|
|
182
278
|
.filter((s) => routedGateIds.includes(s.id))
|
|
183
|
-
.flatMap((s) => s.run({ screenName, cwd: projectRootFromScreenDir(screenDir), featureText, scenarios, universalGaps: gate.universalGaps }));
|
|
279
|
+
.flatMap((s) => s.run({ screenName: catalogScreenName, cwd: projectRootFromScreenDir(screenDir), featureText, scenarios, universalGaps: gate.universalGaps }));
|
|
184
280
|
// Each gate sensor's message carries its own code prefix (VERIFICATION-FAIL / UNIVERSAL / …)
|
|
185
281
|
// → push verbatim.
|
|
186
282
|
for (const f of gateSensorFindings) findings.push(f.message);
|
|
@@ -200,7 +296,11 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
200
296
|
};
|
|
201
297
|
const weakestEntry = Object.entries(axes).sort((a, b) => a[1] - b[1])[0];
|
|
202
298
|
const weakest = { axis: weakestEntry[0], value: weakestEntry[1] };
|
|
203
|
-
|
|
299
|
+
// Inflated when breadth hides a weak axis, OR when businessDepth is high but rests on a
|
|
300
|
+
// denominator collapsed by deferral-to-@manual (TQ-3): the headline depth can't be trusted.
|
|
301
|
+
const depthInflated =
|
|
302
|
+
businessDepth >= 0.9 && depth.deferredBusinessCritical >= depth.businessCriticalTotal && depth.deferredBusinessCritical > 0;
|
|
303
|
+
const inflated = (overall >= 8 && weakest.value < 0.6) || depthInflated;
|
|
204
304
|
if (inflated) {
|
|
205
305
|
findings.push(`SCORE-INFLATED-BY-BREADTH: overall ${Math.round(overall * 10) / 10}/10 but the weakest axis "${weakest.axis}" is ${(weakest.value * 100).toFixed(0)}% — breadth is hiding a weak dimension. Raise "${weakest.axis}" before trusting the headline.`);
|
|
206
306
|
}
|
|
@@ -215,7 +315,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
215
315
|
screen: screenName,
|
|
216
316
|
scenarioCount: scenarios.length,
|
|
217
317
|
gate, depth, claim, taxonomy, balance, duplicates, trace, spec,
|
|
218
|
-
taxonomyMismatch, downstream, manualOracle: manualOracleResult, ledger, calibration,
|
|
318
|
+
taxonomyMismatch, downstream, manualOracle: manualOracleResult, automatableManual: autoManual, ledger, calibration,
|
|
219
319
|
score: {
|
|
220
320
|
overall: Math.round(overall * 10) / 10,
|
|
221
321
|
coverage: Math.round(coverage * 100) / 100,
|