@sun-asterisk/sungen 3.1.2 → 3.2.0-beta.142
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -428
- package/dist/capabilities/builtins.d.ts +31 -0
- package/dist/capabilities/builtins.d.ts.map +1 -0
- package/dist/capabilities/builtins.js +84 -0
- package/dist/capabilities/builtins.js.map +1 -0
- package/dist/capabilities/context-router.d.ts +34 -0
- package/dist/capabilities/context-router.d.ts.map +1 -0
- package/dist/capabilities/context-router.js +49 -0
- package/dist/capabilities/context-router.js.map +1 -0
- package/dist/capabilities/context.d.ts +68 -0
- package/dist/capabilities/context.d.ts.map +1 -0
- package/dist/capabilities/context.js +17 -0
- package/dist/capabilities/context.js.map +1 -0
- package/dist/capabilities/discover.d.ts +2 -0
- package/dist/capabilities/discover.d.ts.map +1 -0
- package/dist/capabilities/discover.js +109 -0
- package/dist/capabilities/discover.js.map +1 -0
- package/dist/capabilities/registry.d.ts +92 -0
- package/dist/capabilities/registry.d.ts.map +1 -0
- package/dist/capabilities/registry.js +43 -0
- package/dist/capabilities/registry.js.map +1 -0
- package/dist/capabilities/sensor.d.ts +52 -0
- package/dist/capabilities/sensor.d.ts.map +1 -0
- package/dist/capabilities/sensor.js +3 -0
- package/dist/capabilities/sensor.js.map +1 -0
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +17 -11
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/capability.d.ts.map +1 -1
- package/dist/cli/commands/capability.js +57 -5
- package/dist/cli/commands/capability.js.map +1 -1
- package/dist/cli/commands/context.d.ts +9 -0
- package/dist/cli/commands/context.d.ts.map +1 -0
- package/dist/cli/commands/context.js +91 -0
- package/dist/cli/commands/context.js.map +1 -0
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +42 -30
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/generate.d.ts.map +1 -1
- package/dist/cli/commands/generate.js +35 -8
- package/dist/cli/commands/generate.js.map +1 -1
- package/dist/cli/commands/ledger.d.ts.map +1 -1
- package/dist/cli/commands/ledger.js +15 -5
- package/dist/cli/commands/ledger.js.map +1 -1
- package/dist/cli/commands/manifest.d.ts.map +1 -1
- package/dist/cli/commands/manifest.js +10 -9
- package/dist/cli/commands/manifest.js.map +1 -1
- package/dist/cli/commands/repair.d.ts +8 -0
- package/dist/cli/commands/repair.d.ts.map +1 -0
- package/dist/cli/commands/repair.js +97 -0
- package/dist/cli/commands/repair.js.map +1 -0
- package/dist/cli/commands/script-check.d.ts.map +1 -1
- package/dist/cli/commands/script-check.js +13 -9
- package/dist/cli/commands/script-check.js.map +1 -1
- package/dist/cli/commands/trace.d.ts.map +1 -1
- package/dist/cli/commands/trace.js +7 -4
- package/dist/cli/commands/trace.js.map +1 -1
- package/dist/cli/index.js +14 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/generators/test-generator/adapters/adapter-interface.d.ts +1 -0
- package/dist/generators/test-generator/adapters/adapter-interface.d.ts.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/playwright-adapter.d.ts +1 -0
- package/dist/generators/test-generator/adapters/playwright/playwright-adapter.d.ts.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/playwright-adapter.js.map +1 -1
- package/dist/generators/test-generator/adapters/playwright/templates/imports.hbs +3 -0
- package/dist/generators/test-generator/code-generator.d.ts +18 -9
- package/dist/generators/test-generator/code-generator.d.ts.map +1 -1
- package/dist/generators/test-generator/code-generator.js +162 -115
- package/dist/generators/test-generator/code-generator.js.map +1 -1
- package/dist/generators/test-generator/patterns/index.d.ts +0 -10
- package/dist/generators/test-generator/patterns/index.d.ts.map +1 -1
- package/dist/generators/test-generator/patterns/index.js +10 -47
- package/dist/generators/test-generator/patterns/index.js.map +1 -1
- package/dist/generators/test-generator/template-engine.d.ts +1 -0
- package/dist/generators/test-generator/template-engine.d.ts.map +1 -1
- package/dist/generators/test-generator/template-engine.js +1 -1
- package/dist/generators/test-generator/template-engine.js.map +1 -1
- package/dist/harness/annotation-overrides.d.ts +11 -0
- package/dist/harness/annotation-overrides.d.ts.map +1 -0
- package/dist/harness/annotation-overrides.js +38 -0
- package/dist/harness/annotation-overrides.js.map +1 -0
- package/dist/harness/audit.d.ts +9 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +140 -10
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/capability-plan.d.ts +14 -0
- package/dist/harness/capability-plan.d.ts.map +1 -1
- package/dist/harness/capability-plan.js +63 -1
- package/dist/harness/capability-plan.js.map +1 -1
- package/dist/harness/catalog/drivers.yaml +35 -12
- package/dist/harness/data-driven-lint.d.ts.map +1 -1
- package/dist/harness/data-driven-lint.js +23 -0
- package/dist/harness/data-driven-lint.js.map +1 -1
- package/dist/harness/flow-check.d.ts +9 -0
- package/dist/harness/flow-check.d.ts.map +1 -1
- package/dist/harness/flow-check.js +13 -6
- package/dist/harness/flow-check.js.map +1 -1
- package/dist/harness/intent.d.ts +6 -0
- package/dist/harness/intent.d.ts.map +1 -1
- package/dist/harness/intent.js +20 -4
- package/dist/harness/intent.js.map +1 -1
- package/dist/harness/ledger.d.ts.map +1 -1
- package/dist/harness/ledger.js +3 -2
- package/dist/harness/ledger.js.map +1 -1
- package/dist/harness/manifest.d.ts.map +1 -1
- package/dist/harness/manifest.js +3 -2
- package/dist/harness/manifest.js.map +1 -1
- package/dist/harness/parse.d.ts +2 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +16 -4
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/quality-gates.js +1 -1
- package/dist/harness/quality-gates.js.map +1 -1
- package/dist/harness/query-catalog.d.ts.map +1 -1
- package/dist/harness/query-catalog.js +0 -0
- package/dist/harness/query-catalog.js.map +1 -1
- package/dist/harness/repair.d.ts +20 -0
- package/dist/harness/repair.d.ts.map +1 -0
- package/dist/harness/repair.js +111 -0
- package/dist/harness/repair.js.map +1 -0
- package/dist/harness/script-check.d.ts +3 -1
- package/dist/harness/script-check.d.ts.map +1 -1
- package/dist/harness/script-check.js +22 -8
- package/dist/harness/script-check.js.map +1 -1
- package/dist/harness/sensors.d.ts +40 -0
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +54 -2
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/trace.d.ts.map +1 -1
- package/dist/harness/trace.js +4 -3
- package/dist/harness/trace.js.map +1 -1
- package/dist/harness/unit-paths.d.ts +3 -0
- package/dist/harness/unit-paths.d.ts.map +1 -0
- package/dist/harness/unit-paths.js +52 -0
- package/dist/harness/unit-paths.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +36 -0
- package/dist/index.js.map +1 -0
- package/dist/orchestrator/ai-rules-updater.d.ts.map +1 -1
- package/dist/orchestrator/ai-rules-updater.js +2 -0
- package/dist/orchestrator/ai-rules-updater.js.map +1 -1
- package/dist/orchestrator/context-discovery.d.ts +12 -0
- package/dist/orchestrator/context-discovery.d.ts.map +1 -0
- package/dist/orchestrator/context-discovery.js +46 -0
- package/dist/orchestrator/context-discovery.js.map +1 -0
- package/dist/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +7 -1
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-create-test.md +10 -5
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-run-test.md +18 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-api-design.md +62 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-gherkin-syntax.md +1 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +2 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +19 -2
- package/dist/orchestrator/templates/ai-instructions/claude-skill-viewpoint.md +14 -0
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-create-test.md +10 -5
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-run-test.md +11 -1
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-api-design.md +62 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-gherkin-syntax.md +1 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +2 -1
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +19 -2
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-viewpoint.md +14 -0
- package/dist/orchestrator/templates/specs-api.d.ts +55 -0
- package/dist/orchestrator/templates/specs-api.d.ts.map +1 -0
- package/dist/orchestrator/templates/specs-api.js +171 -0
- package/dist/orchestrator/templates/specs-api.js.map +1 -0
- package/dist/orchestrator/templates/specs-api.ts +154 -0
- package/dist/orchestrator/templates/specs-db.d.ts +3 -0
- package/dist/orchestrator/templates/specs-db.d.ts.map +1 -1
- package/dist/orchestrator/templates/specs-db.js +78 -1
- package/dist/orchestrator/templates/specs-db.js.map +1 -1
- package/dist/orchestrator/templates/specs-db.ts +78 -1
- package/dist/orchestrator/templates/specs-test-data.ts +2 -1
- package/package.json +7 -30
- package/src/capabilities/builtins.ts +85 -0
- package/src/capabilities/context-router.ts +66 -0
- package/src/capabilities/context.ts +65 -0
- package/src/capabilities/discover.ts +62 -0
- package/src/capabilities/registry.ts +113 -0
- package/src/capabilities/sensor.ts +47 -0
- package/src/cli/commands/audit.ts +15 -9
- package/src/cli/commands/capability.ts +53 -5
- package/src/cli/commands/context.ts +52 -0
- package/src/cli/commands/delivery.ts +40 -31
- package/src/cli/commands/generate.ts +37 -8
- package/src/cli/commands/ledger.ts +13 -5
- package/src/cli/commands/manifest.ts +9 -7
- package/src/cli/commands/repair.ts +57 -0
- package/src/cli/commands/script-check.ts +12 -8
- package/src/cli/commands/trace.ts +7 -4
- package/src/cli/index.ts +14 -1
- package/src/generators/test-generator/adapters/adapter-interface.ts +1 -1
- package/src/generators/test-generator/adapters/playwright/playwright-adapter.ts +1 -1
- package/src/generators/test-generator/adapters/playwright/templates/imports.hbs +3 -0
- package/src/generators/test-generator/code-generator.ts +163 -111
- package/src/generators/test-generator/patterns/index.ts +9 -35
- package/src/generators/test-generator/template-engine.ts +2 -2
- package/src/harness/annotation-overrides.ts +27 -0
- package/src/harness/audit.ts +141 -12
- package/src/harness/capability-plan.ts +51 -1
- package/src/harness/catalog/drivers.yaml +35 -12
- package/src/harness/data-driven-lint.ts +20 -0
- package/src/harness/flow-check.ts +15 -6
- package/src/harness/intent.ts +25 -4
- package/src/harness/ledger.ts +3 -2
- package/src/harness/manifest.ts +3 -2
- package/src/harness/parse.ts +11 -2
- package/src/harness/quality-gates.ts +1 -1
- package/src/harness/query-catalog.ts +0 -0
- package/src/harness/repair.ts +75 -0
- package/src/harness/script-check.ts +25 -8
- package/src/harness/sensors.ts +71 -2
- package/src/harness/trace.ts +4 -3
- package/src/harness/unit-paths.ts +14 -0
- package/src/index.ts +32 -0
- package/src/orchestrator/ai-rules-updater.ts +2 -0
- package/src/orchestrator/context-discovery.ts +50 -0
- package/src/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +7 -1
- package/src/orchestrator/templates/ai-instructions/claude-cmd-create-test.md +10 -5
- package/src/orchestrator/templates/ai-instructions/claude-cmd-run-test.md +18 -1
- package/src/orchestrator/templates/ai-instructions/claude-skill-api-design.md +62 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-gherkin-syntax.md +1 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +2 -1
- package/src/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +19 -2
- package/src/orchestrator/templates/ai-instructions/claude-skill-viewpoint.md +14 -0
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-create-test.md +10 -5
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-run-test.md +11 -1
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-api-design.md +62 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-gherkin-syntax.md +1 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +2 -1
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +19 -2
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-viewpoint.md +14 -0
- package/src/orchestrator/templates/specs-api.ts +154 -0
- package/src/orchestrator/templates/specs-db.ts +78 -1
- package/src/orchestrator/templates/specs-test-data.ts +2 -1
- package/dist/generators/test-generator/patterns/assertion-patterns.d.ts +0 -7
- package/dist/generators/test-generator/patterns/assertion-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/assertion-patterns.js +0 -626
- package/dist/generators/test-generator/patterns/assertion-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/capture-patterns.d.ts +0 -21
- package/dist/generators/test-generator/patterns/capture-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/capture-patterns.js +0 -87
- package/dist/generators/test-generator/patterns/capture-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/database-patterns.d.ts +0 -6
- package/dist/generators/test-generator/patterns/database-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/database-patterns.js +0 -95
- package/dist/generators/test-generator/patterns/database-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/form-patterns.d.ts +0 -6
- package/dist/generators/test-generator/patterns/form-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/form-patterns.js +0 -160
- package/dist/generators/test-generator/patterns/form-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/interaction-patterns.d.ts +0 -6
- package/dist/generators/test-generator/patterns/interaction-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/interaction-patterns.js +0 -433
- package/dist/generators/test-generator/patterns/interaction-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/keyboard-patterns.d.ts +0 -7
- package/dist/generators/test-generator/patterns/keyboard-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/keyboard-patterns.js +0 -47
- package/dist/generators/test-generator/patterns/keyboard-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/navigation-patterns.d.ts +0 -6
- package/dist/generators/test-generator/patterns/navigation-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/navigation-patterns.js +0 -125
- package/dist/generators/test-generator/patterns/navigation-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/scope-patterns.d.ts +0 -7
- package/dist/generators/test-generator/patterns/scope-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/scope-patterns.js +0 -36
- package/dist/generators/test-generator/patterns/scope-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/scroll-patterns.d.ts +0 -7
- package/dist/generators/test-generator/patterns/scroll-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/scroll-patterns.js +0 -25
- package/dist/generators/test-generator/patterns/scroll-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/setup-patterns.d.ts +0 -6
- package/dist/generators/test-generator/patterns/setup-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/setup-patterns.js +0 -72
- package/dist/generators/test-generator/patterns/setup-patterns.js.map +0 -1
- package/dist/generators/test-generator/patterns/table-patterns.d.ts +0 -19
- package/dist/generators/test-generator/patterns/table-patterns.d.ts.map +0 -1
- package/dist/generators/test-generator/patterns/table-patterns.js +0 -239
- package/dist/generators/test-generator/patterns/table-patterns.js.map +0 -1
- package/docs/orchestration-spec.md +0 -267
- package/src/generators/test-generator/patterns/assertion-patterns.ts +0 -691
- package/src/generators/test-generator/patterns/capture-patterns.ts +0 -97
- package/src/generators/test-generator/patterns/database-patterns.ts +0 -96
- package/src/generators/test-generator/patterns/form-patterns.ts +0 -167
- package/src/generators/test-generator/patterns/interaction-patterns.ts +0 -465
- package/src/generators/test-generator/patterns/keyboard-patterns.ts +0 -51
- package/src/generators/test-generator/patterns/navigation-patterns.ts +0 -140
- package/src/generators/test-generator/patterns/scope-patterns.ts +0 -40
- package/src/generators/test-generator/patterns/scroll-patterns.ts +0 -27
- package/src/generators/test-generator/patterns/setup-patterns.ts +0 -76
- package/src/generators/test-generator/patterns/table-patterns.ts +0 -279
package/src/harness/audit.ts
CHANGED
|
@@ -9,16 +9,24 @@
|
|
|
9
9
|
import * as path from 'path';
|
|
10
10
|
import * as fs from 'fs';
|
|
11
11
|
import { loadScenarios, parseViewpointOverview, ScenarioInfo, ViewpointEntry } from './parse';
|
|
12
|
+
import { featureBasename } from './unit-paths';
|
|
12
13
|
import {
|
|
13
|
-
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, coverageBalance, duplicateClusters, traceability, claimProof, taxonomyLint,
|
|
14
|
-
|
|
14
|
+
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, depthThresholdFor, coverageBalance, duplicateClusters, traceability, claimProof, taxonomyLint,
|
|
15
|
+
automatableManual, flowCoveredThemes,
|
|
16
|
+
GateResult, DepthResult, BalanceResult, DuplicateResult, TraceResult, ClaimProofResult, TaxonomyResult, Catalog, AutomatableManualResult,
|
|
15
17
|
} from './sensors';
|
|
18
|
+
import { loadFlowScenarios } from './flow-check';
|
|
19
|
+
import { manualReasonMismatches, MANUAL_REASONS, buildPlan } from './capability-plan';
|
|
20
|
+
import { readCapabilities } from './capability';
|
|
16
21
|
import { readIntent, projectRootFromScreenDir, IntentProfile } from './intent';
|
|
17
22
|
import { getProvenance, Provenance } from './provenance';
|
|
18
23
|
import { specCoverage, SpecCoverageResult, parseSpecClauses } from './spec-coverage';
|
|
19
24
|
import { downstreamScope, manualOracle, readText, DownstreamResult, ManualOracleResult,
|
|
20
25
|
negativeSideEffect, sourceBacked, crossArtifactOwnership } from './quality-gates';
|
|
21
26
|
import { viewpointLedger, parseViewpointItems, LedgerResult } from './viewpoint-ledger';
|
|
27
|
+
import { capabilityRegistry } from '../capabilities/registry';
|
|
28
|
+
import { discoverAndRegisterCapabilities } from '../capabilities/discover';
|
|
29
|
+
import { contextRouter } from '../capabilities/context-router';
|
|
22
30
|
|
|
23
31
|
export interface AuditReport {
|
|
24
32
|
screen: string;
|
|
@@ -33,6 +41,7 @@ export interface AuditReport {
|
|
|
33
41
|
taxonomyMismatch: boolean; // scenarios use IDs not in the project's test-viewpoint.md
|
|
34
42
|
downstream: DownstreamResult; // downstream screens referenced but under-covered
|
|
35
43
|
manualOracle: ManualOracleResult; // @manual scenarios lacking setup/action/oracle
|
|
44
|
+
automatableManual: AutomatableManualResult; // @manual that is actually automatable (deferred, not judgment) — TQ-2
|
|
36
45
|
ledger: LedgerResult; // atomic viewpoint-item coverage (per-bullet status)
|
|
37
46
|
calibration: { // #8 — multi-axis score so a high overall can't hide a weak axis
|
|
38
47
|
axes: Record<string, number>;
|
|
@@ -54,22 +63,84 @@ export interface AuditReport {
|
|
|
54
63
|
spec: SpecCoverageResult; // G2 — spec-clause coverage (FR + validation-trigger matrix)
|
|
55
64
|
}
|
|
56
65
|
|
|
66
|
+
/** The catalog-resolution id for a unit dir (relative to qa/): screen · flows/<flow> · api/<area> · api/flows/<flow>. */
|
|
67
|
+
function catalogIdFromScreenDir(screenDir: string): string {
|
|
68
|
+
const parts = screenDir.split(path.sep);
|
|
69
|
+
const qa = parts.lastIndexOf('qa');
|
|
70
|
+
if (qa >= 0) {
|
|
71
|
+
if (parts[qa + 1] === 'api' && parts[qa + 2] === 'flows' && parts[qa + 3]) return `api/flows/${parts[qa + 3]}`;
|
|
72
|
+
if (parts[qa + 1] === 'api' && parts[qa + 2]) return `api/${parts[qa + 2]}`;
|
|
73
|
+
if (parts[qa + 1] === 'flows' && parts[qa + 2]) return `flows/${parts[qa + 2]}`;
|
|
74
|
+
}
|
|
75
|
+
return path.basename(screenDir);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The capability that owns SCORING for a unit (AO-1): the first path segment of the unit id when it
|
|
80
|
+
* is a registered capability (`api/<area>` → `api`), else the default (UI) capability. Generic — a
|
|
81
|
+
* future `mobile/<x>` or `perf/<x>` unit routes to that capability with no core change. `flows/<flow>`
|
|
82
|
+
* has no `flows` capability → default (UI), which is correct (flows are a UI concept).
|
|
83
|
+
*/
|
|
84
|
+
export function scoringCapabilityFor(catalogScreenName: string, defaultCap: string | undefined): string | undefined {
|
|
85
|
+
const seg = catalogScreenName.split('/')[0];
|
|
86
|
+
return seg && capabilityRegistry.get(seg) ? seg : defaultCap;
|
|
87
|
+
}
|
|
88
|
+
|
|
57
89
|
export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
58
|
-
|
|
90
|
+
// The feature filename is the unit's LAST segment — an api flow (`flows/<flow>`) lives at
|
|
91
|
+
// `<dir>/features/<flow>.feature`, not `features/flows/<flow>.feature` (which found 0 scenarios).
|
|
92
|
+
const featurePath = path.join(screenDir, 'features', `${featureBasename(screenName)}.feature`);
|
|
59
93
|
const viewpointPath = path.join(screenDir, 'requirements', 'test-viewpoint.md');
|
|
94
|
+
// Catalog-resolution id (for the @api/@query gate sensors): the unit's path relative to qa/ —
|
|
95
|
+
// `flows/<flow>`, `api/<area>`, `api/flows/<flow>`, else the bare screen. A bare screen matches
|
|
96
|
+
// the old behaviour (so the audit-sample snapshot is unchanged); flows/api now resolve correctly.
|
|
97
|
+
const catalogScreenName = catalogIdFromScreenDir(screenDir);
|
|
60
98
|
|
|
61
99
|
const specPath = path.join(screenDir, 'requirements', 'spec.md');
|
|
62
100
|
const featureText = fs.existsSync(featurePath) ? fs.readFileSync(featurePath, 'utf-8') : '';
|
|
63
101
|
|
|
64
102
|
const scenarios: ScenarioInfo[] = loadScenarios(featurePath);
|
|
65
103
|
const viewpoints: ViewpointEntry[] = parseViewpointOverview(viewpointPath);
|
|
66
|
-
|
|
104
|
+
// AO-1 — capability-routed scoring: the viewpoint catalog + score-bearing gate are owned by the
|
|
105
|
+
// unit's capability, resolved from the unit id (`api/<area>` → `api`; screen/flow → the default
|
|
106
|
+
// UI capability). A capability that provides no catalog/gate falls back to the in-core UI
|
|
107
|
+
// functions, so UI units — and api units until AO-2 adds the api providers — are byte-identical.
|
|
108
|
+
discoverAndRegisterCapabilities();
|
|
109
|
+
const defaultCap = capabilityRegistry.defaultCapabilityId();
|
|
110
|
+
const scoringCapId = scoringCapabilityFor(catalogScreenName, defaultCap);
|
|
111
|
+
const scoringCap = scoringCapId ? capabilityRegistry.get(scoringCapId) : undefined;
|
|
112
|
+
const catalog = (scoringCap?.viewpoints?.() as Catalog | undefined) || loadCatalog();
|
|
67
113
|
const spec = specCoverage(specPath, scenarios, featureText);
|
|
68
114
|
|
|
69
|
-
const gate = viewpointGate(scenarios, viewpoints, catalog);
|
|
70
115
|
// P3 — intent profile from qa/context.md drives the depth threshold (focus).
|
|
71
116
|
const intent = readIntent(projectRootFromScreenDir(screenDir));
|
|
72
|
-
|
|
117
|
+
// The score-bearing gate (viewpoint coverage + assertion depth) is the scoring capability's
|
|
118
|
+
// `gateProvider`; same functions underneath → byte-identical gate/depth → identical score. Falls
|
|
119
|
+
// back to the in-core functions if the capability provides none.
|
|
120
|
+
// A capability gate may need project context (the API gate resolves endpoint methods from the
|
|
121
|
+
// catalog) + the focus depth threshold (so it scores depth with the SAME bar as the UI gate).
|
|
122
|
+
const capGate = scoringCap?.gateProvider as
|
|
123
|
+
((i: { scenarios: ScenarioInfo[]; viewpoints: ViewpointEntry[]; catalog: Catalog; focus: typeof intent.focus; cwd: string; screenName: string; threshold: number; businessCriticalMethods?: string[] }) => { gate: GateResult; depth: DepthResult }) | undefined;
|
|
124
|
+
const provided = capGate?.({ scenarios, viewpoints, catalog, focus: intent.focus, cwd: projectRootFromScreenDir(screenDir), screenName: catalogScreenName, threshold: depthThresholdFor(intent.focus), businessCriticalMethods: intent.businessCriticalMethods });
|
|
125
|
+
const gate = provided?.gate ?? viewpointGate(scenarios, viewpoints, catalog);
|
|
126
|
+
const depth = provided?.depth ?? assertionDepth(scenarios, dataThemesFor(catalog, gate.pageType), intent.focus);
|
|
127
|
+
|
|
128
|
+
// TQ-4 — deferral-aware coverage credit: an inherently cross-screen theme (cart / detail /
|
|
129
|
+
// filter correctness) belongs in a FLOW, not on the screen. When a flow deeply covers a screen
|
|
130
|
+
// gate gap, credit it to the flow instead of double-counting it as a screen gap. Screens only
|
|
131
|
+
// (a flow/api unit is not credited by sibling flows); mutates the gate before coverage is scored.
|
|
132
|
+
const flowCredits: { theme: string; flow: string }[] = [];
|
|
133
|
+
const isScreenUnit = !/^(flows|api)\//.test(catalogScreenName);
|
|
134
|
+
if (isScreenUnit && gate.gaps.length) {
|
|
135
|
+
const flowScenarios = loadFlowScenarios(projectRootFromScreenDir(screenDir));
|
|
136
|
+
if (flowScenarios.length) {
|
|
137
|
+
for (const c of flowCoveredThemes(gate.gaps, flowScenarios)) {
|
|
138
|
+
const i = gate.gaps.findIndex((g) => g.theme === c.theme);
|
|
139
|
+
if (i >= 0) { gate.gaps.splice(i, 1); gate.themesCovered++; flowCredits.push(c); }
|
|
140
|
+
}
|
|
141
|
+
gate.coverageRatio = gate.themesTotal ? gate.themesCovered / gate.themesTotal : 1;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
73
144
|
const claim = claimProof(scenarios, intent.focus);
|
|
74
145
|
const taxonomy = taxonomyLint(scenarios);
|
|
75
146
|
const balance = coverageBalance(scenarios);
|
|
@@ -80,6 +151,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
80
151
|
// #2 downstream-scope + #4 manual-oracle
|
|
81
152
|
const downstream = downstreamScope(readText(specPath), scenarios);
|
|
82
153
|
const manualOracleResult = manualOracle(featureText);
|
|
154
|
+
const autoManual = automatableManual(scenarios); // TQ-2 — @manual that is really automatable
|
|
83
155
|
const ledger = viewpointLedger(viewpointPath, scenarios, featureText);
|
|
84
156
|
const negSideEffect = negativeSideEffect(scenarios);
|
|
85
157
|
const ownership = crossArtifactOwnership(screenDir, scenarios);
|
|
@@ -97,6 +169,9 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
97
169
|
const overall = (0.4 * coverage + 0.3 * businessDepth + 0.15 * balanceScore + 0.15 * traceScore) * 10;
|
|
98
170
|
|
|
99
171
|
const findings: string[] = [];
|
|
172
|
+
for (const c of flowCredits) {
|
|
173
|
+
findings.push(`COVERED-VIA-FLOW: critical theme "${c.theme}" is not covered on this screen but is deeply covered by flow "${c.flow}" — cross-screen depth correctly owned by the flow, so it is credited (not a screen gap). Verify with \`sungen flow-check\`.`);
|
|
174
|
+
}
|
|
100
175
|
for (const g of gate.gaps) {
|
|
101
176
|
if (g.status === 'shallow') {
|
|
102
177
|
findings.push(`GATE: critical theme "${g.theme}" is covered only by SHALLOW scenarios (no data assertion) → deepen with \`... with {{value}}\` / \`table ... with {{value}}\` (count @manual cross-screen too).`);
|
|
@@ -125,9 +200,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
125
200
|
if (trace.mappedRatio < 0.5) {
|
|
126
201
|
findings.push(`TRACE: ${trace.note}`);
|
|
127
202
|
}
|
|
128
|
-
|
|
129
|
-
findings.push(`UNIVERSAL: missing theme(s): ${gate.universalGaps.join(', ')} (low priority reminder).`);
|
|
130
|
-
}
|
|
203
|
+
// (UNIVERSAL viewpoint-gap finding now emitted by the `ui` gate sensor — see the gate block below.)
|
|
131
204
|
for (const g of spec.triggerGaps) {
|
|
132
205
|
findings.push(`TRIGGER-UNCOVERED: spec validates "${g.constraint}"${g.code ? ` (${g.code})` : ''} on [${g.required.join(', ')}] but scenarios only exercise it on [${g.found.join(', ') || 'none'}] → add a ${g.missing.join(', ')}-trigger scenario for this constraint (don't collapse the trigger × input matrix).`);
|
|
133
206
|
}
|
|
@@ -143,6 +216,40 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
143
216
|
for (const m of manualOracleResult.insufficient.slice(0, 8)) {
|
|
144
217
|
findings.push(`MANUAL-STEPS-INSUFFICIENT: "${m}" — a @manual scenario needs setup · action · observable expected · oracle/tool (not just a one-line note).`);
|
|
145
218
|
}
|
|
219
|
+
// TQ-2 — automatable @manual: deferred (usually cross-screen) but fully DSL-expressible.
|
|
220
|
+
for (const m of autoManual.scenarios.slice(0, 8)) {
|
|
221
|
+
findings.push(`MANUAL-AUTOMATABLE: "${m.name}" is @manual but its steps are fully automatable (a data assertion, no visual/mock/a11y judgment) → generate it as an AUTOMATED flow scenario (or tag @manual:Mx with a real capability reason). Don't leave a non-running @manual duplicate of a flow scenario.`);
|
|
222
|
+
}
|
|
223
|
+
// TQ-9 — manual reason-code mismatch: an explicit @manual:Mx that disagrees with the reason
|
|
224
|
+
// text makes the capability planner recommend the wrong driver. Surface it so the code is fixed.
|
|
225
|
+
for (const mm of manualReasonMismatches(featurePath).slice(0, 8)) {
|
|
226
|
+
const label = MANUAL_REASONS[mm.inferred]?.label ?? (mm.inferred === 'XS' ? 'Cross-screen (→ flow)' : mm.inferred);
|
|
227
|
+
findings.push(`MANUAL-REASON-MISMATCH: "${mm.scenario}" is tagged @manual:${mm.explicit} but its reason reads as ${mm.inferred} (${label}) → fix the code so \`sungen capability plan\` recommends the right driver to automate it.`);
|
|
228
|
+
}
|
|
229
|
+
// TQ-10 — surface the Capability Planner recommendation (recommend-only; never installs). Silenced
|
|
230
|
+
// by `capability_suggestions: off` in qa/context.md. Reuses the planner (trustworthy after TQ-9).
|
|
231
|
+
if (intent.capabilitySuggestions) {
|
|
232
|
+
const plan = buildPlan(screenDir, featureBasename(screenName));
|
|
233
|
+
if (plan.recommendations.length) {
|
|
234
|
+
const recs = plan.recommendations.map((r) => `\`sungen capability add ${r.driver}\` (automates ${r.count})`).join(' · ');
|
|
235
|
+
findings.push(`CAPABILITY-SUGGESTION: ${plan.capabilityManual} @manual scenario(s) are capability-manual (a driver could automate them) — ${recs}. Recommend-only: nothing is installed automatically; the ${plan.judgmentManual} judgment-manual (M6/M8/M9) correctly stay manual.`);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
// TQ-11b — automation-ready (pending capability): @requires:<cap> scenarios whose cap isn't enabled.
|
|
239
|
+
// They are NOT manual (real steps, compiled the moment the cap is added) — surface them distinctly.
|
|
240
|
+
{
|
|
241
|
+
const enabledCaps = new Set(readCapabilities(projectRootFromScreenDir(screenDir)).enabled.map((d) => d.toLowerCase()));
|
|
242
|
+
const pending = scenarios.filter((s) => (s.requiresCaps ?? []).some((c) => !enabledCaps.has(c)));
|
|
243
|
+
if (pending.length) {
|
|
244
|
+
const caps = [...new Set(pending.flatMap((s) => (s.requiresCaps ?? []).filter((c) => !enabledCaps.has(c))))];
|
|
245
|
+
findings.push(`AUTOMATION-READY-PENDING: ${pending.length} scenario(s) are automation-ready but need a capability — \`sungen capability add ${caps.join(' ')}\` to run them. They are skipped (not manual, not a gap) until the driver is enabled.`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
// TQ-3 — businessDepth de-inflation: a high ratio on a tiny denominator because business-critical
|
|
249
|
+
// scenarios were deferred to @manual is misleading. Surface the deferral so 1.0 isn't read as "done".
|
|
250
|
+
if (depth.deferredBusinessCritical > 0 && depth.deferredBusinessCritical >= depth.businessCriticalTotal) {
|
|
251
|
+
findings.push(`DEPTH-DEFERRED: businessDepth ${businessDepth.toFixed(2)} is computed over only ${depth.businessCriticalTotal} on-screen scenario(s); ${depth.deferredBusinessCritical} business-critical scenario(s) are deferred to @manual (excluded from the ratio). Automate them in a flow and verify with \`sungen flow-check\` — this ratio is NOT "all business depth covered".`);
|
|
252
|
+
}
|
|
146
253
|
if (ledger.hasViewpoint && ledger.missing.length) {
|
|
147
254
|
const sample = ledger.missing.slice(0, 6).map((m) => m.id || `"${m.text}"`).join(', ');
|
|
148
255
|
findings.push(`VIEWPOINT-ITEM-MISSING: ${ledger.missing.length}/${ledger.total} atomic viewpoint items have no covering scenario (${(ledger.ratio * 100).toFixed(0)}% covered) — e.g. ${sample}. Cover each item or mark it deferred/spec-gap.`);
|
|
@@ -157,6 +264,24 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
157
264
|
findings.push(`UNSOURCEABLE-SCENARIO: "${u}" doesn't trace to any FR / viewpoint item — link it to a source, or tag it @exploration (not part of the official suite).`);
|
|
158
265
|
}
|
|
159
266
|
|
|
267
|
+
// Capability gate sensors (Capability SPI): the ContextRouter scopes WHICH gate sensors run to
|
|
268
|
+
// the capabilities this feature actually uses — generic ('core') + the default UI + any whose
|
|
269
|
+
// annotation tags appear (e.g. @query). Today core+ui gate sensors are always in scope, so this
|
|
270
|
+
// is behaviour-identical; it bounds the set as capability-specific gate sensors (@api, …) are
|
|
271
|
+
// added. Each runs over the audit context; an 'error' finding fails the gate.
|
|
272
|
+
const featureTags = [
|
|
273
|
+
...(scenarios.some((s) => s.queryRefs && s.queryRefs.length) ? ['@query'] : []),
|
|
274
|
+
...(scenarios.some((s) => s.apiRefs && s.apiRefs.length) ? ['@api'] : []),
|
|
275
|
+
];
|
|
276
|
+
const routedGateIds = contextRouter.route({ target: { kind: 'screen', id: screenName }, artifact: 'feature', tags: featureTags }).gateSensorIds;
|
|
277
|
+
const gateSensorFindings = capabilityRegistry.sensors('gate')
|
|
278
|
+
.filter((s) => routedGateIds.includes(s.id))
|
|
279
|
+
.flatMap((s) => s.run({ screenName: catalogScreenName, cwd: projectRootFromScreenDir(screenDir), featureText, scenarios, universalGaps: gate.universalGaps }));
|
|
280
|
+
// Each gate sensor's message carries its own code prefix (VERIFICATION-FAIL / UNIVERSAL / …)
|
|
281
|
+
// → push verbatim.
|
|
282
|
+
for (const f of gateSensorFindings) findings.push(f.message);
|
|
283
|
+
const gateSensorError = gateSensorFindings.some((f) => f.severity === 'error');
|
|
284
|
+
|
|
160
285
|
// #8 — multi-axis calibration: a high overall must not hide a weak axis.
|
|
161
286
|
const manualCompleteness = manualOracleResult.manualTotal
|
|
162
287
|
? 1 - manualOracleResult.insufficient.length / manualOracleResult.manualTotal : 1;
|
|
@@ -171,7 +296,11 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
171
296
|
};
|
|
172
297
|
const weakestEntry = Object.entries(axes).sort((a, b) => a[1] - b[1])[0];
|
|
173
298
|
const weakest = { axis: weakestEntry[0], value: weakestEntry[1] };
|
|
174
|
-
|
|
299
|
+
// Inflated when breadth hides a weak axis, OR when businessDepth is high but rests on a
|
|
300
|
+
// denominator collapsed by deferral-to-@manual (TQ-3): the headline depth can't be trusted.
|
|
301
|
+
const depthInflated =
|
|
302
|
+
businessDepth >= 0.9 && depth.deferredBusinessCritical >= depth.businessCriticalTotal && depth.deferredBusinessCritical > 0;
|
|
303
|
+
const inflated = (overall >= 8 && weakest.value < 0.6) || depthInflated;
|
|
175
304
|
if (inflated) {
|
|
176
305
|
findings.push(`SCORE-INFLATED-BY-BREADTH: overall ${Math.round(overall * 10) / 10}/10 but the weakest axis "${weakest.axis}" is ${(weakest.value * 100).toFixed(0)}% — breadth is hiding a weak dimension. Raise "${weakest.axis}" before trusting the headline.`);
|
|
177
306
|
}
|
|
@@ -180,13 +309,13 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
180
309
|
// Gate spans coverage (viewpoint themes), depth, claim-proof, spec-clause coverage,
|
|
181
310
|
// AND taxonomy-match (scenarios must use the project's viewpoint IDs when defined).
|
|
182
311
|
const gateStatus: 'PASS' | 'FAIL' =
|
|
183
|
-
gate.gaps.length === 0 && depth.verdict !== 'fail' && claim.verdict !== 'fail' && spec.verdict !== 'fail' && !taxonomyMismatch ? 'PASS' : 'FAIL';
|
|
312
|
+
gate.gaps.length === 0 && depth.verdict !== 'fail' && claim.verdict !== 'fail' && spec.verdict !== 'fail' && !taxonomyMismatch && !gateSensorError ? 'PASS' : 'FAIL';
|
|
184
313
|
|
|
185
314
|
return {
|
|
186
315
|
screen: screenName,
|
|
187
316
|
scenarioCount: scenarios.length,
|
|
188
317
|
gate, depth, claim, taxonomy, balance, duplicates, trace, spec,
|
|
189
|
-
taxonomyMismatch, downstream, manualOracle: manualOracleResult, ledger, calibration,
|
|
318
|
+
taxonomyMismatch, downstream, manualOracle: manualOracleResult, automatableManual: autoManual, ledger, calibration,
|
|
190
319
|
score: {
|
|
191
320
|
overall: Math.round(overall * 10) / 10,
|
|
192
321
|
coverage: Math.round(coverage * 100) / 100,
|
|
@@ -36,7 +36,9 @@ const INFER: { code: string; re: RegExp }[] = [
|
|
|
36
36
|
{ code: 'M3', re: /\b(mock|stub|network|offline|slow network|intercept)\b/i },
|
|
37
37
|
{ code: 'M2', re: /\b(api|endpoint|backend|db|database|server[-\s]?side|via api)\b/i },
|
|
38
38
|
{ code: 'M1', re: /\b(data setup|dataset|seed|test data|empty (category|product|dataset|state)|zero products|forcing an empty|backend\/test data)\b/i },
|
|
39
|
-
|
|
39
|
+
// "email" alone is too greedy (every subscription test mentions it) → require a real external-mail
|
|
40
|
+
// signal (verification mail / OTP / inbox), so an API/DB-oracle test isn't misread as M5.
|
|
41
|
+
{ code: 'M5', re: /\b(external|third[-\s]?party|sandbox|payment gateway|invoice|download|verification (e?mail|link)|otp|mailbox|inbox|e?mail link)\b/i },
|
|
40
42
|
{ code: 'M6', re: /\b(visual|responsive|layout|accessibilit|a11y|keyboard|screen reader|ux|breakpoint)\b/i },
|
|
41
43
|
{ code: 'M7', re: /\b(environment|staging[-\s]?only|infra|env limitation)\b/i },
|
|
42
44
|
{ code: 'M8', re: /\b(not worth|exploratory|one[-\s]?off)\b/i },
|
|
@@ -102,6 +104,54 @@ export function inferReasonCode(tags: string[], reason: string): { code: string;
|
|
|
102
104
|
return { code: 'M9', explicit: false, unclassified: true };
|
|
103
105
|
}
|
|
104
106
|
|
|
107
|
+
/** The reason code inferred ONLY from the reason text, ignoring any explicit @manual:Mx tag. */
|
|
108
|
+
export function inferFromText(reason: string): string | undefined {
|
|
109
|
+
for (const r of INFER) if (r.re.test(reason)) return r.code;
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export interface ReasonMismatch { scenario: string; explicit: string; inferred: string }
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* (TQ-9) @manual scenarios whose explicit `@manual:Mx` disagrees with the code inferred from
|
|
117
|
+
* the reason text — a mis-tag that makes the Capability Planner recommend the wrong driver
|
|
118
|
+
* (e.g. tagged `M1` data-setup but the reason describes a DB/API assertion → `M2`). Advisory:
|
|
119
|
+
* surfaced so the reason code — and therefore the driver suggestion — can be trusted.
|
|
120
|
+
*/
|
|
121
|
+
export function manualReasonMismatches(featurePath: string): ReasonMismatch[] {
|
|
122
|
+
if (!fs.existsSync(featurePath)) return [];
|
|
123
|
+
const lines = fs.readFileSync(featurePath, 'utf-8').split('\n');
|
|
124
|
+
const out: ReasonMismatch[] = [];
|
|
125
|
+
for (let i = 0; i < lines.length; i++) {
|
|
126
|
+
const m = lines[i].match(/^\s*Scenario:\s*(.+)$/);
|
|
127
|
+
if (!m) continue;
|
|
128
|
+
// Tags on the adjacent line(s) above.
|
|
129
|
+
const tags: string[] = [];
|
|
130
|
+
for (let j = i - 1; j >= 0 && j >= i - 4; j--) {
|
|
131
|
+
const l = lines[j].trim();
|
|
132
|
+
if (l === '') break;
|
|
133
|
+
if (/^@/.test(l)) tags.unshift(...l.split(/\s+/).filter((t) => t.startsWith('@')));
|
|
134
|
+
else if (/^#/.test(l)) continue;
|
|
135
|
+
else break;
|
|
136
|
+
}
|
|
137
|
+
if (!tags.some((t) => /^@manual\b/i.test(t))) continue;
|
|
138
|
+
const explicit = explicitCode(tags);
|
|
139
|
+
if (!explicit) continue;
|
|
140
|
+
// FULL reason block — every comment line in the body before the first real step (the
|
|
141
|
+
// driver-relevant signal — "subscribers table", "API/persistence" — is often not line 1).
|
|
142
|
+
const parts: string[] = [];
|
|
143
|
+
for (let k = i + 1; k < lines.length && k <= i + 16; k++) {
|
|
144
|
+
const l = lines[k].trim();
|
|
145
|
+
if (/^#/.test(l)) parts.push(l.replace(/^#+\s*/, ''));
|
|
146
|
+
else if (l === '') continue;
|
|
147
|
+
else break;
|
|
148
|
+
}
|
|
149
|
+
const inferred = inferFromText(parts.join(' '));
|
|
150
|
+
if (inferred && inferred !== explicit) out.push({ scenario: m[1].trim(), explicit, inferred });
|
|
151
|
+
}
|
|
152
|
+
return out;
|
|
153
|
+
}
|
|
154
|
+
|
|
105
155
|
function classifyMode(tags: string[]): string {
|
|
106
156
|
const has = (re: RegExp) => tags.some((t) => re.test(t));
|
|
107
157
|
if (has(/^@manual\b/i)) return 'manual';
|
|
@@ -1,21 +1,37 @@
|
|
|
1
1
|
# Driver Catalog (metadata only — NO driver code is bundled here).
|
|
2
2
|
# Lets Sungen RECOMMEND/RESOLVE a driver that may not be installed yet, and tells
|
|
3
|
-
# `sungen capability add` which package to install. See docs/spec/sungen_phase2a_spec.md
|
|
3
|
+
# `sungen capability add` which package to install. See docs/spec/sungen_phase2a_spec.md
|
|
4
|
+
# and docs/spec/sungen_packaging_spec.md (R5 — the capability SPI + npm packages).
|
|
4
5
|
#
|
|
5
|
-
#
|
|
6
|
-
#
|
|
7
|
-
#
|
|
6
|
+
# Two axes:
|
|
7
|
+
# kind: platform → HOW tests run (runtime/codegen adapter). Pick ONE per project.
|
|
8
|
+
# kind: capability → WHAT extra thing is verified, added on top of a platform.
|
|
9
|
+
# Fields:
|
|
10
|
+
# status: shipped → published as an npm package (R5); planned → not built yet.
|
|
11
|
+
# bundled: true → installed automatically (a dependency of @sun-asterisk/sungen),
|
|
12
|
+
# so `capability add` is unnecessary.
|
|
13
|
+
# unblocks: manual-reason codes (M1–M9) this driver resolves (Phase 2b taxonomy).
|
|
14
|
+
#
|
|
15
|
+
# R5 status: the three real capabilities ship as packages — @sungen/driver-ui (web UI,
|
|
16
|
+
# bundled as the default), @sungen/driver-db, @sungen/driver-api. The web *platform*
|
|
17
|
+
# entry below points at @sungen/driver-ui (the UI step vocabulary + viewpoint gate); the
|
|
18
|
+
# Playwright codegen *adapter* itself is still in-core (Phase 2a). Mobile + the remaining
|
|
19
|
+
# capabilities are planned. See the "Platform axis & mobile evolution" section of the
|
|
20
|
+
# packaging spec for the `sungen init --platform <web|mobile>` roadmap.
|
|
8
21
|
|
|
9
22
|
drivers:
|
|
10
23
|
web:
|
|
11
24
|
kind: platform
|
|
12
|
-
package: "@sungen/driver-
|
|
13
|
-
|
|
14
|
-
|
|
25
|
+
package: "@sungen/driver-ui" # R5: web UI capability (step patterns + viewpoint gate)
|
|
26
|
+
status: shipped
|
|
27
|
+
bundled: true # @sun-asterisk/sungen depends on it → UI works out of the box
|
|
28
|
+
runtime: playwright # codegen adapter still in-core (Phase 2a)
|
|
29
|
+
adapter: web # registry adapter name
|
|
15
30
|
capabilities: ["@ui"]
|
|
16
31
|
mobile:
|
|
17
32
|
kind: platform
|
|
18
33
|
package: "@sungen/driver-mobile"
|
|
34
|
+
status: planned # PoC on the feat/mobile branch (Appium / Flutter)
|
|
19
35
|
runtime: appium
|
|
20
36
|
adapter: mobile
|
|
21
37
|
capabilities: ["@ui"]
|
|
@@ -23,35 +39,42 @@ drivers:
|
|
|
23
39
|
api:
|
|
24
40
|
kind: capability
|
|
25
41
|
package: "@sungen/driver-api"
|
|
42
|
+
status: shipped
|
|
26
43
|
capabilities: ["@api", "@apiAssert", "@hybrid"]
|
|
27
44
|
unblocks: [M2]
|
|
28
|
-
data-factory:
|
|
29
|
-
kind: capability
|
|
30
|
-
package: "@sungen/driver-data-factory"
|
|
31
|
-
capabilities: ["@dataFactory"]
|
|
32
|
-
unblocks: [M1]
|
|
33
45
|
db:
|
|
34
46
|
kind: capability
|
|
35
47
|
package: "@sungen/driver-db"
|
|
48
|
+
status: shipped
|
|
36
49
|
capabilities: ["@dbAssert"]
|
|
37
50
|
unblocks: [M2]
|
|
51
|
+
data-factory:
|
|
52
|
+
kind: capability
|
|
53
|
+
package: "@sungen/driver-data-factory"
|
|
54
|
+
status: planned
|
|
55
|
+
capabilities: ["@dataFactory"]
|
|
56
|
+
unblocks: [M1]
|
|
38
57
|
mock:
|
|
39
58
|
kind: capability
|
|
40
59
|
package: "@sungen/driver-mock"
|
|
60
|
+
status: planned
|
|
41
61
|
capabilities: ["@mock", "@network"]
|
|
42
62
|
unblocks: [M3]
|
|
43
63
|
mail-file:
|
|
44
64
|
kind: capability
|
|
45
65
|
package: "@sungen/driver-mail-file"
|
|
66
|
+
status: planned
|
|
46
67
|
capabilities: ["@mail", "@file"]
|
|
47
68
|
unblocks: [M5]
|
|
48
69
|
contract:
|
|
49
70
|
kind: capability
|
|
50
71
|
package: "@sungen/driver-contract"
|
|
72
|
+
status: planned
|
|
51
73
|
capabilities: ["@contract"]
|
|
52
74
|
unblocks: [M5]
|
|
53
75
|
specialized:
|
|
54
76
|
kind: capability
|
|
55
77
|
package: "@sungen/driver-specialized"
|
|
78
|
+
status: planned
|
|
56
79
|
capabilities: ["@specialized"]
|
|
57
80
|
unblocks: [M6]
|
|
@@ -109,6 +109,26 @@ export function lintDataDriven(screenDir: string, cwd: string = process.cwd()):
|
|
|
109
109
|
}
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
+
// --- orphan test-data: a top-level key never referenced (dead data — generated but not
|
|
113
|
+
// materialized into a scenario). Referenced = a `{{key…}}` in any step, a `@cases:<key>`
|
|
114
|
+
// dataset, or an override value `…={{key…}}` on an @api/@query annotation.
|
|
115
|
+
const usedHeads = new Set<string>();
|
|
116
|
+
const usedDatasets = new Set<string>();
|
|
117
|
+
for (const sc of scenarios) {
|
|
118
|
+
for (const r of collectRefs(sc)) usedHeads.add(r.split(/[.[]/)[0]);
|
|
119
|
+
for (const t of sc.tags || []) {
|
|
120
|
+
const cm = t.match(/^@cases:(.+)$/);
|
|
121
|
+
if (cm) usedDatasets.add(cm[1].trim());
|
|
122
|
+
const om = t.match(/^@(?:api|query):[A-Za-z_]\w*\((.*)\)$/);
|
|
123
|
+
if (om) for (const ref of om[1].matchAll(/\{\{\s*([^}]+?)\s*\}\}/g)) usedHeads.add(ref[1].split(/[.[]/)[0].trim());
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
for (const k of topKeys) {
|
|
127
|
+
if (!usedHeads.has(k) && !usedDatasets.has(k)) {
|
|
128
|
+
warns.push({ message: `test-data key "${k}" is defined but never referenced ({{${k}}}, a @cases dataset, or an override) — dead data: bind it into a scenario or remove it.` });
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
112
132
|
// Catalog-level lint (SELECT-only, params declared/used, datasource present).
|
|
113
133
|
try {
|
|
114
134
|
for (const e of lintCatalog(screenName, null, cwd).errors) warns.push({ message: e });
|
|
@@ -49,17 +49,26 @@ function targetsFromHint(hint: string): string[] {
|
|
|
49
49
|
return after.split(/[^a-z]+/).filter((w) => w.length > 3 && !['home', 'page', 'flow', 'products', 'product', 'result'].includes(w));
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
export
|
|
53
|
-
const screens = listDirs(path.join(cwd, 'qa', 'screens'));
|
|
54
|
-
const flows = (onlyFlow ? [onlyFlow] : listDirs(path.join(cwd, 'qa', 'flows')));
|
|
52
|
+
export interface FlowScenario { flow: string; name: string; haystack: string; deep: boolean }
|
|
55
53
|
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
/** Index every flow's scenarios (name + haystack + whether it carries a data assertion).
|
|
55
|
+
* Shared so the screen audit can credit a cross-screen theme covered deeply by a flow (TQ-4). */
|
|
56
|
+
export function loadFlowScenarios(cwd: string): FlowScenario[] {
|
|
57
|
+
const out: FlowScenario[] = [];
|
|
58
58
|
for (const f of listDirs(path.join(cwd, 'qa', 'flows'))) {
|
|
59
59
|
for (const s of loadScenarios(featurePath(cwd, 'flows', f))) {
|
|
60
|
-
|
|
60
|
+
out.push({ flow: f, name: s.name, haystack: s.haystack, deep: s.hasDataAssertion });
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function buildFlowCheck(cwd: string, onlyFlow?: string): FlowCheckReport {
|
|
67
|
+
const screens = listDirs(path.join(cwd, 'qa', 'screens'));
|
|
68
|
+
const flows = (onlyFlow ? [onlyFlow] : listDirs(path.join(cwd, 'qa', 'flows')));
|
|
69
|
+
|
|
70
|
+
// Index flow scenarios (name + haystack + depth).
|
|
71
|
+
const flowScenarios = loadFlowScenarios(cwd);
|
|
63
72
|
|
|
64
73
|
// A. Deferral integrity (screens).
|
|
65
74
|
const deferrals: Deferral[] = [];
|
package/src/harness/intent.ts
CHANGED
|
@@ -21,18 +21,29 @@ export interface IntentProfile {
|
|
|
21
21
|
focus: IntentFocus;
|
|
22
22
|
riskTier: 'high' | 'normal' | 'low';
|
|
23
23
|
tierScope: 'tier-1' | 'full';
|
|
24
|
+
/** End-user override (AO-6): HTTP methods the API gate treats as business-critical (depth-required).
|
|
25
|
+
* Default (undefined → the gate's POST/PUT/PATCH/DELETE) lets a project mark e.g. GET as critical. */
|
|
26
|
+
businessCriticalMethods?: string[];
|
|
27
|
+
/** TQ-10: surface "enable driver X to automate N @manual" suggestions (recommend-only). Default on;
|
|
28
|
+
* set `capability_suggestions: off` in qa/context.md to silence. */
|
|
29
|
+
capabilitySuggestions: boolean;
|
|
24
30
|
source: 'context.md' | 'default';
|
|
25
31
|
}
|
|
26
32
|
|
|
27
33
|
const DEFAULT_INTENT: IntentProfile = {
|
|
28
|
-
focus: 'functional', riskTier: 'normal', tierScope: 'full', source: 'default',
|
|
34
|
+
focus: 'functional', riskTier: 'normal', tierScope: 'full', capabilitySuggestions: true, source: 'default',
|
|
29
35
|
};
|
|
30
36
|
|
|
31
37
|
const FOCI: IntentFocus[] = ['functional', 'e-commerce', 'security', 'smoke'];
|
|
32
38
|
|
|
33
39
|
/** Resolve project root from a screen/flow dir (…/qa/screens/<name>). */
|
|
34
40
|
export function projectRootFromScreenDir(screenDir: string): string {
|
|
35
|
-
|
|
41
|
+
// The project root is the parent of the `qa/` dir — depth-agnostic, so it works for screens/flows
|
|
42
|
+
// (qa/screens/<x>, 3 deep) AND api flows (qa/api/flows/<flow>, 4 deep). A fixed `../../..` returned
|
|
43
|
+
// `<root>/qa` for the deeper api-flow path, breaking catalog resolution (cwd off by one).
|
|
44
|
+
const parts = screenDir.split(path.sep);
|
|
45
|
+
const qa = parts.lastIndexOf('qa');
|
|
46
|
+
return qa > 0 ? parts.slice(0, qa).join(path.sep) : path.resolve(screenDir, '..', '..', '..');
|
|
36
47
|
}
|
|
37
48
|
|
|
38
49
|
export function readIntent(projectRoot: string): IntentProfile {
|
|
@@ -45,6 +56,13 @@ export function readIntent(projectRoot: string): IntentProfile {
|
|
|
45
56
|
const m = text.match(new RegExp(`(?:^|\\n)\\s*${key}\\s*:\\s*([a-z0-9-]+)`));
|
|
46
57
|
return m?.[1];
|
|
47
58
|
};
|
|
59
|
+
// A comma/space/slash list (e.g. `business_critical_methods: post, put, patch, delete, get`).
|
|
60
|
+
const grabList = (key: string): string[] | undefined => {
|
|
61
|
+
const m = text.match(new RegExp(`(?:^|\\n)\\s*${key}\\s*:\\s*([a-z0-9,\\s/-]+)`));
|
|
62
|
+
if (!m) return undefined;
|
|
63
|
+
const items = m[1].split(/[,\s/]+/).map((s) => s.trim().toUpperCase()).filter(Boolean);
|
|
64
|
+
return items.length ? items : undefined;
|
|
65
|
+
};
|
|
48
66
|
|
|
49
67
|
const focusRaw = grab('focus');
|
|
50
68
|
const focus = (FOCI.includes(focusRaw as IntentFocus) ? focusRaw : DEFAULT_INTENT.focus) as IntentFocus;
|
|
@@ -53,6 +71,9 @@ export function readIntent(projectRoot: string): IntentProfile {
|
|
|
53
71
|
const scope = grab('tier_scope');
|
|
54
72
|
const tierScope = (['tier-1', 'full'].includes(scope as string) ? scope : DEFAULT_INTENT.tierScope) as IntentProfile['tierScope'];
|
|
55
73
|
|
|
56
|
-
const
|
|
57
|
-
|
|
74
|
+
const businessCriticalMethods = grabList('business_critical_methods');
|
|
75
|
+
const capRaw = grab('capability_suggestions');
|
|
76
|
+
const capabilitySuggestions = capRaw !== 'off'; // default on; only an explicit `off` silences it
|
|
77
|
+
const found = focusRaw || risk || scope || businessCriticalMethods || capRaw;
|
|
78
|
+
return { focus, riskTier, tierScope, businessCriticalMethods, capabilitySuggestions, source: found ? 'context.md' : 'default' };
|
|
58
79
|
}
|
package/src/harness/ledger.ts
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import * as fs from 'fs';
|
|
15
15
|
import * as path from 'path';
|
|
16
|
+
import { reportSlug } from './unit-paths';
|
|
16
17
|
|
|
17
18
|
export interface LedgerEvent {
|
|
18
19
|
ts: string;
|
|
@@ -60,7 +61,7 @@ export function latestRunEvents(events: LedgerEvent[]): LedgerEvent[] {
|
|
|
60
61
|
}
|
|
61
62
|
|
|
62
63
|
function ledgerPath(screen: string): string {
|
|
63
|
-
return path.join(process.cwd(), '.sungen', 'ledger', `${screen}.jsonl`);
|
|
64
|
+
return path.join(process.cwd(), '.sungen', 'ledger', `${reportSlug(screen)}.jsonl`);
|
|
64
65
|
}
|
|
65
66
|
|
|
66
67
|
export function recordEvent(screen: string, ev: Omit<LedgerEvent, 'ts'> & { ts?: string }): string {
|
|
@@ -117,7 +118,7 @@ export function buildReport(screen: string, opts: { allRuns?: boolean } = {}): L
|
|
|
117
118
|
// Pull audit signals if present
|
|
118
119
|
let coveredCritical: number | null = null;
|
|
119
120
|
let scenarioCount: number | null = null;
|
|
120
|
-
const auditPath = path.join(process.cwd(), '.sungen', 'reports', `${screen}-audit.json`);
|
|
121
|
+
const auditPath = path.join(process.cwd(), '.sungen', 'reports', `${reportSlug(screen)}-audit.json`);
|
|
121
122
|
if (fs.existsSync(auditPath)) {
|
|
122
123
|
try {
|
|
123
124
|
const a = JSON.parse(fs.readFileSync(auditPath, 'utf-8'));
|
package/src/harness/manifest.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
import * as fs from 'fs';
|
|
12
12
|
import * as path from 'path';
|
|
13
13
|
import { createHash } from 'crypto';
|
|
14
|
+
import { featureBasename, reportSlug } from './unit-paths';
|
|
14
15
|
|
|
15
16
|
export interface SpecSection { name: string; hash: string }
|
|
16
17
|
export interface ManifestEntry { scenario: string; vpCode?: string; section: string; specHash: string }
|
|
@@ -97,7 +98,7 @@ function parseFeatureSections(featurePath: string): { scenario: string; vpCode?:
|
|
|
97
98
|
|
|
98
99
|
export function buildManifest(screenDir: string, screenName: string): Manifest {
|
|
99
100
|
const specPath = path.join(screenDir, 'requirements', 'spec.md');
|
|
100
|
-
const featurePath = path.join(screenDir, 'features', `${screenName}.feature`);
|
|
101
|
+
const featurePath = path.join(screenDir, 'features', `${featureBasename(screenName)}.feature`);
|
|
101
102
|
const specSections = parseSpecSections(specPath);
|
|
102
103
|
const specMap: Record<string, string> = {};
|
|
103
104
|
for (const s of specSections) specMap[s.name] = s.hash;
|
|
@@ -159,7 +160,7 @@ export function diffManifest(screenDir: string, screenName: string, manifest: Ma
|
|
|
159
160
|
}
|
|
160
161
|
|
|
161
162
|
export function manifestPath(screenName: string): string {
|
|
162
|
-
return path.join(process.cwd(), '.sungen', 'manifest', `${screenName}.json`);
|
|
163
|
+
return path.join(process.cwd(), '.sungen', 'manifest', `${reportSlug(screenName)}.json`);
|
|
163
164
|
}
|
|
164
165
|
export function loadManifest(screenName: string): Manifest | null {
|
|
165
166
|
const p = manifestPath(screenName);
|
package/src/harness/parse.ts
CHANGED
|
@@ -32,6 +32,8 @@ export interface ScenarioInfo {
|
|
|
32
32
|
vpId?: string; // raw leading ID token of the title (project's scheme: VP0-001, MS-HP-001, VP-LIST-001)
|
|
33
33
|
casesDataset?: string; // @cases:<dataset> — data-driven; one scenario expands to N row-tests
|
|
34
34
|
queryRefs?: string[]; // named queries referenced by this scenario (inline `query [name]` + @query: tags)
|
|
35
|
+
apiRefs?: string[]; // named API endpoints referenced by this scenario (@api: tags)
|
|
36
|
+
requiresCaps?: string[]; // @requires:<cap> — automation-ready but needs an opt-in driver (TQ-11)
|
|
35
37
|
}
|
|
36
38
|
|
|
37
39
|
/** Format-tolerant: is this token an ID (project's scheme), not a prose word?
|
|
@@ -102,12 +104,17 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
102
104
|
const manual = tags.includes('@manual');
|
|
103
105
|
const casesTag = tags.find((t) => t.startsWith('@cases:'));
|
|
104
106
|
const casesDataset = casesTag ? casesTag.slice('@cases:'.length).trim() : undefined;
|
|
105
|
-
// Named-query references: @query:<name> tags + inline `query [name]` step refs.
|
|
107
|
+
// Named-query references: @query:<name>[(overrides)] tags + inline `query [name]` step refs.
|
|
106
108
|
const queryRefs = new Set<string>();
|
|
107
|
-
for (const t of tags) if (t.startsWith('@query:')) { const
|
|
109
|
+
for (const t of tags) if (t.startsWith('@query:')) { const m = t.slice('@query:'.length).match(/^([A-Za-z_][A-Za-z0-9_]*)/); if (m) queryRefs.add(m[1]); }
|
|
108
110
|
for (const step of (sc.steps as ParsedStep[]) || []) {
|
|
109
111
|
for (const m of (step.text || '').matchAll(/\bquery\s+\[([A-Za-z_][A-Za-z0-9_]*)\]/gi)) queryRefs.add(m[1]);
|
|
110
112
|
}
|
|
113
|
+
// Named-API references: @api:<name>[(overrides)] tags.
|
|
114
|
+
const apiRefs = new Set<string>();
|
|
115
|
+
for (const t of tags) if (t.startsWith('@api:')) { const m = t.slice('@api:'.length).match(/^([A-Za-z_][A-Za-z0-9_]*)/); if (m) apiRefs.add(m[1]); }
|
|
116
|
+
// @requires:<cap> — automation-ready, needs an opt-in driver (TQ-11).
|
|
117
|
+
const requiresCaps = tags.filter((t) => /^@requires:/i.test(t)).map((t) => t.slice('@requires:'.length).trim().toLowerCase()).filter(Boolean);
|
|
111
118
|
let priority: Priority = 'unknown';
|
|
112
119
|
for (const t of tags) if (PRIORITY_TAGS[t]) priority = PRIORITY_TAGS[t];
|
|
113
120
|
|
|
@@ -164,6 +171,8 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
164
171
|
vpId,
|
|
165
172
|
casesDataset,
|
|
166
173
|
queryRefs: queryRefs.size ? [...queryRefs] : undefined,
|
|
174
|
+
apiRefs: apiRefs.size ? [...apiRefs] : undefined,
|
|
175
|
+
requiresCaps: requiresCaps.length ? requiresCaps : undefined,
|
|
167
176
|
};
|
|
168
177
|
}
|
|
169
178
|
|
|
@@ -85,7 +85,7 @@ export function negativeSideEffect(scenarios: ScenarioInfo[]): string[] {
|
|
|
85
85
|
for (const s of scenarios) {
|
|
86
86
|
if (s.manual) continue; // @manual is a legitimate deferral (oracle checked by #4 manual-oracle)
|
|
87
87
|
if (!NEG_TITLE.test(s.name)) continue;
|
|
88
|
-
const proven = /\bcount\b|tohavecount|table with|is hidden|are hidden|not complete|message is hidden/.test(s.stepsText);
|
|
88
|
+
const proven = /\bcount\b|ok_count|status_counts|tohavecount|table with|is hidden|are hidden|not complete|message is hidden/.test(s.stepsText);
|
|
89
89
|
if (!proven) flagged.push(s.name.slice(0, 80));
|
|
90
90
|
}
|
|
91
91
|
return flagged;
|
|
Binary file
|