@sun-asterisk/sungen 3.1.2-beta.99 → 3.2.0-beta.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capabilities/context.d.ts +17 -0
- package/dist/capabilities/context.d.ts.map +1 -1
- package/dist/capabilities/discover.d.ts.map +1 -1
- package/dist/capabilities/discover.js +70 -9
- package/dist/capabilities/discover.js.map +1 -1
- package/dist/capabilities/registry.d.ts +3 -1
- package/dist/capabilities/registry.d.ts.map +1 -1
- package/dist/capabilities/registry.js.map +1 -1
- package/dist/capabilities/sensor.d.ts +3 -0
- package/dist/capabilities/sensor.d.ts.map +1 -1
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +17 -11
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/capability.d.ts.map +1 -1
- package/dist/cli/commands/capability.js +57 -5
- package/dist/cli/commands/capability.js.map +1 -1
- package/dist/cli/commands/context.d.ts +9 -0
- package/dist/cli/commands/context.d.ts.map +1 -0
- package/dist/cli/commands/context.js +91 -0
- package/dist/cli/commands/context.js.map +1 -0
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +42 -30
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/generate.d.ts.map +1 -1
- package/dist/cli/commands/generate.js +28 -5
- package/dist/cli/commands/generate.js.map +1 -1
- package/dist/cli/commands/ledger.d.ts.map +1 -1
- package/dist/cli/commands/ledger.js +15 -5
- package/dist/cli/commands/ledger.js.map +1 -1
- package/dist/cli/commands/manifest.d.ts.map +1 -1
- package/dist/cli/commands/manifest.js +10 -9
- package/dist/cli/commands/manifest.js.map +1 -1
- package/dist/cli/commands/repair.d.ts +8 -0
- package/dist/cli/commands/repair.d.ts.map +1 -0
- package/dist/cli/commands/repair.js +97 -0
- package/dist/cli/commands/repair.js.map +1 -0
- package/dist/cli/commands/script-check.d.ts.map +1 -1
- package/dist/cli/commands/script-check.js +13 -9
- package/dist/cli/commands/script-check.js.map +1 -1
- package/dist/cli/commands/trace.d.ts.map +1 -1
- package/dist/cli/commands/trace.js +7 -4
- package/dist/cli/commands/trace.js.map +1 -1
- package/dist/cli/index.js +4 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/generators/test-generator/code-generator.d.ts +7 -0
- package/dist/generators/test-generator/code-generator.d.ts.map +1 -1
- package/dist/generators/test-generator/code-generator.js +112 -42
- package/dist/generators/test-generator/code-generator.js.map +1 -1
- package/dist/harness/annotation-overrides.d.ts +3 -1
- package/dist/harness/annotation-overrides.d.ts.map +1 -1
- package/dist/harness/annotation-overrides.js +3 -1
- package/dist/harness/annotation-overrides.js.map +1 -1
- package/dist/harness/audit.d.ts +9 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +114 -12
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/capability-plan.d.ts +14 -0
- package/dist/harness/capability-plan.d.ts.map +1 -1
- package/dist/harness/capability-plan.js +63 -1
- package/dist/harness/capability-plan.js.map +1 -1
- package/dist/harness/data-driven-lint.d.ts.map +1 -1
- package/dist/harness/data-driven-lint.js +23 -0
- package/dist/harness/data-driven-lint.js.map +1 -1
- package/dist/harness/flow-check.d.ts +9 -0
- package/dist/harness/flow-check.d.ts.map +1 -1
- package/dist/harness/flow-check.js +13 -6
- package/dist/harness/flow-check.js.map +1 -1
- package/dist/harness/intent.d.ts +6 -0
- package/dist/harness/intent.d.ts.map +1 -1
- package/dist/harness/intent.js +20 -4
- package/dist/harness/intent.js.map +1 -1
- package/dist/harness/ledger.d.ts.map +1 -1
- package/dist/harness/ledger.js +3 -2
- package/dist/harness/ledger.js.map +1 -1
- package/dist/harness/manifest.d.ts.map +1 -1
- package/dist/harness/manifest.js +3 -2
- package/dist/harness/manifest.js.map +1 -1
- package/dist/harness/parse.d.ts +1 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +3 -0
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/quality-gates.js +1 -1
- package/dist/harness/quality-gates.js.map +1 -1
- package/dist/harness/query-catalog.d.ts.map +1 -1
- package/dist/harness/query-catalog.js +0 -0
- package/dist/harness/query-catalog.js.map +1 -1
- package/dist/harness/repair.d.ts +20 -0
- package/dist/harness/repair.d.ts.map +1 -0
- package/dist/harness/repair.js +111 -0
- package/dist/harness/repair.js.map +1 -0
- package/dist/harness/script-check.d.ts +3 -1
- package/dist/harness/script-check.d.ts.map +1 -1
- package/dist/harness/script-check.js +22 -8
- package/dist/harness/script-check.js.map +1 -1
- package/dist/harness/sensors.d.ts +40 -0
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +54 -2
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/trace.d.ts.map +1 -1
- package/dist/harness/trace.js +4 -3
- package/dist/harness/trace.js.map +1 -1
- package/dist/harness/unit-paths.d.ts +3 -0
- package/dist/harness/unit-paths.d.ts.map +1 -0
- package/dist/harness/unit-paths.js +52 -0
- package/dist/harness/unit-paths.js.map +1 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/orchestrator/ai-rules-updater.d.ts.map +1 -1
- package/dist/orchestrator/ai-rules-updater.js +2 -0
- package/dist/orchestrator/ai-rules-updater.js.map +1 -1
- package/dist/orchestrator/context-discovery.d.ts +12 -0
- package/dist/orchestrator/context-discovery.d.ts.map +1 -0
- package/dist/orchestrator/context-discovery.js +46 -0
- package/dist/orchestrator/context-discovery.js.map +1 -0
- package/dist/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +7 -1
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-create-test.md +6 -2
- package/dist/orchestrator/templates/ai-instructions/claude-cmd-run-test.md +18 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-api-design.md +62 -0
- package/dist/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +2 -1
- package/dist/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +16 -2
- package/dist/orchestrator/templates/ai-instructions/claude-skill-viewpoint.md +14 -0
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-create-test.md +6 -2
- package/dist/orchestrator/templates/ai-instructions/copilot-cmd-run-test.md +11 -1
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-api-design.md +62 -0
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +2 -1
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +16 -2
- package/dist/orchestrator/templates/ai-instructions/github-skill-sungen-viewpoint.md +14 -0
- package/dist/orchestrator/templates/specs-api.d.ts +38 -2
- package/dist/orchestrator/templates/specs-api.d.ts.map +1 -1
- package/dist/orchestrator/templates/specs-api.js +65 -22
- package/dist/orchestrator/templates/specs-api.js.map +1 -1
- package/dist/orchestrator/templates/specs-api.ts +71 -18
- package/dist/orchestrator/templates/specs-db.d.ts +3 -0
- package/dist/orchestrator/templates/specs-db.d.ts.map +1 -1
- package/dist/orchestrator/templates/specs-db.js +78 -1
- package/dist/orchestrator/templates/specs-db.js.map +1 -1
- package/dist/orchestrator/templates/specs-db.ts +78 -1
- package/dist/orchestrator/templates/specs-test-data.ts +2 -1
- package/package.json +2 -2
- package/src/capabilities/context.ts +19 -0
- package/src/capabilities/discover.ts +27 -7
- package/src/capabilities/registry.ts +3 -1
- package/src/capabilities/sensor.ts +1 -1
- package/src/cli/commands/audit.ts +15 -9
- package/src/cli/commands/capability.ts +53 -5
- package/src/cli/commands/context.ts +52 -0
- package/src/cli/commands/delivery.ts +40 -31
- package/src/cli/commands/generate.ts +30 -5
- package/src/cli/commands/ledger.ts +13 -5
- package/src/cli/commands/manifest.ts +9 -7
- package/src/cli/commands/repair.ts +57 -0
- package/src/cli/commands/script-check.ts +12 -8
- package/src/cli/commands/trace.ts +7 -4
- package/src/cli/index.ts +4 -0
- package/src/generators/test-generator/code-generator.ts +115 -40
- package/src/harness/annotation-overrides.ts +3 -1
- package/src/harness/audit.ts +115 -15
- package/src/harness/capability-plan.ts +51 -1
- package/src/harness/data-driven-lint.ts +20 -0
- package/src/harness/flow-check.ts +15 -6
- package/src/harness/intent.ts +25 -4
- package/src/harness/ledger.ts +3 -2
- package/src/harness/manifest.ts +3 -2
- package/src/harness/parse.ts +4 -0
- package/src/harness/quality-gates.ts +1 -1
- package/src/harness/query-catalog.ts +0 -0
- package/src/harness/repair.ts +75 -0
- package/src/harness/script-check.ts +25 -8
- package/src/harness/sensors.ts +71 -2
- package/src/harness/trace.ts +4 -3
- package/src/harness/unit-paths.ts +14 -0
- package/src/index.ts +4 -2
- package/src/orchestrator/ai-rules-updater.ts +2 -0
- package/src/orchestrator/context-discovery.ts +50 -0
- package/src/orchestrator/templates/ai-instructions/claude-agent-reviewer.md +7 -1
- package/src/orchestrator/templates/ai-instructions/claude-cmd-create-test.md +6 -2
- package/src/orchestrator/templates/ai-instructions/claude-cmd-run-test.md +18 -1
- package/src/orchestrator/templates/ai-instructions/claude-skill-api-design.md +62 -0
- package/src/orchestrator/templates/ai-instructions/claude-skill-harness-audit.md +2 -1
- package/src/orchestrator/templates/ai-instructions/claude-skill-tc-generation.md +16 -2
- package/src/orchestrator/templates/ai-instructions/claude-skill-viewpoint.md +14 -0
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-create-test.md +6 -2
- package/src/orchestrator/templates/ai-instructions/copilot-cmd-run-test.md +11 -1
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-api-design.md +62 -0
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-harness-audit.md +2 -1
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-tc-generation.md +16 -2
- package/src/orchestrator/templates/ai-instructions/github-skill-sungen-viewpoint.md +14 -0
- package/src/orchestrator/templates/specs-api.ts +71 -18
- package/src/orchestrator/templates/specs-db.ts +78 -1
- package/src/orchestrator/templates/specs-test-data.ts +2 -1
|
@@ -36,7 +36,9 @@ const INFER: { code: string; re: RegExp }[] = [
|
|
|
36
36
|
{ code: 'M3', re: /\b(mock|stub|network|offline|slow network|intercept)\b/i },
|
|
37
37
|
{ code: 'M2', re: /\b(api|endpoint|backend|db|database|server[-\s]?side|via api)\b/i },
|
|
38
38
|
{ code: 'M1', re: /\b(data setup|dataset|seed|test data|empty (category|product|dataset|state)|zero products|forcing an empty|backend\/test data)\b/i },
|
|
39
|
-
|
|
39
|
+
// "email" alone is too greedy (every subscription test mentions it) → require a real external-mail
|
|
40
|
+
// signal (verification mail / OTP / inbox), so an API/DB-oracle test isn't misread as M5.
|
|
41
|
+
{ code: 'M5', re: /\b(external|third[-\s]?party|sandbox|payment gateway|invoice|download|verification (e?mail|link)|otp|mailbox|inbox|e?mail link)\b/i },
|
|
40
42
|
{ code: 'M6', re: /\b(visual|responsive|layout|accessibilit|a11y|keyboard|screen reader|ux|breakpoint)\b/i },
|
|
41
43
|
{ code: 'M7', re: /\b(environment|staging[-\s]?only|infra|env limitation)\b/i },
|
|
42
44
|
{ code: 'M8', re: /\b(not worth|exploratory|one[-\s]?off)\b/i },
|
|
@@ -102,6 +104,54 @@ export function inferReasonCode(tags: string[], reason: string): { code: string;
|
|
|
102
104
|
return { code: 'M9', explicit: false, unclassified: true };
|
|
103
105
|
}
|
|
104
106
|
|
|
107
|
+
/** The reason code inferred ONLY from the reason text, ignoring any explicit @manual:Mx tag. */
|
|
108
|
+
export function inferFromText(reason: string): string | undefined {
|
|
109
|
+
for (const r of INFER) if (r.re.test(reason)) return r.code;
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export interface ReasonMismatch { scenario: string; explicit: string; inferred: string }
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* (TQ-9) @manual scenarios whose explicit `@manual:Mx` disagrees with the code inferred from
|
|
117
|
+
* the reason text — a mis-tag that makes the Capability Planner recommend the wrong driver
|
|
118
|
+
* (e.g. tagged `M1` data-setup but the reason describes a DB/API assertion → `M2`). Advisory:
|
|
119
|
+
* surfaced so the reason code — and therefore the driver suggestion — can be trusted.
|
|
120
|
+
*/
|
|
121
|
+
export function manualReasonMismatches(featurePath: string): ReasonMismatch[] {
|
|
122
|
+
if (!fs.existsSync(featurePath)) return [];
|
|
123
|
+
const lines = fs.readFileSync(featurePath, 'utf-8').split('\n');
|
|
124
|
+
const out: ReasonMismatch[] = [];
|
|
125
|
+
for (let i = 0; i < lines.length; i++) {
|
|
126
|
+
const m = lines[i].match(/^\s*Scenario:\s*(.+)$/);
|
|
127
|
+
if (!m) continue;
|
|
128
|
+
// Tags on the adjacent line(s) above.
|
|
129
|
+
const tags: string[] = [];
|
|
130
|
+
for (let j = i - 1; j >= 0 && j >= i - 4; j--) {
|
|
131
|
+
const l = lines[j].trim();
|
|
132
|
+
if (l === '') break;
|
|
133
|
+
if (/^@/.test(l)) tags.unshift(...l.split(/\s+/).filter((t) => t.startsWith('@')));
|
|
134
|
+
else if (/^#/.test(l)) continue;
|
|
135
|
+
else break;
|
|
136
|
+
}
|
|
137
|
+
if (!tags.some((t) => /^@manual\b/i.test(t))) continue;
|
|
138
|
+
const explicit = explicitCode(tags);
|
|
139
|
+
if (!explicit) continue;
|
|
140
|
+
// FULL reason block — every comment line in the body before the first real step (the
|
|
141
|
+
// driver-relevant signal — "subscribers table", "API/persistence" — is often not line 1).
|
|
142
|
+
const parts: string[] = [];
|
|
143
|
+
for (let k = i + 1; k < lines.length && k <= i + 16; k++) {
|
|
144
|
+
const l = lines[k].trim();
|
|
145
|
+
if (/^#/.test(l)) parts.push(l.replace(/^#+\s*/, ''));
|
|
146
|
+
else if (l === '') continue;
|
|
147
|
+
else break;
|
|
148
|
+
}
|
|
149
|
+
const inferred = inferFromText(parts.join(' '));
|
|
150
|
+
if (inferred && inferred !== explicit) out.push({ scenario: m[1].trim(), explicit, inferred });
|
|
151
|
+
}
|
|
152
|
+
return out;
|
|
153
|
+
}
|
|
154
|
+
|
|
105
155
|
function classifyMode(tags: string[]): string {
|
|
106
156
|
const has = (re: RegExp) => tags.some((t) => re.test(t));
|
|
107
157
|
if (has(/^@manual\b/i)) return 'manual';
|
|
@@ -109,6 +109,26 @@ export function lintDataDriven(screenDir: string, cwd: string = process.cwd()):
|
|
|
109
109
|
}
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
+
// --- orphan test-data: a top-level key never referenced (dead data — generated but not
|
|
113
|
+
// materialized into a scenario). Referenced = a `{{key…}}` in any step, a `@cases:<key>`
|
|
114
|
+
// dataset, or an override value `…={{key…}}` on an @api/@query annotation.
|
|
115
|
+
const usedHeads = new Set<string>();
|
|
116
|
+
const usedDatasets = new Set<string>();
|
|
117
|
+
for (const sc of scenarios) {
|
|
118
|
+
for (const r of collectRefs(sc)) usedHeads.add(r.split(/[.[]/)[0]);
|
|
119
|
+
for (const t of sc.tags || []) {
|
|
120
|
+
const cm = t.match(/^@cases:(.+)$/);
|
|
121
|
+
if (cm) usedDatasets.add(cm[1].trim());
|
|
122
|
+
const om = t.match(/^@(?:api|query):[A-Za-z_]\w*\((.*)\)$/);
|
|
123
|
+
if (om) for (const ref of om[1].matchAll(/\{\{\s*([^}]+?)\s*\}\}/g)) usedHeads.add(ref[1].split(/[.[]/)[0].trim());
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
for (const k of topKeys) {
|
|
127
|
+
if (!usedHeads.has(k) && !usedDatasets.has(k)) {
|
|
128
|
+
warns.push({ message: `test-data key "${k}" is defined but never referenced ({{${k}}}, a @cases dataset, or an override) — dead data: bind it into a scenario or remove it.` });
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
112
132
|
// Catalog-level lint (SELECT-only, params declared/used, datasource present).
|
|
113
133
|
try {
|
|
114
134
|
for (const e of lintCatalog(screenName, null, cwd).errors) warns.push({ message: e });
|
|
@@ -49,17 +49,26 @@ function targetsFromHint(hint: string): string[] {
|
|
|
49
49
|
return after.split(/[^a-z]+/).filter((w) => w.length > 3 && !['home', 'page', 'flow', 'products', 'product', 'result'].includes(w));
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
export
|
|
53
|
-
const screens = listDirs(path.join(cwd, 'qa', 'screens'));
|
|
54
|
-
const flows = (onlyFlow ? [onlyFlow] : listDirs(path.join(cwd, 'qa', 'flows')));
|
|
52
|
+
export interface FlowScenario { flow: string; name: string; haystack: string; deep: boolean }
|
|
55
53
|
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
/** Index every flow's scenarios (name + haystack + whether it carries a data assertion).
|
|
55
|
+
* Shared so the screen audit can credit a cross-screen theme covered deeply by a flow (TQ-4). */
|
|
56
|
+
export function loadFlowScenarios(cwd: string): FlowScenario[] {
|
|
57
|
+
const out: FlowScenario[] = [];
|
|
58
58
|
for (const f of listDirs(path.join(cwd, 'qa', 'flows'))) {
|
|
59
59
|
for (const s of loadScenarios(featurePath(cwd, 'flows', f))) {
|
|
60
|
-
|
|
60
|
+
out.push({ flow: f, name: s.name, haystack: s.haystack, deep: s.hasDataAssertion });
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function buildFlowCheck(cwd: string, onlyFlow?: string): FlowCheckReport {
|
|
67
|
+
const screens = listDirs(path.join(cwd, 'qa', 'screens'));
|
|
68
|
+
const flows = (onlyFlow ? [onlyFlow] : listDirs(path.join(cwd, 'qa', 'flows')));
|
|
69
|
+
|
|
70
|
+
// Index flow scenarios (name + haystack + depth).
|
|
71
|
+
const flowScenarios = loadFlowScenarios(cwd);
|
|
63
72
|
|
|
64
73
|
// A. Deferral integrity (screens).
|
|
65
74
|
const deferrals: Deferral[] = [];
|
package/src/harness/intent.ts
CHANGED
|
@@ -21,18 +21,29 @@ export interface IntentProfile {
|
|
|
21
21
|
focus: IntentFocus;
|
|
22
22
|
riskTier: 'high' | 'normal' | 'low';
|
|
23
23
|
tierScope: 'tier-1' | 'full';
|
|
24
|
+
/** End-user override (AO-6): HTTP methods the API gate treats as business-critical (depth-required).
|
|
25
|
+
* Default (undefined → the gate's POST/PUT/PATCH/DELETE) lets a project mark e.g. GET as critical. */
|
|
26
|
+
businessCriticalMethods?: string[];
|
|
27
|
+
/** TQ-10: surface "enable driver X to automate N @manual" suggestions (recommend-only). Default on;
|
|
28
|
+
* set `capability_suggestions: off` in qa/context.md to silence. */
|
|
29
|
+
capabilitySuggestions: boolean;
|
|
24
30
|
source: 'context.md' | 'default';
|
|
25
31
|
}
|
|
26
32
|
|
|
27
33
|
const DEFAULT_INTENT: IntentProfile = {
|
|
28
|
-
focus: 'functional', riskTier: 'normal', tierScope: 'full', source: 'default',
|
|
34
|
+
focus: 'functional', riskTier: 'normal', tierScope: 'full', capabilitySuggestions: true, source: 'default',
|
|
29
35
|
};
|
|
30
36
|
|
|
31
37
|
const FOCI: IntentFocus[] = ['functional', 'e-commerce', 'security', 'smoke'];
|
|
32
38
|
|
|
33
39
|
/** Resolve project root from a screen/flow dir (…/qa/screens/<name>). */
|
|
34
40
|
export function projectRootFromScreenDir(screenDir: string): string {
|
|
35
|
-
|
|
41
|
+
// The project root is the parent of the `qa/` dir — depth-agnostic, so it works for screens/flows
|
|
42
|
+
// (qa/screens/<x>, 3 deep) AND api flows (qa/api/flows/<flow>, 4 deep). A fixed `../../..` returned
|
|
43
|
+
// `<root>/qa` for the deeper api-flow path, breaking catalog resolution (cwd off by one).
|
|
44
|
+
const parts = screenDir.split(path.sep);
|
|
45
|
+
const qa = parts.lastIndexOf('qa');
|
|
46
|
+
return qa > 0 ? parts.slice(0, qa).join(path.sep) : path.resolve(screenDir, '..', '..', '..');
|
|
36
47
|
}
|
|
37
48
|
|
|
38
49
|
export function readIntent(projectRoot: string): IntentProfile {
|
|
@@ -45,6 +56,13 @@ export function readIntent(projectRoot: string): IntentProfile {
|
|
|
45
56
|
const m = text.match(new RegExp(`(?:^|\\n)\\s*${key}\\s*:\\s*([a-z0-9-]+)`));
|
|
46
57
|
return m?.[1];
|
|
47
58
|
};
|
|
59
|
+
// A comma/space/slash list (e.g. `business_critical_methods: post, put, patch, delete, get`).
|
|
60
|
+
const grabList = (key: string): string[] | undefined => {
|
|
61
|
+
const m = text.match(new RegExp(`(?:^|\\n)\\s*${key}\\s*:\\s*([a-z0-9,\\s/-]+)`));
|
|
62
|
+
if (!m) return undefined;
|
|
63
|
+
const items = m[1].split(/[,\s/]+/).map((s) => s.trim().toUpperCase()).filter(Boolean);
|
|
64
|
+
return items.length ? items : undefined;
|
|
65
|
+
};
|
|
48
66
|
|
|
49
67
|
const focusRaw = grab('focus');
|
|
50
68
|
const focus = (FOCI.includes(focusRaw as IntentFocus) ? focusRaw : DEFAULT_INTENT.focus) as IntentFocus;
|
|
@@ -53,6 +71,9 @@ export function readIntent(projectRoot: string): IntentProfile {
|
|
|
53
71
|
const scope = grab('tier_scope');
|
|
54
72
|
const tierScope = (['tier-1', 'full'].includes(scope as string) ? scope : DEFAULT_INTENT.tierScope) as IntentProfile['tierScope'];
|
|
55
73
|
|
|
56
|
-
const
|
|
57
|
-
|
|
74
|
+
const businessCriticalMethods = grabList('business_critical_methods');
|
|
75
|
+
const capRaw = grab('capability_suggestions');
|
|
76
|
+
const capabilitySuggestions = capRaw !== 'off'; // default on; only an explicit `off` silences it
|
|
77
|
+
const found = focusRaw || risk || scope || businessCriticalMethods || capRaw;
|
|
78
|
+
return { focus, riskTier, tierScope, businessCriticalMethods, capabilitySuggestions, source: found ? 'context.md' : 'default' };
|
|
58
79
|
}
|
package/src/harness/ledger.ts
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import * as fs from 'fs';
|
|
15
15
|
import * as path from 'path';
|
|
16
|
+
import { reportSlug } from './unit-paths';
|
|
16
17
|
|
|
17
18
|
export interface LedgerEvent {
|
|
18
19
|
ts: string;
|
|
@@ -60,7 +61,7 @@ export function latestRunEvents(events: LedgerEvent[]): LedgerEvent[] {
|
|
|
60
61
|
}
|
|
61
62
|
|
|
62
63
|
function ledgerPath(screen: string): string {
|
|
63
|
-
return path.join(process.cwd(), '.sungen', 'ledger', `${screen}.jsonl`);
|
|
64
|
+
return path.join(process.cwd(), '.sungen', 'ledger', `${reportSlug(screen)}.jsonl`);
|
|
64
65
|
}
|
|
65
66
|
|
|
66
67
|
export function recordEvent(screen: string, ev: Omit<LedgerEvent, 'ts'> & { ts?: string }): string {
|
|
@@ -117,7 +118,7 @@ export function buildReport(screen: string, opts: { allRuns?: boolean } = {}): L
|
|
|
117
118
|
// Pull audit signals if present
|
|
118
119
|
let coveredCritical: number | null = null;
|
|
119
120
|
let scenarioCount: number | null = null;
|
|
120
|
-
const auditPath = path.join(process.cwd(), '.sungen', 'reports', `${screen}-audit.json`);
|
|
121
|
+
const auditPath = path.join(process.cwd(), '.sungen', 'reports', `${reportSlug(screen)}-audit.json`);
|
|
121
122
|
if (fs.existsSync(auditPath)) {
|
|
122
123
|
try {
|
|
123
124
|
const a = JSON.parse(fs.readFileSync(auditPath, 'utf-8'));
|
package/src/harness/manifest.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
import * as fs from 'fs';
|
|
12
12
|
import * as path from 'path';
|
|
13
13
|
import { createHash } from 'crypto';
|
|
14
|
+
import { featureBasename, reportSlug } from './unit-paths';
|
|
14
15
|
|
|
15
16
|
export interface SpecSection { name: string; hash: string }
|
|
16
17
|
export interface ManifestEntry { scenario: string; vpCode?: string; section: string; specHash: string }
|
|
@@ -97,7 +98,7 @@ function parseFeatureSections(featurePath: string): { scenario: string; vpCode?:
|
|
|
97
98
|
|
|
98
99
|
export function buildManifest(screenDir: string, screenName: string): Manifest {
|
|
99
100
|
const specPath = path.join(screenDir, 'requirements', 'spec.md');
|
|
100
|
-
const featurePath = path.join(screenDir, 'features', `${screenName}.feature`);
|
|
101
|
+
const featurePath = path.join(screenDir, 'features', `${featureBasename(screenName)}.feature`);
|
|
101
102
|
const specSections = parseSpecSections(specPath);
|
|
102
103
|
const specMap: Record<string, string> = {};
|
|
103
104
|
for (const s of specSections) specMap[s.name] = s.hash;
|
|
@@ -159,7 +160,7 @@ export function diffManifest(screenDir: string, screenName: string, manifest: Ma
|
|
|
159
160
|
}
|
|
160
161
|
|
|
161
162
|
export function manifestPath(screenName: string): string {
|
|
162
|
-
return path.join(process.cwd(), '.sungen', 'manifest', `${screenName}.json`);
|
|
163
|
+
return path.join(process.cwd(), '.sungen', 'manifest', `${reportSlug(screenName)}.json`);
|
|
163
164
|
}
|
|
164
165
|
export function loadManifest(screenName: string): Manifest | null {
|
|
165
166
|
const p = manifestPath(screenName);
|
package/src/harness/parse.ts
CHANGED
|
@@ -33,6 +33,7 @@ export interface ScenarioInfo {
|
|
|
33
33
|
casesDataset?: string; // @cases:<dataset> — data-driven; one scenario expands to N row-tests
|
|
34
34
|
queryRefs?: string[]; // named queries referenced by this scenario (inline `query [name]` + @query: tags)
|
|
35
35
|
apiRefs?: string[]; // named API endpoints referenced by this scenario (@api: tags)
|
|
36
|
+
requiresCaps?: string[]; // @requires:<cap> — automation-ready but needs an opt-in driver (TQ-11)
|
|
36
37
|
}
|
|
37
38
|
|
|
38
39
|
/** Format-tolerant: is this token an ID (project's scheme), not a prose word?
|
|
@@ -112,6 +113,8 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
112
113
|
// Named-API references: @api:<name>[(overrides)] tags.
|
|
113
114
|
const apiRefs = new Set<string>();
|
|
114
115
|
for (const t of tags) if (t.startsWith('@api:')) { const m = t.slice('@api:'.length).match(/^([A-Za-z_][A-Za-z0-9_]*)/); if (m) apiRefs.add(m[1]); }
|
|
116
|
+
// @requires:<cap> — automation-ready, needs an opt-in driver (TQ-11).
|
|
117
|
+
const requiresCaps = tags.filter((t) => /^@requires:/i.test(t)).map((t) => t.slice('@requires:'.length).trim().toLowerCase()).filter(Boolean);
|
|
115
118
|
let priority: Priority = 'unknown';
|
|
116
119
|
for (const t of tags) if (PRIORITY_TAGS[t]) priority = PRIORITY_TAGS[t];
|
|
117
120
|
|
|
@@ -169,6 +172,7 @@ function classifyScenario(sc: ParsedScenario): ScenarioInfo {
|
|
|
169
172
|
casesDataset,
|
|
170
173
|
queryRefs: queryRefs.size ? [...queryRefs] : undefined,
|
|
171
174
|
apiRefs: apiRefs.size ? [...apiRefs] : undefined,
|
|
175
|
+
requiresCaps: requiresCaps.length ? requiresCaps : undefined,
|
|
172
176
|
};
|
|
173
177
|
}
|
|
174
178
|
|
|
@@ -85,7 +85,7 @@ export function negativeSideEffect(scenarios: ScenarioInfo[]): string[] {
|
|
|
85
85
|
for (const s of scenarios) {
|
|
86
86
|
if (s.manual) continue; // @manual is a legitimate deferral (oracle checked by #4 manual-oracle)
|
|
87
87
|
if (!NEG_TITLE.test(s.name)) continue;
|
|
88
|
-
const proven = /\bcount\b|tohavecount|table with|is hidden|are hidden|not complete|message is hidden/.test(s.stepsText);
|
|
88
|
+
const proven = /\bcount\b|ok_count|status_counts|tohavecount|table with|is hidden|are hidden|not complete|message is hidden/.test(s.stepsText);
|
|
89
89
|
if (!proven) flagged.push(s.name.slice(0, 80));
|
|
90
90
|
}
|
|
91
91
|
return flagged;
|
|
Binary file
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Repair planner (#343) — the consumer of the `repair` capability SPI.
|
|
3
|
+
*
|
|
4
|
+
* Gathers the unit-capability's fix rules and matches them against the audit findings (always) and
|
|
5
|
+
* the latest Playwright failures (best-effort), turning them into a concrete fix plan. Deterministic:
|
|
6
|
+
* the AI repair loop and a human get the same proposals. Backs `sungen repair`.
|
|
7
|
+
*/
|
|
8
|
+
import * as fs from 'fs';
|
|
9
|
+
import * as path from 'path';
|
|
10
|
+
import { capabilityRegistry } from '../capabilities/registry';
|
|
11
|
+
import { discoverAndRegisterCapabilities } from '../capabilities/discover';
|
|
12
|
+
import { scoringCapabilityFor } from './audit';
|
|
13
|
+
|
|
14
|
+
export interface RepairProposal { source: 'audit' | 'runtime'; signal: string; ruleId: string; fix: string }
|
|
15
|
+
export interface RepairPlan {
|
|
16
|
+
capability: string | undefined;
|
|
17
|
+
rulesAvailable: number;
|
|
18
|
+
proposals: RepairProposal[];
|
|
19
|
+
unmatched: string[]; // findings/failures with no matching rule (need a human)
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Collect failure messages from a Playwright JSON result file (best-effort, defensive). */
|
|
23
|
+
function failuresFromResult(file: string): string[] {
|
|
24
|
+
const out: string[] = [];
|
|
25
|
+
try {
|
|
26
|
+
const r = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
27
|
+
const visit = (suite: any) => {
|
|
28
|
+
for (const sp of suite.specs ?? []) {
|
|
29
|
+
for (const t of sp.tests ?? []) {
|
|
30
|
+
for (const res of t.results ?? []) {
|
|
31
|
+
if (res.status === 'failed' || res.status === 'timedOut') {
|
|
32
|
+
const msg = res.error?.message || res.errors?.[0]?.message || res.status;
|
|
33
|
+
out.push(`${sp.title}: ${String(msg).split('\n')[0].slice(0, 200)}`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
for (const s of suite.suites ?? []) visit(s);
|
|
39
|
+
};
|
|
40
|
+
for (const s of r.suites ?? []) visit(s);
|
|
41
|
+
} catch { /* missing/!json → no runtime signals */ }
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Build the repair plan for a unit.
|
|
47
|
+
* @param unitId capability-resolution id (`api/<area>`, `flows/<flow>`, or a screen)
|
|
48
|
+
* @param reportName the bare name used for `.sungen/reports/<name>-audit.json` (+ test-result)
|
|
49
|
+
* @param generatedDir the unit's specs/generated dir (for runtime failures); optional
|
|
50
|
+
*/
|
|
51
|
+
export function planRepair(unitId: string, reportName: string, cwd: string, generatedDir?: string): RepairPlan {
|
|
52
|
+
discoverAndRegisterCapabilities();
|
|
53
|
+
const capId = scoringCapabilityFor(unitId, capabilityRegistry.defaultCapabilityId());
|
|
54
|
+
const rules = (capId ? capabilityRegistry.get(capId)?.repair?.rules : undefined) ?? [];
|
|
55
|
+
|
|
56
|
+
const signals: { source: 'audit' | 'runtime'; text: string }[] = [];
|
|
57
|
+
const auditPath = path.join(cwd, '.sungen', 'reports', `${reportName}-audit.json`);
|
|
58
|
+
if (fs.existsSync(auditPath)) {
|
|
59
|
+
try { for (const f of JSON.parse(fs.readFileSync(auditPath, 'utf8')).findings ?? []) signals.push({ source: 'audit', text: String(f) }); } catch { /* ignore */ }
|
|
60
|
+
}
|
|
61
|
+
if (generatedDir && fs.existsSync(generatedDir)) {
|
|
62
|
+
for (const f of fs.readdirSync(generatedDir)) {
|
|
63
|
+
if (/test-result.*\.json$/.test(f)) for (const msg of failuresFromResult(path.join(generatedDir, f))) signals.push({ source: 'runtime', text: msg });
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const proposals: RepairProposal[] = [];
|
|
68
|
+
const unmatched: string[] = [];
|
|
69
|
+
for (const s of signals) {
|
|
70
|
+
const rule = rules.find((r) => r.match.test(s.text));
|
|
71
|
+
if (rule) proposals.push({ source: s.source, signal: s.text, ruleId: rule.id, fix: rule.fix });
|
|
72
|
+
else unmatched.push(s.text);
|
|
73
|
+
}
|
|
74
|
+
return { capability: capId, rulesAvailable: rules.length, proposals, unmatched };
|
|
75
|
+
}
|
|
@@ -16,6 +16,7 @@ import * as fs from 'fs';
|
|
|
16
16
|
import * as path from 'path';
|
|
17
17
|
import * as os from 'os';
|
|
18
18
|
import { loadScenarios, ScenarioInfo } from './parse';
|
|
19
|
+
import { featureBasename } from './unit-paths';
|
|
19
20
|
|
|
20
21
|
export interface ScriptCheckResult {
|
|
21
22
|
screen: string;
|
|
@@ -67,6 +68,9 @@ export function analyzeFaithfulness(specSrc: string, automatedTitles: Set<string
|
|
|
67
68
|
const hollowSteps: { test: string; step: string }[] = [];
|
|
68
69
|
for (const blk of extractTestBlocks(specSrc)) {
|
|
69
70
|
if (!automatedTitles.has(blk.title)) continue; // only non-@manual scenarios
|
|
71
|
+
// TQ-11 — a capability-pending @requires scenario compiles to a `test.skip(true, …)` stub:
|
|
72
|
+
// it intentionally proves nothing here (it runs once the driver is added), so it is not a bypass.
|
|
73
|
+
if (blk.body.some((l) => /\btest\.skip\(\s*true\b/.test(l))) continue;
|
|
70
74
|
const body = blk.body;
|
|
71
75
|
// An assertion is a Playwright `expect(...)` OR a Data Driver DB assertion
|
|
72
76
|
// (`db.assertRow/assertNoRow/assertCount/...`) — a DB check is a real oracle, so a
|
|
@@ -106,9 +110,18 @@ function normalize(src: string): string {
|
|
|
106
110
|
.trim();
|
|
107
111
|
}
|
|
108
112
|
|
|
109
|
-
|
|
113
|
+
/** The unit kind — drives the generated-spec subdir + the qa source dir. */
|
|
114
|
+
export type UnitKind = 'screen' | 'flow' | 'api';
|
|
115
|
+
|
|
116
|
+
/** Generated-spec subdir for a unit: screen → <name>, flow → flows/<name>, api → api/<name>. */
|
|
117
|
+
function specSubdir(dir: string, name: string, kind: UnitKind): string {
|
|
118
|
+
return kind === 'flow' ? path.join(dir, 'flows', name) : kind === 'api' ? path.join(dir, 'api', name) : path.join(dir, name);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function findSpec(dir: string, name: string, kind: UnitKind): string | null {
|
|
110
122
|
// Screens compile to <dir>/<name>/<feature>.spec.ts
|
|
111
123
|
// Flows compile to <dir>/flows/<name>/<feature>.spec.ts
|
|
124
|
+
// Api compile to <dir>/api/<name>/<feature>.spec.ts
|
|
112
125
|
// Scope the search to THIS target's own subdir — otherwise the first spec of
|
|
113
126
|
// ANY other screen/flow is returned, which (for an uncompiled flow) falsely
|
|
114
127
|
// reports the wrong screen's tests as drift.
|
|
@@ -121,19 +134,19 @@ function findSpec(dir: string, name: string, flowMode: boolean): string | null {
|
|
|
121
134
|
else if (e.name.endsWith('.spec.ts')) hits.push(p);
|
|
122
135
|
}
|
|
123
136
|
};
|
|
124
|
-
const scoped =
|
|
137
|
+
const scoped = specSubdir(dir, name, kind);
|
|
125
138
|
if (!fs.existsSync(scoped)) return null; // no spec for this target (e.g. not compiled yet)
|
|
126
139
|
walk(scoped);
|
|
127
140
|
return hits[0] ?? null;
|
|
128
141
|
}
|
|
129
142
|
|
|
130
|
-
export async function runScriptCheck(screenDir: string, screenName: string,
|
|
131
|
-
const featurePath = path.join(screenDir, 'features', `${screenName}.feature`);
|
|
143
|
+
export async function runScriptCheck(screenDir: string, screenName: string, kind: UnitKind): Promise<ScriptCheckResult> {
|
|
144
|
+
const featurePath = path.join(screenDir, 'features', `${featureBasename(screenName)}.feature`);
|
|
132
145
|
const scenarios = loadScenarios(featurePath);
|
|
133
146
|
const automated = scenarios.filter((s) => !s.manual);
|
|
134
147
|
const manual = scenarios.filter((s) => s.manual);
|
|
135
148
|
|
|
136
|
-
const committedSpec = findSpec(path.join(process.cwd(), 'specs', 'generated'), screenName,
|
|
149
|
+
const committedSpec = findSpec(path.join(process.cwd(), 'specs', 'generated'), screenName, kind);
|
|
137
150
|
|
|
138
151
|
const findings: string[] = [];
|
|
139
152
|
let specTitles: string[] = [];
|
|
@@ -167,10 +180,14 @@ export async function runScriptCheck(screenDir: string, screenName: string, flow
|
|
|
167
180
|
try {
|
|
168
181
|
const { CodeGenerator } = require('../generators/test-generator/code-generator');
|
|
169
182
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'sungen-scriptcheck-'));
|
|
170
|
-
const qaSourceDir = path.join(process.cwd(), 'qa',
|
|
171
|
-
|
|
183
|
+
const qaSourceDir = path.join(process.cwd(), 'qa', kind === 'flow' ? 'flows' : kind === 'api' ? 'api' : 'screens');
|
|
184
|
+
// api units derive their unit id (api/<area>) from the feature path — like `generate --api`;
|
|
185
|
+
// screen/flow pass screenName + flowMode explicitly (unchanged → byte-identical regenerate).
|
|
186
|
+
const gen = kind === 'api'
|
|
187
|
+
? new CodeGenerator({ framework: 'playwright', runtimeData: true })
|
|
188
|
+
: new CodeGenerator({ framework: 'playwright', screenName, runtimeData: true, flowMode: kind === 'flow' });
|
|
172
189
|
await gen.generateAllTests(qaSourceDir, tmp, [featurePath]);
|
|
173
|
-
const fresh = findSpec(tmp, screenName,
|
|
190
|
+
const fresh = findSpec(tmp, screenName, kind);
|
|
174
191
|
if (fresh) {
|
|
175
192
|
const a = normalize(specSrc);
|
|
176
193
|
const b = normalize(fs.readFileSync(fresh, 'utf-8'));
|
package/src/harness/sensors.ts
CHANGED
|
@@ -111,6 +111,11 @@ export interface DepthResult {
|
|
|
111
111
|
businessCriticalShallow: number; // = depth-required scenarios that are shallow
|
|
112
112
|
bcDepthRatio: number; // fraction of depth-required scenarios with a real data assertion
|
|
113
113
|
shallowBusinessCritical: { name: string; category?: string }[];
|
|
114
|
+
// @manual scenarios that would be business-critical if automated (match a data-theme).
|
|
115
|
+
// They are EXCLUDED from bcDepthRatio, so deferring them to @manual collapses the
|
|
116
|
+
// denominator and inflates the ratio toward 1.0 — reported so a high ratio on a tiny
|
|
117
|
+
// denominator isn't misread as "all good" (TQ-3).
|
|
118
|
+
deferredBusinessCritical: number;
|
|
114
119
|
// Depth-as-Gate (harness-roadmap P1)
|
|
115
120
|
focus: string; // intent focus driving the threshold
|
|
116
121
|
threshold: number; // required bcDepthRatio for this focus
|
|
@@ -124,6 +129,16 @@ const DEPTH_THRESHOLDS: Record<string, number> = {
|
|
|
124
129
|
};
|
|
125
130
|
const WARN_ONLY_FOCUS = new Set(['smoke']);
|
|
126
131
|
|
|
132
|
+
/** The required businessDepth ratio for a focus (default `functional` = 0.7). Shared so a capability
|
|
133
|
+
* gate (e.g. the API gate, which computes its own depth) uses the SAME thresholds as the UI gate. */
|
|
134
|
+
export function depthThresholdFor(focus: string): number {
|
|
135
|
+
return DEPTH_THRESHOLDS[focus] ?? DEPTH_THRESHOLDS.functional;
|
|
136
|
+
}
|
|
137
|
+
/** Whether a depth miss only WARNs (vs FAILs) for a focus (smoke). */
|
|
138
|
+
export function depthWarnOnly(focus: string): boolean {
|
|
139
|
+
return WARN_ONLY_FOCUS.has(focus);
|
|
140
|
+
}
|
|
141
|
+
|
|
127
142
|
/**
|
|
128
143
|
* Depth = do DATA-correctness scenarios actually assert DATA (not just visibility)?
|
|
129
144
|
* "Depth-required" is CATALOG-DRIVEN: only scenarios matching a theme whose
|
|
@@ -151,6 +166,8 @@ export function assertionDepth(
|
|
|
151
166
|
|
|
152
167
|
const required = nonManual.filter(isDepthRequired);
|
|
153
168
|
const reqShallow = required.filter((s) => s.shallow);
|
|
169
|
+
// Business-critical scenarios deferred to @manual (match a data-theme but excluded above).
|
|
170
|
+
const deferredBusinessCritical = scenarios.filter((s) => s.manual && isDepthRequired(s)).length;
|
|
154
171
|
// No data-theme scenarios on this screen → depth is not the binding constraint
|
|
155
172
|
// (the viewpoint gate already flags missing data themes). Don't double-penalize.
|
|
156
173
|
const ratio = required.length ? 1 - reqShallow.length / required.length : 1;
|
|
@@ -167,12 +184,64 @@ export function assertionDepth(
|
|
|
167
184
|
businessCriticalShallow: reqShallow.length,
|
|
168
185
|
bcDepthRatio: ratio,
|
|
169
186
|
shallowBusinessCritical: reqShallow.map((s) => ({ name: s.name, category: s.category })),
|
|
187
|
+
deferredBusinessCritical,
|
|
170
188
|
focus,
|
|
171
189
|
threshold,
|
|
172
190
|
verdict,
|
|
173
191
|
};
|
|
174
192
|
}
|
|
175
193
|
|
|
194
|
+
// ---------- Sensor 2b: Automatable-@manual (TQ-2) ----------
|
|
195
|
+
|
|
196
|
+
export interface AutomatableManualResult {
|
|
197
|
+
manualTotal: number; // all @manual scenarios
|
|
198
|
+
automatable: number; // @manual that are actually automatable
|
|
199
|
+
scenarios: { name: string; category?: string }[]; // the automatable ones (to surface)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// Genuine-judgment markers (M6/M8/M9 territory): visual/responsive/a11y/mock/network/
|
|
203
|
+
// external/empty-state — these legitimately stay @manual (or need a future driver).
|
|
204
|
+
const JUDGMENT_MARKER =
|
|
205
|
+
/\b(visual|responsive|layout|breakpoint|mobile|tablet|viewport|accessib|a11y|keyboard|screen reader|focus order|\baria\b|empty[- ]?(state|product|list|category|cart)|no[- ]?result|missing (image|product|data)|placeholder|fallback|slow|failing|offline|network|loading|spinner|external|new tab|video tutorial|email|mailbox|download|payment gateway|exploratory|not worth)\b/;
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Automatable-@manual (TQ-2) — a `@manual` scenario whose steps are fully DSL-expressible
|
|
209
|
+
* (it carries a real data assertion) and shows no genuine-judgment marker is *automatable*:
|
|
210
|
+
* it was deferred (typically cross-screen → a flow) rather than truly un-automatable. Leaving
|
|
211
|
+
* it `@manual` creates a non-running duplicate AND inflates businessDepth (it's excluded from
|
|
212
|
+
* the ratio). The UI analog of the API driver's `api-manual-automatable`.
|
|
213
|
+
*/
|
|
214
|
+
export function automatableManual(scenarios: ScenarioInfo[]): AutomatableManualResult {
|
|
215
|
+
const manual = scenarios.filter((s) => s.manual);
|
|
216
|
+
const automatable = manual.filter((s) => s.hasDataAssertion && !JUDGMENT_MARKER.test(s.haystack));
|
|
217
|
+
return {
|
|
218
|
+
manualTotal: manual.length,
|
|
219
|
+
automatable: automatable.length,
|
|
220
|
+
scenarios: automatable.map((s) => ({ name: s.name, category: s.category })),
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// ---------- TQ-4: deferral-aware coverage credit ----------
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Which of the given gate gap-themes are deeply covered by a FLOW scenario (a cross-screen
|
|
228
|
+
* deferral the flow actually fulfils). Returns theme → covering flow. The screen audit uses
|
|
229
|
+
* this to credit an inherently-cross-screen theme to the flow that owns it, instead of
|
|
230
|
+
* double-counting it as a screen gap. A flow scenario covers a theme when its haystack hits
|
|
231
|
+
* the theme keywords AND it carries a data assertion (`deep`).
|
|
232
|
+
*/
|
|
233
|
+
export function flowCoveredThemes(
|
|
234
|
+
gaps: { theme: string; keywords: string[] }[],
|
|
235
|
+
flowScenarios: { flow: string; haystack: string; deep: boolean }[],
|
|
236
|
+
): { theme: string; flow: string }[] {
|
|
237
|
+
const out: { theme: string; flow: string }[] = [];
|
|
238
|
+
for (const g of gaps) {
|
|
239
|
+
const hit = flowScenarios.find((s) => s.deep && g.keywords.some((k) => s.haystack.includes(k.toLowerCase())));
|
|
240
|
+
if (hit) out.push({ theme: g.theme, flow: hit.flow });
|
|
241
|
+
}
|
|
242
|
+
return out;
|
|
243
|
+
}
|
|
244
|
+
|
|
176
245
|
/** Collect data-correctness themes (depth.requires) for a page-type + universal. */
|
|
177
246
|
export function dataThemesFor(catalog: Catalog, pageType: string | null): CatalogTheme[] {
|
|
178
247
|
const themes: CatalogTheme[] = [];
|
|
@@ -384,8 +453,8 @@ const CLAIM_RULES: ClaimRule[] = [
|
|
|
384
453
|
// "double-click does not create two orders" — not a per-feature keyword.
|
|
385
454
|
claim: 'no-side-effect/no-duplicate',
|
|
386
455
|
title: /(?=.*\b(submit|sen[dt]|resend|resubmit|re-?fire|re-?issue|re-?post|repost|create|charge|order|payment|\bpay\b|email|request|\botp\b|insert|register|book|duplicate|double[- ]?submit|again|twice)\b)(?=.*(\bno\b|\bnot\b|n['’]t\b|\bnever\b|\bwithout\b|\bcannot\b|prevent|block|avoid|reject|disabl|\bdeny\b|denies|\bkhông\b|\bchưa\b))/i,
|
|
387
|
-
proof: /\bcount\b|row with \{\{|table with|tohavecount|is hidden|are hidden|not complete|no longer/,
|
|
388
|
-
need: 'a record/request-count proof (count stays at one, e.g. `User see [Table] row with {{count}}`) or @manual with a request-count oracle',
|
|
456
|
+
proof: /\bcount\b|ok_count|status_counts|row with \{\{|table with|tohavecount|is hidden|are hidden|not complete|no longer/,
|
|
457
|
+
need: 'a record/request-count proof (count stays at one, e.g. `User see [Table] row with {{count}}`, an API `{{name.ok_count}}` invariant, or a `@query` DB count) or @manual with a request-count oracle',
|
|
389
458
|
hint: 'a "does-not-happen / does-not-repeat" claim about a state-changing action is NOT proven by a terminal `see [...] page` — that page is identical whether or not the action (re-)fired. Prove the side-effect count is unchanged, or mark @manual with a setup→action→assert-no-duplicate oracle.',
|
|
390
459
|
severity: 'fail',
|
|
391
460
|
},
|
package/src/harness/trace.ts
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import * as fs from 'fs';
|
|
15
15
|
import * as path from 'path';
|
|
16
|
+
import { reportSlug } from './unit-paths';
|
|
16
17
|
import { segmentRuns, latestRunEvents, LedgerEvent } from './ledger';
|
|
17
18
|
|
|
18
19
|
interface ManualItem { scenario: string; reason: string }
|
|
@@ -22,7 +23,7 @@ function readJson(p: string): any | null {
|
|
|
22
23
|
}
|
|
23
24
|
|
|
24
25
|
function readLedger(screen: string): any[] {
|
|
25
|
-
const p = path.join(process.cwd(), '.sungen', 'ledger', `${screen}.jsonl`);
|
|
26
|
+
const p = path.join(process.cwd(), '.sungen', 'ledger', `${reportSlug(screen)}.jsonl`);
|
|
26
27
|
if (!fs.existsSync(p)) return [];
|
|
27
28
|
return fs.readFileSync(p, 'utf-8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
|
28
29
|
}
|
|
@@ -76,7 +77,7 @@ export function buildTrace(screenDir: string, screenName: string): TraceReport {
|
|
|
76
77
|
const recordedSteps = [...new Set(ledger.map((e) => e.step.replace(/:\d+$/, '')))];
|
|
77
78
|
const missingSteps = EXPECTED_PHASES.filter((p) => !recordedSteps.includes(p));
|
|
78
79
|
|
|
79
|
-
const auditRaw = readJson(path.join(process.cwd(), '.sungen', 'reports', `${screenName}-audit.json`));
|
|
80
|
+
const auditRaw = readJson(path.join(process.cwd(), '.sungen', 'reports', `${reportSlug(screenName)}-audit.json`));
|
|
80
81
|
let audit: TraceReport['audit'] = null;
|
|
81
82
|
if (auditRaw) {
|
|
82
83
|
const subs: Record<string, number> = {
|
|
@@ -91,7 +92,7 @@ export function buildTrace(screenDir: string, screenName: string): TraceReport {
|
|
|
91
92
|
};
|
|
92
93
|
}
|
|
93
94
|
|
|
94
|
-
const scRaw = readJson(path.join(process.cwd(), '.sungen', 'reports', `${screenName}-script-check.json`));
|
|
95
|
+
const scRaw = readJson(path.join(process.cwd(), '.sungen', 'reports', `${reportSlug(screenName)}-script-check.json`));
|
|
95
96
|
const drift = scRaw ? scRaw.drift : null;
|
|
96
97
|
|
|
97
98
|
const manual = parseManual(path.join(screenDir, 'features', `${screenName}.feature`));
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit-path helpers (api-flow fix). A unit id may be a bare name (`orders`, `login`) or a nested
|
|
3
|
+
* api-flow id (`flows/<flow>`). Two derivations the harness/CLI need:
|
|
4
|
+
* - featureBasename: the `.feature` filename — the LAST path segment (`flows/x` → `x`), so
|
|
5
|
+
* `<dir>/features/<basename>.feature` resolves (the bug: the full id looked for
|
|
6
|
+
* `features/flows/x.feature` → 0 scenarios).
|
|
7
|
+
* - reportSlug: a flat key for `.sungen/reports/<slug>-*.json` + `.sungen/ledger/<slug>.jsonl`
|
|
8
|
+
* (`flows/x` → `flows-x`), so artifacts never nest under a `flows/` subdir and read/write agree.
|
|
9
|
+
* Bare names (no slash) are unchanged by both → no regression for screens/flows/areas.
|
|
10
|
+
*/
|
|
11
|
+
import * as path from 'path';
|
|
12
|
+
|
|
13
|
+
export const featureBasename = (unit: string): string => path.basename(unit);
|
|
14
|
+
export const reportSlug = (unit: string): string => unit.replace(/[\\/]+/g, '-');
|
package/src/index.ts
CHANGED
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
export { capabilityRegistry, CapabilityRegistry } from './capabilities/registry';
|
|
9
9
|
export type { CapabilityDescriptor } from './capabilities/registry';
|
|
10
10
|
export type { Sensor, SensorFinding, AdvisoryScanInput, GateInput } from './capabilities/sensor';
|
|
11
|
-
export type { Context, DiscoveryProvider, ContextMapper, GenerationUnit } from './capabilities/context';
|
|
11
|
+
export type { Context, DiscoveryProvider, ContextMapper, GenerationUnit, RepairProvider, RepairRule } from './capabilities/context';
|
|
12
|
+
export { discoverUnitContext } from './orchestrator/context-discovery';
|
|
13
|
+
export type { DiscoveredContext } from './orchestrator/context-discovery';
|
|
12
14
|
|
|
13
15
|
// --- Step-pattern authoring (a driver contributes step patterns via its descriptor) ---
|
|
14
16
|
export type { PatternContext, StepPattern, StepTemplateData } from './generators/test-generator/patterns/types';
|
|
@@ -25,6 +27,6 @@ export type { QueryEntry } from './harness/query-catalog';
|
|
|
25
27
|
|
|
26
28
|
// --- Shared harness: viewpoint catalog + coverage gate / assertion depth ---
|
|
27
29
|
// (the UI capability's gateProvider composes these; they also back core's ingest + audit fallback)
|
|
28
|
-
export { loadCatalog, viewpointGate, assertionDepth, dataThemesFor } from './harness/sensors';
|
|
30
|
+
export { loadCatalog, viewpointGate, assertionDepth, dataThemesFor, depthThresholdFor, depthWarnOnly } from './harness/sensors';
|
|
29
31
|
export type { Catalog, GateResult, DepthResult } from './harness/sensors';
|
|
30
32
|
export type { ScenarioInfo, ViewpointEntry } from './harness/parse';
|
|
@@ -47,6 +47,7 @@ export const AI_RULES_FILE_MAPPING: [string, string][] = [
|
|
|
47
47
|
['claude-skill-selector-fix.md', '.claude/skills/sungen-selector-fix/SKILL.md'],
|
|
48
48
|
['claude-skill-tc-review.md', '.claude/skills/sungen-tc-review/SKILL.md'],
|
|
49
49
|
['claude-skill-harness-audit.md', '.claude/skills/sungen-harness-audit/SKILL.md'],
|
|
50
|
+
['claude-skill-api-design.md', '.claude/skills/sungen-api-design/SKILL.md'],
|
|
50
51
|
['claude-skill-ingest-legacy.md', '.claude/skills/sungen-ingest-legacy/SKILL.md'],
|
|
51
52
|
['claude-skill-viewpoint.md', '.claude/skills/sungen-viewpoint/SKILL.md'],
|
|
52
53
|
['claude-skill-viewpoint-group-a-data-entry.md', '.claude/skills/sungen-viewpoint/group-a-data-entry.md'],
|
|
@@ -79,6 +80,7 @@ export const AI_RULES_FILE_MAPPING: [string, string][] = [
|
|
|
79
80
|
['github-skill-sungen-selector-fix.md', '.github/skills/sungen-selector-fix/SKILL.md'],
|
|
80
81
|
['github-skill-sungen-tc-review.md', '.github/skills/sungen-tc-review/SKILL.md'],
|
|
81
82
|
['github-skill-sungen-harness-audit.md', '.github/skills/sungen-harness-audit/SKILL.md'],
|
|
83
|
+
['github-skill-sungen-api-design.md', '.github/skills/sungen-api-design/SKILL.md'],
|
|
82
84
|
['github-skill-sungen-ingest-legacy.md', '.github/skills/sungen-ingest-legacy/SKILL.md'],
|
|
83
85
|
['github-skill-sungen-viewpoint.md', '.github/skills/sungen-viewpoint/SKILL.md'],
|
|
84
86
|
['github-skill-sungen-viewpoint-group-a-data-entry.md', '.github/skills/sungen-viewpoint/group-a-data-entry.md'],
|