@sun-asterisk/sungen 3.2.20 → 3.2.21-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +8 -0
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +15 -0
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/trace.d.ts.map +1 -1
- package/dist/cli/commands/trace.js +9 -0
- package/dist/cli/commands/trace.js.map +1 -1
- package/dist/exporters/api-testcase-formatter.d.ts +8 -0
- package/dist/exporters/api-testcase-formatter.d.ts.map +1 -1
- package/dist/exporters/api-testcase-formatter.js +44 -22
- package/dist/exporters/api-testcase-formatter.js.map +1 -1
- package/dist/exporters/matrix/build.d.ts +14 -2
- package/dist/exporters/matrix/build.d.ts.map +1 -1
- package/dist/exporters/matrix/build.js +62 -2
- package/dist/exporters/matrix/build.js.map +1 -1
- package/dist/exporters/matrix/export.d.ts +6 -0
- package/dist/exporters/matrix/export.d.ts.map +1 -1
- package/dist/exporters/matrix/export.js +20 -1
- package/dist/exporters/matrix/export.js.map +1 -1
- package/dist/exporters/matrix/gates.js +7 -1
- package/dist/exporters/matrix/gates.js.map +1 -1
- package/dist/exporters/matrix/render-xlsx.d.ts.map +1 -1
- package/dist/exporters/matrix/render-xlsx.js +44 -1
- package/dist/exporters/matrix/render-xlsx.js.map +1 -1
- package/dist/exporters/matrix/types.d.ts +13 -0
- package/dist/exporters/matrix/types.d.ts.map +1 -1
- package/dist/exporters/matrix/types.js.map +1 -1
- package/dist/exporters/playwright-report-parser.d.ts.map +1 -1
- package/dist/exporters/playwright-report-parser.js +1 -0
- package/dist/exporters/playwright-report-parser.js.map +1 -1
- package/dist/exporters/types.d.ts +2 -0
- package/dist/exporters/types.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.d.ts +6 -1
- package/dist/generators/test-generator/diagnostics.d.ts.map +1 -1
- package/dist/generators/test-generator/diagnostics.js +5 -0
- package/dist/generators/test-generator/diagnostics.js.map +1 -1
- package/dist/generators/test-generator/patterns/index.d.ts.map +1 -1
- package/dist/generators/test-generator/patterns/index.js +27 -19
- package/dist/generators/test-generator/patterns/index.js.map +1 -1
- package/dist/generators/test-generator/step-mapper.d.ts.map +1 -1
- package/dist/generators/test-generator/step-mapper.js +22 -0
- package/dist/generators/test-generator/step-mapper.js.map +1 -1
- package/dist/harness/audit.d.ts +2 -0
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +91 -9
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-contract.d.ts +71 -0
- package/dist/harness/flow-contract.d.ts.map +1 -0
- package/dist/harness/flow-contract.js +235 -0
- package/dist/harness/flow-contract.js.map +1 -0
- package/dist/harness/flow-plan.d.ts +3 -0
- package/dist/harness/flow-plan.d.ts.map +1 -1
- package/dist/harness/flow-plan.js +6 -2
- package/dist/harness/flow-plan.js.map +1 -1
- package/dist/harness/parse.d.ts +5 -0
- package/dist/harness/parse.d.ts.map +1 -1
- package/dist/harness/parse.js +29 -1
- package/dist/harness/parse.js.map +1 -1
- package/dist/harness/perf.d.ts +40 -0
- package/dist/harness/perf.d.ts.map +1 -0
- package/dist/harness/perf.js +136 -0
- package/dist/harness/perf.js.map +1 -0
- package/dist/harness/script-check.d.ts.map +1 -1
- package/dist/harness/script-check.js +17 -4
- package/dist/harness/script-check.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +13 -1
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/spec-coverage.d.ts.map +1 -1
- package/dist/harness/spec-coverage.js +29 -5
- package/dist/harness/spec-coverage.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/add-flow.md +46 -3
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +10 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +52 -16
- package/dist/orchestrator/templates/qa-context.md +14 -1
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +8 -0
- package/src/cli/commands/delivery.ts +14 -0
- package/src/cli/commands/trace.ts +9 -0
- package/src/exporters/api-testcase-formatter.ts +46 -20
- package/src/exporters/matrix/build.ts +69 -4
- package/src/exporters/matrix/export.ts +28 -1
- package/src/exporters/matrix/gates.ts +7 -1
- package/src/exporters/matrix/render-xlsx.ts +45 -1
- package/src/exporters/matrix/types.ts +13 -0
- package/src/exporters/playwright-report-parser.ts +2 -0
- package/src/exporters/types.ts +2 -0
- package/src/generators/test-generator/diagnostics.ts +6 -1
- package/src/generators/test-generator/patterns/index.ts +30 -24
- package/src/generators/test-generator/step-mapper.ts +22 -0
- package/src/harness/audit.ts +94 -10
- package/src/harness/flow-contract.ts +229 -0
- package/src/harness/flow-plan.ts +10 -3
- package/src/harness/parse.ts +31 -1
- package/src/harness/perf.ts +112 -0
- package/src/harness/script-check.ts +18 -4
- package/src/harness/sensors.ts +13 -1
- package/src/harness/spec-coverage.ts +26 -4
- package/src/orchestrator/templates/ai-src/commands/add-flow.md +46 -3
- package/src/orchestrator/templates/ai-src/commands/create-test.md +10 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +52 -16
- package/src/orchestrator/templates/qa-context.md +14 -1
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* is structural: there is no independent parent result input).
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import { FeatureMetadata, PlaywrightResult } from '../types';
|
|
8
|
+
import { ApiCatalogEntry, FeatureMetadata, PlaywrightResult } from '../types';
|
|
9
|
+
import { formatApiCallLines, parseApiCalls } from '../api-testcase-formatter';
|
|
9
10
|
import { bucketTexts } from './step-buckets';
|
|
10
11
|
import { MergedScenario } from '../scenario-merger';
|
|
11
12
|
import {
|
|
@@ -31,6 +32,8 @@ import {
|
|
|
31
32
|
MAX_VARIANTS_PER_ITEM,
|
|
32
33
|
} from './types';
|
|
33
34
|
import { runGates } from './gates';
|
|
35
|
+
import { capabilityRegistry } from '../../capabilities/registry';
|
|
36
|
+
import { discoverAndRegisterCapabilities } from '../../capabilities/discover';
|
|
34
37
|
|
|
35
38
|
// ---------------------------------------------------------------------------
|
|
36
39
|
// Variant derivation
|
|
@@ -41,6 +44,27 @@ function normalizeShape(text: string): string {
|
|
|
41
44
|
return text.replace(/\{\{[^}]*\}\}/g, '{{*}}').replace(/\s+/g, ' ').trim();
|
|
42
45
|
}
|
|
43
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Is this scenario verified against MOCKED responses, and against which named deps? (#580)
|
|
49
|
+
* The mock capability owns its grammar: prefer the registry's `annotations` + `detectsStep`
|
|
50
|
+
* (driver-mock exports isMockStep — install AND call-count forms); fall back to literal
|
|
51
|
+
* twins of that grammar when the driver is not installed on this machine.
|
|
52
|
+
*/
|
|
53
|
+
export function detectMocked(tags: string[], stepTexts: string[]): { mocked: boolean; mockDeps: string[] } {
|
|
54
|
+
discoverAndRegisterCapabilities();
|
|
55
|
+
const cap = capabilityRegistry.get('mock');
|
|
56
|
+
const tagForms = cap?.annotations ?? ['@mock', '@network'];
|
|
57
|
+
const tagged = tags.some((t) => tagForms.some((a) => t === a || t.startsWith(`${a}:`)));
|
|
58
|
+
const isMockStep = cap?.detectsStep ?? ((t: string) =>
|
|
59
|
+
/\[[^\]]+\]\s+mock\s+is\s+active\b/i.test(t) ||
|
|
60
|
+
/\[[^\]]+\]\s+mock\s+called\s+(?:\{\{[^}]+\}\}|\d+)\s+times?\b/i.test(t));
|
|
61
|
+
const mockDeps = Array.from(new Set(stepTexts
|
|
62
|
+
.filter((t) => isMockStep(t))
|
|
63
|
+
.map((t) => (t.match(/\[([^\]]+)\]\s+mock\b/i) || [])[1])
|
|
64
|
+
.filter((x): x is string => Boolean(x))));
|
|
65
|
+
return { mocked: tagged || mockDeps.length > 0, mockDeps };
|
|
66
|
+
}
|
|
67
|
+
|
|
44
68
|
function deriveLayers(tags: string[]): MatrixLayer[] {
|
|
45
69
|
const layers = new Set<MatrixLayer>(['ui']);
|
|
46
70
|
if (tags.some((t) => t === '@api' || t.startsWith('@api:') || t === '@requires:api')) layers.add('api');
|
|
@@ -95,6 +119,8 @@ export interface BuildInputs {
|
|
|
95
119
|
transformerVersion: string;
|
|
96
120
|
/** requirements/spec.md content — source of the requirement-id inventory (FR/TR/NFR). */
|
|
97
121
|
specText?: string;
|
|
122
|
+
/** apis.yaml endpoint catalog (api units) — renders `@api:` calls as request blocks. */
|
|
123
|
+
apiCatalog?: Record<string, ApiCatalogEntry>;
|
|
98
124
|
}
|
|
99
125
|
|
|
100
126
|
/**
|
|
@@ -102,11 +128,12 @@ export interface BuildInputs {
|
|
|
102
128
|
* `@cases` dataset row). Exported separately so the gates + the map-proposal
|
|
103
129
|
* support tooling see the same universe the builder does.
|
|
104
130
|
*/
|
|
105
|
-
export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' | 'testData' | 'results'>): CoverageVariant[] {
|
|
131
|
+
export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' | 'testData' | 'results' | 'apiCatalog'>): CoverageVariant[] {
|
|
106
132
|
const { merged, results } = inputs;
|
|
107
133
|
// Test-data values may cross-reference other keys (email_padded: " {{valid_email}} ") —
|
|
108
134
|
// resolve one level so display cells never leak a template token (review B-04).
|
|
109
135
|
const testData = resolveCrossRefs(inputs.testData);
|
|
136
|
+
const apiCatalog = inputs.apiCatalog ?? {};
|
|
110
137
|
const variants: CoverageVariant[] = [];
|
|
111
138
|
|
|
112
139
|
for (const m of merged) {
|
|
@@ -141,11 +168,27 @@ export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' |
|
|
|
141
168
|
tags,
|
|
142
169
|
);
|
|
143
170
|
|
|
171
|
+
// `@api:<name>(args)` invocations — the runtime's precondition codegen (driver-api):
|
|
172
|
+
// each call sends its catalog request BEFORE the scenario's own steps and binds
|
|
173
|
+
// {{<name>.*}}. Cell placement follows what the tester actually does:
|
|
174
|
+
// - pure api scenario (no When steps): the calls ARE the procedure → trigger,
|
|
175
|
+
// rendered as Method/API/Header/Body blocks (the legacy Steps-cell language) —
|
|
176
|
+
// the page-load fallback would misdescribe an HTTP request;
|
|
177
|
+
// - hybrid (@api + own When steps): the calls are setup → precondition.
|
|
178
|
+
// Blocks keep raw `{{var}}` tokens here; renderCells' sub resolves them per data
|
|
179
|
+
// context (so @cases rows render their own arg values).
|
|
180
|
+
const apiCalls = parseApiCalls(tags);
|
|
181
|
+
const apiCallsAreTrigger = apiCalls.length > 0 && rawTrigger.length === 0;
|
|
182
|
+
const apiRequestLines = apiCalls.length > 0 ? formatApiCallLines(apiCalls, apiCatalog, (s) => s) : [];
|
|
183
|
+
// Shape entry per call: endpoint identity + arg KEYS (values are data, not shape).
|
|
184
|
+
const apiCallShapes = apiCalls.map((c) => `api:${c.name}(${Object.keys(c.args).join(',')})`);
|
|
185
|
+
|
|
144
186
|
const preconditionProfile = [
|
|
145
187
|
authRole ?? '-',
|
|
146
188
|
m.feature.extendsName ?? '-',
|
|
147
189
|
...bucketTexts(m.feature.orderedSteps, 'given').map(normalizeShape),
|
|
148
190
|
...(manual?.preconditions ?? []).map(normalizeShape),
|
|
191
|
+
...(apiCallsAreTrigger ? [] : apiCallShapes),
|
|
149
192
|
].join(' | ');
|
|
150
193
|
// Display precondition = auth state + Background Given (shared start state) +
|
|
151
194
|
// the scenario's own Given steps + manual Setup lines — deduplicated (a manual
|
|
@@ -155,18 +198,36 @@ export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' |
|
|
|
155
198
|
...bucketTexts(inputs.feature.backgroundOrderedSteps, 'given').map(renderPrecondition),
|
|
156
199
|
...bucketTexts(m.feature.orderedSteps, 'given').map(renderPrecondition),
|
|
157
200
|
...(manual?.preconditions ?? []).map((t) => renderSetupInstruction(t)),
|
|
201
|
+
...(apiCallsAreTrigger ? [] : apiRequestLines),
|
|
158
202
|
]));
|
|
159
203
|
|
|
204
|
+
// Mocked-run detection (#580): a pass against a scripted response must never read as a
|
|
205
|
+
// pass against the real dependency. The GRAMMAR authority is the mock capability itself
|
|
206
|
+
// (driver-mock declares `annotations` and exports `detectsStep`) — consult it instead of
|
|
207
|
+
// re-writing its regexes here, so new tag/step forms are picked up automatically. The
|
|
208
|
+
// literal fallback keeps delivery honest on machines WITHOUT the driver installed
|
|
209
|
+
// (delivery reads only feature text + results, it never compiles). Background steps are
|
|
210
|
+
// scanned too — a Background install mocks every scenario in the feature, and the
|
|
211
|
+
// driver's own spec warns that per-scenario tracking misses them.
|
|
212
|
+
const scenarioStepTexts = [
|
|
213
|
+
...inputs.feature.backgroundOrderedSteps.map((st) => st.text),
|
|
214
|
+
...m.feature.orderedSteps.map((st) => st.text),
|
|
215
|
+
];
|
|
216
|
+
const { mocked, mockDeps } = detectMocked(tags, scenarioStepTexts);
|
|
217
|
+
|
|
160
218
|
const base = {
|
|
161
219
|
vpId,
|
|
162
220
|
vpCategory,
|
|
163
221
|
priority: extractPriority(tags),
|
|
164
222
|
mode: mode as 'auto' | 'manual',
|
|
165
223
|
manualReason: manualReason(tags),
|
|
224
|
+
mocked,
|
|
225
|
+
...(mockDeps.length ? { mockDeps } : {}),
|
|
166
226
|
layers: deriveLayers(tags),
|
|
167
227
|
traces: tags.filter((t) => t.startsWith('@spec:')).map((t) => t.slice('@spec:'.length)),
|
|
168
228
|
runtimeVars: [...runtimeVars],
|
|
169
|
-
|
|
229
|
+
apiCalls: apiCalls.length,
|
|
230
|
+
triggerShape: [...rawTrigger.map(normalizeShape), ...(apiCallsAreTrigger ? apiCallShapes : [])],
|
|
170
231
|
oracleShape: rawOracle.map(normalizeShape),
|
|
171
232
|
preconditionProfile,
|
|
172
233
|
precondition,
|
|
@@ -196,7 +257,11 @@ export function deriveVariants(inputs: Pick<BuildInputs, 'feature' | 'merged' |
|
|
|
196
257
|
});
|
|
197
258
|
return { trigger, oracle, precondition: preconditionOut, verification: verificationOut };
|
|
198
259
|
}
|
|
199
|
-
|
|
260
|
+
// A pure api scenario's procedure is its request blocks (not renderAction —
|
|
261
|
+
// Method/API/Header/Body lines are already reader-facing, not step DSL).
|
|
262
|
+
const trigger = apiCallsAreTrigger
|
|
263
|
+
? apiRequestLines.map(sub)
|
|
264
|
+
: rawTrigger.map((s) => renderAction(sub(s)));
|
|
200
265
|
const oracle = rawOracle.map((s) => renderExpected(sub(s)));
|
|
201
266
|
return {
|
|
202
267
|
// A pure-render check (Given + Then only) is checked on page load — say so
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import * as fs from 'fs';
|
|
11
|
+
import { loadApiCatalog } from '../api-catalog-loader';
|
|
11
12
|
import { parseFeatureMetadata } from '../feature-parser';
|
|
12
13
|
import { parseSpecFile } from '../spec-parser';
|
|
13
14
|
import { loadTestData } from '../test-data-resolver';
|
|
@@ -16,17 +17,25 @@ import { mergeFeatureAndSpec } from '../scenario-merger';
|
|
|
16
17
|
import { getPackageVersion } from '../package-info';
|
|
17
18
|
import { writeCsv } from '../csv-exporter';
|
|
18
19
|
import { writeXlsx } from '../xlsx-exporter';
|
|
20
|
+
import { loadPerfConfig, perfVerdict } from '../../harness/perf';
|
|
19
21
|
import { loadDeliveryMap, writeDeliveryMap } from './map-loader';
|
|
20
22
|
import { mapContentFingerprint } from './fingerprint';
|
|
21
23
|
import { buildMatrix, deriveVariants } from './build';
|
|
22
24
|
import { renderMatrixXlsx } from './render-xlsx';
|
|
23
25
|
import { renderMatrixCsv } from './render-csv';
|
|
24
26
|
import { DeliveryMap, MatrixFinding, MatrixModel } from './types';
|
|
27
|
+
import { ApiCatalogEntry } from '../types';
|
|
25
28
|
|
|
26
29
|
export interface MatrixTargetPaths {
|
|
27
30
|
cwd: string;
|
|
28
31
|
/** Feature basename — names the deliverable files. */
|
|
29
32
|
unit: string;
|
|
33
|
+
/** Unit kind — 'api' loads the endpoint catalog so cells render request blocks.
|
|
34
|
+
* Optional for back-compat: absent means non-api (no catalog). */
|
|
35
|
+
kind?: 'screen' | 'flow' | 'api';
|
|
36
|
+
/** Area/screen directory name (differs from `unit` for sub-features) — keys the
|
|
37
|
+
* unit-level apis.yaml path. Defaults to `unit`. */
|
|
38
|
+
area?: string;
|
|
30
39
|
/** Display label (flow/… prefix for flows). */
|
|
31
40
|
label: string;
|
|
32
41
|
featureFile: string;
|
|
@@ -44,6 +53,11 @@ function readSpecText(paths: MatrixTargetPaths): string {
|
|
|
44
53
|
: '';
|
|
45
54
|
}
|
|
46
55
|
|
|
56
|
+
/** apis.yaml endpoint catalog for an api unit ({} for screen/flow — loader no-ops). */
|
|
57
|
+
function catalogFor(paths: MatrixTargetPaths): Record<string, ApiCatalogEntry> {
|
|
58
|
+
return loadApiCatalog(paths.cwd, paths.kind ?? 'screen', paths.area ?? paths.unit);
|
|
59
|
+
}
|
|
60
|
+
|
|
47
61
|
export interface MatrixLoadResult {
|
|
48
62
|
model?: MatrixModel;
|
|
49
63
|
map?: DeliveryMap;
|
|
@@ -74,7 +88,18 @@ export function loadMatrixModel(paths: MatrixTargetPaths): MatrixLoadResult {
|
|
|
74
88
|
map,
|
|
75
89
|
transformerVersion: getPackageVersion(),
|
|
76
90
|
specText: readSpecText(paths),
|
|
91
|
+
apiCatalog: catalogFor(paths),
|
|
77
92
|
});
|
|
93
|
+
// Perf verdict (#569): budgets are config (qa/perf.yaml), samples are the run's
|
|
94
|
+
// per-test durations. Advisory — reported here because delivery is where runs
|
|
95
|
+
// are read; the audit's score stays a pure function of the design artifacts.
|
|
96
|
+
const perfConfig = loadPerfConfig(paths.cwd);
|
|
97
|
+
if (perfConfig && results) {
|
|
98
|
+
const samples = [...results.values()]
|
|
99
|
+
.filter((r) => (r.status === 'passed' || r.status === 'failed' || r.status === 'timedOut') && typeof r.duration === 'number')
|
|
100
|
+
.map((r) => ({ title: r.testTitle, ms: r.duration as number }));
|
|
101
|
+
model.perf = perfVerdict(perfConfig, paths.unit, samples);
|
|
102
|
+
}
|
|
78
103
|
return { model, map, mapErrors: [] };
|
|
79
104
|
}
|
|
80
105
|
|
|
@@ -95,15 +120,17 @@ export function approveMatrix(paths: MatrixTargetPaths, groupIds?: string[]): {
|
|
|
95
120
|
const merged = mergeFeatureAndSpec(feature, spec);
|
|
96
121
|
const testData = fs.existsSync(paths.testDataFile) ? loadTestData(paths.testDataFile, process.env.SUNGEN_ENV ?? null) : null;
|
|
97
122
|
|
|
123
|
+
const apiCatalog = catalogFor(paths);
|
|
98
124
|
const model = buildMatrix({
|
|
99
125
|
unit: paths.unit, feature, merged, testData, results: null, map,
|
|
100
126
|
transformerVersion: getPackageVersion(),
|
|
101
127
|
specText: readSpecText(paths),
|
|
128
|
+
apiCatalog,
|
|
102
129
|
});
|
|
103
130
|
const blocking = model.findings.filter((f) => f.severity === 'error');
|
|
104
131
|
if (blocking.length > 0) return { findings: blocking, approved: [] };
|
|
105
132
|
|
|
106
|
-
const variants = deriveVariants({ feature, merged, testData, results: null });
|
|
133
|
+
const variants = deriveVariants({ feature, merged, testData, results: null, apiCatalog });
|
|
107
134
|
const byRef = new Map(variants.map((v) => [v.ref, v] as const));
|
|
108
135
|
const byVp = new Map<string, typeof variants>();
|
|
109
136
|
for (const v of variants) {
|
|
@@ -248,7 +248,13 @@ function gateDExecutability(ctx: GateContext, findings: MatrixFinding[]): void {
|
|
|
248
248
|
for (const g of map.groups) {
|
|
249
249
|
const vs = g.variants.flatMap((ref) => expandMapRef(ref, variantsByVp));
|
|
250
250
|
for (const v of vs) {
|
|
251
|
-
|
|
251
|
+
// An api-first variant (`@api:` calls, no UI steps) legitimately has no UI start
|
|
252
|
+
// state — the request blocks in its trigger (Method/API/Header/Body) carry
|
|
253
|
+
// everything a tester needs. Demanding a Background/Given here blocked 100% of
|
|
254
|
+
// api-first units (field report 3.2.19).
|
|
255
|
+
const hasPrecondition = v.precondition.length > 0
|
|
256
|
+
|| bucketTexts(feature.backgroundOrderedSteps, 'given').length > 0
|
|
257
|
+
|| v.apiCalls > 0;
|
|
252
258
|
if (!hasPrecondition) {
|
|
253
259
|
findings.push({
|
|
254
260
|
gate: 'D', severity: 'error', ref: v.ref,
|
|
@@ -224,7 +224,7 @@ function addMatrixSheet(wb: ExcelJS.Workbook, model: MatrixModel, sungenVersion?
|
|
|
224
224
|
// parent's steps + these read as one continuous procedure.
|
|
225
225
|
numbered(item.triggerDeltas[v.ref] ?? [], item.trigger.length + 1),
|
|
226
226
|
expectedWithVerification(v.oracle, v.verification),
|
|
227
|
-
v.mode === 'manual' ? `Manual${v.manualReason ? ` (${v.manualReason})` : ''}` : 'Auto',
|
|
227
|
+
v.mode === 'manual' ? `Manual${v.manualReason ? ` (${v.manualReason})` : ''}` : (v.mocked ? 'Auto · Mocked' : 'Auto'),
|
|
228
228
|
v.traces.join(', '),
|
|
229
229
|
variantState(v),
|
|
230
230
|
isoDate(v.result?.startTime),
|
|
@@ -315,6 +315,50 @@ function addCoverageSheet(wb: ExcelJS.Workbook, model: MatrixModel, sungenVersio
|
|
|
315
315
|
|
|
316
316
|
let rowIdx = 7;
|
|
317
317
|
|
|
318
|
+
// --- Performance budget (#569) — advisory: config (qa/perf.yaml) vs measured runs.
|
|
319
|
+
if (model.perf) {
|
|
320
|
+
const p = model.perf;
|
|
321
|
+
dataCell(ws.getRow(rowIdx).getCell(2), 'Performance budget', { bold: true });
|
|
322
|
+
rowIdx++;
|
|
323
|
+
const r = ws.getRow(rowIdx);
|
|
324
|
+
dataCell(r.getCell(2), `p${p.percentile} ${p.metric.replace('_ms', ' time')}`);
|
|
325
|
+
dataCell(r.getCell(3), p.pass ? 'PASS' : 'OVER BUDGET', { bold: true });
|
|
326
|
+
dataCell(r.getCell(4), `measured ${p.measuredMs}ms`);
|
|
327
|
+
dataCell(r.getCell(5), `budget ${p.budgetMs}ms`);
|
|
328
|
+
dataCell(r.getCell(6), `${p.samples} runs`);
|
|
329
|
+
if (!p.pass && p.slowest.length) dataCell(r.getCell(8), `slowest: ${p.slowest.map((x) => `${x.ms}ms ${x.title.slice(0, 40)}`).join(' · ')}`);
|
|
330
|
+
rowIdx += 2;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// --- External Dependency Tracking (#580, ISTQB system-test practice): every case that
|
|
334
|
+
// ran against a MOCK is listed centrally. Its Result stays in the canonical vocabulary
|
|
335
|
+
// (the roll-up formulas depend on it); THIS table is where "Pass (Mocked) ≠ Pass (Real)"
|
|
336
|
+
// is visible — a case verified only against mocks is not Done until re-verified real.
|
|
337
|
+
{
|
|
338
|
+
const mockedVars = model.items.flatMap((it) => it.variants.filter((v) => v.mocked).map((v) => ({ it, v })));
|
|
339
|
+
if (mockedVars.length > 0) {
|
|
340
|
+
dataCell(ws.getRow(rowIdx).getCell(2), 'External Dependency Tracking — mock-verified cases (Pass here = Pass (Mocked), not Pass (Real))', { bold: true });
|
|
341
|
+
rowIdx++;
|
|
342
|
+
const hd = ws.getRow(rowIdx);
|
|
343
|
+
dataCell(hd.getCell(2), 'Variant');
|
|
344
|
+
dataCell(hd.getCell(3), 'Mocked dependency');
|
|
345
|
+
dataCell(hd.getCell(4), 'Delivery item');
|
|
346
|
+
dataCell(hd.getCell(5), 'Result');
|
|
347
|
+
dataCell(hd.getCell(8), 'Real-environment re-verify');
|
|
348
|
+
rowIdx++;
|
|
349
|
+
for (const { it, v } of mockedVars) {
|
|
350
|
+
const r = ws.getRow(rowIdx);
|
|
351
|
+
dataCell(r.getCell(2), v.ref);
|
|
352
|
+
dataCell(r.getCell(3), (v.mockDeps ?? []).join(', ') || '(scenario-level @mock)');
|
|
353
|
+
dataCell(r.getCell(4), it.id);
|
|
354
|
+
dataCell(r.getCell(5), variantState(v));
|
|
355
|
+
dataCell(r.getCell(8), '');
|
|
356
|
+
rowIdx++;
|
|
357
|
+
}
|
|
358
|
+
rowIdx++;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
318
362
|
// --- Requirement coverage (review §6): every requirement id gets ONE explicit status.
|
|
319
363
|
if (model.requirements.length > 0) {
|
|
320
364
|
dataCell(ws.getRow(rowIdx).getCell(2), 'Requirement coverage — every id has an explicit status', { bold: true });
|
|
@@ -92,12 +92,23 @@ export interface CoverageVariant {
|
|
|
92
92
|
/** 'auto' | 'manual'; manual keeps its @manual:Mx reason. */
|
|
93
93
|
mode: 'auto' | 'manual';
|
|
94
94
|
manualReason?: string;
|
|
95
|
+
/** The scenario runs against MOCKED network responses (@mock / `[x] mock is active`) —
|
|
96
|
+
* a pass verifies the handling logic, not the real dependency (Pass (Mocked) ≠ Pass (Real),
|
|
97
|
+
* #580: ISTQB external-dependency tracking). */
|
|
98
|
+
mocked?: boolean;
|
|
99
|
+
/** Named mocked dependencies (`[name] mock is active` refs), for the tracking table. */
|
|
100
|
+
mockDeps?: string[];
|
|
95
101
|
layers: MatrixLayer[];
|
|
96
102
|
/** @spec:* requirement traces. */
|
|
97
103
|
traces: string[];
|
|
98
104
|
/** Variables this scenario produces at RUN TIME (`remember … as {{v}}`, capability
|
|
99
105
|
* bindings) — they never exist in test-data, so Gate D must not demand them. */
|
|
100
106
|
runtimeVars: string[];
|
|
107
|
+
/** Count of `@api:<name>` invocations. An api-first variant's start state lives in
|
|
108
|
+
* its request blocks (Method/API/Header/Body), so Gate D's precondition demand is
|
|
109
|
+
* waived when this is > 0 — requiring a Background/Given blocked EVERY api-first
|
|
110
|
+
* unit (field report 3.2.19). */
|
|
111
|
+
apiCalls: number;
|
|
101
112
|
/** Condition summary: differentiating dimension shown on the sub-row. */
|
|
102
113
|
condition: string;
|
|
103
114
|
/** Resolved data pairs (test-data values / dataset-row columns), `key: value` lines. */
|
|
@@ -220,6 +231,8 @@ export interface MatrixModel {
|
|
|
220
231
|
requirements: RequirementCoverage[];
|
|
221
232
|
findings: MatrixFinding[];
|
|
222
233
|
manifest: MatrixManifest;
|
|
234
|
+
/** Perf verdict (#569) — present when qa/perf.yaml sets a budget AND the run produced durations. */
|
|
235
|
+
perf?: import('../../harness/perf').PerfVerdict | null;
|
|
223
236
|
}
|
|
224
237
|
|
|
225
238
|
/** Complexity warning threshold (rules draft Gate I; raised for compact grouping) — echoed in the manifest. */
|
|
@@ -54,6 +54,7 @@ interface PlaywrightSpec {
|
|
|
54
54
|
projectId?: string;
|
|
55
55
|
results?: Array<{
|
|
56
56
|
status?: string; // 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'
|
|
57
|
+
duration?: number; // ms — Playwright emits this on every result (#569 perf budgets)
|
|
57
58
|
startTime?: string;
|
|
58
59
|
error?: { message?: string; stack?: string };
|
|
59
60
|
attachments?: Array<{ name?: string; path?: string; contentType?: string }>;
|
|
@@ -120,6 +121,7 @@ export function loadPlaywrightReport(reportPath: string): Map<string, Playwright
|
|
|
120
121
|
result.set(stripped, {
|
|
121
122
|
testTitle: stripped,
|
|
122
123
|
status,
|
|
124
|
+
duration: typeof res?.duration === 'number' ? res.duration : undefined,
|
|
123
125
|
startTime: res?.startTime,
|
|
124
126
|
error: errorMsg,
|
|
125
127
|
tracePath: trace,
|
package/src/exporters/types.ts
CHANGED
|
@@ -121,6 +121,8 @@ export interface SpecFileData {
|
|
|
121
121
|
export interface PlaywrightResult {
|
|
122
122
|
testTitle: string;
|
|
123
123
|
status: 'passed' | 'failed' | 'skipped' | 'timedOut' | 'interrupted' | 'unknown';
|
|
124
|
+
/** Wall-clock run time in ms (from Playwright's results[].duration) — perf budgets (#569). */
|
|
125
|
+
duration?: number;
|
|
124
126
|
startTime?: string;
|
|
125
127
|
error?: string;
|
|
126
128
|
tracePath?: string;
|
|
@@ -35,12 +35,17 @@
|
|
|
35
35
|
* nothing for the query half to assert, so the step checks the PATH only. Not an
|
|
36
36
|
* error (the value may legitimately vary per environment), but the step asserts
|
|
37
37
|
* strictly less than it appears to
|
|
38
|
+
* SG-W015 — `see [X] card/item with {{n}}` where the value is a bare number: this compiles
|
|
39
|
+
* to a TEXT assertion (toHaveText('3')), NOT a count — an author reaching for
|
|
40
|
+
* "3 cards are shown" gets an assertion that can only pass if the card's text
|
|
41
|
+
* IS the digit. Count semantics exist on row/table ("see [T] table with {{n}}",
|
|
42
|
+
* "see [Ref] row with {{n}}") and lists ("should have count") only
|
|
38
43
|
*
|
|
39
44
|
* Diagnostics never block generation by themselves; `sungen generate --strict`
|
|
40
45
|
* turns any collected diagnostic into a non-zero exit.
|
|
41
46
|
*/
|
|
42
47
|
|
|
43
|
-
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007' | 'SG-W008' | 'SG-W009' | 'SG-W010' | 'SG-W011' | 'SG-W012' | 'SG-W013' | 'SG-W014';
|
|
48
|
+
export type DiagnosticCode = 'SG-W001' | 'SG-W002' | 'SG-W003' | 'SG-W005' | 'SG-W006' | 'SG-W007' | 'SG-W008' | 'SG-W009' | 'SG-W010' | 'SG-W011' | 'SG-W012' | 'SG-W013' | 'SG-W014' | 'SG-W015';
|
|
44
49
|
|
|
45
50
|
export interface Diagnostic {
|
|
46
51
|
code: DiagnosticCode;
|
|
@@ -73,34 +73,40 @@ export class PatternRegistry {
|
|
|
73
73
|
* Prefers resolver (returns template data) over generator (renders directly).
|
|
74
74
|
*/
|
|
75
75
|
generateStep(step: ParsedStep, context: PatternContext): MappedStep | null {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
76
|
+
// Matchers see only the STEP; some decisions need context (a resolver reading the
|
|
77
|
+
// test-data value, e.g. row-count's numeric check). A resolver may therefore return
|
|
78
|
+
// null to say "not mine after all" — the search continues with the next matching
|
|
79
|
+
// pattern in priority order instead of crashing or silently dropping the step.
|
|
80
|
+
const matchStep: ParsedStep = { ...step, text: maskRefsForMatching(step.text) };
|
|
81
|
+
for (const pattern of this.patterns) {
|
|
82
|
+
if (!this.matchesPattern(matchStep, pattern.matcher)) continue;
|
|
83
|
+
|
|
84
|
+
// Prefer resolver (framework-agnostic) over generator (legacy)
|
|
85
|
+
if (pattern.resolver) {
|
|
86
|
+
const resolved = pattern.resolver(step, context);
|
|
87
|
+
if (!resolved) continue; // resolver declined — fall through
|
|
88
|
+
|
|
89
|
+
// Auto-inject parent scoping if step has parentRef
|
|
90
|
+
// Skip for table-* patterns — they resolve the table name internally from step text
|
|
91
|
+
if (step.parentRef && step.parentType && !pattern.name.startsWith('table-')) {
|
|
92
|
+
resolved.data.parentLocator = PatternRegistry.resolveParentLocator(
|
|
93
|
+
step.parentRef, step.parentType, context
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const code = context.templateEngine.renderStep(resolved.templateName, resolved.data);
|
|
98
|
+
return {
|
|
99
|
+
code,
|
|
100
|
+
comment: resolved.comment,
|
|
101
|
+
};
|
|
91
102
|
}
|
|
92
103
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
comment: resolved.comment,
|
|
97
|
-
};
|
|
98
|
-
}
|
|
104
|
+
if (pattern.generator) {
|
|
105
|
+
return pattern.generator(step, context);
|
|
106
|
+
}
|
|
99
107
|
|
|
100
|
-
|
|
101
|
-
return pattern.generator(step, context);
|
|
108
|
+
return null;
|
|
102
109
|
}
|
|
103
|
-
|
|
104
110
|
return null;
|
|
105
111
|
}
|
|
106
112
|
|
|
@@ -175,6 +175,28 @@ export class StepMapper {
|
|
|
175
175
|
this.lastPrimaryKeyword = keyword;
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
+
// SG-W015 — `see [X] card with {{n}}` where n is a bare number compiles to a TEXT
|
|
179
|
+
// assertion (toHaveText('3')), not a count. An author reaching for "3 cards are shown"
|
|
180
|
+
// gets an assertion that can only pass if the card's TEXT is the digit — a wrong test
|
|
181
|
+
// with no error. Count semantics live on row/table ("see [T] table with {{n}}",
|
|
182
|
+
// "see [Ref] row with {{n}}") and lists ("should have count"); on the container types
|
|
183
|
+
// below, a bare-number text expectation is almost never what the author meant. A text
|
|
184
|
+
// element asserting a numeric readout ("[Row Quantity] text with 4") stays silent.
|
|
185
|
+
const COUNT_MISREAD_TYPES = new Set(['card', 'item', 'items', 'cards', 'option', 'options']);
|
|
186
|
+
if (this.diagnostics && step.dataRef && step.selectorRef &&
|
|
187
|
+
COUNT_MISREAD_TYPES.has((step.elementType || '').toLowerCase()) &&
|
|
188
|
+
/\bsees?\s+\[/.test(step.text) && /\bwith\b/.test(step.text)) {
|
|
189
|
+
// peekBaseValue reads the literal from test-data even in runtime-data mode,
|
|
190
|
+
// where resolveData returns an opaque marker (never numeric-looking).
|
|
191
|
+
let resolvedValue: string | undefined;
|
|
192
|
+
try { resolvedValue = this.dataResolver.peekBaseValue(step.dataRef, this.featureName); } catch { /* unresolved at compile time → cannot judge */ }
|
|
193
|
+
if (resolvedValue !== undefined && /^\d+$/.test(resolvedValue.trim())) {
|
|
194
|
+
this.diagnostics.add('SG-W015',
|
|
195
|
+
`\`see [${step.selectorRef}] ${step.elementType} with {{${step.dataRef}}}\` compiles to a TEXT assertion toHaveText('${resolvedValue.trim()}'), NOT a count of ${step.elementType}s. For a count use \`see [Ref] row with {{n}}\` / \`see [T] table with {{n}}\` (or a list "should have count"); if '${resolvedValue.trim()}' really is the ${step.elementType}'s text, assert it on a text/label element instead`,
|
|
196
|
+
{ feature: this.featureName, step: step.text });
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
178
200
|
// SG-W007 — a two-step-form browser-alert handler (`click [OK] alert`) written AFTER
|
|
179
201
|
// an action step registers its page.once('dialog') listener too late: a dialog
|
|
180
202
|
// triggered by the earlier action auto-dismisses with no error (issue #472).
|