@sungen/driver-data-factory 3.2.12-beta.8 → 3.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/common/experience/auth-account-states.yaml +19 -0
- package/common/field-types/email.yaml +3 -0
- package/package.json +2 -2
- package/src/boundary.ts +59 -0
- package/src/catalog-loader.ts +7 -2
- package/src/cli.ts +33 -4
- package/src/coverage.ts +165 -0
- package/src/crosscheck.ts +26 -2
- package/src/detect.ts +122 -0
- package/src/generator.ts +18 -1
- package/src/index.ts +8 -4
- package/src/sensor.ts +20 -4
- package/src/state.ts +47 -0
- package/src/types.ts +15 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Experience checklist: authentication credential / account states — from real login-testing experience.
|
|
2
|
+
# These are LOGIN/AUTH *scenarios*, NOT field-format data: each case is a credential pair whose ACCOUNT
|
|
3
|
+
# is in a particular state, with an expected auth outcome. Most need a domain-state PRECONDITION (a
|
|
4
|
+
# seeded account in that state) — declare it with a state-recipe (`sungen data state`) so it renders a
|
|
5
|
+
# @manual:data-setup Background. Data Factory carries this knowledge; create-test's login viewpoint
|
|
6
|
+
# generates the scenarios. Confirm the exact outcome/message code with the spec/BA — never assume.
|
|
7
|
+
experience: auth-account-states
|
|
8
|
+
description: Credential/account states a login (email+password or credential form) screen must cover.
|
|
9
|
+
applies_when: "a login / sign-in / authentication screen (credential form)"
|
|
10
|
+
cases:
|
|
11
|
+
- { id: CHK-AUTH-01, state: active, expect: "login succeeds → authenticated destination", note: "happy path — a valid ACTIVE account" }
|
|
12
|
+
- { id: CHK-AUTH-02, state: wrong-password, expect: "auth fails with a generic error (no user-enumeration)" }
|
|
13
|
+
- { id: CHK-AUTH-03, state: non-existent-account, expect: "auth fails with the SAME generic error as wrong-password (don't reveal which is wrong)" }
|
|
14
|
+
- { id: CHK-AUTH-04, state: soft-deleted, expect: "login REJECTED — a soft-deleted account must NOT authenticate", note: "real-world bug precedent: soft-deleted rows still passing auth because the query didn't filter deleted_at" }
|
|
15
|
+
- { id: CHK-AUTH-05, state: locked-or-suspended, expect: "login blocked with a locked/suspended message (not the generic auth error)" }
|
|
16
|
+
- { id: CHK-AUTH-06, state: unverified-email, expect: "login blocked or routed to verification, per spec" }
|
|
17
|
+
- { id: CHK-AUTH-07, state: password-expired, expect: "routed to password reset / forced change, per spec" }
|
|
18
|
+
- { id: CHK-AUTH-08, state: valid-with-surrounding-whitespace, expect: "email trimmed then login succeeds (pair a valid account email with leading/trailing spaces)", note: "trim behaviour — see the email field-type CHK-EMAIL-WS" }
|
|
19
|
+
oracle: "Each non-happy state needs a seeded account IN that state (a domain-state precondition / fixture). The 'wrong-password' and 'non-existent' outcomes should be indistinguishable (anti-enumeration). soft-deleted / locked / unverified / expired must each be blocked with the correct, spec-defined outcome."
|
|
@@ -16,6 +16,9 @@ boundary:
|
|
|
16
16
|
- { id: CHK-EMAIL-B1, gen: "local part = maxLocal", note: "local part = max (64)" }
|
|
17
17
|
- { id: CHK-EMAIL-B2, gen: "local part = maxLocal-1", note: "local part = max-1" }
|
|
18
18
|
- { id: CHK-EMAIL-B3, gen: "total = maxTotal", note: "total length = max (255)" }
|
|
19
|
+
# min-length side (total = minLength / minLength±1) is auto-completed by the BVA pass from the
|
|
20
|
+
# field-map's `minLength` — see ensureBvaComplete; no gen entry needed here.
|
|
21
|
+
- { id: CHK-EMAIL-WS, value: " user@example.com ", validity: valid, note: "leading/trailing whitespace — server MUST trim then accept (pair with a valid password); if the spec rejects instead, override validity to invalid" }
|
|
19
22
|
invalid:
|
|
20
23
|
- { id: CHK-EMAIL-I1, value: "user@", reason: "missing domain", errorRef: M13 }
|
|
21
24
|
- { id: CHK-EMAIL-I2, value: "user@.com", reason: "dot in wrong position", errorRef: M13 }
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sungen/driver-data-factory",
|
|
3
|
-
"version": "3.2.12
|
|
3
|
+
"version": "3.2.12",
|
|
4
4
|
"description": "Sungen Data Factory capability — standardized test-data generation from a field-type + security knowledge catalog (valid/boundary/invalid + traceability), the .overwrite merge layer, and (opt-in) @dataFactory runtime seeding. Design-time knowledge is runtime-free. Plugs into @sun-asterisk/sungen via the capability SPI.",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"author": "eqe team (engineer & quality) — Sun Asterisk",
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@sun-asterisk/sungen": "3.2.12
|
|
17
|
+
"@sun-asterisk/sungen": "3.2.12",
|
|
18
18
|
"yaml": "^2.4.0"
|
|
19
19
|
},
|
|
20
20
|
"files": [
|
package/src/boundary.ts
CHANGED
|
@@ -216,9 +216,22 @@ export function satisfiesEmail(value: string, c: Record<string, any>): boolean {
|
|
|
216
216
|
const local = value.slice(0, at);
|
|
217
217
|
if (Number.isFinite(c.maxLocal) && local.length > c.maxLocal) return false;
|
|
218
218
|
if (Number.isFinite(c.maxTotal) && value.length > c.maxTotal) return false;
|
|
219
|
+
// Total-length MIN (a spec often sets an email min length, e.g. 5) — the oracle must enforce it so a
|
|
220
|
+
// below-min email is correctly INVALID (was ignored → the min boundary silently missing).
|
|
221
|
+
const minTotal = c.minLength ?? c.minTotal;
|
|
222
|
+
if (Number.isFinite(minTotal) && value.length < minTotal) return false;
|
|
219
223
|
return true;
|
|
220
224
|
}
|
|
221
225
|
|
|
226
|
+
/** Build an email of exactly `n` total chars (for the total-length MIN boundary — small n). */
|
|
227
|
+
export function emailOfTotalLength(n: number): string | null {
|
|
228
|
+
if (!Number.isFinite(n) || n < 1) return null;
|
|
229
|
+
if (n < 3) return 'a'.repeat(n); // too short to be a real address → malformed (below-min case)
|
|
230
|
+
const rest = n - 2; // chars after "a@"
|
|
231
|
+
if (rest < 3) return 'a@' + 'b'.repeat(rest); // e.g. n=4 → "a@bb" (no dot → also malformed, below-min)
|
|
232
|
+
return 'a@' + 'b'.repeat(rest - 2) + '.c'; // e.g. n=5 → "a@b.c", n=6 → "a@bb.c"
|
|
233
|
+
}
|
|
234
|
+
|
|
222
235
|
function satisfiesLength(type: string, value: string, c: Record<string, any>): boolean {
|
|
223
236
|
const min = c.minLength ?? c.length;
|
|
224
237
|
const max = c.maxLength ?? c.length;
|
|
@@ -258,3 +271,49 @@ export function satisfiesConstraints(type: string, value: string | null, c: Reco
|
|
|
258
271
|
}
|
|
259
272
|
return null;
|
|
260
273
|
}
|
|
274
|
+
|
|
275
|
+
// ---- BVA completeness: every bounded dimension gets its full min-1/min/min+1/max-1/max/max+1 set ----
|
|
276
|
+
|
|
277
|
+
export interface BvaExtra { label: string; value: string; validity: Validity; group: 'boundary' | 'invalid'; }
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Ensure the canonical Boundary-Value-Analysis set is present for EVERY bounded dimension declared in
|
|
281
|
+
* the constraints — even one the field-type's catalog `gen:` entries don't exercise. This closes the
|
|
282
|
+
* QA-reported gap: email declared `minLength: 5` but the catalog only modelled maxLocal/maxTotal, so
|
|
283
|
+
* the min side (length 4 = min-1 invalid, 5 = min, 6 = min+1) was silently missing. Emits only the
|
|
284
|
+
* MISSING points (dedup by exact value), so a type whose catalog already covers a dimension (string /
|
|
285
|
+
* password / number) yields nothing — no churn.
|
|
286
|
+
*/
|
|
287
|
+
export function ensureBvaComplete(type: string, c: Record<string, any>, existing: Array<string | null>): BvaExtra[] {
|
|
288
|
+
const have = new Set(existing.filter((v): v is string => v != null));
|
|
289
|
+
const out: BvaExtra[] = [];
|
|
290
|
+
const push = (label: string, value: string | null) => {
|
|
291
|
+
if (value == null || have.has(value)) return;
|
|
292
|
+
const ok = satisfiesConstraints(type, value, c);
|
|
293
|
+
const validity: Validity = ok === false ? 'invalid' : 'valid';
|
|
294
|
+
out.push({ label, value, validity, group: validity === 'invalid' ? 'invalid' : 'boundary' });
|
|
295
|
+
have.add(value);
|
|
296
|
+
};
|
|
297
|
+
if (type === 'email') {
|
|
298
|
+
const minTotal = c.minLength ?? c.minTotal;
|
|
299
|
+
if (Number.isFinite(minTotal)) {
|
|
300
|
+
push('minTotal-1', emailOfTotalLength((minTotal as number) - 1));
|
|
301
|
+
push('minTotal', emailOfTotalLength(minTotal as number));
|
|
302
|
+
push('minTotal+1', emailOfTotalLength((minTotal as number) + 1));
|
|
303
|
+
}
|
|
304
|
+
return out;
|
|
305
|
+
}
|
|
306
|
+
if (STR_TYPES.has(type)) {
|
|
307
|
+
const min = c.minLength ?? c.length, max = c.maxLength ?? c.length;
|
|
308
|
+
if (Number.isFinite(min)) { push('minLen-1', fillString(type, min - 1)); push('minLen', fillString(type, min)); push('minLen+1', fillString(type, min + 1)); }
|
|
309
|
+
if (Number.isFinite(max)) { push('maxLen-1', fillString(type, max - 1)); push('maxLen', fillString(type, max)); push('maxLen+1', fillString(type, max + 1)); }
|
|
310
|
+
return out;
|
|
311
|
+
}
|
|
312
|
+
if (NUM_TYPES.has(type)) {
|
|
313
|
+
const min = c.min, max = c.max, st = step(type, c);
|
|
314
|
+
if (Number.isFinite(min)) { push('min-1', fmtNum(type, c, min - st)); push('min', fmtNum(type, c, min)); push('min+1', fmtNum(type, c, min + st)); }
|
|
315
|
+
if (Number.isFinite(max)) { push('max-1', fmtNum(type, c, max - st)); push('max', fmtNum(type, c, max)); push('max+1', fmtNum(type, c, max + st)); }
|
|
316
|
+
return out;
|
|
317
|
+
}
|
|
318
|
+
return out;
|
|
319
|
+
}
|
package/src/catalog-loader.ts
CHANGED
|
@@ -14,7 +14,7 @@ import { readFileSync, readdirSync, existsSync, statSync } from 'fs';
|
|
|
14
14
|
import { join } from 'path';
|
|
15
15
|
import { createHash } from 'crypto';
|
|
16
16
|
import { parse as parseYaml } from 'yaml';
|
|
17
|
-
import type { Catalog, FieldType, SecurityBank, Technique, DataPoint, OverwriteWarning, CatalogProvenance, FieldTypeProvenance } from './types';
|
|
17
|
+
import type { Catalog, FieldType, SecurityBank, Technique, Experience, DataPoint, OverwriteWarning, CatalogProvenance, FieldTypeProvenance } from './types';
|
|
18
18
|
|
|
19
19
|
/** Resolve the packaged `common/` dir (works from src/ in dev via tsx and from dist/ when built). */
|
|
20
20
|
export function commonDir(): string {
|
|
@@ -49,7 +49,11 @@ export function loadShippedCatalog(root = commonDir()): Catalog {
|
|
|
49
49
|
for (const { data } of readYamlFilesDeep(join(root, 'techniques'))) {
|
|
50
50
|
if (data?.technique) techniques[data.technique] = data as Technique;
|
|
51
51
|
}
|
|
52
|
-
|
|
52
|
+
const experience: Record<string, Experience> = {};
|
|
53
|
+
for (const { data } of readYamlFilesDeep(join(root, 'experience'))) {
|
|
54
|
+
if (data?.experience) experience[data.experience] = data as Experience;
|
|
55
|
+
}
|
|
56
|
+
return { fieldTypes, security, techniques, experience };
|
|
53
57
|
}
|
|
54
58
|
|
|
55
59
|
/**
|
|
@@ -130,6 +134,7 @@ export function loadEffectiveCatalog(projectDir: string, root = commonDir()): {
|
|
|
130
134
|
fieldTypes: { ...shipped.fieldTypes },
|
|
131
135
|
security: { ...shipped.security },
|
|
132
136
|
techniques: { ...shipped.techniques },
|
|
137
|
+
experience: { ...shipped.experience },
|
|
133
138
|
};
|
|
134
139
|
|
|
135
140
|
for (const [type, ov] of Object.entries(overlayFieldTypes)) {
|
package/src/cli.ts
CHANGED
|
@@ -13,8 +13,10 @@ import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
|
|
|
13
13
|
import { loadEffectiveCatalog, describeProvenance } from './catalog-loader';
|
|
14
14
|
import { expandFieldMap, renderTestData, type FieldMap } from './generator';
|
|
15
15
|
import { buildRequest, fingerprint } from './contract';
|
|
16
|
-
import { loadStateRecipes, validateStateRecipe, buildStatePlan, renderPreconditionBlock, hasDatasource } from './state';
|
|
16
|
+
import { loadStateRecipes, validateStateRecipe, buildStatePlan, renderPreconditionBlock, hasDatasource, scaffoldRecipe, renderRecipeYaml } from './state';
|
|
17
17
|
import { crosscheckUnit, crosscheckShared } from './crosscheck';
|
|
18
|
+
import { coverageUnit } from './coverage';
|
|
19
|
+
import { detectPreconditionSignals } from './detect';
|
|
18
20
|
import { lintFieldMap, summarize } from './lint';
|
|
19
21
|
import { validateGenerated } from './validate';
|
|
20
22
|
import { generateFixture } from './fixture';
|
|
@@ -237,21 +239,23 @@ export function registerDataFactoryCommand(program: any): void {
|
|
|
237
239
|
|
|
238
240
|
data
|
|
239
241
|
.command('crosscheck')
|
|
240
|
-
.description('Cross-artifact Gate
|
|
242
|
+
.description('Cross-artifact + coverage Gate: testcase ↔ data agree (no dangling {{var}}, shared datasets identical) AND the authored data COVERS the required BVA + the login experience checklist')
|
|
241
243
|
.option('--screen <name>', 'Cross-check one unit (default: all field-maps + all shared datasets)')
|
|
242
244
|
.action((opts: { screen?: string }) => {
|
|
243
245
|
const cwd = process.cwd();
|
|
244
246
|
ensureDataFactoryReadme(cwd);
|
|
245
247
|
const names = opts.screen ? [opts.screen] : listFieldMaps(cwd);
|
|
248
|
+
const { catalog } = loadEffectiveCatalog(cwd);
|
|
246
249
|
const findings = [
|
|
247
250
|
...names.flatMap((n) => crosscheckUnit(cwd, n)),
|
|
248
251
|
...(opts.screen ? [] : crosscheckShared(cwd)),
|
|
252
|
+
...names.flatMap((n) => coverageUnit(cwd, n, catalog)),
|
|
249
253
|
];
|
|
250
254
|
const errors = findings.filter((f) => f.severity === 'error').length;
|
|
251
255
|
const warns = findings.filter((f) => f.severity === 'warn').length;
|
|
252
256
|
const badge = errors ? '✗' : warns ? '⚠' : '✓';
|
|
253
257
|
console.log(`${badge} crosscheck: ${errors} error · ${warns} warn (${names.length} unit(s))`);
|
|
254
|
-
for (const f of findings) console.log(` ${f.severity === 'error' ? '✗' : f.severity === 'warn' ? '⚠' : '·'} [${f.unit}] ${f.code}: ${f.message}`);
|
|
258
|
+
for (const f of findings) console.log(` ${f.severity === 'error' ? '✗' : f.severity === 'warn' ? '⚠' : '·'} [${(f as any).unit}] ${f.code}: ${f.message}`);
|
|
255
259
|
if (errors) process.exitCode = 1;
|
|
256
260
|
});
|
|
257
261
|
|
|
@@ -259,9 +263,34 @@ export function registerDataFactoryCommand(program: any): void {
|
|
|
259
263
|
.command('state')
|
|
260
264
|
.description('Domain/stateful data (M3-A Layer 0): validate the state-recipes in recipes.yaml and render each as a standardized manual precondition block (+ a resolved .state.yaml plan)')
|
|
261
265
|
.option('--recipe <name>', 'Only this recipe (default: all recipes with a `resources`/`shared` graph)')
|
|
262
|
-
.
|
|
266
|
+
.option('--scaffold <unit>', 'Auto-draft a state-recipe SKELETON from the unit\'s spec.md precondition signals (Layer-C auto-draft), then review/fill it')
|
|
267
|
+
.option('--force', 'With --scaffold: overwrite an existing recipe of the same name')
|
|
268
|
+
.action((opts: { recipe?: string; scaffold?: string; force?: boolean }) => {
|
|
263
269
|
const cwd = process.cwd();
|
|
264
270
|
ensureDataFactoryReadme(cwd);
|
|
271
|
+
|
|
272
|
+
// --scaffold: auto-draft a recipe skeleton from the unit's spec precondition signals.
|
|
273
|
+
if (opts.scaffold) {
|
|
274
|
+
const unit = opts.scaffold;
|
|
275
|
+
const specPath = ['screens', 'flows'].map((k) => join(cwd, 'qa', k, unit, 'requirements', 'spec.md')).find(existsSync);
|
|
276
|
+
const specText = specPath ? readFileSync(specPath, 'utf8') : '';
|
|
277
|
+
const signals = detectPreconditionSignals(specText);
|
|
278
|
+
const recipe = scaffoldRecipe(unit, signals);
|
|
279
|
+
if (!recipe) { console.log(`No stateful-precondition signals found in ${unit}'s spec — nothing to scaffold (a screen may simply have no precondition).`); return; }
|
|
280
|
+
const recipesPath = join(FIELDMAP_DIR(cwd), 'recipes.yaml');
|
|
281
|
+
const doc: any = existsSync(recipesPath) ? (parseYaml(readFileSync(recipesPath, 'utf8')) || {}) : {};
|
|
282
|
+
doc.recipes = doc.recipes || {};
|
|
283
|
+
if (doc.recipes[recipe.name] && !opts.force) { console.error(`✗ recipe "${recipe.name}" already exists — edit it, or pass --force to overwrite the scaffold.`); process.exitCode = 1; return; }
|
|
284
|
+
// Splice the human-friendly (commented) block rather than a bare re-serialize, to preserve TODO hints.
|
|
285
|
+
const header = existsSync(recipesPath) ? readFileSync(recipesPath, 'utf8').replace(/\s*$/, '') + '\n' : 'recipes:\n';
|
|
286
|
+
const block = renderRecipeYaml(recipe);
|
|
287
|
+
writeFileSync(recipesPath, (header.includes('recipes:') ? header : 'recipes:\n' + header) + block, 'utf8');
|
|
288
|
+
console.log(` ✓ scaffolded recipe "${recipe.name}" → ${recipesPath.replace(cwd + '/', '')} (${signals.length} signal(s))`);
|
|
289
|
+
console.log(` → REVIEW: name each entity, set desiredState/bindings, confirm domain values with QA, then \`sungen data state --recipe ${recipe.name}\`.`);
|
|
290
|
+
for (const s of signals) console.log(` · signal: ${s}`);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
|
|
265
294
|
let recipes = loadStateRecipes(cwd);
|
|
266
295
|
if (opts.recipe) recipes = recipes.filter((r) => r.name === opts.recipe);
|
|
267
296
|
if (recipes.length === 0) { console.error('No state-recipes (recipes with a `resources` or `shared` graph) in qa/data-factory/recipes.yaml.'); process.exitCode = 1; return; }
|
package/src/coverage.ts
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Data Factory — coverage check (does the AUTHORED test-data actually cover the standardized set?).
|
|
3
|
+
*
|
|
4
|
+
* `data validate` checks the DF-generated field block is internally correct; `data crosscheck` checks
|
|
5
|
+
* testcase ↔ data agree. But `/sungen:create-test` often AUTHORS login/flow test-data by hand (seeded
|
|
6
|
+
* accounts + `@cases`) instead of merging `data gen`'s output — so the deterministic BVA the generator
|
|
7
|
+
* would produce (e.g. email length 4/5/6 for `minLength: 5`) and the experience checklist (soft-deleted
|
|
8
|
+
* login…) can be silently missed. This check reconciles the authored values against BOTH:
|
|
9
|
+
* - the BVA the field-map's constraints REQUIRE (per bounded dimension), and
|
|
10
|
+
* - the auth-account-states experience checklist (for a login/auth unit).
|
|
11
|
+
* It matches fuzzily (authored keys are free-form): length for length dims, numeric value for numeric
|
|
12
|
+
* dims, keyword for experience. Advisory (warn) — a visible nudge the create-test loop must address or
|
|
13
|
+
* the QA waives; never blocks. Pure + deterministic.
|
|
14
|
+
*/
|
|
15
|
+
import { readFileSync, existsSync, readdirSync } from 'fs';
|
|
16
|
+
import { join } from 'path';
|
|
17
|
+
import { parse as parseYaml } from 'yaml';
|
|
18
|
+
import type { Catalog } from './types';
|
|
19
|
+
import { expandField, type FieldMap, type FieldDef } from './generator';
|
|
20
|
+
|
|
21
|
+
export interface CoverageFinding {
|
|
22
|
+
unit: string;
|
|
23
|
+
severity: 'warn' | 'info';
|
|
24
|
+
code: 'BVA_UNCOVERED' | 'EXPERIENCE_UNCOVERED';
|
|
25
|
+
field?: string;
|
|
26
|
+
message: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const STR_LEN_TYPES = new Set(['email', 'string', 'password', 'phone', 'postcode', 'katakana']);
|
|
30
|
+
const NUM_TYPES = new Set(['number-integer', 'number-float', 'currency']);
|
|
31
|
+
|
|
32
|
+
function unitTestDataPath(cwd: string, name: string): string | null {
|
|
33
|
+
for (const kind of ['screens', 'flows']) {
|
|
34
|
+
const p = join(cwd, 'qa', kind, name, 'test-data', `${name}.yaml`);
|
|
35
|
+
if (existsSync(p)) return p;
|
|
36
|
+
}
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
function unitFeatureText(cwd: string, name: string): string {
|
|
40
|
+
for (const kind of ['screens', 'flows']) {
|
|
41
|
+
const dir = join(cwd, 'qa', kind, name, 'features');
|
|
42
|
+
if (!existsSync(dir)) continue;
|
|
43
|
+
try { return readdirSync(dir).filter((f) => f.endsWith('.feature')).map((f) => readFileSync(join(dir, f), 'utf8')).join('\n'); } catch { return ''; }
|
|
44
|
+
}
|
|
45
|
+
return '';
|
|
46
|
+
}
|
|
47
|
+
function loadYaml(p: string): Record<string, any> {
|
|
48
|
+
if (!existsSync(p)) return {};
|
|
49
|
+
try { return (parseYaml(readFileSync(p, 'utf8')) as Record<string, any>) || {}; } catch { return {}; }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Every string value anywhere in the test-data (top-level scalars + nested + `@cases` rows). */
|
|
53
|
+
function collectStrings(data: Record<string, any>): string[] {
|
|
54
|
+
const out: string[] = [];
|
|
55
|
+
const walk = (v: unknown) => {
|
|
56
|
+
if (typeof v === 'string') out.push(v);
|
|
57
|
+
else if (Array.isArray(v)) v.forEach(walk);
|
|
58
|
+
else if (v && typeof v === 'object') Object.values(v as Record<string, unknown>).forEach(walk);
|
|
59
|
+
};
|
|
60
|
+
for (const [k, v] of Object.entries(data)) { if (k.startsWith('_')) continue; walk(v); }
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Short human label for a boundary point (from its note / id). */
|
|
65
|
+
function pointLabel(p: { id: string; note?: string }): string {
|
|
66
|
+
if (p.note) { const tail = p.note.split(/·|:/).pop()?.trim(); if (tail) return tail; }
|
|
67
|
+
return p.id.replace(/^CHK-[A-Z0-9]+-/, '');
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** BVA coverage for one field: which required boundary points are not represented among authored values. */
|
|
71
|
+
function bvaGaps(field: FieldDef, catalog: Catalog, authored: string[]): string[] {
|
|
72
|
+
if (!field.type) return [];
|
|
73
|
+
const ef = expandField(field, catalog);
|
|
74
|
+
const synth = ef.points.filter((p) => p.synthesized && p.value != null);
|
|
75
|
+
if (!synth.length) return [];
|
|
76
|
+
const gaps: string[] = [];
|
|
77
|
+
const seen = new Set<string>();
|
|
78
|
+
if (STR_LEN_TYPES.has(field.type)) {
|
|
79
|
+
// Type-aware filter so an unrelated string of the same LENGTH doesn't falsely "cover" the boundary
|
|
80
|
+
// (e.g. a 5-char URL must not count as covering an email `minLength`-5 case). Email → values with
|
|
81
|
+
// an `@`; katakana → values with kana; others (generic string) stay loose.
|
|
82
|
+
const plausible =
|
|
83
|
+
field.type === 'email' ? authored.filter((s) => s.includes('@'))
|
|
84
|
+
: field.type === 'katakana' ? authored.filter((s) => /[゠-ヿヲ-゚]/.test(s))
|
|
85
|
+
: authored;
|
|
86
|
+
const authoredLens = new Set(plausible.map((s) => s.length));
|
|
87
|
+
for (const p of synth) {
|
|
88
|
+
const len = (p.value as string).length;
|
|
89
|
+
const key = `len:${len}`;
|
|
90
|
+
if (seen.has(key) || authoredLens.has(len)) { seen.add(key); continue; }
|
|
91
|
+
seen.add(key);
|
|
92
|
+
gaps.push(`${pointLabel(p)} (length ${len}, ${p.expectedValidity})`);
|
|
93
|
+
}
|
|
94
|
+
} else if (NUM_TYPES.has(field.type)) {
|
|
95
|
+
const authoredNums = new Set(authored.map((s) => Number(s)).filter((n) => Number.isFinite(n)));
|
|
96
|
+
for (const p of synth) {
|
|
97
|
+
const n = Number(p.value);
|
|
98
|
+
const key = `num:${n}`;
|
|
99
|
+
if (seen.has(key) || authoredNums.has(n)) { seen.add(key); continue; }
|
|
100
|
+
seen.add(key);
|
|
101
|
+
gaps.push(`${pointLabel(p)} (value ${p.value}, ${p.expectedValidity})`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return gaps;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Keyword patterns proving an auth-account-state is covered somewhere in the data/feature. */
|
|
108
|
+
const STATE_COVERED: Record<string, RegExp> = {
|
|
109
|
+
active: /valid|active|success|happy/i,
|
|
110
|
+
'wrong-password': /wrong.?pass|password.?mismatch|incorrect.?pass|bad.?pass/i,
|
|
111
|
+
'non-existent-account': /unregister|non.?exist|not.*(exist|found|in nj)|no account/i,
|
|
112
|
+
'soft-deleted': /soft.?delet|deleted.?account|marked.?deleted|deleted_at/i,
|
|
113
|
+
'locked-or-suspended': /lock|suspend|inactive|disabled account/i,
|
|
114
|
+
'unverified-email': /unverif|not.?verified|pending.?verif|email.?verification/i,
|
|
115
|
+
'password-expired': /expir/i,
|
|
116
|
+
// valid-with-surrounding-whitespace is handled by a VALUE-level check (an actual email with
|
|
117
|
+
// leading/trailing space), not this blob regex — the word "trim" alone (from the empty-trim case)
|
|
118
|
+
// must NOT count as covering the valid-after-trim case.
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
/** Coverage check for one unit — BVA gaps per field + auth-experience gaps for a login/auth unit. */
|
|
122
|
+
export function coverageUnit(cwd: string, name: string, catalog: Catalog): CoverageFinding[] {
|
|
123
|
+
const fmPath = join(cwd, 'qa', 'data-factory', `${name}.fields.yaml`);
|
|
124
|
+
if (!existsSync(fmPath)) return [];
|
|
125
|
+
const map = loadYaml(fmPath) as FieldMap;
|
|
126
|
+
if (!map || !Array.isArray(map.fields)) return [];
|
|
127
|
+
const tdPath = unitTestDataPath(cwd, name);
|
|
128
|
+
const data = tdPath ? loadYaml(tdPath) : {};
|
|
129
|
+
const authored = collectStrings(data);
|
|
130
|
+
const feature = unitFeatureText(cwd, name);
|
|
131
|
+
const blob = (authored.join('\n') + '\n' + JSON.stringify(data) + '\n' + feature).toLowerCase();
|
|
132
|
+
const out: CoverageFinding[] = [];
|
|
133
|
+
|
|
134
|
+
// 1. BVA coverage per field (flatten nested groups to leaves).
|
|
135
|
+
const leaves: FieldDef[] = [];
|
|
136
|
+
const walk = (fs: FieldDef[]) => { for (const f of fs) { if (f.fields) walk(f.fields); else leaves.push(f); } };
|
|
137
|
+
walk(map.fields);
|
|
138
|
+
for (const f of leaves) {
|
|
139
|
+
const gaps = bvaGaps(f, catalog, authored);
|
|
140
|
+
if (gaps.length) {
|
|
141
|
+
out.push({ unit: name, field: f.name, severity: 'warn', code: 'BVA_UNCOVERED',
|
|
142
|
+
message: `field "${f.name}" (${f.type}): ${gaps.length} boundary case(s) not represented in test-data — ${gaps.join('; ')}. The field-map's constraints require them (e.g. minLength → a below-min & at-min value); author them or waive with a reason.` });
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// 2. Auth-experience coverage — only for a login/auth unit (has an email + a password field).
|
|
147
|
+
const isLogin = leaves.some((f) => f.type === 'email') && leaves.some((f) => f.type === 'password');
|
|
148
|
+
const exp = catalog.experience?.['auth-account-states'];
|
|
149
|
+
if (isLogin && exp) {
|
|
150
|
+
// An actual email value carrying leading/trailing whitespace = the valid-after-trim case covered.
|
|
151
|
+
const hasWhitespaceEmail = authored.some((s) => s.includes('@') && s !== s.trim());
|
|
152
|
+
const missing: string[] = [];
|
|
153
|
+
for (const c of exp.cases || []) {
|
|
154
|
+
const state = String((c as any).state || c.id);
|
|
155
|
+
if (state === 'valid-with-surrounding-whitespace') { if (!hasWhitespaceEmail) missing.push(state); continue; }
|
|
156
|
+
const re = STATE_COVERED[state];
|
|
157
|
+
if (re && !re.test(blob)) missing.push(state);
|
|
158
|
+
}
|
|
159
|
+
if (missing.length) {
|
|
160
|
+
out.push({ unit: name, severity: 'warn', code: 'EXPERIENCE_UNCOVERED',
|
|
161
|
+
message: `login unit: auth-account-states not covered — ${missing.join(', ')}. Each needs a credential-state scenario (a seeded account in that state → the correct rejection/route). Add them or waive per spec (some may be N/A).` });
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return out;
|
|
165
|
+
}
|
package/src/crosscheck.ts
CHANGED
|
@@ -61,6 +61,29 @@ function loadYaml(path: string): Record<string, any> {
|
|
|
61
61
|
try { return (parseYaml(readFileSync(path, 'utf8')) as Record<string, any>) || {}; } catch { return {}; }
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
+
/**
|
|
65
|
+
* Column names supplied per-row by every `@cases:<dataset>` referenced in the feature (where the
|
|
66
|
+
* dataset is a top-level LIST in test-data). A `{{col}}` inside a data-driven scenario resolves to
|
|
67
|
+
* the current row, not a top-level key — so these are valid roots, not dangling. Mirrors the
|
|
68
|
+
* compiler's data-driven lint, which is @cases-aware; without this, any @cases scenario that reads
|
|
69
|
+
* a row column (`{{input}}`, `{{code}}`, …) would be a false-positive DANGLING_VAR.
|
|
70
|
+
*/
|
|
71
|
+
export function casesColumns(featureText: string, data: Record<string, any>): Set<string> {
|
|
72
|
+
const cols = new Set<string>();
|
|
73
|
+
const re = /@cases:([A-Za-z0-9_.-]+)/g;
|
|
74
|
+
let m: RegExpExecArray | null;
|
|
75
|
+
while ((m = re.exec(featureText))) {
|
|
76
|
+
const ds = data[m[1]];
|
|
77
|
+
if (!Array.isArray(ds)) continue;
|
|
78
|
+
for (const row of ds) {
|
|
79
|
+
if (row && typeof row === 'object' && !Array.isArray(row)) {
|
|
80
|
+
for (const k of Object.keys(row)) cols.add(k);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return cols;
|
|
85
|
+
}
|
|
86
|
+
|
|
64
87
|
/** Cross-check one unit's feature ↔ test-data. */
|
|
65
88
|
export function crosscheckUnit(cwd: string, name: string): CrossFinding[] {
|
|
66
89
|
const out: CrossFinding[] = [];
|
|
@@ -70,10 +93,11 @@ export function crosscheckUnit(cwd: string, name: string): CrossFinding[] {
|
|
|
70
93
|
if (!feature) return out;
|
|
71
94
|
const data = loadYaml(paths.testData);
|
|
72
95
|
const topKeys = new Set(Object.keys(data).filter((k) => !k.startsWith('_')));
|
|
96
|
+
const caseCols = casesColumns(feature, data); // @cases row columns resolve per-row, not as top-level keys
|
|
73
97
|
|
|
74
|
-
// 1. Dangling vars — a {{var}} whose root key is absent from test-data.
|
|
98
|
+
// 1. Dangling vars — a {{var}} whose root key is absent from test-data (and not a @cases column).
|
|
75
99
|
for (const v of referencedVars(feature)) {
|
|
76
|
-
if (!topKeys.has(v)) {
|
|
100
|
+
if (!topKeys.has(v) && !caseCols.has(v)) {
|
|
77
101
|
out.push({ unit: name, severity: 'error', code: 'DANGLING_VAR', message: `scenario references {{${v}}} but no "${v}" key exists in test-data/${name}.yaml` });
|
|
78
102
|
}
|
|
79
103
|
}
|
package/src/detect.ts
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Data Factory — auto-detection (Layer A of the auto-orchestration design).
|
|
3
|
+
*
|
|
4
|
+
* Makes Sungen NOTICE what test-data work a unit needs, without QA having to know to run a command:
|
|
5
|
+
* - a unit that HAS input fields but has NO field-map → its data was never standardized;
|
|
6
|
+
* - a spec that describes a STATEFUL PRECONDITION (a required entity/state) with no state-recipe and
|
|
7
|
+
* no Background/@manual precondition → the stateful setup is undeclared.
|
|
8
|
+
* Both are surfaced as advisory findings at `sungen generate`/`sungen audit` (the DF advisory sensor).
|
|
9
|
+
* Deterministic heuristics over spec.md + the feature; no runtime, no LLM. Advisory only — a nudge +
|
|
10
|
+
* an auto-draft prompt, never a hard fail (the fuzzy precondition scan must not block).
|
|
11
|
+
*/
|
|
12
|
+
import { join } from 'path';
|
|
13
|
+
import { existsSync, readFileSync, readdirSync } from 'fs';
|
|
14
|
+
import { loadStateRecipes } from './state';
|
|
15
|
+
|
|
16
|
+
export interface DetectFinding {
|
|
17
|
+
code: 'UNSTANDARDIZED' | 'PRECONDITION_UNDECLARED';
|
|
18
|
+
scenario: string;
|
|
19
|
+
message: string;
|
|
20
|
+
severity: 'warn' | 'info';
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function readIfExists(path: string): string {
|
|
24
|
+
try { return existsSync(path) ? readFileSync(path, 'utf8') : ''; } catch { return ''; }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function readSpec(dir: string): string {
|
|
28
|
+
return readIfExists(join(dir, 'requirements', 'spec.md'));
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function readFeature(dir: string): string {
|
|
32
|
+
const fdir = join(dir, 'features');
|
|
33
|
+
if (!existsSync(fdir)) return '';
|
|
34
|
+
try {
|
|
35
|
+
return readdirSync(fdir).filter((f) => f.endsWith('.feature')).map((f) => readIfExists(join(fdir, f))).join('\n');
|
|
36
|
+
} catch { return ''; }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Does the unit take user input? Strongest signal: a Gherkin `fill [X] field` / `type` step; plus a
|
|
41
|
+
* spec field table row typed as input/select/textarea. A pure navigation / list / capture screen has
|
|
42
|
+
* none → we must NOT nag it ("no input fields → nothing to generate").
|
|
43
|
+
*/
|
|
44
|
+
export function hasInputFields(specText: string, featureText: string): boolean {
|
|
45
|
+
if (/\b(fill|type|enter|input|select|choose)\b\s+\[[^\]]+\]\s*(field|input|dropdown|textbox)?/i.test(featureText)) return true;
|
|
46
|
+
if (/\|\s*(input|select|textarea|checkbox|radio|combobox)\b/i.test(specText)) return true;
|
|
47
|
+
if (/\binput\s*\(|\bselect\s*\(|\btextarea\b/i.test(specText)) return true;
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** A field-map exists for this unit. */
|
|
52
|
+
export function hasFieldMap(cwd: string, name: string): boolean {
|
|
53
|
+
return existsSync(join(cwd, 'qa', 'data-factory', `${name}.fields.yaml`));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Precondition signal phrases — a spec sentence hinting the test needs a pre-existing entity/state. */
|
|
57
|
+
const PRECONDITION_SIGNALS = [
|
|
58
|
+
'requires', 'required before', 'precondition', 'pre-condition', 'prerequisite',
|
|
59
|
+
'logged in', 'authenticated', 'signed in', 'must be logged',
|
|
60
|
+
'already exists', 'already registered', 'existing ', 'pre-existing', 'must exist',
|
|
61
|
+
'reset token', 'verification', 'provisional account', 'session-storage', 'session storage',
|
|
62
|
+
'buffer', 'seeded', 'must have', 'previously', 'valid token', 'auth token',
|
|
63
|
+
];
|
|
64
|
+
|
|
65
|
+
/** Extract spec sentences that look like a stateful precondition (deduped, capped). */
|
|
66
|
+
export function detectPreconditionSignals(specText: string): string[] {
|
|
67
|
+
if (!specText) return [];
|
|
68
|
+
const hits: string[] = [];
|
|
69
|
+
const seen = new Set<string>();
|
|
70
|
+
// Split into sentence-ish units (lines / bullet items / sentences).
|
|
71
|
+
const units = specText.split(/\n|(?<=[.。])\s+/).map((s) => s.replace(/\s+/g, ' ').trim()).filter(Boolean);
|
|
72
|
+
for (const u of units) {
|
|
73
|
+
const low = u.toLowerCase();
|
|
74
|
+
// Skip markdown headings/table separators, and spec-template meta lines (US labels, rationale)
|
|
75
|
+
// that mention auth/priority only descriptively — not an actual precondition.
|
|
76
|
+
if (/^#{1,6}\s|^\|?\s*-{3,}/.test(u)) continue;
|
|
77
|
+
if (/^\*\*(why|independent test|acceptance|business context|target users|priority)\b/i.test(u)) continue;
|
|
78
|
+
if (/\bpriority\b/i.test(low) && /\bp[0-9]\b/i.test(low)) continue;
|
|
79
|
+
const sig = PRECONDITION_SIGNALS.find((s) => low.includes(s));
|
|
80
|
+
if (sig) {
|
|
81
|
+
const key = low.slice(0, 60);
|
|
82
|
+
if (!seen.has(key)) { seen.add(key); hits.push(u.length > 160 ? u.slice(0, 157) + '…' : u); }
|
|
83
|
+
}
|
|
84
|
+
if (hits.length >= 4) break;
|
|
85
|
+
}
|
|
86
|
+
return hits;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Is a stateful precondition already declared — a state-recipe with resources, or a Background/@manual? */
|
|
90
|
+
export function hasStateDeclaration(cwd: string, featureText: string): boolean {
|
|
91
|
+
if (/^\s*Background:/m.test(featureText) || /@manual\b/.test(featureText)) return true;
|
|
92
|
+
return loadStateRecipes(cwd).some((r) => (r.resources && r.resources.length) || r.shared);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Run Layer-A detection for one unit dir. Returns advisory findings (possibly empty). */
|
|
96
|
+
export function detectUnit(cwd: string, dir: string, name: string): DetectFinding[] {
|
|
97
|
+
const spec = readSpec(dir);
|
|
98
|
+
const feature = readFeature(dir);
|
|
99
|
+
const out: DetectFinding[] = [];
|
|
100
|
+
|
|
101
|
+
// 1. Inputs exist but never standardized.
|
|
102
|
+
if (!hasFieldMap(cwd, name) && hasInputFields(spec, feature)) {
|
|
103
|
+
out.push({
|
|
104
|
+
code: 'UNSTANDARDIZED',
|
|
105
|
+
scenario: name,
|
|
106
|
+
severity: 'warn',
|
|
107
|
+
message: `DATA-FACTORY-UNSTANDARDIZED: ${name} has input fields but no standardized test-data (no qa/data-factory/${name}.fields.yaml) — run \`/sungen:create-data-test ${name}\` (or it is authored inline by \`/sungen:create-test\`) so values are standardized valid/boundary/invalid, not hand-invented.`,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// 2. Stateful precondition described in the spec but not declared anywhere.
|
|
112
|
+
const signals = detectPreconditionSignals(spec);
|
|
113
|
+
if (signals.length && !hasStateDeclaration(cwd, feature)) {
|
|
114
|
+
out.push({
|
|
115
|
+
code: 'PRECONDITION_UNDECLARED',
|
|
116
|
+
scenario: name,
|
|
117
|
+
severity: 'info',
|
|
118
|
+
message: `DATA-FACTORY-PRECONDITION-UNDECLARED: ${name}'s spec describes a stateful precondition but the test declares none (no state-recipe, no Background/@manual). Declare it with a state-recipe + \`sungen data state\` so it renders a @manual:data-setup Background. Signals: ${signals.map((s) => `"${s}"`).join('; ')}`,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
return out;
|
|
122
|
+
}
|
package/src/generator.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* same output (determinism is an asset). Nothing here reads a live page or an LLM.
|
|
11
11
|
*/
|
|
12
12
|
import type { Catalog, FieldType, DataPoint, SelectionContext } from './types';
|
|
13
|
-
import { expandBoundaryHint, satisfiesConstraints, type Validity } from './boundary';
|
|
13
|
+
import { expandBoundaryHint, satisfiesConstraints, ensureBvaComplete, type Validity } from './boundary';
|
|
14
14
|
import { evaluateWhen, fieldFacts, inProfile } from './applicability';
|
|
15
15
|
|
|
16
16
|
export interface FieldDef {
|
|
@@ -265,6 +265,23 @@ export function expandField(field: FieldDef, catalog: Catalog, ctx: SelectionCon
|
|
|
265
265
|
}
|
|
266
266
|
}
|
|
267
267
|
|
|
268
|
+
// BVA completeness: for every bounded dimension in the constraints, ensure the full
|
|
269
|
+
// min-1/min/min+1/max-1/max/max+1 set exists — auto-adding only what the catalog left out (e.g. an
|
|
270
|
+
// email `minLength` the catalog didn't model). Only in-range points are `boundary`; out-of-range are
|
|
271
|
+
// `invalid`. Appended after the catalog cases so it never overrides an authored one.
|
|
272
|
+
const idCode = type.toUpperCase().replace(/[^A-Z0-9]/g, '');
|
|
273
|
+
for (const ex of ensureBvaComplete(type, c, points.map((p) => p.value))) {
|
|
274
|
+
points.push({
|
|
275
|
+
id: `CHK-${idCode}-BVA-${ex.label}`,
|
|
276
|
+
group: ex.group,
|
|
277
|
+
value: ex.value,
|
|
278
|
+
expectedValidity: ex.validity,
|
|
279
|
+
synthesized: true,
|
|
280
|
+
note: `BVA (auto): ${ex.label}`,
|
|
281
|
+
...(ex.validity === 'invalid' ? { reason: `out of bounds (${ex.label})` } : {}),
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
|
|
268
285
|
return {
|
|
269
286
|
name: field.name,
|
|
270
287
|
type: field.type,
|
package/src/index.ts
CHANGED
|
@@ -25,21 +25,25 @@ export { lintFieldMap, summarize } from './lint';
|
|
|
25
25
|
export type { LintIssue } from './lint';
|
|
26
26
|
export { validateGenerated, validatePoints } from './validate';
|
|
27
27
|
export type { ValidationFinding } from './validate';
|
|
28
|
-
export { expandBoundaryHint, satisfiesConstraints } from './boundary';
|
|
29
|
-
export type { Validity, BoundaryPoint } from './boundary';
|
|
28
|
+
export { expandBoundaryHint, satisfiesConstraints, ensureBvaComplete, emailOfTotalLength } from './boundary';
|
|
29
|
+
export type { Validity, BoundaryPoint, BvaExtra } from './boundary';
|
|
30
30
|
export { evaluateWhen, inProfile, fieldFacts } from './applicability';
|
|
31
31
|
export { generate, buildRequest, validateRequest, fingerprint, GENERATOR_VERSION, SCHEMA_VERSION } from './contract';
|
|
32
32
|
export type { DataFactoryRequest, DataFactoryResult } from './contract';
|
|
33
|
-
export { loadStateRecipes, validateStateRecipe, buildStatePlan, renderPreconditionBlock, topoSort, hasDatasource } from './state';
|
|
33
|
+
export { loadStateRecipes, validateStateRecipe, buildStatePlan, renderPreconditionBlock, topoSort, hasDatasource, scaffoldRecipe, renderRecipeYaml, guessProvenance } from './state';
|
|
34
34
|
export type { StateRecipe, StateResource, StatePlan, StateFinding, Provenance, Lifecycle } from './state';
|
|
35
35
|
export { crosscheckUnit, crosscheckShared, crosscheckAll, referencedVars } from './crosscheck';
|
|
36
36
|
export type { CrossFinding } from './crosscheck';
|
|
37
|
+
export { detectUnit, hasInputFields, hasFieldMap, detectPreconditionSignals, hasStateDeclaration } from './detect';
|
|
38
|
+
export type { DetectFinding } from './detect';
|
|
39
|
+
export { coverageUnit } from './coverage';
|
|
40
|
+
export type { CoverageFinding } from './coverage';
|
|
37
41
|
export { registerDataFactoryCommand } from './cli';
|
|
38
42
|
export { generateFixture } from './fixture';
|
|
39
43
|
export type { FixtureResult, FixtureExpected, ExpectedRow } from './fixture';
|
|
40
44
|
export { dataFactoryCoverageSensor } from './sensor';
|
|
41
45
|
export { dfPreconditionCodegen } from './precondition';
|
|
42
|
-
export type { Catalog, FieldType, SecurityBank, Technique, DataPoint, OverwriteWarning, CatalogProvenance, FieldTypeProvenance, SelectionContext, Profile } from './types';
|
|
46
|
+
export type { Catalog, FieldType, SecurityBank, Technique, Experience, DataPoint, OverwriteWarning, CatalogProvenance, FieldTypeProvenance, SelectionContext, Profile } from './types';
|
|
43
47
|
|
|
44
48
|
/** Register the Data Factory capability. */
|
|
45
49
|
export function register(registry: CapabilityRegistry): void {
|
package/src/sensor.ts
CHANGED
|
@@ -5,7 +5,10 @@
|
|
|
5
5
|
* coverage lint as `sungen data lint` and surfaces the gaps in the generate output — so weak
|
|
6
6
|
* test-data coverage (missing boundary/invalid, unresolved values, unmapped error codes, blind
|
|
7
7
|
* spots) is visible in the standard harness flow, not only when you remember to run `data lint`.
|
|
8
|
-
*
|
|
8
|
+
*
|
|
9
|
+
* Auto-detection (Layer A): even with NO field-map, it NOTICES what's missing — a unit with input
|
|
10
|
+
* fields but no standardized data, or a spec that describes a stateful precondition the test never
|
|
11
|
+
* declares — so Sungen nudges QA instead of staying silent (was: "no field-map → nothing to say").
|
|
9
12
|
*
|
|
10
13
|
* Advisory, not gate: the ContextRouter scopes gate sensors by the annotation tags a feature carries,
|
|
11
14
|
* and test-data generation is a design-time concern with no `.feature` tag — so gating here would
|
|
@@ -18,6 +21,7 @@ import type { Sensor } from '@sun-asterisk/sungen';
|
|
|
18
21
|
import { loadEffectiveCatalog } from './catalog-loader';
|
|
19
22
|
import { lintFieldMap, summarize } from './lint';
|
|
20
23
|
import { validateGenerated } from './validate';
|
|
24
|
+
import { detectUnit } from './detect';
|
|
21
25
|
import type { FieldMap } from './generator';
|
|
22
26
|
|
|
23
27
|
export const dataFactoryCoverageSensor: Sensor<{ dir: string; cwd: string }> = {
|
|
@@ -26,11 +30,21 @@ export const dataFactoryCoverageSensor: Sensor<{ dir: string; cwd: string }> = {
|
|
|
26
30
|
kind: 'advisory',
|
|
27
31
|
run: ({ dir, cwd }) => {
|
|
28
32
|
const name = basename(dir);
|
|
33
|
+
// Layer A auto-detection — runs regardless of whether a field-map exists, so a unit that needs
|
|
34
|
+
// standardization or a declared precondition is NOTICED, not silently skipped.
|
|
35
|
+
const detected = detectUnit(cwd, dir, name).map((d) => ({
|
|
36
|
+
sensorId: 'data-factory-coverage',
|
|
37
|
+
capability: 'data-factory',
|
|
38
|
+
scenario: d.scenario,
|
|
39
|
+
message: d.message,
|
|
40
|
+
severity: d.severity,
|
|
41
|
+
}));
|
|
42
|
+
|
|
29
43
|
const fieldMapPath = join(cwd, 'qa', 'data-factory', `${name}.fields.yaml`);
|
|
30
|
-
if (!existsSync(fieldMapPath)) return
|
|
44
|
+
if (!existsSync(fieldMapPath)) return detected; // no field-map: report only what detection noticed
|
|
31
45
|
let map: FieldMap;
|
|
32
|
-
try { map = parseYaml(readFileSync(fieldMapPath, 'utf8')) as FieldMap; } catch { return
|
|
33
|
-
if (!map || !Array.isArray(map.fields)) return
|
|
46
|
+
try { map = parseYaml(readFileSync(fieldMapPath, 'utf8')) as FieldMap; } catch { return detected; }
|
|
47
|
+
if (!map || !Array.isArray(map.fields)) return detected;
|
|
34
48
|
|
|
35
49
|
const { catalog, warnings } = loadEffectiveCatalog(cwd);
|
|
36
50
|
const { issues } = lintFieldMap(map, catalog);
|
|
@@ -71,6 +85,8 @@ export const dataFactoryCoverageSensor: Sensor<{ dir: string; cwd: string }> = {
|
|
|
71
85
|
severity: s.errors ? 'warn' : 'info',
|
|
72
86
|
});
|
|
73
87
|
}
|
|
88
|
+
// Detection findings (e.g. an undeclared stateful precondition) are still relevant with a field-map.
|
|
89
|
+
findings.push(...detected);
|
|
74
90
|
return findings;
|
|
75
91
|
},
|
|
76
92
|
};
|
package/src/state.ts
CHANGED
|
@@ -171,6 +171,53 @@ export function hasDatasource(cwd: string): boolean {
|
|
|
171
171
|
return existsSync(join(cwd, 'datasources.yaml')) || existsSync(join(cwd, 'qa', 'datasources.yaml'));
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
+
/** Guess a provenance from a precondition sentence — best-effort; QA confirms. */
|
|
175
|
+
export function guessProvenance(signal: string): Provenance {
|
|
176
|
+
const s = signal.toLowerCase();
|
|
177
|
+
if (/session-storage|session storage|local ?storage|cookie|buffer/.test(s)) return 'client-storage';
|
|
178
|
+
if (/token|reset|verification|otp|invitation|seeded/.test(s)) return 'fixture';
|
|
179
|
+
if (/logged in|authenticated|signed in|auth token|session/.test(s)) return 'server-context';
|
|
180
|
+
if (/already|existing|registered|must exist|previously|prior/.test(s)) return 'prior-flow';
|
|
181
|
+
return 'manual';
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Auto-scaffold a state-recipe SKELETON from a spec's precondition sentences (Layer-C auto-draft).
|
|
186
|
+
* Deterministic: one resource per detected signal, provenance guessed, entity left as a TODO for the
|
|
187
|
+
* AI/QA to fill (never fabricates a domain entity). Returns null when no precondition signal is found.
|
|
188
|
+
*/
|
|
189
|
+
export function scaffoldRecipe(unit: string, signals: string[]): StateRecipe | null {
|
|
190
|
+
if (!signals.length) return null;
|
|
191
|
+
const resources: StateResource[] = signals.slice(0, 4).map((sig, i) => ({
|
|
192
|
+
ref: `r${i + 1}`,
|
|
193
|
+
entity: 'TODO', // ← name the real entity (from the signal); do not invent domain values
|
|
194
|
+
provenance: guessProvenance(sig),
|
|
195
|
+
lifecycle: 'reusable',
|
|
196
|
+
note: sig,
|
|
197
|
+
}));
|
|
198
|
+
return {
|
|
199
|
+
name: `${unit}_precondition`,
|
|
200
|
+
description: `stateful precondition for ${unit} — AUTO-SCAFFOLDED from spec; REVIEW: name each entity, set desiredState/bindings, confirm with QA`,
|
|
201
|
+
resources,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Serialize a scaffolded recipe as a YAML block to splice under `recipes:` (2-space indented). */
|
|
206
|
+
export function renderRecipeYaml(recipe: StateRecipe): string {
|
|
207
|
+
const lines: string[] = [` ${recipe.name}:`];
|
|
208
|
+
if (recipe.description) lines.push(` description: "${recipe.description}"`);
|
|
209
|
+
lines.push(' resources:');
|
|
210
|
+
for (const r of recipe.resources || []) {
|
|
211
|
+
lines.push(` - ref: ${r.ref}`);
|
|
212
|
+
lines.push(` entity: ${r.entity} # ← name the real entity`);
|
|
213
|
+
lines.push(` provenance: ${r.provenance} # guessed — confirm (prior-flow|fixture|client-storage|server-context|manual|api|db)`);
|
|
214
|
+
if (r.lifecycle) lines.push(` lifecycle: ${r.lifecycle}`);
|
|
215
|
+
if (r.note) lines.push(` note: ${JSON.stringify(r.note)}`);
|
|
216
|
+
}
|
|
217
|
+
lines.push(' # bindings: { someId: "${r1.id}" } # ← wire ids the test needs');
|
|
218
|
+
return lines.join('\n') + '\n';
|
|
219
|
+
}
|
|
220
|
+
|
|
174
221
|
/** Render the precondition block for weaving into a Gherkin Background (with the @manual tag if needed). */
|
|
175
222
|
export function renderPreconditionBlock(plan: StatePlan): string {
|
|
176
223
|
const tag = plan.manual ? '# @manual:data-setup — required state (auto-provisions when an api/db datasource is added)' : '# required state (Layer 1: auto-provisioned via the referenced endpoints)';
|
package/src/types.ts
CHANGED
|
@@ -75,10 +75,25 @@ export interface Technique {
|
|
|
75
75
|
[k: string]: unknown;
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
+
/**
|
|
79
|
+
* A domain-EXPERIENCE checklist — cases that come from real-world testing experience, not from a
|
|
80
|
+
* field's format (e.g. the credential/account states a login screen must cover: soft-deleted, locked,
|
|
81
|
+
* unverified…). Each case is a SCENARIO that usually needs a domain-state precondition, so it is
|
|
82
|
+
* knowledge for the login/auth viewpoint + a state-recipe — not a field-level value.
|
|
83
|
+
*/
|
|
84
|
+
export interface Experience {
|
|
85
|
+
experience: string;
|
|
86
|
+
description?: string;
|
|
87
|
+
applies_when?: string;
|
|
88
|
+
cases: Array<Record<string, unknown> & { id: string; expect?: string }>;
|
|
89
|
+
oracle?: string;
|
|
90
|
+
}
|
|
91
|
+
|
|
78
92
|
export interface Catalog {
|
|
79
93
|
fieldTypes: Record<string, FieldType>;
|
|
80
94
|
security: Record<string, SecurityBank>;
|
|
81
95
|
techniques: Record<string, Technique>;
|
|
96
|
+
experience: Record<string, Experience>;
|
|
82
97
|
}
|
|
83
98
|
|
|
84
99
|
/** Named case-selection profiles (which groups/categories a run wants). */
|