@sun-asterisk/sungen 3.2.24-beta.3 → 3.2.24-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/audit.d.ts.map +1 -1
- package/dist/cli/commands/audit.js +3 -1
- package/dist/cli/commands/audit.js.map +1 -1
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +56 -4
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/flow-contract.d.ts +16 -1
- package/dist/harness/flow-contract.d.ts.map +1 -1
- package/dist/harness/flow-contract.js +8 -1
- package/dist/harness/flow-contract.js.map +1 -1
- package/dist/harness/next-step.d.ts +10 -0
- package/dist/harness/next-step.d.ts.map +1 -1
- package/dist/harness/next-step.js +36 -3
- package/dist/harness/next-step.js.map +1 -1
- package/dist/harness/sensors.d.ts.map +1 -1
- package/dist/harness/sensors.js +13 -0
- package/dist/harness/sensors.js.map +1 -1
- package/dist/harness/viewpoint-ledger.d.ts +35 -0
- package/dist/harness/viewpoint-ledger.d.ts.map +1 -1
- package/dist/harness/viewpoint-ledger.js +40 -0
- package/dist/harness/viewpoint-ledger.js.map +1 -1
- package/dist/orchestrator/flow-manager.d.ts +11 -0
- package/dist/orchestrator/flow-manager.d.ts.map +1 -1
- package/dist/orchestrator/flow-manager.js +48 -2
- package/dist/orchestrator/flow-manager.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +17 -0
- package/dist/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +26 -0
- package/package.json +3 -3
- package/src/cli/commands/audit.ts +3 -1
- package/src/harness/audit.ts +54 -5
- package/src/harness/flow-contract.ts +24 -3
- package/src/harness/next-step.ts +43 -3
- package/src/harness/sensors.ts +13 -0
- package/src/harness/viewpoint-ledger.ts +65 -0
- package/src/orchestrator/flow-manager.ts +52 -2
- package/src/orchestrator/templates/ai-src/commands/create-test.md +17 -0
- package/src/orchestrator/templates/ai-src/skills/sungen-tc-generation/SKILL.md +26 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sun-asterisk/sungen",
|
|
3
|
-
"version": "3.2.24-beta.
|
|
3
|
+
"version": "3.2.24-beta.5",
|
|
4
4
|
"description": "Deterministic E2E Test Compiler - Gherkin + Selectors → Playwright tests",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
@@ -39,8 +39,8 @@
|
|
|
39
39
|
"@babel/types": "^7.28.5",
|
|
40
40
|
"@cucumber/gherkin": "^37.0.0",
|
|
41
41
|
"@cucumber/messages": "^31.0.0",
|
|
42
|
-
"@sungen/driver-data-factory": "3.2.24-beta.
|
|
43
|
-
"@sungen/driver-ui": "3.2.24-beta.
|
|
42
|
+
"@sungen/driver-data-factory": "3.2.24-beta.5",
|
|
43
|
+
"@sungen/driver-ui": "3.2.24-beta.5",
|
|
44
44
|
"chalk": "^5.6.2",
|
|
45
45
|
"commander": "^14.0.2",
|
|
46
46
|
"dotenv": "^17.2.3",
|
|
@@ -51,7 +51,9 @@ function render(r: AuditReport): void {
|
|
|
51
51
|
// axis read as a contradiction.
|
|
52
52
|
const inv = r.flowInventory;
|
|
53
53
|
if (inv) {
|
|
54
|
-
|
|
54
|
+
const auto = inv.covered.filter((c) => c.automated).length;
|
|
55
|
+
L(` flows: ${inv.covered.length}/${inv.covered.length + inv.uncovered.length} declared covered, ${auto} with automated evidence`
|
|
56
|
+
+ (inv.manualOnly.length ? ` (manual-only: ${inv.manualOnly.map((f) => f.id).join(', ')})` : '')
|
|
55
57
|
+ (inv.accounted.length ? ` (${inv.accounted.map((f) => `${f.id}=${f.status}`).join(', ')})` : ''));
|
|
56
58
|
}
|
|
57
59
|
L(` handoffs asserted: ${fq.handoffs.asserted}/${fq.handoffs.total}${fq.offGoal.length ? ` ⚠ off-goal scenarios: ${fq.offGoal.length} (${fq.offGoalCategories.join(', ')})` : ''}`);
|
package/src/harness/audit.ts
CHANGED
|
@@ -25,7 +25,7 @@ import { getProvenance, Provenance } from './provenance';
|
|
|
25
25
|
import { specCoverage, SpecCoverageResult, parseSpecClauses, restatedRequirementSources } from './spec-coverage';
|
|
26
26
|
import { downstreamScope, manualOracle, readText, DownstreamResult, ManualOracleResult,
|
|
27
27
|
negativeSideEffect, sourceBacked, crossArtifactOwnership, isolationRisk, serialCascadeRisk } from './quality-gates';
|
|
28
|
-
import { viewpointLedger, parseViewpointItems, browserGestureSubstitutions, LedgerResult } from './viewpoint-ledger';
|
|
28
|
+
import { viewpointLedger, parseViewpointItems, browserGestureSubstitutions, viewpointIntegrity, LedgerResult } from './viewpoint-ledger';
|
|
29
29
|
import { capabilityRegistry } from '../capabilities/registry';
|
|
30
30
|
import { discoverAndRegisterCapabilities } from '../capabilities/discover';
|
|
31
31
|
import { contextRouter } from '../capabilities/context-router';
|
|
@@ -304,6 +304,15 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
304
304
|
// what a create-test run did, silently dropping the performance viewpoint on
|
|
305
305
|
// the way. While the declaration is unconfirmed, neither axis is evidence.
|
|
306
306
|
const viewpointMoved = viewpointBaseline.status === 'changed';
|
|
307
|
+
// #618 — is there a YARDSTICK at all? `atomicLedger` and `traceability` are both measured
|
|
308
|
+
// against test-viewpoint.md, and four more sensors read it (the ledger, the declaration-integrity
|
|
309
|
+
// checks, the browser-gesture check, the continuity check). When the file is absent every one of
|
|
310
|
+
// them goes quiet — and worse, the checks that DERIVE from it keep talking: scenarios read as
|
|
311
|
+
// unsourceable because half their source contract is missing, `trace.mappedRatio` is 0 because
|
|
312
|
+
// there is nothing to map to, and the calibration axis reports a manufactured 0% that then trips
|
|
313
|
+
// SCORE-INFLATED-BY-BREADTH with advice nobody can act on. One absent file produced nine
|
|
314
|
+
// misleading findings and not one naming the file. So: say it once, and stop concluding.
|
|
315
|
+
const hasYardstick = ledger.hasViewpoint && (viewpoints.length > 0 || ledger.total > 0);
|
|
307
316
|
// 4. A flow with a contract is measured on ITS coverage: journey phases (HP/ER/EH
|
|
308
317
|
// automated) — not a screen's page-type themes. Same weight, same criticality,
|
|
309
318
|
// so a flow can finally EARN the coverage axis instead of always losing it to
|
|
@@ -315,7 +324,11 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
315
324
|
// "weakest: flowCoverage 100% — fix it to lift the number", which is not fixable advice (#595).
|
|
316
325
|
const axisDefs: Array<{ key: string; value: number; weight: number; applicable: boolean; critical: boolean; byDesign?: boolean }> = [
|
|
317
326
|
flowScored
|
|
318
|
-
|
|
327
|
+
// A declared flow claimed only by @manual scenarios is ACCOUNTED FOR but nothing runs, so
|
|
328
|
+
// it cannot count the same as an automated one — `covered` was collapsing three very
|
|
329
|
+
// different states into one number (#599). Half credit: the design work is real, the
|
|
330
|
+
// regression evidence is not.
|
|
331
|
+
? { key: 'flowCoverage', value: inventory ? Math.min((inventory.ratio + inventory.automatedRatio) / 2, flowQ.phaseRatio) : flowQ.phaseRatio, weight: 0.22, applicable: true, critical: true }
|
|
319
332
|
: { key: 'coverage', value: coverage, weight: 0.22, applicable: !!gate.pageType && gate.themesTotal > 0, critical: true },
|
|
320
333
|
{ key: 'specFR', value: specRatio, weight: 0.15, applicable: spec.hasSpec && spec.frTotal > 0, critical: true },
|
|
321
334
|
{ key: 'atomicLedger', value: ledger.ratio, weight: 0.13, applicable: ledger.hasViewpoint && ledger.total > 0 && !viewpointMoved, critical: true },
|
|
@@ -416,6 +429,27 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
416
429
|
if (!flowQ.contract!.minimalGuarantee) {
|
|
417
430
|
findings.push('FLOW-GUARANTEE-MISSING: the contract declares no `minimalGuarantee:` — the postcondition that must hold in EVERY flow, including failure ("no second account for the same email", "no verification mail on a rejected submit"). Exception/guard scenarios have nothing to assert against without it; add it (and `successGuarantee:` for the goal-reached side).');
|
|
418
431
|
}
|
|
432
|
+
// #599 — integrity of the DECLARATION itself: three ways it can be internally wrong while
|
|
433
|
+
// every coverage number still reads well.
|
|
434
|
+
{
|
|
435
|
+
const testDataDir = path.join(screenDir, 'test-data');
|
|
436
|
+
const extra: Array<{ label: string; text: string }> = [];
|
|
437
|
+
try {
|
|
438
|
+
for (const f of fs.readdirSync(testDataDir).filter((x) => x.endsWith('.yaml'))) {
|
|
439
|
+
extra.push({ label: `test-data/${f}`, text: readTextFile(path.join(testDataDir, f)) ?? '' });
|
|
440
|
+
}
|
|
441
|
+
} catch { /* api units and fresh scaffolds carry none */ }
|
|
442
|
+
const vi = viewpointIntegrity(viewpointPath, scenarios.map((sc) => sc.vpId ?? '').filter(Boolean), extra);
|
|
443
|
+
for (const d of vi.duplicateIds.slice(0, 4)) {
|
|
444
|
+
findings.push(`VIEWPOINT-ID-REUSED: "${d.id}" labels ${d.count} different claims (${d.texts.map((t) => `"${t.slice(0, 44)}…"`).join(' · ')}). The ledger then counts ${d.count} items while traceability sees ONE id, so both the coverage arithmetic and "which test proves this?" go wrong — and neither number looks suspicious. Give each atomic claim its own id; several ids may share a flow.`);
|
|
445
|
+
}
|
|
446
|
+
for (const d of vi.danglingRefs.slice(0, 5)) {
|
|
447
|
+
findings.push(`TRACE-REF-DANGLING: ${d.where} cites "${d.ref}", which no scenario carries. Nothing breaks at runtime, so a wrong pointer survives every refactor and quietly misleads the next maintainer. Fix the reference, or the scenario id it was meant to name.`);
|
|
448
|
+
}
|
|
449
|
+
if (vi.uniformPriority) {
|
|
450
|
+
findings.push(`VIEWPOINT-PRIORITY-UNIFORM: all ${vi.uniformPriority.rows} rows of the Priority Viewpoints table are "${vi.uniformPriority.levels[0]}". That is a formality, not a priority: nothing can be selected for a smoke run and the release conversation has no input. Rank by business impact and risk — the basic flow and the data-integrity guards are not the same bet as an exit path a screen suite already covers.`);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
419
453
|
// #595 — a viewpoint item naming a browser-level gesture, answered with an in-app control.
|
|
420
454
|
for (const g of browserGestureSubstitutions(viewpointPath, scenarios).slice(0, 4)) {
|
|
421
455
|
findings.push(`VIEWPOINT-GESTURE-SUBSTITUTED: the viewpoint asks for "${g.gesture}" ("${g.item.slice(0, 80)}…") but the suite exercises an in-app control instead. They are different mechanisms — an app button runs the screen's own re-hydrate logic, browser history does not, and a defect that only shows on history navigation lives in exactly that gap. The step vocabulary has no reload/back step (#593), so defer it with \`@manual\` naming the gesture rather than substituting silently.`);
|
|
@@ -436,6 +470,9 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
436
470
|
for (const f of inventory.uncovered) {
|
|
437
471
|
findings.push(`FLOW-UNCOVERED: declared flow ${f.id}${f.branchFrom ? ` (branches from ${f.branchFrom})` : ''} has status \`covered\` but no scenario carries its id — write it, or change its status to \`deferred\` / \`pending-clarification\` / \`out-of-scope\` WITH a reason. Silence is the one option the inventory removes.`);
|
|
438
472
|
}
|
|
473
|
+
if (inventory.manualOnly.length > 0) {
|
|
474
|
+
findings.push(`FLOW-MANUAL-ONLY: ${inventory.manualOnly.length} declared flow(s) are claimed only by @manual scenarios — ${inventory.manualOnly.map((f) => f.id).join(', ')}. They are accounted for, but NOTHING RUNS for them: a regression run would not notice if any of these broke. \`status: covered\` says the flow was thought about; it does not say the suite can catch a regression in it, so flowCoverage credits them at half. Either automate them (check the capability plan for a driver that would) or say plainly in the delivery that this flow's evidence is a manual procedure.`);
|
|
475
|
+
}
|
|
439
476
|
for (const u of inventory.undeclared.slice(0, 6)) {
|
|
440
477
|
findings.push(`FLOW-UNDECLARED: "${u.scenario}" claims flow id ${u.id}, which the contract's inventory does not declare — a phase id invented for one scenario inflates the flow count without adding branch coverage. Either declare ${u.id} as a real flow (branch point + own outcome), or fold the scenario into the flow it actually belongs to.`);
|
|
441
478
|
}
|
|
@@ -537,7 +574,13 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
537
574
|
} else if (balance.imbalanced && !flowScored) {
|
|
538
575
|
findings.push(`BALANCE: ${balance.note} Stop expanding secondary viewpoints until business-core gaps are filled.`);
|
|
539
576
|
}
|
|
540
|
-
if (
|
|
577
|
+
if (!hasYardstick) {
|
|
578
|
+
const lost = [
|
|
579
|
+
'atomicLedger', 'traceability',
|
|
580
|
+
...(unsourced.length ? [`${unsourced.length} scenario(s) could not be traced to a source`] : []),
|
|
581
|
+
];
|
|
582
|
+
findings.push(`VIEWPOINT-MISSING: \`requirements/test-viewpoint.md\` ${ledger.hasViewpoint ? 'declares no viewpoints' : 'does not exist'}, so this unit has no yardstick. Two scored axes are excluded (atomicLedger, traceability) and four sensors cannot run: the atomic ledger, the declaration-integrity checks (reused ids, dangling refs, uniform priority), the browser-gesture check and the two-way continuity check. Nothing here is a pass — it is unmeasured [${lost.join(', ')}]. A flow contract does NOT substitute: the contract declares the use case, the viewpoint declares the CLAIMS the suite is scored against, and only the second is independent of what was generated. Author it (\`sungen add-flow\` scaffolds a template), then \`sungen audit --screen ${screenName} --accept-viewpoint\`.`);
|
|
583
|
+
} else if (trace.mappedRatio < 0.5) {
|
|
541
584
|
findings.push(`TRACE: ${trace.note}`);
|
|
542
585
|
}
|
|
543
586
|
// (UNIVERSAL viewpoint-gap finding now emitted by the `ui` gate sensor — see the gate block below.)
|
|
@@ -637,7 +680,10 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
637
680
|
findings.push(`FLOW-EXHAUSTIVE-INPUTS: ${c.scenarios.length} same-shape scenarios differ only in data (${c.scenarios.slice(0, 3).join(' | ')}${c.scenarios.length > 3 ? ' …' : ''}) — a system-test flow uses REPRESENTATIVE inputs (one valid + one that triggers the exception branch); the full EP/boundary matrix belongs to the owning screen (or a @cases dataset there). Keep one representative per branch here.`);
|
|
638
681
|
}
|
|
639
682
|
}
|
|
640
|
-
|
|
683
|
+
// Reported per scenario only when BOTH halves of the source contract exist. With no viewpoint,
|
|
684
|
+
// "doesn't trace to any FR / viewpoint item" is true by construction for anything that does not
|
|
685
|
+
// happen to cite an FR — the count is folded into VIEWPOINT-MISSING instead of six accusations.
|
|
686
|
+
for (const u of (hasYardstick ? unsourced : []).slice(0, 6)) {
|
|
641
687
|
findings.push(`UNSOURCEABLE-SCENARIO: "${u}" doesn't trace to any FR / viewpoint item — link it to a source, or tag it @exploration (not part of the official suite).`);
|
|
642
688
|
}
|
|
643
689
|
|
|
@@ -669,7 +715,10 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
669
715
|
for (const a of scored) axes[a.key] = Math.round(a.value * 100) / 100;
|
|
670
716
|
// Measured but not weighted: they drive findings, not the headline.
|
|
671
717
|
axes.manualOracle = Math.round(manualCompleteness * 100) / 100;
|
|
672
|
-
|
|
718
|
+
// Only a real mismatch scores 0. Without a yardstick `trace.mappedRatio` is 0 because there is
|
|
719
|
+
// nothing to map to, and reporting that as the weakest axis sent authors to "raise taxonomy" —
|
|
720
|
+
// an axis that was never measured.
|
|
721
|
+
if (hasYardstick) axes.taxonomy = taxonomyMismatch ? 0 : Math.round(trace.mappedRatio * 100) / 100;
|
|
673
722
|
const notApplicable = axisDefs.filter((a) => !a.applicable).map((a) => a.key);
|
|
674
723
|
const weakestEntry = Object.entries(axes).sort((a, b) => a[1] - b[1])[0];
|
|
675
724
|
const weakest = { axis: weakestEntry[0], value: weakestEntry[1] };
|
|
@@ -235,8 +235,22 @@ export function flowIdOf(s: ScenarioInfo, declaredPhases: string[]): string | nu
|
|
|
235
235
|
}
|
|
236
236
|
|
|
237
237
|
export interface InventoryResult {
|
|
238
|
-
/**
|
|
239
|
-
|
|
238
|
+
/**
|
|
239
|
+
* Declared → the scenarios claiming it. A declared flow with none is a NAMED gap.
|
|
240
|
+
* `automated` is false when EVERY scenario claiming the flow is `@manual`: the flow is
|
|
241
|
+
* accounted for, but nothing runs — see `manualOnly` (#599).
|
|
242
|
+
*/
|
|
243
|
+
covered: Array<{ id: string; scenarios: string[]; automated: boolean }>;
|
|
244
|
+
/**
|
|
245
|
+
* Declared `covered` and claimed only by `@manual` scenarios. `covered` used to mean three
|
|
246
|
+
* different things at once — a flow with three automated scenarios, one with a single manual
|
|
247
|
+
* procedure, and one whose oracle is still an open question all read as `covered`, and
|
|
248
|
+
* `flowCoverage` reported 100% over the mix. A reader cannot tell which flows would actually
|
|
249
|
+
* fail a regression run.
|
|
250
|
+
*/
|
|
251
|
+
manualOnly: FlowDecl[];
|
|
252
|
+
/** Covered declared flows with at least one AUTOMATED scenario / flows that ought to be. */
|
|
253
|
+
automatedRatio: number;
|
|
240
254
|
/** Declared `covered` but nothing written — the silent-absence case, now named. */
|
|
241
255
|
uncovered: FlowDecl[];
|
|
242
256
|
/** Declared with a non-covered status, carried into the report so it stays visible. */
|
|
@@ -271,9 +285,16 @@ export function flowInventory(contract: FlowContract, scenarios: ScenarioInfo[])
|
|
|
271
285
|
}
|
|
272
286
|
const expected = contract.flows.filter((f) => f.status === 'covered');
|
|
273
287
|
const covered = expected.filter((f) => (byId.get(f.id) ?? []).length > 0)
|
|
274
|
-
.map((f) => ({
|
|
288
|
+
.map((f) => ({
|
|
289
|
+
id: f.id,
|
|
290
|
+
scenarios: byId.get(f.id)!,
|
|
291
|
+
automated: scenarios.some((sc) => flowIdOf(sc, contract.phases) === f.id && !sc.manual),
|
|
292
|
+
}));
|
|
293
|
+
const manualOnly = covered.filter((c) => !c.automated).map((c) => expected.find((f) => f.id === c.id)!);
|
|
275
294
|
return {
|
|
276
295
|
covered,
|
|
296
|
+
manualOnly,
|
|
297
|
+
automatedRatio: expected.length ? covered.filter((c) => c.automated).length / expected.length : 1,
|
|
277
298
|
uncovered: expected.filter((f) => (byId.get(f.id) ?? []).length === 0),
|
|
278
299
|
accounted: contract.flows.filter((f) => f.status !== 'covered'),
|
|
279
300
|
undeclared,
|
package/src/harness/next-step.ts
CHANGED
|
@@ -18,6 +18,7 @@ import * as fs from 'fs';
|
|
|
18
18
|
import * as path from 'path';
|
|
19
19
|
import { reportSlug, featureFilesFor } from './unit-paths';
|
|
20
20
|
import { readTextFile } from './read-text';
|
|
21
|
+
import { checkAssetsDrift } from '../orchestrator/assets-drift';
|
|
21
22
|
|
|
22
23
|
export interface NextStep {
|
|
23
24
|
/** The command to run, as the operator would type it. */
|
|
@@ -47,6 +48,13 @@ export interface UnitState {
|
|
|
47
48
|
findings: string[];
|
|
48
49
|
viewpointBaselineStatus?: string;
|
|
49
50
|
};
|
|
51
|
+
/**
|
|
52
|
+
* The project's AI commands/skills were written by a different sungen build. This belongs in
|
|
53
|
+
* "what to do next" because it silently voids every fix that ships as an INSTRUCTION: a HARD
|
|
54
|
+
* RULE added to `create-test` reaches nobody until `sungen update` runs, and the existing
|
|
55
|
+
* warning is one grey stderr line at the top of every command (#620).
|
|
56
|
+
*/
|
|
57
|
+
assetsDrift?: { recordedVersion: string; runningVersion: string };
|
|
50
58
|
steps: NextStep[];
|
|
51
59
|
}
|
|
52
60
|
|
|
@@ -63,7 +71,7 @@ function readJson(p: string): Record<string, unknown> | null {
|
|
|
63
71
|
function countScenarios(unitDir: string, unit: string): number {
|
|
64
72
|
let n = 0;
|
|
65
73
|
for (const f of featureFilesFor(unitDir, unit)) {
|
|
66
|
-
const text = readTextFile(f)
|
|
74
|
+
const text = fs.existsSync(f) ? readTextFile(f) : '';
|
|
67
75
|
n += (text.match(/^\s*Scenario(?: Outline)?:/gm) ?? []).length;
|
|
68
76
|
}
|
|
69
77
|
return n;
|
|
@@ -84,9 +92,12 @@ export function readUnitState(
|
|
|
84
92
|
try {
|
|
85
93
|
selectorFiles = fs.readdirSync(selectorsDir).filter((f) => f.endsWith('.yaml'));
|
|
86
94
|
} catch { /* api units carry no selectors */ }
|
|
87
|
-
|
|
95
|
+
// `readTextFile` throws on a missing file, and every path here is legitimately absent at some
|
|
96
|
+
// point in a unit's life — the resolver exists to report exactly that, so it must not crash on it.
|
|
97
|
+
const read = (p: string): string => (fs.existsSync(p) ? readTextFile(p) : '');
|
|
98
|
+
const selectorText = selectorFiles.map((f) => read(path.join(selectorsDir, f))).join('\n');
|
|
88
99
|
const contractPath = path.join(req, 'flow-contract.yaml');
|
|
89
|
-
const contractText =
|
|
100
|
+
const contractText = read(contractPath);
|
|
90
101
|
|
|
91
102
|
const specDirs = [
|
|
92
103
|
path.join(projectRoot, 'specs', 'generated', 'flows', unit),
|
|
@@ -125,6 +136,12 @@ export function readUnitState(
|
|
|
125
136
|
selectorsArePlaceholder: selectorFiles.length > 0
|
|
126
137
|
&& (/TODO|PLACEHOLDER|REPLACE ME/i.test(selectorText) || selectorText.trim().length < 40),
|
|
127
138
|
compiled, hasResults, hasDeliverable,
|
|
139
|
+
assetsDrift: (() => {
|
|
140
|
+
const d = checkAssetsDrift(projectRoot);
|
|
141
|
+
return d && d.recordedVersion !== d.runningVersion
|
|
142
|
+
? { recordedVersion: d.recordedVersion, runningVersion: d.runningVersion }
|
|
143
|
+
: undefined;
|
|
144
|
+
})(),
|
|
128
145
|
audit: audit
|
|
129
146
|
? {
|
|
130
147
|
overall: typeof score.overall === 'number' ? score.overall : 0,
|
|
@@ -163,6 +180,29 @@ export function deriveSteps(s: UnitState): NextStep[] {
|
|
|
163
180
|
});
|
|
164
181
|
return out;
|
|
165
182
|
}
|
|
183
|
+
// Stale AI assets come FIRST, because they invalidate the advice below: a HARD RULE added to
|
|
184
|
+
// `create-test` reaches nobody until `sungen update` runs, so the run that follows will repeat
|
|
185
|
+
// whatever the old instructions said. Upgrading the package alone does not refresh them, and the
|
|
186
|
+
// existing warning is one grey stderr line among a command's whole output (#620).
|
|
187
|
+
if (s.assetsDrift) {
|
|
188
|
+
out.push({
|
|
189
|
+
kind: 'do',
|
|
190
|
+
command: 'sungen update',
|
|
191
|
+
because: `this project's AI commands/skills were written by sungen ${s.assetsDrift.recordedVersion} and you are running ${s.assetsDrift.runningVersion} — upgrading the package does NOT refresh them, so any rule added since ${s.assetsDrift.recordedVersion} is not in effect for the next run`,
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
// The yardstick. `atomicLedger` and `traceability` are measured against test-viewpoint.md and
|
|
195
|
+
// four sensors read it, so its absence is not cosmetic — and it can go missing from a unit that
|
|
196
|
+
// already HAS a spec (a run deleted it), which the "no source at all" branch above never sees.
|
|
197
|
+
if (!s.hasViewpoint) {
|
|
198
|
+
out.push({
|
|
199
|
+
kind: 'blocked',
|
|
200
|
+
command: s.kind === 'flow'
|
|
201
|
+
? `sungen add-flow --flow ${s.unit} --path <start-url> # repairs the missing scaffold; never overwrites`
|
|
202
|
+
: `write qa/screens/${s.unit}/requirements/test-viewpoint.md`,
|
|
203
|
+
because: 'requirements/test-viewpoint.md is missing — it is the yardstick `atomicLedger` and `traceability` are measured against, and four sensors read it, so the audit reports VIEWPOINT-MISSING and those axes are excluded. A flow contract does not substitute: it declares the use case, the viewpoint declares the claims',
|
|
204
|
+
});
|
|
205
|
+
}
|
|
166
206
|
if (s.kind === 'flow' && !s.hasContract) {
|
|
167
207
|
out.push({
|
|
168
208
|
kind: 'blocked',
|
package/src/harness/sensors.ts
CHANGED
|
@@ -588,6 +588,19 @@ interface ClaimRule {
|
|
|
588
588
|
|
|
589
589
|
// Ordered by specificity; the first matching rule per scenario wins.
|
|
590
590
|
const CLAIM_RULES: ClaimRule[] = [
|
|
591
|
+
{
|
|
592
|
+
// A continuity claim is an EXACTNESS claim. `text contains {{v}}` passes when the UI
|
|
593
|
+
// concatenates, prefixes or truncates — which is exactly the defect "unchanged" exists to
|
|
594
|
+
// catch, so proving it with a substring proves the opposite of what the title says. Found on
|
|
595
|
+
// a real basic flow whose seven confirmation rows all used `contains` while claiming the six
|
|
596
|
+
// values "survive unchanged" (#599).
|
|
597
|
+
claim: 'unchanged/preserved',
|
|
598
|
+
title: /\b(unchanged|unmodified|un-?edited|identical|preserved|intact|verbatim|as entered|without (?:any )?(?:change|modification))\b/,
|
|
599
|
+
proof: /^(?![\s\S]*\bcontains \{\{)/,
|
|
600
|
+
need: 'exact-value assertions (`see [X] text with {{v}}` / `field with {{v}}`) for the values the title says are unchanged',
|
|
601
|
+
hint: 'swap `contains {{v}}` for `with {{v}}` on the continuity rows. A substring match cannot distinguish "unchanged" from "changed by adding a prefix" — and if the screen genuinely renders extra text around the value, say so in the title instead of claiming exactness.',
|
|
602
|
+
severity: 'warn',
|
|
603
|
+
},
|
|
591
604
|
{
|
|
592
605
|
claim: 'all/every/each',
|
|
593
606
|
title: /\b(all|every|each)\b/,
|
|
@@ -136,3 +136,68 @@ export function browserGestureSubstitutions(
|
|
|
136
136
|
}
|
|
137
137
|
return out;
|
|
138
138
|
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Traceability integrity of the DECLARATION itself — three ways a viewpoint document can be
|
|
142
|
+
* internally wrong while every coverage number still reads well (#599).
|
|
143
|
+
*
|
|
144
|
+
* These are cheap and deterministic, and each was found on a real generated document:
|
|
145
|
+
*
|
|
146
|
+
* - **A reused item id.** `FL-BF-001` labelled four different atomic claims. The ledger then
|
|
147
|
+
* counts four items while traceability sees one id, so coverage arithmetic and "which test
|
|
148
|
+
* proves this?" both go wrong — and neither number looks suspicious.
|
|
149
|
+
* - **A dangling scenario reference.** A `test-data` comment cited `VP-LOGIC-EF07-001` when the
|
|
150
|
+
* suite spells it `VP-SEC-EF07-001`. Nothing breaks at runtime, so the wrong pointer survives
|
|
151
|
+
* every refactor and quietly misleads the next maintainer.
|
|
152
|
+
* - **A uniform priority table.** Every declared phase marked `High` is not a priority, it is a
|
|
153
|
+
* formality: nothing can be selected for a smoke run, and the release conversation has no
|
|
154
|
+
* input. A real ranking has at least two levels.
|
|
155
|
+
*/
|
|
156
|
+
export interface ViewpointIntegrity {
|
|
157
|
+
duplicateIds: Array<{ id: string; count: number; texts: string[] }>;
|
|
158
|
+
danglingRefs: Array<{ ref: string; where: string }>;
|
|
159
|
+
uniformPriority: { levels: string[]; rows: number } | null;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** `VP-…`/`FL-…` ids as cited in prose — the shape scenario titles use. */
|
|
163
|
+
const CITED_ID = /\b((?:VP|FL)-[A-Z][A-Z0-9]*(?:-[A-Z0-9]+)*-\d+)\b/g;
|
|
164
|
+
|
|
165
|
+
export function viewpointIntegrity(
|
|
166
|
+
viewpointPath: string, scenarioIds: string[], extraSources: Array<{ label: string; text: string }> = [],
|
|
167
|
+
): ViewpointIntegrity {
|
|
168
|
+
const items = parseViewpointItems(viewpointPath);
|
|
169
|
+
const byId = new Map<string, string[]>();
|
|
170
|
+
for (const it of items) {
|
|
171
|
+
if (!it.id) continue;
|
|
172
|
+
byId.set(it.id, [...(byId.get(it.id) ?? []), it.text]);
|
|
173
|
+
}
|
|
174
|
+
const duplicateIds = [...byId.entries()]
|
|
175
|
+
.filter(([, texts]) => texts.length > 1)
|
|
176
|
+
.map(([id, texts]) => ({ id, count: texts.length, texts }))
|
|
177
|
+
.sort((a, b) => b.count - a.count);
|
|
178
|
+
|
|
179
|
+
// A cited id must name a scenario that exists. Compared case-insensitively; the viewpoint's own
|
|
180
|
+
// declaration ids are NOT scenario references, so only the extra sources (test-data, spec) are
|
|
181
|
+
// scanned — a viewpoint legitimately declares ids the suite then implements.
|
|
182
|
+
const known = new Set(scenarioIds.map((s) => s.toUpperCase()));
|
|
183
|
+
const danglingRefs: Array<{ ref: string; where: string }> = [];
|
|
184
|
+
for (const src of extraSources) {
|
|
185
|
+
for (const m of src.text.matchAll(CITED_ID)) {
|
|
186
|
+
const ref = m[1].toUpperCase();
|
|
187
|
+
if (known.has(ref)) continue;
|
|
188
|
+
if (danglingRefs.some((d) => d.ref === ref && d.where === src.label)) continue;
|
|
189
|
+
danglingRefs.push({ ref: m[1], where: src.label });
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// The Priority-Viewpoints table: `| <id> | <priority> | <reason> |`.
|
|
194
|
+
// `readTextFile` throws on a missing file, and a unit may legitimately have no viewpoint yet.
|
|
195
|
+
const vpText = fs.existsSync(viewpointPath) ? readTextFile(viewpointPath) : '';
|
|
196
|
+
const rows = [...vpText
|
|
197
|
+
.matchAll(/^\|\s*((?:VP|FL)[A-Z0-9._-]*)\s*\|\s*(critical|high|medium|normal|low|deferred)\s*\|/gim)]
|
|
198
|
+
.map((m) => m[2].toLowerCase());
|
|
199
|
+
const levels = [...new Set(rows)];
|
|
200
|
+
const uniformPriority = rows.length >= 3 && levels.length === 1 ? { levels, rows: rows.length } : null;
|
|
201
|
+
|
|
202
|
+
return { duplicateIds, danglingRefs, uniformPriority };
|
|
203
|
+
}
|
|
@@ -59,8 +59,26 @@ export class FlowManager {
|
|
|
59
59
|
const testDataPath = path.join(testDataDir, `${flowName}.yaml`);
|
|
60
60
|
|
|
61
61
|
if (fs.existsSync(flowDir)) {
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
// Refusing outright left no way back when a REQUIREMENT file went missing: a run deleted
|
|
63
|
+
// `requirements/test-viewpoint.md`, `add-flow` said "already exists", and the only remaining
|
|
64
|
+
// route was writing the scaffold by hand. The viewpoint is the yardstick two scored axes are
|
|
65
|
+
// measured against, so losing it silently costs real coverage evidence (#620).
|
|
66
|
+
//
|
|
67
|
+
// Repair CREATES ONLY WHAT IS MISSING — it never overwrites, so a filled spec or viewpoint is
|
|
68
|
+
// safe. Anything already present is reported as skipped.
|
|
69
|
+
const restored = this.restoreMissingScaffolds(options, flowName, isMobile, {
|
|
70
|
+
requirementsDir, featuresDir, selectorsDir, testDataDir, requirementsUiDir,
|
|
71
|
+
featurePath, selectorPath, testDataPath,
|
|
72
|
+
});
|
|
73
|
+
if (restored.length === 0) {
|
|
74
|
+
console.error(`Error: Flow "${options.name}" already exists at ${flowDir} — and every scaffold file is present, so there is nothing to repair.`);
|
|
75
|
+
process.exit(1);
|
|
76
|
+
}
|
|
77
|
+
console.log(`Flow "${options.name}" already exists — restored ${restored.length} missing file(s):`);
|
|
78
|
+
for (const f of restored) console.log(` + ${path.relative(this.cwd, f)}`);
|
|
79
|
+
console.log('');
|
|
80
|
+
console.log('Nothing else was touched: repair only creates what is absent, never overwrites.');
|
|
81
|
+
return;
|
|
64
82
|
}
|
|
65
83
|
|
|
66
84
|
console.log(`Creating flow: ${options.name}\n`);
|
|
@@ -172,6 +190,38 @@ export class FlowManager {
|
|
|
172
190
|
.replace(/-+/g, '-');
|
|
173
191
|
}
|
|
174
192
|
|
|
193
|
+
/**
|
|
194
|
+
* Re-create only the scaffold files that are ABSENT from an existing flow.
|
|
195
|
+
*
|
|
196
|
+
* The requirement files are not decoration: `requirements/test-viewpoint.md` is the yardstick
|
|
197
|
+
* `atomicLedger` and `traceability` are measured against, and four sensors read it. When a run
|
|
198
|
+
* removed it, `add-flow` refused ("already exists") and there was no CLI route back — so the
|
|
199
|
+
* yardstick stayed gone and the audit's two viewpoint axes stayed excluded (#620).
|
|
200
|
+
*
|
|
201
|
+
* Never overwrites: a filled spec or viewpoint is an INPUT the QA owns.
|
|
202
|
+
*/
|
|
203
|
+
private restoreMissingScaffolds(
|
|
204
|
+
options: FlowOptions, flowName: string, isMobile: boolean,
|
|
205
|
+
paths: {
|
|
206
|
+
requirementsDir: string; featuresDir: string; selectorsDir: string; testDataDir: string;
|
|
207
|
+
requirementsUiDir: string; featurePath: string; selectorPath: string; testDataPath: string;
|
|
208
|
+
},
|
|
209
|
+
): string[] {
|
|
210
|
+
const restored: string[] = [];
|
|
211
|
+
for (const d of [paths.featuresDir, paths.selectorsDir, paths.testDataDir, paths.requirementsUiDir]) {
|
|
212
|
+
if (!fs.existsSync(d)) fs.mkdirSync(d, { recursive: true });
|
|
213
|
+
}
|
|
214
|
+
const ensure = (file: string, body: () => string): void => {
|
|
215
|
+
if (fs.existsSync(file)) return;
|
|
216
|
+
fs.writeFileSync(file, body(), 'utf-8');
|
|
217
|
+
restored.push(file);
|
|
218
|
+
};
|
|
219
|
+
ensure(path.join(paths.requirementsDir, 'spec.md'), () => this.generateSpecTemplate(options, flowName, isMobile));
|
|
220
|
+
ensure(path.join(paths.requirementsDir, 'test-viewpoint.md'), () => this.generateViewpointTemplate(options));
|
|
221
|
+
ensure(paths.featurePath, () => this.generateFeatureTemplate(options, flowName, isMobile));
|
|
222
|
+
return restored;
|
|
223
|
+
}
|
|
224
|
+
|
|
175
225
|
private generateSpecTemplate(options: FlowOptions, flowName: string, isMobile: boolean): string {
|
|
176
226
|
if (isMobile) {
|
|
177
227
|
return `# ${options.name} Flow Specification
|
|
@@ -64,6 +64,23 @@ restated from screen specs the project does not hold, say so: the audit reports
|
|
|
64
64
|
`SPEC-RESTATED-UNVERIFIED` because `specFR 100%` over a hand-copied list certifies the copy, not
|
|
65
65
|
the source.
|
|
66
66
|
|
|
67
|
+
**The contract does NOT replace `test-viewpoint.md` — author BOTH.** They answer different
|
|
68
|
+
questions and only one of them is a yardstick:
|
|
69
|
+
|
|
70
|
+
| | declares | scored against it |
|
|
71
|
+
|---|---|---|
|
|
72
|
+
| `flow-contract.yaml` | the USE CASE — actor · trigger · goal · precondition · both guarantees · the flow inventory | `flowCoverage` |
|
|
73
|
+
| `test-viewpoint.md` | the CLAIMS the suite must prove — cross-screen concerns, edge cases, design decisions, open questions, priorities | `atomicLedger` · `traceability` |
|
|
74
|
+
|
|
75
|
+
If `requirements/test-viewpoint.md` is missing, **write it before generating** (`add-flow` scaffolds
|
|
76
|
+
a template; a run that deletes or replaces it has removed the yardstick). Without it two scored
|
|
77
|
+
axes are excluded and four sensors cannot run at all — the atomic ledger, the
|
|
78
|
+
declaration-integrity checks, the browser-gesture check and the two-way continuity check — so the
|
|
79
|
+
audit reports `VIEWPOINT-MISSING` and the result is *unmeasured*, not passing. And the reason the
|
|
80
|
+
contract cannot stand in: the viewpoint is the one artifact that must stay **independent of what
|
|
81
|
+
you generated**. That is why a filled one is an INPUT you never rewrite, and why authoring it
|
|
82
|
+
FIRST — from the spec, before the scenarios exist — is the only order that keeps it honest.
|
|
83
|
+
|
|
67
84
|
---
|
|
68
85
|
|
|
69
86
|
## Platform detection (do this FIRST)
|
|
@@ -719,6 +719,32 @@ missing flow. `sungen audit` then measures `flowCoverage` per DECLARED FLOW and
|
|
|
719
719
|
`FLOW-UNCOVERED` (declared, nobody wrote it), `FLOW-UNDECLARED` (a scenario claiming a flow id the
|
|
720
720
|
inventory never declares), `FLOW-STATUS-UNREASONED` and `FLOW-INVENTORY-MISSING`.
|
|
721
721
|
|
|
722
|
+
**One id per atomic CLAIM in the viewpoint.** The flow id groups; the viewpoint id identifies.
|
|
723
|
+
Writing `FL-BF-001` on four different claims makes the ledger count four items while traceability
|
|
724
|
+
sees one, so the coverage arithmetic and "which test proves this?" are both wrong and neither
|
|
725
|
+
number looks suspicious (`VIEWPOINT-ID-REUSED`). Number them within the flow —
|
|
726
|
+
`FL-BF-001`, `FL-BF-002`, … — and never cite an id no scenario carries (`TRACE-REF-DANGLING`
|
|
727
|
+
checks the ids mentioned in `test-data/*.yaml` too).
|
|
728
|
+
|
|
729
|
+
**Rank the priorities; do not stamp them.** A Priority-Viewpoints table where every row says
|
|
730
|
+
`High` is a formality, not a priority: nothing can be selected for a smoke run and the release
|
|
731
|
+
conversation has no input (`VIEWPOINT-PRIORITY-UNIFORM`). Rank by business impact and risk — the
|
|
732
|
+
basic flow, data integrity and the auth/context guards are High; a recoverable back-navigation or
|
|
733
|
+
a network blip is usually Normal; an exit path a screen suite already owns is Low.
|
|
734
|
+
|
|
735
|
+
**A continuity claim is an EXACTNESS claim.** A title saying values are *unchanged / preserved /
|
|
736
|
+
intact* must be proven with `see [X] text with {{v}}`, never `contains {{v}}`: a substring match
|
|
737
|
+
passes when the UI concatenates, prefixes or truncates — exactly the defect "unchanged" exists to
|
|
738
|
+
catch, so `contains` proves the opposite of the claim (`CLAIM-WEAK`). If the screen genuinely
|
|
739
|
+
renders extra text around the value, change the title instead of weakening the oracle.
|
|
740
|
+
|
|
741
|
+
**`status: covered` is about the DESIGN, not the evidence.** A flow whose only scenarios are
|
|
742
|
+
`@manual` is accounted for and still has nothing that runs: a regression suite would not notice if
|
|
743
|
+
it broke. The audit reports `FLOW-MANUAL-ONLY` and credits those flows at half in `flowCoverage`,
|
|
744
|
+
so do not treat a manual procedure as equivalent to automation. Check the capability plan for a
|
|
745
|
+
driver that would automate it; if none can, say so plainly in the delivery rather than letting
|
|
746
|
+
`covered` imply a passing test.
|
|
747
|
+
|
|
722
748
|
**One flow, one id — a viewpoint is not a flow.** Several scenarios may prove one flow: give them
|
|
723
749
|
the same flow id and different sequence numbers (`VP-VAL-EF01-001`, `VP-VAL-EF01-002`), never a
|
|
724
750
|
fresh flow id per assertion. Two shapes the audit reports as `FLOW-PHASE-MISFILED`: an `EF` that
|