@sun-asterisk/sungen 3.3.0-alpha.5 → 3.3.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/next.d.ts +3 -2
- package/dist/cli/commands/next.d.ts.map +1 -1
- package/dist/cli/commands/next.js +7 -1
- package/dist/cli/commands/next.js.map +1 -1
- package/dist/cli/commands/triage.d.ts +2 -1
- package/dist/cli/commands/triage.d.ts.map +1 -1
- package/dist/cli/commands/triage.js +42 -2
- package/dist/cli/commands/triage.js.map +1 -1
- package/dist/decisions/decision-schema.d.ts +2 -1
- package/dist/decisions/decision-schema.d.ts.map +1 -1
- package/dist/decisions/decision-schema.js +2 -1
- package/dist/decisions/decision-schema.js.map +1 -1
- package/dist/graph/manual-triage-decisions.d.ts +19 -0
- package/dist/graph/manual-triage-decisions.d.ts.map +1 -0
- package/dist/graph/manual-triage-decisions.js +53 -0
- package/dist/graph/manual-triage-decisions.js.map +1 -0
- package/dist/graph/nodes/test-case-nodes.d.ts +3 -0
- package/dist/graph/nodes/test-case-nodes.d.ts.map +1 -1
- package/dist/graph/nodes/test-case-nodes.js +18 -1
- package/dist/graph/nodes/test-case-nodes.js.map +1 -1
- package/dist/graph/test-case-results.d.ts +1 -0
- package/dist/graph/test-case-results.d.ts.map +1 -1
- package/dist/graph/test-case-results.js +27 -3
- package/dist/graph/test-case-results.js.map +1 -1
- package/dist/graph/test-case-satisfied.d.ts +5 -4
- package/dist/graph/test-case-satisfied.d.ts.map +1 -1
- package/dist/graph/test-case-satisfied.js +4 -0
- package/dist/graph/test-case-satisfied.js.map +1 -1
- package/dist/graph/test-case-state-shape.d.ts +7 -1
- package/dist/graph/test-case-state-shape.d.ts.map +1 -1
- package/dist/graph/unit-results-freshness.d.ts +29 -0
- package/dist/graph/unit-results-freshness.d.ts.map +1 -0
- package/dist/graph/unit-results-freshness.js +213 -0
- package/dist/graph/unit-results-freshness.js.map +1 -0
- package/dist/harness/audit.d.ts +3 -2
- package/dist/harness/audit.d.ts.map +1 -1
- package/dist/harness/audit.js +4 -17
- package/dist/harness/audit.js.map +1 -1
- package/dist/harness/defect-draft.d.ts.map +1 -1
- package/dist/harness/defect-draft.js +12 -2
- package/dist/harness/defect-draft.js.map +1 -1
- package/dist/harness/defects.d.ts +14 -0
- package/dist/harness/defects.d.ts.map +1 -1
- package/dist/harness/defects.js +14 -10
- package/dist/harness/defects.js.map +1 -1
- package/dist/harness/flow-check.d.ts +9 -0
- package/dist/harness/flow-check.d.ts.map +1 -1
- package/dist/harness/flow-check.js +11 -0
- package/dist/harness/flow-check.js.map +1 -1
- package/dist/harness/gates/audit-gate-verdict.d.ts.map +1 -1
- package/dist/harness/gates/audit-gate-verdict.js +2 -1
- package/dist/harness/gates/audit-gate-verdict.js.map +1 -1
- package/dist/harness/gates/audit-input-hash.d.ts +2 -0
- package/dist/harness/gates/audit-input-hash.d.ts.map +1 -1
- package/dist/harness/gates/audit-input-hash.js +30 -2
- package/dist/harness/gates/audit-input-hash.js.map +1 -1
- package/dist/harness/gates/audit-report-verification.js +1 -1
- package/dist/harness/gates/audit-report-verification.js.map +1 -1
- package/dist/harness/gates/delivery-checklist.d.ts +3 -0
- package/dist/harness/gates/delivery-checklist.d.ts.map +1 -1
- package/dist/harness/gates/delivery-checklist.js +28 -3
- package/dist/harness/gates/delivery-checklist.js.map +1 -1
- package/dist/harness/gates/in-run-decision-check.d.ts.map +1 -1
- package/dist/harness/gates/in-run-decision-check.js +14 -1
- package/dist/harness/gates/in-run-decision-check.js.map +1 -1
- package/dist/harness/next-step.d.ts +3 -0
- package/dist/harness/next-step.d.ts.map +1 -1
- package/dist/harness/next-step.js +7 -0
- package/dist/harness/next-step.js.map +1 -1
- package/dist/harness/triage-manual-classification.d.ts +32 -0
- package/dist/harness/triage-manual-classification.d.ts.map +1 -0
- package/dist/harness/triage-manual-classification.js +56 -0
- package/dist/harness/triage-manual-classification.js.map +1 -0
- package/dist/harness/unit-paths.d.ts +8 -0
- package/dist/harness/unit-paths.d.ts.map +1 -1
- package/dist/harness/unit-paths.js +23 -1
- package/dist/harness/unit-paths.js.map +1 -1
- package/dist/hooks/guard-tty-spoof-rule.d.ts +2 -1
- package/dist/hooks/guard-tty-spoof-rule.d.ts.map +1 -1
- package/dist/hooks/guard-tty-spoof-rule.js +3 -2
- package/dist/hooks/guard-tty-spoof-rule.js.map +1 -1
- package/package.json +3 -3
- package/src/cli/commands/next.ts +9 -3
- package/src/cli/commands/triage.ts +38 -2
- package/src/decisions/decision-schema.ts +2 -1
- package/src/graph/manual-triage-decisions.ts +55 -0
- package/src/graph/nodes/test-case-nodes.ts +19 -1
- package/src/graph/test-case-results.ts +25 -4
- package/src/graph/test-case-satisfied.ts +7 -4
- package/src/graph/test-case-state-shape.ts +7 -2
- package/src/graph/unit-results-freshness.ts +165 -0
- package/src/harness/audit.ts +4 -13
- package/src/harness/defect-draft.ts +13 -2
- package/src/harness/defects.ts +22 -9
- package/src/harness/flow-check.ts +11 -0
- package/src/harness/gates/audit-gate-verdict.ts +2 -1
- package/src/harness/gates/audit-input-hash.ts +27 -3
- package/src/harness/gates/audit-report-verification.ts +2 -2
- package/src/harness/gates/delivery-checklist.ts +26 -4
- package/src/harness/gates/in-run-decision-check.ts +15 -1
- package/src/harness/next-step.ts +7 -0
- package/src/harness/triage-manual-classification.ts +76 -0
- package/src/harness/unit-paths.ts +19 -0
- package/src/hooks/guard-tty-spoof-rule.ts +5 -3
|
@@ -7,15 +7,21 @@
|
|
|
7
7
|
* stale the run manifest (`sungen changed`) selects the scenario: its steps/tags, a selector or a
|
|
8
8
|
* test-data key it uses, or a project input changed since the recorded run
|
|
9
9
|
* triage the rows of `.sungen/triage/<unit>.json` for the case, only when that triage is not
|
|
10
|
-
* older than the case's result files (a re-run voids it)
|
|
10
|
+
* older than the case's result files (a re-run voids it) — unless a manual classification
|
|
11
|
+
* in force judged this very red (manual-triage-decisions.ts, #928): that one wins
|
|
12
|
+
* evidence a red case's hash of its red final attempts (file, title, status, start, duration,
|
|
13
|
+
* messages): what a manual classification is bound to
|
|
11
14
|
*/
|
|
15
|
+
import { createHash } from 'crypto';
|
|
12
16
|
import * as fs from 'fs';
|
|
13
17
|
import * as path from 'path';
|
|
18
|
+
import { canonicalJson } from '../decisions/decision-identity';
|
|
14
19
|
import { loadPlaywrightReport } from '../exporters/playwright-report-parser';
|
|
15
20
|
import type { PlaywrightResult } from '../exporters/types';
|
|
16
21
|
import { buildRunManifest, readRunManifest, selectChanged } from '../harness/run-manifest';
|
|
17
22
|
import { readTriageReport } from '../harness/triage-unit';
|
|
18
23
|
import type { TriageClass } from '../harness/triage';
|
|
24
|
+
import { manualClassificationFor, manualClassifications } from './manual-triage-decisions';
|
|
19
25
|
import type { FeatureCase } from './test-case-identity';
|
|
20
26
|
import type { TestCaseResult, TestCaseState } from './test-case-state-shape';
|
|
21
27
|
import { generatedDirs, RESULT_FILE } from './unit-artifact-probes';
|
|
@@ -24,7 +30,7 @@ const RED = new Set<PlaywrightResult['status']>(['failed', 'timedOut', 'interrup
|
|
|
24
30
|
/** The class a case shows when its rows disagree: what needs a human first. */
|
|
25
31
|
const TRIAGE_PRECEDENCE: TriageClass[] = ['product-candidate', 'unclassified', 'test-defect', 'environment', 'known-defect', 'blocked-capability', 'flaky'];
|
|
26
32
|
|
|
27
|
-
export interface CaseOutcome { result: TestCaseResult; failedResults: number; resultFiles: string[]; stale?: string; triage?: TestCaseState['triage'] }
|
|
33
|
+
export interface CaseOutcome { result: TestCaseResult; failedResults: number; resultFiles: string[]; stale?: string; triage?: TestCaseState['triage']; evidence?: string }
|
|
28
34
|
|
|
29
35
|
const ownsTitle = (key: string, title: string): boolean => key === title || key.endsWith(` > ${title}`)
|
|
30
36
|
|| key.startsWith(`${title} — `) || key.includes(` > ${title} — `);
|
|
@@ -63,16 +69,28 @@ function triageRows(projectRoot: string, unitId: string): { at: number; rows: Ar
|
|
|
63
69
|
return { at: report && typeof report.generatedAt === 'string' ? Date.parse(report.generatedAt) : NaN, rows };
|
|
64
70
|
}
|
|
65
71
|
|
|
72
|
+
/** The red final attempts of one case, hashed: the same red reads the same; a re-run (new start, duration) does not. */
|
|
73
|
+
function redEvidence(projectRoot: string, reds: Array<{ file: string; key: string; r: PlaywrightResult }>): string {
|
|
74
|
+
const rows = reds.map(({ file, key, r }) => ({
|
|
75
|
+
file: path.relative(projectRoot, file).split(path.sep).join('/'), key, status: r.status,
|
|
76
|
+
startTime: r.startTime ?? null, duration: r.duration ?? null, messages: r.errorMessages ?? (r.error ? [r.error] : []),
|
|
77
|
+
})).sort((a, b) => (`${a.file}\n${a.key}` < `${b.file}\n${b.key}` ? -1 : 1));
|
|
78
|
+
return createHash('sha256').update(canonicalJson(rows)).digest('hex');
|
|
79
|
+
}
|
|
80
|
+
|
|
66
81
|
export function caseOutcomes(projectRoot: string, unitDir: string, unit: string, unitId: string, cases: FeatureCase[]): CaseOutcome[] {
|
|
67
82
|
const byFeature = resultsByFeature(projectRoot, unit, unitId);
|
|
68
83
|
const stale = staleReasons(projectRoot, unitDir, unit);
|
|
69
84
|
const triage = triageRows(projectRoot, unitId);
|
|
85
|
+
const manual = manualClassifications(projectRoot, unitDir);
|
|
70
86
|
return cases.map((c) => {
|
|
71
87
|
const statuses: PlaywrightResult['status'][] = [];
|
|
72
88
|
const files: string[] = [];
|
|
89
|
+
const reds: Array<{ file: string; key: string; r: PlaywrightResult }> = [];
|
|
73
90
|
for (const { file, report } of byFeature.get(c.featureBase) ?? []) {
|
|
74
|
-
const mine = [...report.entries()].filter(([key]) => ownsTitle(key, c.title))
|
|
75
|
-
if (mine.length) { statuses.push(...mine); files.push(file); }
|
|
91
|
+
const mine = [...report.entries()].filter(([key]) => ownsTitle(key, c.title));
|
|
92
|
+
if (mine.length) { statuses.push(...mine.map(([, r]) => r.status)); files.push(file); }
|
|
93
|
+
for (const [key, r] of mine) if (RED.has(r.status)) reds.push({ file, key, r });
|
|
76
94
|
}
|
|
77
95
|
const result: TestCaseResult = statuses.some((s) => RED.has(s)) ? 'failed'
|
|
78
96
|
: statuses.includes('passed') ? 'passed' : statuses.includes('skipped') ? 'skipped' : 'never-run';
|
|
@@ -81,10 +99,13 @@ export function caseOutcomes(projectRoot: string, unitDir: string, unit: string,
|
|
|
81
99
|
resultFiles: files.map((f) => path.relative(projectRoot, f).split(path.sep).join('/')),
|
|
82
100
|
};
|
|
83
101
|
if (result !== 'never-run' && stale[c.title]) out.stale = stale[c.title];
|
|
102
|
+
if (result === 'failed') out.evidence = redEvidence(projectRoot, reds);
|
|
84
103
|
const current = !Number.isNaN(triage.at) && files.every((f) => mtime(f) <= triage.at);
|
|
85
104
|
const rows = current ? triage.rows.filter((r) => rowOwns(r.scenario, c.title)) : [];
|
|
86
105
|
const worst = TRIAGE_PRECEDENCE.find((cls) => rows.some((r) => r.class === cls));
|
|
87
106
|
if (worst) out.triage = { class: worst, ruleId: rows.find((r) => r.class === worst)!.ruleId };
|
|
107
|
+
const byHand = manualClassificationFor(manual, c.id, out.evidence);
|
|
108
|
+
if (byHand) out.triage = { class: byHand.class, ruleId: 'manual', decision: byHand.decision };
|
|
88
109
|
return out;
|
|
89
110
|
});
|
|
90
111
|
}
|
|
@@ -8,14 +8,15 @@
|
|
|
8
8
|
* trigger — what the case needs otherwise, first match wins:
|
|
9
9
|
* never run → run · result older than the scenario (run manifest) → rerun-stale ·
|
|
10
10
|
* red with no current triage, or left `unclassified` by it → triage (a classification
|
|
11
|
-
* is still owed) · triaged product-candidate → a QA decision
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* is still owed) · triaged product-candidate → a QA decision · triaged environment →
|
|
12
|
+
* rerun-environment (once it is up) · triaged test-defect → fix-test (#928)
|
|
13
|
+
* A pass — Pass (Mocked) included — needs nothing. The class may be `sungen triage`'s or a manual
|
|
14
|
+
* classification in force (manual-triage-decisions.ts); test-case-results.ts decides which.
|
|
14
15
|
*/
|
|
15
16
|
import type { TestCaseState } from './test-case-state-shape';
|
|
16
17
|
|
|
17
18
|
export type TestCaseExemption = 'manual' | 'requires' | 'known-defect' | 'deferred' | 'blocked-capability';
|
|
18
|
-
export type TestCaseNeed = 'run' | 'rerun-stale' | 'triage' | 'product-candidate';
|
|
19
|
+
export type TestCaseNeed = 'run' | 'rerun-stale' | 'triage' | 'product-candidate' | 'rerun-environment' | 'fix-test';
|
|
19
20
|
|
|
20
21
|
const tagged = (tc: TestCaseState, tag: string): boolean => tc.tags.some((t) => t === tag || t.startsWith(`${tag}:`));
|
|
21
22
|
|
|
@@ -36,6 +37,8 @@ export function triggerOf(tc: TestCaseState): TestCaseNeed | null {
|
|
|
36
37
|
if (tc.result === 'failed') {
|
|
37
38
|
if (!tc.triage || tc.triage.class === 'unclassified') return 'triage';
|
|
38
39
|
if (tc.triage.class === 'product-candidate') return 'product-candidate';
|
|
40
|
+
if (tc.triage.class === 'environment') return 'rerun-environment';
|
|
41
|
+
if (tc.triage.class === 'test-defect') return 'fix-test';
|
|
39
42
|
}
|
|
40
43
|
return null;
|
|
41
44
|
}
|
|
@@ -52,8 +52,13 @@ export interface TestCaseState {
|
|
|
52
52
|
resultFiles: string[];
|
|
53
53
|
/** Set when the scenario changed after the run that produced its result (run manifest). */
|
|
54
54
|
stale?: string;
|
|
55
|
-
/**
|
|
56
|
-
|
|
55
|
+
/**
|
|
56
|
+
* The class of a red case: a manual classification in force (`ruleId: 'manual'` and its decision id,
|
|
57
|
+
* #928), else the current `sungen triage` row — absent when neither has judged this result.
|
|
58
|
+
*/
|
|
59
|
+
triage?: { class: TriageClass; ruleId: string; decision?: string };
|
|
60
|
+
/** Hash of the red final attempts the case accounts for (failed only) — a manual classification is bound to it; a re-run changes it. */
|
|
61
|
+
evidence?: string;
|
|
57
62
|
/** Spec anchors the case was generated from (manifest), and whether they moved since. */
|
|
58
63
|
anchors: Array<{ anchor: string; change: 'keep' | 'regenerate' | 'retire' | 'unknown' }>;
|
|
59
64
|
findings: LocatedFinding[];
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which run a unit's results come from, and whether they still describe the suite (#942) — so `next`
|
|
3
|
+
* and the delivery checklist never read a months-old results JSON in silence. Read-only, like the rest
|
|
4
|
+
* of the graph's facts.
|
|
5
|
+
*
|
|
6
|
+
* run each result file's run start, as Playwright records it (`stats.startTime`); a file
|
|
7
|
+
* without one is dated by its own change time, and says so
|
|
8
|
+
* older an input changed after the results it feeds were recorded:
|
|
9
|
+
* per feature its `.feature` and compiled `<base>.spec.ts` ↔ its `<base>-test-result*.json`
|
|
10
|
+
* unit-wide requirements/spec.md ↔ the oldest result file
|
|
11
|
+
* Not contracts.lock.yaml: approving the contract after a run is the normal order, and it
|
|
12
|
+
* changes no test — the contract check judges the lock, not the results.
|
|
13
|
+
* A file's "change time" is its last commit when git tracks it unchanged, else its file
|
|
14
|
+
* time. Results are "recorded" at the later of their run start and their own change time.
|
|
15
|
+
* So a fresh checkout (every file the checkout's time) and results committed in the same
|
|
16
|
+
* commit as the edit they ran against read fresh; an edit, or a commit, after them does not.
|
|
17
|
+
* env SUNGEN_ENV names an env no result file is for: readers fall back to another env's results
|
|
18
|
+
* Advisory: it states what it sees and warns; it decides no step and changes no exit code.
|
|
19
|
+
*/
|
|
20
|
+
import { execFileSync } from 'child_process';
|
|
21
|
+
import * as fs from 'fs';
|
|
22
|
+
import * as path from 'path';
|
|
23
|
+
import { catalogIdFromScreenDir, featureFilesFor } from '../harness/unit-paths';
|
|
24
|
+
import { generatedDirs, RESULT_FILE } from './unit-artifact-probes';
|
|
25
|
+
|
|
26
|
+
export interface ResultsFreshness {
|
|
27
|
+
files: Array<{ file: string; runAt: string; source: 'run' | 'file' }>;
|
|
28
|
+
/** The oldest result file's run — what the counts are at least as old as. */
|
|
29
|
+
oldestRunAt: string;
|
|
30
|
+
newestRunAt: string;
|
|
31
|
+
/** How the oldest run is dated: Playwright's run start, or (none recorded) the file's change time. */
|
|
32
|
+
runAtSource: 'run' | 'file';
|
|
33
|
+
/** Inputs changed after the results they feed were recorded. */
|
|
34
|
+
newerInputs: Array<{ file: string; changedAt: string; source: 'commit' | 'file' }>;
|
|
35
|
+
/** SUNGEN_ENV, when no result file is for it. */
|
|
36
|
+
missingEnv?: string;
|
|
37
|
+
/** One line: what makes these results untrustworthy — absent when nothing does. */
|
|
38
|
+
warning?: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
type Change = { ms: number; source: 'commit' | 'file' };
|
|
42
|
+
/** `next` also runs inside the Claude Code hooks (10 s budget, the contract's git calls included): a slow git dates by file time instead. */
|
|
43
|
+
const GIT_TIMEOUT_MS = 2_000;
|
|
44
|
+
/** Commit times have one-second resolution: an edit and its results in one commit tie. */
|
|
45
|
+
const TIE_MS = 1_000;
|
|
46
|
+
const posix = (p: string): string => p.split(path.sep).join('/');
|
|
47
|
+
const real = (p: string): string => { try { return fs.realpathSync(p); } catch { return path.resolve(p); } };
|
|
48
|
+
const iso = (ms: number): string => new Date(ms).toISOString();
|
|
49
|
+
|
|
50
|
+
/** project root → its git work tree (null outside one), asked once per process: `next` reads every unit. */
|
|
51
|
+
const workTrees = new Map<string, string | null>();
|
|
52
|
+
|
|
53
|
+
function git(cwd: string, args: string[]): string | null {
|
|
54
|
+
try { return execFileSync('git', args, { cwd, stdio: ['ignore', 'pipe', 'ignore'], timeout: GIT_TIMEOUT_MS, maxBuffer: 16 * 1024 * 1024 }).toString('utf8'); } catch { return null; }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* When each file last changed: its last commit when git tracks it and it is unchanged, else its file
|
|
59
|
+
* time (no git, uncommitted, untracked, ignored, or git failed). Two git calls for the whole set.
|
|
60
|
+
*/
|
|
61
|
+
function changeTimes(projectRoot: string, files: string[]): (abs: string) => Change | null {
|
|
62
|
+
const mtime = (abs: string): Change | null => { try { return { ms: fs.statSync(abs).mtimeMs, source: 'file' }; } catch { return null; } };
|
|
63
|
+
if (!workTrees.has(projectRoot)) workTrees.set(projectRoot, git(projectRoot, ['rev-parse', '--show-toplevel'])?.trim() || null);
|
|
64
|
+
const top = workTrees.get(projectRoot);
|
|
65
|
+
if (!top) return mtime;
|
|
66
|
+
const topReal = real(top);
|
|
67
|
+
const rels = files.map((f) => posix(path.relative(topReal, real(f)))).filter((r) => r && !r.startsWith('..'));
|
|
68
|
+
if (!rels.length) return mtime;
|
|
69
|
+
const status = git(topReal, ['status', '--porcelain=v1', '-z', '--untracked-files=all', '--', ...rels]);
|
|
70
|
+
const log = git(topReal, ['log', '-n', '500', '--format=%x01%ct', '--name-only', '--', ...rels]);
|
|
71
|
+
if (status === null || log === null) return mtime;
|
|
72
|
+
const dirty = new Set<string>();
|
|
73
|
+
const entries = status.split('\0');
|
|
74
|
+
for (let i = 0; i < entries.length; i++) {
|
|
75
|
+
const e = entries[i];
|
|
76
|
+
if (e.length < 4) continue;
|
|
77
|
+
dirty.add(e.slice(3));
|
|
78
|
+
if (e[0] === 'R' || e[0] === 'C') i++; // a rename/copy carries its source path next
|
|
79
|
+
}
|
|
80
|
+
const committed = new Map<string, number>();
|
|
81
|
+
let at = 0;
|
|
82
|
+
for (const line of log.split('\n')) {
|
|
83
|
+
if (line.startsWith('\x01')) at = Number(line.slice(1)) * 1000;
|
|
84
|
+
else if (line.trim() && at && !committed.has(line.trim())) committed.set(line.trim(), at);
|
|
85
|
+
}
|
|
86
|
+
return (abs) => {
|
|
87
|
+
const rel = posix(path.relative(topReal, real(abs)));
|
|
88
|
+
const t = committed.get(rel);
|
|
89
|
+
return t !== undefined && !dirty.has(rel) && fs.existsSync(abs) ? { ms: t, source: 'commit' } : mtime(abs);
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function runStartOf(file: string): number | null {
|
|
94
|
+
try {
|
|
95
|
+
const t = (JSON.parse(fs.readFileSync(file, 'utf8')) as { stats?: { startTime?: unknown } })?.stats?.startTime;
|
|
96
|
+
const ms = typeof t === 'string' ? Date.parse(t) : NaN;
|
|
97
|
+
return Number.isNaN(ms) ? null : ms;
|
|
98
|
+
} catch { return null; }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Every result file of the unit, in the directories its readers look in. */
|
|
102
|
+
function resultFilesOf(projectRoot: string, unitDir: string, unit: string): string[] {
|
|
103
|
+
const dirs = [...new Set([...generatedDirs(projectRoot, unit), path.join(projectRoot, 'specs', 'generated', catalogIdFromScreenDir(unitDir))])];
|
|
104
|
+
return dirs.flatMap((d) => { try { return fs.readdirSync(d).filter((f) => RESULT_FILE.test(f)).sort().map((f) => path.join(d, f)); } catch { return []; } });
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** The unit's results, dated, and what they are older than — undefined when the unit has none. */
|
|
108
|
+
export function resultsFreshness(projectRoot: string, unitDir: string, unit: string, env: string | undefined = process.env.SUNGEN_ENV): ResultsFreshness | undefined {
|
|
109
|
+
const results = resultFilesOf(projectRoot, unitDir, unit);
|
|
110
|
+
if (!results.length) return undefined;
|
|
111
|
+
const features = featureFilesFor(unitDir, unit).filter((f) => fs.existsSync(f));
|
|
112
|
+
const specsOf = (base: string): string[] => [...new Set(results.map((r) => path.join(path.dirname(r), `${base}.spec.ts`)))].filter((f) => fs.existsSync(f));
|
|
113
|
+
const unitWide = [path.join(unitDir, 'requirements', 'spec.md')].filter((f) => fs.existsSync(f));
|
|
114
|
+
const baseOf = (f: string): string => path.basename(f).replace(RESULT_FILE, '');
|
|
115
|
+
const bases = [...new Set(results.map(baseOf))];
|
|
116
|
+
const changed = changeTimes(projectRoot, [...results, ...features, ...bases.flatMap(specsOf), ...unitWide]);
|
|
117
|
+
const rel = (f: string): string => posix(path.relative(projectRoot, f));
|
|
118
|
+
|
|
119
|
+
const dated = results.map((file) => {
|
|
120
|
+
const start = runStartOf(file);
|
|
121
|
+
const own = changed(file)?.ms ?? 0;
|
|
122
|
+
return { file, base: baseOf(file), runAt: start ?? own, source: (start !== null ? 'run' : 'file') as 'run' | 'file', recordedAt: Math.max(start ?? 0, own) };
|
|
123
|
+
});
|
|
124
|
+
const oldest = dated.reduce((a, b) => (b.runAt < a.runAt ? b : a));
|
|
125
|
+
const newest = dated.reduce((a, b) => (b.runAt > a.runAt ? b : a));
|
|
126
|
+
const recordedFor = (files: typeof dated): number => Math.min(...files.map((d) => d.recordedAt));
|
|
127
|
+
|
|
128
|
+
const newerInputs: ResultsFreshness['newerInputs'] = [];
|
|
129
|
+
const judge = (input: string, recordedAt: number): void => {
|
|
130
|
+
const c = changed(input);
|
|
131
|
+
if (c && c.ms > recordedAt + TIE_MS) newerInputs.push({ file: rel(input), changedAt: iso(c.ms), source: c.source });
|
|
132
|
+
};
|
|
133
|
+
for (const base of bases) {
|
|
134
|
+
const feeds = recordedFor(dated.filter((d) => d.base === base));
|
|
135
|
+
for (const f of [...features.filter((x) => path.basename(x, '.feature') === base), ...specsOf(base)]) judge(f, feeds);
|
|
136
|
+
}
|
|
137
|
+
for (const f of unitWide) judge(f, recordedFor(dated));
|
|
138
|
+
|
|
139
|
+
const variants = results.map((f) => path.basename(f).replace(/^.*-test-result\.?/, '').replace(/\.?json$/, ''));
|
|
140
|
+
const missingEnv = env && !variants.some((v) => v === env || v.endsWith(`.${env}`)) ? env : undefined;
|
|
141
|
+
const problems = [
|
|
142
|
+
...(newerInputs.length ? [`results are older than ${newerInputs.slice(0, 3).map((i) => `${i.file} (changed ${shortDate(i.changedAt)})`).join(', ')}${newerInputs.length > 3 ? ', …' : ''} — re-run the tests before trusting the counts`] : []),
|
|
143
|
+
...(missingEnv ? [`no results for SUNGEN_ENV=${missingEnv} — readers fall back to ${rel(oldest.file)}`] : []),
|
|
144
|
+
];
|
|
145
|
+
return {
|
|
146
|
+
files: dated.map((d) => ({ file: rel(d.file), runAt: iso(d.runAt), source: d.source })),
|
|
147
|
+
oldestRunAt: iso(oldest.runAt), newestRunAt: iso(newest.runAt), runAtSource: oldest.source,
|
|
148
|
+
newerInputs, ...(missingEnv ? { missingEnv } : {}), ...(problems.length ? { warning: problems.join('; ') } : {}),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** `2026-08-21 03:42Z` */
|
|
153
|
+
export const shortDate = (isoTime: string): string => `${isoTime.slice(0, 16).replace('T', ' ')}Z`;
|
|
154
|
+
|
|
155
|
+
/** `run 2026-08-21 03:42Z (65 days ago)` — the oldest run, and the newest when they are days apart. */
|
|
156
|
+
export function describeResultsRun(f: ResultsFreshness, now = Date.now()): string {
|
|
157
|
+
// A file neither the JSON nor the file system could date reads as epoch 0 — "20000 days ago" is noise.
|
|
158
|
+
if (!(Date.parse(f.oldestRunAt) > 0)) return 'run time unknown (no run start in the JSON, no file time)';
|
|
159
|
+
const days = Math.floor((now - Date.parse(f.oldestRunAt)) / 86_400_000);
|
|
160
|
+
const age = days <= 0 ? 'today' : days === 1 ? '1 day ago' : `${days} days ago`;
|
|
161
|
+
const span = Date.parse(f.newestRunAt) - Date.parse(f.oldestRunAt) >= 86_400_000 ? ` to ${shortDate(f.newestRunAt)}` : '';
|
|
162
|
+
return f.runAtSource === 'run'
|
|
163
|
+
? `run ${shortDate(f.oldestRunAt)} (${age})${span}`
|
|
164
|
+
: `file time ${shortDate(f.oldestRunAt)} (${age}; the JSON records no run start)${span}`;
|
|
165
|
+
}
|
package/src/harness/audit.ts
CHANGED
|
@@ -11,7 +11,7 @@ import * as fs from 'fs';
|
|
|
11
11
|
import { loadUnitScenarios, readUnitFeatureText, parseViewpointOverview, ScenarioInfo, ViewpointEntry } from './parse';
|
|
12
12
|
import { checkViewpointBaseline, ViewpointBaseline } from './viewpoint-baseline';
|
|
13
13
|
import { flowQuality, statefulDepthFor, continuityGaps, flowInventory, misfiledPhases, FlowQualityResult, InventoryResult } from './flow-contract';
|
|
14
|
-
import { featureFilesFor } from './unit-paths';
|
|
14
|
+
import { catalogIdFromScreenDir, featureFilesFor, isScreenUnitId } from './unit-paths';
|
|
15
15
|
import {
|
|
16
16
|
loadCatalog, viewpointGate, assertionDepth, dataThemesFor, depthThresholdFor, coverageBalance, duplicateClusters, scenarioIdReuse, ReusedScenarioId, traceability, claimProof, taxonomyLint,
|
|
17
17
|
automatableManual, flowCoveredThemes, flowRegressionDepth, oracleStrength, declaredPageType,
|
|
@@ -129,17 +129,8 @@ export function projectUnitDirs(projectRoot: string): string[] {
|
|
|
129
129
|
return out;
|
|
130
130
|
}
|
|
131
131
|
|
|
132
|
-
/** The catalog-resolution id for a unit dir
|
|
133
|
-
export
|
|
134
|
-
const parts = screenDir.split(path.sep);
|
|
135
|
-
const qa = parts.lastIndexOf('qa');
|
|
136
|
-
if (qa >= 0) {
|
|
137
|
-
if (parts[qa + 1] === 'api' && parts[qa + 2] === 'flows' && parts[qa + 3]) return `api/flows/${parts[qa + 3]}`;
|
|
138
|
-
if (parts[qa + 1] === 'api' && parts[qa + 2]) return `api/${parts[qa + 2]}`;
|
|
139
|
-
if (parts[qa + 1] === 'flows' && parts[qa + 2]) return `flows/${parts[qa + 2]}`;
|
|
140
|
-
}
|
|
141
|
-
return path.basename(screenDir);
|
|
142
|
-
}
|
|
132
|
+
/** The catalog-resolution id for a unit dir — lives in unit-paths.ts (the inputsHash reads it too); kept exported here. */
|
|
133
|
+
export { catalogIdFromScreenDir };
|
|
143
134
|
|
|
144
135
|
/**
|
|
145
136
|
* The capability that owns SCORING for a unit (AO-1): the first path segment of the unit id when it
|
|
@@ -246,7 +237,7 @@ export function runAudit(screenDir: string, screenName: string): AuditReport {
|
|
|
246
237
|
// gate gap, credit it to the flow instead of double-counting it as a screen gap. Screens only
|
|
247
238
|
// (a flow/api unit is not credited by sibling flows); mutates the gate before coverage is scored.
|
|
248
239
|
const flowCredits: { theme: string; flow: string }[] = [];
|
|
249
|
-
const isScreenUnit =
|
|
240
|
+
const isScreenUnit = isScreenUnitId(catalogScreenName);
|
|
250
241
|
if (isScreenUnit && gate.gaps.length) {
|
|
251
242
|
const flowScenarios = loadFlowScenarios(projectRootFromScreenDir(screenDir));
|
|
252
243
|
if (flowScenarios.length) {
|
|
@@ -20,6 +20,7 @@ import { readCapabilities } from './capability';
|
|
|
20
20
|
import { defectsDir, nextDefectId, readDefects } from './defects';
|
|
21
21
|
import type { TriageRow } from './triage';
|
|
22
22
|
import { readTriageReport, unitSecrets } from './triage-unit';
|
|
23
|
+
import { testCaseStates } from '../graph/test-case-states';
|
|
23
24
|
|
|
24
25
|
export interface DraftResult {
|
|
25
26
|
/** `sungen triage` has not run for this unit. */
|
|
@@ -89,14 +90,24 @@ function render(cwd: string, unit: ResolvedUnit, id: string, row: TriageRow, sc:
|
|
|
89
90
|
return redactSecrets(body, sc?.secrets ?? []);
|
|
90
91
|
}
|
|
91
92
|
|
|
93
|
+
/**
|
|
94
|
+
* The class a triage row stands for now: a classification by hand in force for its case wins (#928) —
|
|
95
|
+
* a red classified `product` by hand is drafted, one a human classified test-defect is not.
|
|
96
|
+
*/
|
|
97
|
+
function effectiveClass(cwd: string, unit: ResolvedUnit): (row: TriageRow) => TriageRow['class'] {
|
|
98
|
+
const byHand = testCaseStates(cwd, unit.unitId, { unitDir: unit.dir, auditFindings: [] }).filter((c) => c.triage?.ruleId === 'manual');
|
|
99
|
+
return (row) => byHand.find((c) => row.scenario === c.title || row.scenario.startsWith(`${c.title} — `))?.triage?.class ?? row.class;
|
|
100
|
+
}
|
|
101
|
+
|
|
92
102
|
export function draftDefects(cwd: string, unit: ResolvedUnit, opts: { scenario?: string } = {}): DraftResult {
|
|
93
103
|
const triage = readTriageReport(cwd, unit.unitId);
|
|
94
104
|
if (!triage) return { missingTriage: true, written: [], skipped: [] };
|
|
95
105
|
const recorded = new Map<string, string>();
|
|
96
106
|
for (const r of readDefects(cwd)) if (r.units.includes(unit.unitId)) for (const s of r.scenarios) recorded.set(s, r.id);
|
|
97
107
|
|
|
98
|
-
const
|
|
99
|
-
|
|
108
|
+
const classOf = effectiveClass(cwd, unit);
|
|
109
|
+
const wanted = triage.rows.map((r): TriageRow => (classOf(r) === r.class ? r : { ...r, class: classOf(r), ruleId: 'manual' }))
|
|
110
|
+
.filter((r) => r.class === 'product-candidate' && (!opts.scenario || r.scenario.includes(opts.scenario) || r.ids.includes(opts.scenario)));
|
|
100
111
|
const scenarios = unitScenarios(cwd, unit);
|
|
101
112
|
const result: DraftResult = { written: [], skipped: [] };
|
|
102
113
|
const seen = new Set<string>();
|
package/src/harness/defects.ts
CHANGED
|
@@ -66,24 +66,37 @@ export function readDefects(cwd: string): DefectRecord[] {
|
|
|
66
66
|
return out;
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
+
/**
|
|
70
|
+
* Everything the defect-link findings of one unit read off qa/defects/ — and so what the audit's
|
|
71
|
+
* inputsHash covers there (#927): the unit's `@known-defect:<id>` tags with no record, in tag order,
|
|
72
|
+
* and the unit's OPEN records with the scenarios they name. Title, severity and prose are not read,
|
|
73
|
+
* so editing them never asks for a re-audit.
|
|
74
|
+
*/
|
|
75
|
+
export interface DefectLinkInputs { missing: string[]; open: Array<{ id: string; scenarios: string[] }> }
|
|
76
|
+
|
|
77
|
+
export function defectLinkInputs(cwd: string, unitId: string, featureText: string): DefectLinkInputs {
|
|
78
|
+
const records = readDefects(cwd);
|
|
79
|
+
const known = new Set(records.map((r) => r.id));
|
|
80
|
+
const tagged = [...new Set([...featureText.matchAll(KNOWN_DEFECT_TAG)].map((m) => m[1]))];
|
|
81
|
+
return {
|
|
82
|
+
missing: tagged.filter((id) => !known.has(id)),
|
|
83
|
+
open: records.filter((r) => r.status === 'open' && r.units.includes(unitId)).map((r) => ({ id: r.id, scenarios: r.scenarios })),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
69
87
|
/**
|
|
70
88
|
* Advisory audit findings for one unit (never scored):
|
|
71
89
|
* DEFECT-REF-MISSING a `@known-defect:DEF-NNN` tag with no record file.
|
|
72
90
|
* DEFECT-ORPHAN (info) an OPEN record of this unit names a scenario the unit no longer has.
|
|
73
91
|
*/
|
|
74
92
|
export function defectLinkFindings(cwd: string, unitId: string, featureText: string, scenarioNames: string[]): string[] {
|
|
75
|
-
const
|
|
93
|
+
const { missing, open } = defectLinkInputs(cwd, unitId, featureText);
|
|
76
94
|
const findings: string[] = [];
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
for (const id of tagged) {
|
|
80
|
-
if (!known.has(id)) {
|
|
81
|
-
findings.push(`DEFECT-REF-MISSING: \`@known-defect:${id}\` has no record in qa/defects/ — draft it with \`sungen defect draft --screen ${unitId}\`, or correct the id.`);
|
|
82
|
-
}
|
|
95
|
+
for (const id of missing) {
|
|
96
|
+
findings.push(`DEFECT-REF-MISSING: \`@known-defect:${id}\` has no record in qa/defects/ — draft it with \`sungen defect draft --screen ${unitId}\`, or correct the id.`);
|
|
83
97
|
}
|
|
84
98
|
const names = new Set(scenarioNames);
|
|
85
|
-
for (const r of
|
|
86
|
-
if (r.status !== 'open' || !r.units.includes(unitId)) continue;
|
|
99
|
+
for (const r of open) {
|
|
87
100
|
for (const s of r.scenarios) {
|
|
88
101
|
if (!names.has(s)) {
|
|
89
102
|
findings.push(`DEFECT-ORPHAN (info): ${r.id} is open and names scenario "${s}", which ${unitId} no longer has — update the record's scenarios, or close it.`);
|
|
@@ -70,6 +70,17 @@ export function loadFlowScenarios(cwd: string): FlowScenario[] {
|
|
|
70
70
|
return out;
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
/**
|
|
74
|
+
* What a screen audit reads from its sibling flows (#927): the scenarios that can credit a gate theme
|
|
75
|
+
* — only deep ones do (flowCoveredThemes) — by flow and haystack. The screen's inputsHash covers them,
|
|
76
|
+
* sorted so the hash does not depend on directory order; a shallow scenario or prose moves nothing.
|
|
77
|
+
*/
|
|
78
|
+
export function flowCreditInputs(cwd: string): Array<{ flow: string; haystack: string }> {
|
|
79
|
+
const key = (s: { flow: string; haystack: string }): string => `${s.flow}\n${s.haystack}`;
|
|
80
|
+
return loadFlowScenarios(cwd).filter((s) => s.deep).map(({ flow, haystack }) => ({ flow, haystack }))
|
|
81
|
+
.sort((a, b) => (key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0));
|
|
82
|
+
}
|
|
83
|
+
|
|
73
84
|
export function buildFlowCheck(cwd: string, onlyFlow?: string): FlowCheckReport {
|
|
74
85
|
const screens = listDirs(path.join(cwd, 'qa', 'screens'));
|
|
75
86
|
const flows = (onlyFlow ? [onlyFlow] : listDirs(path.join(cwd, 'qa', 'flows')));
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
*
|
|
6
6
|
* gateOverrides[] every override touching the unit, applied / ineffective / rejected
|
|
7
7
|
* decisionsHash the overrides it was judged under (stale when they change)
|
|
8
|
-
* inputsHash .feature + spec + viewpoint + decisionsHash
|
|
8
|
+
* inputsHash .feature + spec + viewpoint + … + decisionsHash, and the defect records and
|
|
9
|
+
* crediting flows it reads (audit-input-hash.ts, #927)
|
|
9
10
|
* exitCode 0 on PASS, 2 on FAIL — what `sungen audit` exits with
|
|
10
11
|
*
|
|
11
12
|
* The decision layer adds three hard terms: a rejected override (gate.override-rejected) and a
|
|
@@ -12,14 +12,23 @@
|
|
|
12
12
|
* requirements/flow-contract.yaml · requirements/spec_figma.md · requirements/ui-inventory.md ·
|
|
13
13
|
* requirements/ui/* · qa/context.md (intent focus → thresholds) · qa/capabilities.yaml (platform,
|
|
14
14
|
* enabled drivers → gate sensors)
|
|
15
|
-
*
|
|
15
|
+
* and what it reads outside the unit (#927), as the audit reads it — not the files whole, so a
|
|
16
|
+
* re-audit is asked for only when the verdict can move:
|
|
17
|
+
* defects qa/defects/: the unit's `@known-defect:<id>` tags with no record, and its OPEN records
|
|
18
|
+
* with their scenarios (defectLinkInputs — DEFECT-REF-MISSING, DEFECT-ORPHAN)
|
|
19
|
+
* flowCredit a SCREEN's sibling flows: the deep scenarios that can credit a gate theme
|
|
20
|
+
* (flowCreditInputs — TQ-4, COVERED-VIA-FLOW); a flow or api unit is never credited
|
|
21
|
+
* Each is part of the hash only when it holds something, so a project with neither keeps the hash it
|
|
22
|
+
* had; a report written before #927 in a project with either reads stale once.
|
|
16
23
|
*/
|
|
17
24
|
import { createHash } from 'crypto';
|
|
18
25
|
import * as fs from 'fs';
|
|
19
26
|
import * as path from 'path';
|
|
20
27
|
import { canonicalJson } from '../../decisions/decision-identity';
|
|
28
|
+
import { defectLinkInputs } from '../defects';
|
|
29
|
+
import { flowCreditInputs } from '../flow-check';
|
|
21
30
|
import { projectRootFromScreenDir } from '../intent';
|
|
22
|
-
import { featureFilesFor } from '../unit-paths';
|
|
31
|
+
import { catalogIdFromScreenDir, featureFilesFor, isScreenUnitId } from '../unit-paths';
|
|
23
32
|
|
|
24
33
|
const sha256 = (s: string | Buffer): string => createHash('sha256').update(s).digest('hex');
|
|
25
34
|
|
|
@@ -47,9 +56,24 @@ export function auditInputFiles(unitDir: string, unitName: string): string[] {
|
|
|
47
56
|
return candidates.filter((f) => fs.existsSync(f));
|
|
48
57
|
}
|
|
49
58
|
|
|
59
|
+
/** What the audit reads outside the unit — each key only when it holds something. */
|
|
60
|
+
function crossUnitInputs(root: string, unitDir: string, unitName: string): Record<string, unknown> {
|
|
61
|
+
const unitId = catalogIdFromScreenDir(unitDir);
|
|
62
|
+
const featureText = featureFilesFor(unitDir, unitName).map((f) => { try { return fs.readFileSync(f, 'utf8'); } catch { return ''; } }).join('\n');
|
|
63
|
+
const defects = defectLinkInputs(root, unitId, featureText);
|
|
64
|
+
const flowCredit = isScreenUnitId(unitId) ? flowCreditInputs(root) : [];
|
|
65
|
+
return { ...(defects.missing.length || defects.open.length ? { defects } : {}), ...(flowCredit.length ? { flowCredit } : {}) };
|
|
66
|
+
}
|
|
67
|
+
|
|
50
68
|
export function auditInputsHash(unitDir: string, unitName: string, decisionsHash: string): string {
|
|
51
69
|
const root = projectRootFromScreenDir(unitDir);
|
|
52
70
|
const inputs = auditInputFiles(unitDir, unitName)
|
|
53
71
|
.map((f) => ({ file: path.relative(root, f).split(path.sep).join('/'), sha: fileSha(f) }));
|
|
54
|
-
return sha256(canonicalJson({ schema: 2, inputs, decisionsHash }));
|
|
72
|
+
return sha256(canonicalJson({ schema: 2, inputs, decisionsHash, ...crossUnitInputs(root, unitDir, unitName) }));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** The stale reason when the inputsHash moved — it names every kind of input, the ones outside the unit too. */
|
|
76
|
+
export function auditInputsChangedReason(unitDir: string): string {
|
|
77
|
+
const outside = isScreenUnitId(catalogIdFromScreenDir(unitDir)) ? 'the defect records and sibling flows it reads' : 'the defect records it reads';
|
|
78
|
+
return `an input the audit judged (.feature, spec, viewpoint, context, design, or ${outside}) changed since this audit`;
|
|
55
79
|
}
|
|
@@ -28,7 +28,7 @@ import * as path from 'path';
|
|
|
28
28
|
import { resolveUnitCandidates } from '../../cli/resolve-unit';
|
|
29
29
|
import { reportSlug } from '../unit-paths';
|
|
30
30
|
import { getProvenance } from '../provenance';
|
|
31
|
-
import { auditInputsHash } from './audit-input-hash';
|
|
31
|
+
import { auditInputsChangedReason, auditInputsHash } from './audit-input-hash';
|
|
32
32
|
import { AUDIT_EXIT_CODE } from './audit-gate-verdict';
|
|
33
33
|
import { recomputeAuditGateStatus, termSourcesOf } from './audit-gate-terms';
|
|
34
34
|
import { EMPTY_DECISIONS_HASH } from './decisions-hash';
|
|
@@ -166,7 +166,7 @@ export function verifyAuditReport(projectRoot: string, unit: string): AuditVerif
|
|
|
166
166
|
if (r.decisionsHash !== current) stale.push({ code: 'decisions-changed', message: 'the gate-override decisions changed since this audit' });
|
|
167
167
|
if (!unitDir) return result('invalid', [{ code: 'unit-unresolved', message: `"${unit}" does not name exactly one unit, so the audit's inputs cannot be checked` }]);
|
|
168
168
|
if (r.inputsHash !== auditInputsHash(unitDir, unit, r.decisionsHash as string)) {
|
|
169
|
-
stale.push({ code: 'inputs-changed', message:
|
|
169
|
+
stale.push({ code: 'inputs-changed', message: auditInputsChangedReason(unitDir) });
|
|
170
170
|
}
|
|
171
171
|
if (stale.length) return result('stale', stale);
|
|
172
172
|
const tampered = verdictProblem(r, overrides);
|
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
* script-check runScriptCheck: OK | FAIL (`sungen script-check` 0 | 2)
|
|
7
7
|
* contract check buildContractVerdict: pass | review | fail (`sungen contract check` 0 | 0 | 2)
|
|
8
8
|
* decisions evaluateDeliveryDecisions: problems, in-run decisions (what this delivery checks)
|
|
9
|
-
*
|
|
9
|
+
* results resultsFreshness: the run the counts come from, not older than the .feature,
|
|
10
|
+
* compiled spec or spec.md, and present for SUNGEN_ENV (#942)
|
|
11
|
+
* triage testCaseStates: every red case classified, candidates decided — naming the classified
|
|
12
|
+
* reds still awaiting a test fix or an environment re-run
|
|
10
13
|
* A box that cannot apply is SKIPPED with its reason — never absent, never ticked. The checklist only
|
|
11
14
|
* reports: the gates before it decide whether the delivery runs, so no exit code moves because of it.
|
|
12
15
|
*/
|
|
@@ -15,6 +18,7 @@ import { readTextOrNull } from '../../contract/contract-git';
|
|
|
15
18
|
import { lockPathFor } from '../../contract/contracts-lock-file';
|
|
16
19
|
import { testCaseNeed } from '../../graph/test-case-satisfied';
|
|
17
20
|
import { testCaseStates } from '../../graph/test-case-states';
|
|
21
|
+
import { describeResultsRun, resultsFreshness } from '../../graph/unit-results-freshness';
|
|
18
22
|
import { runScriptCheck, UnitKind } from '../script-check';
|
|
19
23
|
import { auditCounts, auditNotCountedReason, verifyAuditReport } from './audit-report-verification';
|
|
20
24
|
import { evaluateDeliveryDecisions } from './delivery-decision-gate';
|
|
@@ -87,7 +91,21 @@ function decisionsBox(projectRoot: string, u: ChecklistUnit, accepted?: InRunAcc
|
|
|
87
91
|
}
|
|
88
92
|
}
|
|
89
93
|
|
|
90
|
-
|
|
94
|
+
/** Which run the delivered counts come from — and whether the suite moved on since (#942). */
|
|
95
|
+
export function resultsBox(projectRoot: string, u: ChecklistUnit): ChecklistBox {
|
|
96
|
+
try {
|
|
97
|
+
const f = resultsFreshness(projectRoot, u.unitDir, u.unitName);
|
|
98
|
+
if (!f) return box('results', 'skipped', 'no run results: the unit has not run, every case reads Pending');
|
|
99
|
+
const run = `${describeResultsRun(f)}${f.files.length > 1 ? `, ${f.files.length} files` : ''}`;
|
|
100
|
+
return f.warning
|
|
101
|
+
? box('results', 'fail', `${run} — ${f.warning}`)
|
|
102
|
+
: box('results', 'pass', `${run} — not older than the .feature, compiled spec or spec.md`);
|
|
103
|
+
} catch (e) {
|
|
104
|
+
return box('results', 'fail', `could not be dated: ${why(e)}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export function triageBox(projectRoot: string, u: ChecklistUnit): ChecklistBox {
|
|
91
109
|
try {
|
|
92
110
|
const cases = testCaseStates(projectRoot, u.unitName, { unitDir: u.unitDir });
|
|
93
111
|
if (!cases.some((c) => c.result !== 'never-run')) return box('triage', 'skipped', 'no run results to classify');
|
|
@@ -97,14 +115,18 @@ function triageBox(projectRoot: string, u: ChecklistUnit): ChecklistBox {
|
|
|
97
115
|
if (owed.length) return box('triage', 'fail', `${owed.length} red test case(s) not classified — run \`sungen triage ${flag} ${u.unitName}\``);
|
|
98
116
|
if (decide.length) return box('triage', 'fail', `${decide.length} product-defect candidate(s) await a QA decision (\`sungen defect draft ${flag} ${u.unitName}\`)`);
|
|
99
117
|
const reds = cases.filter((c) => c.result === 'failed').length;
|
|
100
|
-
|
|
118
|
+
// Classified is not done: a test defect still needs its fix, an environment red its re-run (#928).
|
|
119
|
+
const waiting = cases.filter((c) => testCaseNeed(c) === 'fix-test' || testCaseNeed(c) === 'rerun-environment');
|
|
120
|
+
const named = waiting.length
|
|
121
|
+
? ` — ${waiting.length} awaiting a fix or a re-run (${waiting.slice(0, 4).map((c) => `${c.id}: ${c.triage?.class}`).join(', ')}${waiting.length > 4 ? ', …' : ''})` : '';
|
|
122
|
+
return box('triage', 'pass', reds ? `${reds} red test case(s), each classified or exempt${named}` : 'no red test case');
|
|
101
123
|
} catch (e) {
|
|
102
124
|
return box('triage', 'fail', `could not be read: ${why(e)}`);
|
|
103
125
|
}
|
|
104
126
|
}
|
|
105
127
|
|
|
106
128
|
export async function unitChecklist(projectRoot: string, u: ChecklistUnit, accepted?: InRunAcceptance): Promise<ChecklistBox[]> {
|
|
107
|
-
return [auditBox(projectRoot, u), await scriptCheckBox(u), contractBox(projectRoot, u), decisionsBox(projectRoot, u, accepted), triageBox(projectRoot, u)];
|
|
129
|
+
return [auditBox(projectRoot, u), await scriptCheckBox(u), contractBox(projectRoot, u), decisionsBox(projectRoot, u, accepted), resultsBox(projectRoot, u), triageBox(projectRoot, u)];
|
|
108
130
|
}
|
|
109
131
|
|
|
110
132
|
const MARK: Record<BoxState, string> = { pass: '[x]', fail: '[ ]', skipped: '[-]' };
|
|
@@ -18,6 +18,12 @@
|
|
|
18
18
|
* file does not vouch for — absent, uncommitted, different, introduced beyond the commits the walk
|
|
19
19
|
* reads (#924) or at a shallow clone's boundary — is judged as any other; in the last two cases an
|
|
20
20
|
* in-run finding says why.
|
|
21
|
+
*
|
|
22
|
+
* A manual triage classification that claims a human typed it (`triage-classification` with
|
|
23
|
+
* `humanConfirmed`, #928) is a claim the entry makes about itself — a script can write one. What backs
|
|
24
|
+
* it is review, so it is in-run while git HEAD does not hold it (and, with no git, always: nothing
|
|
25
|
+
* shows it was reviewed). Judged by HEAD membership only: it is always recorded after the run it
|
|
26
|
+
* judges, so its commit time says nothing.
|
|
21
27
|
*/
|
|
22
28
|
import * as fs from 'fs';
|
|
23
29
|
import * as path from 'path';
|
|
@@ -58,6 +64,9 @@ export function runStartOf(resultsPath: string | null): string | null {
|
|
|
58
64
|
}
|
|
59
65
|
|
|
60
66
|
const isInRunKind = (kind: string): boolean => (IN_RUN_KINDS as readonly string[]).includes(kind);
|
|
67
|
+
/** A triage classification claiming a human typed it (#928): reviewed only once git HEAD holds it. */
|
|
68
|
+
const claimsHumanTriage = (e: { kind: string; payload?: Record<string, unknown> }): boolean =>
|
|
69
|
+
e.kind === 'triage-classification' && e.payload?.humanConfirmed === true;
|
|
61
70
|
|
|
62
71
|
/** In-run decisions among `scopes`, judged against one run start (null: HEAD membership only) and the committed set. */
|
|
63
72
|
export function inRunDecisions(projectRoot: string, scopes: string[], runStart: string | null, committed: Set<string> | null): InRunDecision[] {
|
|
@@ -66,10 +75,15 @@ export function inRunDecisions(projectRoot: string, scopes: string[], runStart:
|
|
|
66
75
|
const migratedReview = committed ? migratedWaiverReview(projectRoot) : null;
|
|
67
76
|
for (const scope of scopes) {
|
|
68
77
|
for (const e of readDecisions(projectRoot, scope).entries) {
|
|
78
|
+
const rel = path.relative(projectRoot, e.path).split(path.sep).join('/');
|
|
79
|
+
if (claimsHumanTriage(e) && !committed?.has(rel)) {
|
|
80
|
+
out.push({ id: e.id, kind: e.kind, path: rel, visibility: e.visibility, why: [committed
|
|
81
|
+
? 'a classification by hand that claims a human typed it, not in git HEAD (never committed through review)'
|
|
82
|
+
: 'a classification by hand that claims a human typed it, and no git to show it was committed through review'] });
|
|
83
|
+
}
|
|
69
84
|
if (!isInRunKind(e.kind)) continue;
|
|
70
85
|
// A migrated waiver re-confirmed on a terminal is replaced: the new decision is judged on its own.
|
|
71
86
|
if (e.superseded_by && e.source?.where === 'migrated') continue;
|
|
72
|
-
const rel = path.relative(projectRoot, e.path).split(path.sep).join('/');
|
|
73
87
|
const why: string[] = [];
|
|
74
88
|
const migrated = migratedReview?.reviewOf(scope, e) ?? null;
|
|
75
89
|
if (migrated?.at) {
|