@sun-asterisk/sungen 3.3.0-alpha.2 → 3.3.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/contract.d.ts +3 -0
- package/dist/cli/commands/contract.d.ts.map +1 -1
- package/dist/cli/commands/contract.js +34 -4
- package/dist/cli/commands/contract.js.map +1 -1
- package/dist/contract/accept-opaque-steps.d.ts +3 -0
- package/dist/contract/accept-opaque-steps.d.ts.map +1 -0
- package/dist/contract/accept-opaque-steps.js +114 -0
- package/dist/contract/accept-opaque-steps.js.map +1 -0
- package/dist/contract/approve-contract.d.ts +8 -1
- package/dist/contract/approve-contract.d.ts.map +1 -1
- package/dist/contract/approve-contract.js +6 -0
- package/dist/contract/approve-contract.js.map +1 -1
- package/dist/contract/build-contract-verdict.js +2 -2
- package/dist/contract/build-contract-verdict.js.map +1 -1
- package/dist/contract/build-trace-conformance.d.ts +5 -1
- package/dist/contract/build-trace-conformance.d.ts.map +1 -1
- package/dist/contract/build-trace-conformance.js +9 -4
- package/dist/contract/build-trace-conformance.js.map +1 -1
- package/dist/contract/classify-contract-change.d.ts +3 -2
- package/dist/contract/classify-contract-change.d.ts.map +1 -1
- package/dist/contract/classify-contract-change.js.map +1 -1
- package/dist/contract/classify-trace-conformance.d.ts +18 -3
- package/dist/contract/classify-trace-conformance.d.ts.map +1 -1
- package/dist/contract/classify-trace-conformance.js +64 -17
- package/dist/contract/classify-trace-conformance.js.map +1 -1
- package/dist/contract/contract-schema.d.ts +21 -0
- package/dist/contract/contract-schema.d.ts.map +1 -1
- package/dist/contract/contract-schema.js +25 -3
- package/dist/contract/contract-schema.js.map +1 -1
- package/dist/contract/contracts-lock-file.d.ts.map +1 -1
- package/dist/contract/contracts-lock-file.js +1 -0
- package/dist/contract/contracts-lock-file.js.map +1 -1
- package/dist/contract/project-contract-blockers.d.ts.map +1 -1
- package/dist/contract/project-contract-blockers.js +4 -2
- package/dist/contract/project-contract-blockers.js.map +1 -1
- package/dist/contract/record-contract-approval.d.ts +16 -1
- package/dist/contract/record-contract-approval.d.ts.map +1 -1
- package/dist/contract/record-contract-approval.js +29 -1
- package/dist/contract/record-contract-approval.js.map +1 -1
- package/dist/contract/render-free-gherkin.d.ts +2 -0
- package/dist/contract/render-free-gherkin.d.ts.map +1 -1
- package/dist/contract/render-free-gherkin.js +6 -0
- package/dist/contract/render-free-gherkin.js.map +1 -1
- package/dist/contract/trace/collect-trace-evidence.d.ts +8 -3
- package/dist/contract/trace/collect-trace-evidence.d.ts.map +1 -1
- package/dist/contract/trace/collect-trace-evidence.js +65 -34
- package/dist/contract/trace/collect-trace-evidence.js.map +1 -1
- package/dist/contract/trace/read-file-once.d.ts +20 -0
- package/dist/contract/trace/read-file-once.d.ts.map +1 -0
- package/dist/contract/trace/read-file-once.js +135 -0
- package/dist/contract/trace/read-file-once.js.map +1 -0
- package/dist/contract/trace/select-final-attempt.d.ts +2 -2
- package/dist/contract/trace/select-final-attempt.d.ts.map +1 -1
- package/dist/contract/trace/select-final-attempt.js +3 -3
- package/dist/contract/trace/select-final-attempt.js.map +1 -1
- package/dist/contract/trace/trace-file-reader.d.ts +7 -1
- package/dist/contract/trace/trace-file-reader.d.ts.map +1 -1
- package/dist/contract/trace/trace-file-reader.js +17 -41
- package/dist/contract/trace/trace-file-reader.js.map +1 -1
- package/dist/contract/trace/trace-run-record.d.ts +2 -2
- package/dist/contract/trace/trace-run-record.d.ts.map +1 -1
- package/dist/contract/trace/trace-run-record.js +3 -3
- package/dist/contract/trace/trace-run-record.js.map +1 -1
- package/dist/contract/verify-approval-decisions.d.ts.map +1 -1
- package/dist/contract/verify-approval-decisions.js +9 -4
- package/dist/contract/verify-approval-decisions.js.map +1 -1
- package/dist/decisions/git-head-trust.d.ts +12 -0
- package/dist/decisions/git-head-trust.d.ts.map +1 -1
- package/dist/decisions/git-head-trust.js +52 -0
- package/dist/decisions/git-head-trust.js.map +1 -1
- package/dist/decisions/legacy-waiver-migration.d.ts +6 -1
- package/dist/decisions/legacy-waiver-migration.d.ts.map +1 -1
- package/dist/decisions/legacy-waiver-migration.js +8 -2
- package/dist/decisions/legacy-waiver-migration.js.map +1 -1
- package/dist/harness/gates/in-run-decision-check.d.ts.map +1 -1
- package/dist/harness/gates/in-run-decision-check.js +19 -11
- package/dist/harness/gates/in-run-decision-check.js.map +1 -1
- package/dist/harness/gates/migrated-waiver-import-bounds.d.ts +13 -0
- package/dist/harness/gates/migrated-waiver-import-bounds.d.ts.map +1 -0
- package/dist/harness/gates/migrated-waiver-import-bounds.js +141 -0
- package/dist/harness/gates/migrated-waiver-import-bounds.js.map +1 -0
- package/dist/harness/gates/migrated-waiver-review.d.ts +14 -4
- package/dist/harness/gates/migrated-waiver-review.d.ts.map +1 -1
- package/dist/harness/gates/migrated-waiver-review.js +78 -80
- package/dist/harness/gates/migrated-waiver-review.js.map +1 -1
- package/package.json +3 -3
- package/src/cli/commands/contract.ts +35 -4
- package/src/contract/accept-opaque-steps.ts +114 -0
- package/src/contract/approve-contract.ts +10 -3
- package/src/contract/build-contract-verdict.ts +2 -2
- package/src/contract/build-trace-conformance.ts +8 -3
- package/src/contract/classify-contract-change.ts +3 -3
- package/src/contract/classify-trace-conformance.ts +76 -22
- package/src/contract/contract-schema.ts +31 -2
- package/src/contract/contracts-lock-file.ts +1 -0
- package/src/contract/project-contract-blockers.ts +4 -2
- package/src/contract/record-contract-approval.ts +41 -1
- package/src/contract/render-free-gherkin.ts +5 -0
- package/src/contract/trace/collect-trace-evidence.ts +66 -29
- package/src/contract/trace/read-file-once.ts +95 -0
- package/src/contract/trace/select-final-attempt.ts +3 -3
- package/src/contract/trace/trace-file-reader.ts +17 -6
- package/src/contract/trace/trace-run-record.ts +3 -3
- package/src/contract/verify-approval-decisions.ts +9 -5
- package/src/decisions/git-head-trust.ts +44 -0
- package/src/decisions/legacy-waiver-migration.ts +9 -2
- package/src/harness/gates/in-run-decision-check.ts +16 -12
- package/src/harness/gates/migrated-waiver-import-bounds.ts +103 -0
- package/src/harness/gates/migrated-waiver-review.ts +90 -37
|
@@ -11,25 +11,31 @@
|
|
|
11
11
|
* 5. read everything AGAIN: if the feature or the lock moved while the human was reading, what
|
|
12
12
|
* they confirmed is not what would be written — refuse (L5)
|
|
13
13
|
* 6. record the decision and the lock (record-contract-approval.ts)
|
|
14
|
+
*
|
|
15
|
+
* `--accept-opaque` (with `--scenario`) is the other human act of this command: accepting the reviewed
|
|
16
|
+
* opaque trace steps of one approved free / impl:code scenario — accept-opaque-steps.ts (#923).
|
|
14
17
|
*/
|
|
15
18
|
import { ConfirmationIO, formatCommand, requireTypedConfirmation } from '../decisions/human-confirmation';
|
|
16
19
|
import { gitUserName } from '../decisions/decision-source';
|
|
17
20
|
import { classifyScenario } from './classify-contract-change';
|
|
18
21
|
import { canonicalJson } from './contract-hash';
|
|
19
|
-
import {
|
|
22
|
+
import { acceptOpaqueSteps } from './accept-opaque-steps';
|
|
23
|
+
import { ContractScenario, isHumanApproved, OpaqueStep } from './contract-schema';
|
|
20
24
|
import { resolveContractUnit } from './contract-units';
|
|
21
25
|
import { lockPathFor, ReadLock, readLockFile } from './contracts-lock-file';
|
|
22
26
|
import { dropProposedTags } from './free-feature-blocks';
|
|
23
27
|
import { contractFeatureFiles, loadContractScenarios } from './normalize-feature-steps';
|
|
24
28
|
import { ApprovalWrite, checkApprovalSecrets, recordContractApproval } from './record-contract-approval';
|
|
25
29
|
|
|
26
|
-
export interface ApproveInput { projectRoot: string; unit: string; scenario?: string; reason?: string; io: ConfirmationIO }
|
|
30
|
+
export interface ApproveInput { projectRoot: string; unit: string; scenario?: string; reason?: string; io: ConfirmationIO; acceptOpaque?: boolean }
|
|
27
31
|
export interface ApproveResult {
|
|
28
|
-
status: 'approved' | 'nothing-to-approve';
|
|
32
|
+
status: 'approved' | 'nothing-to-approve' | 'accepted' | 'nothing-to-accept';
|
|
29
33
|
lockPath: string;
|
|
30
34
|
decisionId?: string;
|
|
31
35
|
approved: string[];
|
|
32
36
|
removed: string[];
|
|
37
|
+
/** `--accept-opaque`: the scenario's opaque steps, all accepted by the new decision. */
|
|
38
|
+
accepted?: OpaqueStep[];
|
|
33
39
|
}
|
|
34
40
|
|
|
35
41
|
interface Plan { write: ApprovalWrite; lock: ReadLock | null; fingerprint: string }
|
|
@@ -62,6 +68,7 @@ function describe(sc: ContractScenario, lock: ReadLock | null): string {
|
|
|
62
68
|
}
|
|
63
69
|
|
|
64
70
|
export async function approveContract(i: ApproveInput): Promise<ApproveResult> {
|
|
71
|
+
if (i.acceptOpaque) return acceptOpaqueSteps(i);
|
|
65
72
|
const before = plan(i);
|
|
66
73
|
const { approve, removed } = before.write;
|
|
67
74
|
const lockPath = lockPathFor(before.write.unitDir);
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* (added · changed · removed vs HEAD, tampered, unverified), or an APPROVED free /
|
|
10
10
|
* impl:code scenario with no admissible trace (unproven, also past the deadline) exit 2
|
|
11
11
|
* review any hash-changed (changed, not weaker — QA re-approves), opaque (a check no code
|
|
12
|
-
* can compare, or a target identity that is only `low`), or inconclusive (an
|
|
12
|
+
* can compare and QA has not accepted, or a target identity that is only `low`), or inconclusive (an
|
|
13
13
|
* approved free scenario's final attempt did not pass); blocks delivery exit 0
|
|
14
14
|
* pass only match / strengthened / proposed / nondeterministic exit 0
|
|
15
15
|
*
|
|
@@ -64,7 +64,7 @@ function withTrace(results: ScenarioResult[], outcomes: Map<string, TraceOutcome
|
|
|
64
64
|
const o = outcomes.get(r.id);
|
|
65
65
|
if (!o) return r;
|
|
66
66
|
const cls = severityOf(o.class) > severityOf(r.class) ? o.class : r.class;
|
|
67
|
-
return { ...r, class: cls, findings: [...r.findings, ...o.findings], trace: { hashes: o.hashes, runs: o.runs } };
|
|
67
|
+
return { ...r, class: cls, findings: [...r.findings, ...o.findings], trace: { hashes: o.hashes, runs: o.runs, ...(o.opaque.length ? { opaque: o.opaque } : {}) } };
|
|
68
68
|
});
|
|
69
69
|
}
|
|
70
70
|
|
|
@@ -38,11 +38,16 @@ const sha = (s: string | Buffer): string => createHash('sha256').update(s).diges
|
|
|
38
38
|
const why = (e: unknown): string => (e instanceof Error ? e.message : String(e)).split('\n')[0].slice(0, 300);
|
|
39
39
|
|
|
40
40
|
export const MOBILE_NOT_SUPPORTED = 'trace conformance is web-only: free / impl:code scenarios of a mobile project are not supported yet (not judged)';
|
|
41
|
-
/** The guard
|
|
41
|
+
/** The Claude Code guard's deadline for the trace part of one unit's verdict (its hook has ~5 s in all). */
|
|
42
|
+
export const GUARD_TRACE_BUDGET_MS = 3_000;
|
|
43
|
+
/** The default deadline (`next`, the guard) — override: SUNGEN_TRACE_VERDICT_BUDGET_MS. */
|
|
42
44
|
export function defaultTraceBudgetMs(env: Record<string, string | undefined> = process.env): number {
|
|
43
45
|
const v = Number(env.SUNGEN_TRACE_VERDICT_BUDGET_MS);
|
|
44
|
-
return Number.isFinite(v) && v > 0 ? v :
|
|
46
|
+
return Number.isFinite(v) && v > 0 ? v : GUARD_TRACE_BUDGET_MS;
|
|
45
47
|
}
|
|
48
|
+
/** Inside the guard path the override may only SHORTEN the deadline (#926): the hook cannot wait longer. */
|
|
49
|
+
export const guardTraceBudgetMs = (env: Record<string, string | undefined> = process.env): number =>
|
|
50
|
+
Math.min(defaultTraceBudgetMs(env), GUARD_TRACE_BUDGET_MS);
|
|
46
51
|
|
|
47
52
|
export function traceConformanceForUnit(
|
|
48
53
|
projectRoot: string, unit: { name: string; dir: string }, current: ContractScenario[], lock: ContractLock | null,
|
|
@@ -61,7 +66,7 @@ export function traceConformanceForUnit(
|
|
|
61
66
|
let setupError = '';
|
|
62
67
|
try {
|
|
63
68
|
const notBefore = Math.max(mtimeMs(lockPathFor(unit.dir)), ...contractFeatureFiles(unit.dir, unit.name).map(mtimeMs));
|
|
64
|
-
evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), notBefore, deadline);
|
|
69
|
+
evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), notBefore, deadline, new Map(traced.map((s) => [s.id, s.source.name])));
|
|
65
70
|
index = loadTestDataIndex(unit.dir, unit.name);
|
|
66
71
|
} catch (e) { setupError = `trace conformance could not run: ${why(e)}`; }
|
|
67
72
|
const { cache, problem } = readIdentityCache(projectRoot);
|
|
@@ -14,12 +14,12 @@
|
|
|
14
14
|
* still `hash-changed`: equal hashes are the only road to `match`.
|
|
15
15
|
*/
|
|
16
16
|
import { sameJson } from './contract-hash';
|
|
17
|
-
import { ContractScenario, ContractStep, FindingClass, LadderClass, LockEntry, RULE_OF, ScenarioClass, TraceClass } from './contract-schema';
|
|
17
|
+
import { ContractScenario, ContractStep, FindingClass, LadderClass, LockEntry, OpaqueStep, RULE_OF, ScenarioClass, TraceClass } from './contract-schema';
|
|
18
18
|
import { compareAsserts } from './oracle-strength';
|
|
19
19
|
|
|
20
20
|
export interface ContractFinding { class: FindingClass; rule?: string; detail: string; step?: number }
|
|
21
|
-
/** What a free / impl:code scenario's traces fingerprinted to (DOS-14.6) — output only. */
|
|
22
|
-
export interface TraceSummary { hashes: string[]; runs: string[] }
|
|
21
|
+
/** What a free / impl:code scenario's traces fingerprinted to (DOS-14.6), and their opaque steps (#923) — output only. */
|
|
22
|
+
export interface TraceSummary { hashes: string[]; runs: string[]; opaque?: OpaqueStep[] }
|
|
23
23
|
export interface ScenarioResult { id: string; class: ScenarioClass; findings: ContractFinding[]; file?: string; trace?: TraceSummary }
|
|
24
24
|
|
|
25
25
|
const SEVERITY: Record<ScenarioClass, number> = {
|
|
@@ -7,26 +7,43 @@
|
|
|
7
7
|
* with that step — identically (fine), weaker (hard), stronger (strengthened), changed
|
|
8
8
|
* (review); an approved check with no group or no check in its group → assertion-lost
|
|
9
9
|
* overlays a step whose target identity is `low` → opaque (review, never match, red-team #4);
|
|
10
|
-
* an opaque step → opaque with its verbatim text and file:line (contract.opaque-step)
|
|
10
|
+
* an opaque step → opaque with its verbatim text and file:line (contract.opaque-step) —
|
|
11
|
+
* unless the approved entry lists it as accepted by QA at its sha (#923, opaqueStepSha):
|
|
12
|
+
* then a note, not a finding. impl:code: a test.step whose only candidates are opaque is
|
|
13
|
+
* left to the overlay (judged on the ladder it could only read "changed"), which names the
|
|
14
|
+
* approved step the opaque one replaces
|
|
11
15
|
* runs no admissible trace → unproven (all inconclusive → inconclusive) when APPROVED, a
|
|
12
16
|
* warning while proposed (M5, M6); a trace refused beside an admissible one → a warning;
|
|
13
17
|
* several admissible traces that fingerprint differently → nondeterministic (worst wins)
|
|
14
18
|
*/
|
|
15
19
|
import { classifyScenario, ContractFinding, finding, ScenarioResult, severityOf } from './classify-contract-change';
|
|
16
|
-
import { sameJson } from './contract-hash';
|
|
17
|
-
import type { ContractStep, Implementation, LockEntry, ScenarioClass } from './contract-schema';
|
|
20
|
+
import { canonicalJson, sameJson, sha256Hex } from './contract-hash';
|
|
21
|
+
import type { AcceptedOpaqueStep, ContractStep, Implementation, LockEntry, OpaqueStep, ScenarioClass } from './contract-schema';
|
|
18
22
|
import { compareAsserts } from './oracle-strength';
|
|
23
|
+
import { clip, RenderedStep } from './render-free-gherkin';
|
|
19
24
|
import type { TraceRun } from './trace/collect-trace-evidence';
|
|
20
25
|
import type { TraceFingerprint } from './trace/trace-fingerprint';
|
|
21
26
|
|
|
22
|
-
/**
|
|
23
|
-
|
|
27
|
+
/**
|
|
28
|
+
* `warnings` never change the verdict: a not-yet-approved scenario's trace problems, unused traces, an
|
|
29
|
+
* accepted opaque step. `opaque`: every opaque step of the admissible traces, accepted or not.
|
|
30
|
+
*/
|
|
31
|
+
export interface TraceOutcome { class: ScenarioClass; findings: ContractFinding[]; hashes: string[]; runs: string[]; warnings: string[]; opaque: OpaqueStep[] }
|
|
24
32
|
|
|
25
33
|
/** What an unjudgeable scenario becomes: hard when QA locked it (approved), a warning while it is proposed. */
|
|
26
34
|
export function notJudged(id: string, approved: boolean, cls: 'unproven' | 'inconclusive', reasons: string[]): TraceOutcome {
|
|
27
35
|
return approved
|
|
28
|
-
? { class: cls, findings: reasons.map((r) => finding(cls, r)), hashes: [], runs: [], warnings: [] }
|
|
29
|
-
: { class: 'proposed', findings: [], hashes: [], runs: [], warnings: reasons.map((r) => `${id} (not approved yet, so a warning): ${r}`) };
|
|
36
|
+
? { class: cls, findings: reasons.map((r) => finding(cls, r)), hashes: [], runs: [], warnings: [], opaque: [] }
|
|
37
|
+
: { class: 'proposed', findings: [], hashes: [], runs: [], warnings: reasons.map((r) => `${id} (not approved yet, so a warning): ${r}`), opaque: [] };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The sha a QA acceptance binds to (#923): the opaque step as rendered — its verbatim line or request,
|
|
42
|
+
* check or step, the `test.step` it sits in — and as normalized, with the implementation path. Any
|
|
43
|
+
* change to the step gives another sha, so the old acceptance no longer covers it.
|
|
44
|
+
*/
|
|
45
|
+
export function opaqueStepSha(implementation: Implementation, r: RenderedStep, step: ContractStep | undefined): string {
|
|
46
|
+
return sha256Hex(canonicalJson({ implementation, kind: r.kind, group: r.group ?? null, verbatim: r.opaque?.verbatim ?? null, step: step ?? null })).slice(0, 16);
|
|
30
47
|
}
|
|
31
48
|
|
|
32
49
|
/** How a low target identity opens its finding — `next` reads it to offer `contract check --trace`. */
|
|
@@ -35,19 +52,41 @@ export const LOW_IDENTITY_FINDING = 'target identity is low';
|
|
|
35
52
|
const worstOf = (findings: ContractFinding[], start: ScenarioClass = 'match'): ScenarioClass =>
|
|
36
53
|
findings.reduce<ScenarioClass>((acc, f) => (severityOf(f.class as ScenarioClass) > severityOf(acc) ? f.class as ScenarioClass : acc), start);
|
|
37
54
|
|
|
38
|
-
|
|
39
|
-
|
|
55
|
+
type Judged = { findings: ContractFinding[]; opaque: OpaqueStep[] };
|
|
56
|
+
|
|
57
|
+
/** An approved step as a reader sees it: `see [My Account] heading with text "My Account"`. */
|
|
58
|
+
function approvedStepText(s: ContractStep): string {
|
|
59
|
+
const refs = [s.target?.ref, ...(s.target?.also ?? [])].filter((r): r is string => r !== undefined);
|
|
60
|
+
const values = (s.kind === 'assert' ? s.expected : s.value)?.split(' | ') ?? [];
|
|
61
|
+
let r = 0;
|
|
62
|
+
let v = 0;
|
|
63
|
+
return s.verb.replace(/\[\]/g, () => `[${refs[r++] ?? ''}]`).replace(/\{\}/g, () => (v < values.length ? `"${values[v++]}"` : '{}'));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function overlays(
|
|
67
|
+
fp: TraceFingerprint, implementation: Implementation, accepted: readonly AcceptedOpaqueStep[], used?: Set<number>, replaces?: ReadonlyMap<number, string>,
|
|
68
|
+
): Judged {
|
|
69
|
+
const findings: ContractFinding[] = [];
|
|
70
|
+
const opaque: OpaqueStep[] = [];
|
|
40
71
|
fp.rendered.forEach((r, i) => {
|
|
41
72
|
if (used && !used.has(i)) return;
|
|
42
|
-
if (r.opaque)
|
|
43
|
-
|
|
73
|
+
if (r.opaque) {
|
|
74
|
+
const sha = opaqueStepSha(implementation, r, fp.steps[i]);
|
|
75
|
+
const isAccepted = accepted.some((a) => a.step === i + 1 && a.sha === sha);
|
|
76
|
+
const instead = replaces?.get(i);
|
|
77
|
+
opaque.push({ step: i + 1, sha, kind: r.kind, verbatim: r.opaque.verbatim, ...(r.opaque.where ? { where: r.opaque.where } : {}), ...(instead ? { replaces: instead } : {}), accepted: isAccepted });
|
|
78
|
+
if (!isAccepted) {
|
|
79
|
+
findings.push(finding('opaque', `"${r.opaque.verbatim}"${r.opaque.where ? ` at ${r.opaque.where}` : ''} — ${r.kind === 'assert' ? 'a check' : 'a step'} sungen cannot compare; QA reviews it${instead ? ` — replaces approved ${instead}` : ''}`, i + 1));
|
|
80
|
+
}
|
|
81
|
+
} else if (r.low) findings.push(finding('opaque', `${LOW_IDENTITY_FINDING} (${r.low}) — cannot prove "${r.sentence}" acts on the approved element`, i + 1));
|
|
44
82
|
});
|
|
45
|
-
return
|
|
83
|
+
return { findings, opaque };
|
|
46
84
|
}
|
|
47
85
|
|
|
48
|
-
function implCodeFindings(entry: LockEntry, fp: TraceFingerprint): { findings: ContractFinding[]; used: Set<number> } {
|
|
86
|
+
function implCodeFindings(entry: LockEntry, fp: TraceFingerprint): { findings: ContractFinding[]; used: Set<number>; replaces: Map<number, string> } {
|
|
49
87
|
const findings: ContractFinding[] = [];
|
|
50
88
|
const used = new Set<number>();
|
|
89
|
+
const replaces = new Map<number, string>();
|
|
51
90
|
const taken = new Set<number>();
|
|
52
91
|
entry.steps.forEach((locked, i) => {
|
|
53
92
|
const g = fp.groups.findIndex((x, gi) => !taken.has(gi) && sameJson(x.titleStep, locked));
|
|
@@ -68,6 +107,13 @@ function implCodeFindings(entry: LockEntry, fp: TraceFingerprint): { findings: C
|
|
|
68
107
|
return;
|
|
69
108
|
}
|
|
70
109
|
candidates.forEach((m) => used.add(m));
|
|
110
|
+
// Only steps sungen cannot compare implement this approved step: the overlay sends them to QA's
|
|
111
|
+
// review (#923), naming the approved step. On the ladder they could only read "changed", which
|
|
112
|
+
// re-approving never clears.
|
|
113
|
+
if (candidates.every((m) => fp.rendered[m].opaque)) {
|
|
114
|
+
for (const m of candidates) replaces.set(m, `step ${i + 1} "${clip(approvedStepText(locked))}"`);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
71
117
|
if (locked.kind === 'action') { findings.push(finding('hash-changed', `the code for "${locked.verb}" does something else: "${fp.rendered[candidates[0]].sentence}"`, i + 1)); return; }
|
|
72
118
|
const judged = candidates.map((m) => compareAsserts(locked, fp.steps[m] as ContractStep));
|
|
73
119
|
const rank = (k: string): number => (k === 'oracle-weakened' ? 0 : k === 'review' ? 1 : 2);
|
|
@@ -75,18 +121,21 @@ function implCodeFindings(entry: LockEntry, fp: TraceFingerprint): { findings: C
|
|
|
75
121
|
findings.push(finding(best.class === 'review' ? 'hash-changed' : best.class, best.detail, i + 1));
|
|
76
122
|
});
|
|
77
123
|
fp.groups.forEach((g, gi) => { if (!taken.has(gi)) findings.push(finding('hash-changed', `test.step "${g.title}" implements no approved step`)); });
|
|
78
|
-
return { findings, used };
|
|
124
|
+
return { findings, used, replaces };
|
|
79
125
|
}
|
|
80
126
|
|
|
81
|
-
function classifyOne(id: string, implementation: Implementation, entry: LockEntry, fp: TraceFingerprint):
|
|
127
|
+
function classifyOne(id: string, implementation: Implementation, entry: LockEntry, fp: TraceFingerprint): Judged {
|
|
128
|
+
const accepted = entry.acceptedOpaque ?? [];
|
|
82
129
|
if (implementation === 'impl:code') {
|
|
83
|
-
const { findings, used } = implCodeFindings(entry, fp);
|
|
84
|
-
|
|
130
|
+
const { findings, used, replaces } = implCodeFindings(entry, fp);
|
|
131
|
+
const o = overlays(fp, implementation, accepted, used, replaces);
|
|
132
|
+
return { findings: [...findings, ...o.findings], opaque: o.opaque };
|
|
85
133
|
}
|
|
86
134
|
const ladder: ScenarioResult = classifyScenario(id, entry, {
|
|
87
135
|
id, implementation, steps: fp.steps, hash: fp.hash, source: { file: '', name: id, hasId: true, lines: [], outline: false },
|
|
88
136
|
});
|
|
89
|
-
|
|
137
|
+
const o = overlays(fp, implementation, accepted);
|
|
138
|
+
return { findings: [...ladder.findings, ...o.findings], opaque: o.opaque };
|
|
90
139
|
}
|
|
91
140
|
|
|
92
141
|
const runLabel = (r: TraceRun): string => `${r.record?.runId ?? '?'}#${r.result?.retry ?? '?'}`;
|
|
@@ -104,13 +153,18 @@ export function classifyTraceRuns(
|
|
|
104
153
|
}
|
|
105
154
|
// A trace that may not speak, beside one that may: the admissible one judges, the other is a warning.
|
|
106
155
|
const warnings = others.map((r) => `${id}: another trace of this scenario was not used — ${r.reason ?? r.status}`);
|
|
107
|
-
if (!entry) return { class: 'proposed', findings: [], hashes, runs: labels, warnings };
|
|
156
|
+
if (!entry) return { class: 'proposed', findings: [], hashes, runs: labels, warnings, opaque: [] };
|
|
108
157
|
const perRun = admissible.map((r) => classifyOne(id, implementation, entry, fingerprints.get(r)!));
|
|
109
|
-
const findings = perRun.
|
|
110
|
-
|
|
158
|
+
const findings = perRun.flatMap((p) => p.findings);
|
|
159
|
+
const opaque = [...new Map(perRun.flatMap((p) => p.opaque).map((o) => [`${o.step}:${o.sha}`, o] as const)).values()]
|
|
160
|
+
.sort((a, b) => a.step - b.step || a.sha.localeCompare(b.sha));
|
|
161
|
+
for (const o of opaque.filter((x) => x.accepted)) {
|
|
162
|
+
warnings.push(`${id}: step ${o.step} — opaque ${o.kind === 'assert' ? 'check' : 'step'} "${clip(o.verbatim)}" accepted by QA (decision ${entry.decisionId}), not compared`);
|
|
163
|
+
}
|
|
164
|
+
let cls = worstOf(findings);
|
|
111
165
|
if (new Set(hashes).size > 1) {
|
|
112
166
|
findings.push(finding('nondeterministic', `${new Set(hashes).size} different fingerprints across ${admissible.length} runs — the runs took different branches`));
|
|
113
167
|
if (severityOf(cls) <= severityOf('strengthened')) cls = 'nondeterministic';
|
|
114
168
|
}
|
|
115
|
-
return { class: cls, findings, hashes, runs: labels, warnings };
|
|
169
|
+
return { class: cls, findings, hashes, runs: labels, warnings, opaque };
|
|
116
170
|
}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* step {kind: action|assert, verb, target?:{ref, also?, parent?, nth?}, value?, matcher?, expected?, negated?, table?}
|
|
5
5
|
* lock qa/<kind>/<unit>/contracts.lock.yaml → {schema: 1, scenarios: {<scenario id>: LockEntry}}
|
|
6
|
-
* entry {hash, steps, implementation, approvedAt, decisionId, humanApproved?}
|
|
6
|
+
* entry {hash, steps, implementation, approvedAt, decisionId, humanApproved?, acceptedOpaque?}
|
|
7
7
|
*
|
|
8
8
|
* A stored entry is checked the way the decision layer checks its records: a wrong type or an
|
|
9
9
|
* UNKNOWN key is an error, because a field outside the hash could carry meaning the hash does not
|
|
@@ -49,6 +49,9 @@ export interface ContractScenario {
|
|
|
49
49
|
source: { file: string; name: string; hasId: boolean; lines: number[]; outline: boolean };
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
/** A reviewed opaque trace step QA accepted (#923): its 1-based position in the trace's steps and its sha (opaqueStepSha). */
|
|
53
|
+
export interface AcceptedOpaqueStep { step: number; sha: string }
|
|
54
|
+
|
|
52
55
|
export interface LockEntry {
|
|
53
56
|
hash: string;
|
|
54
57
|
steps: ContractStep[];
|
|
@@ -56,8 +59,16 @@ export interface LockEntry {
|
|
|
56
59
|
approvedAt: string;
|
|
57
60
|
decisionId: string;
|
|
58
61
|
humanApproved?: true;
|
|
62
|
+
/** Opaque trace steps a QA accepted on a terminal (`contract approve --accept-opaque`) — vouched for by `decisionId`. */
|
|
63
|
+
acceptedOpaque?: AcceptedOpaqueStep[];
|
|
59
64
|
}
|
|
60
65
|
|
|
66
|
+
/**
|
|
67
|
+
* One opaque step of a scenario's trace(s), as `contract check` reports it and `--accept-opaque` shows it.
|
|
68
|
+
* `replaces` (impl:code): the approved step it stands in for — `step 5 "see [Title] heading"`.
|
|
69
|
+
*/
|
|
70
|
+
export interface OpaqueStep { step: number; sha: string; kind: 'action' | 'assert'; verbatim: string; where?: string; replaces?: string; accepted: boolean }
|
|
71
|
+
|
|
61
72
|
export const LOCK_SCHEMA = 1;
|
|
62
73
|
export interface ContractLock { schema: typeof LOCK_SCHEMA; scenarios: Record<string, LockEntry> }
|
|
63
74
|
|
|
@@ -88,8 +99,10 @@ export const RULE_OF: Readonly<Partial<Record<FindingClass, string>>> = {
|
|
|
88
99
|
|
|
89
100
|
const STEP_KEYS = new Set(['kind', 'verb', 'target', 'value', 'matcher', 'expected', 'negated', 'table']);
|
|
90
101
|
const TARGET_KEYS = new Set(['ref', 'also', 'parent', 'nth']);
|
|
91
|
-
const ENTRY_KEYS = new Set(['hash', 'steps', 'implementation', 'approvedAt', 'decisionId', 'humanApproved']);
|
|
102
|
+
const ENTRY_KEYS = new Set(['hash', 'steps', 'implementation', 'approvedAt', 'decisionId', 'humanApproved', 'acceptedOpaque']);
|
|
103
|
+
const ACCEPTED_KEYS = new Set(['step', 'sha']);
|
|
92
104
|
const HASH_RE = /^[0-9a-f]{12}$/;
|
|
105
|
+
export const OPAQUE_SHA_RE = /^[0-9a-f]{16}$/;
|
|
93
106
|
|
|
94
107
|
const isObject = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
95
108
|
const isStringList = (v: unknown): v is string[] => Array.isArray(v) && v.every((x) => typeof x === 'string');
|
|
@@ -118,6 +131,18 @@ export function stepErrors(s: unknown, at: string): string[] {
|
|
|
118
131
|
return e;
|
|
119
132
|
}
|
|
120
133
|
|
|
134
|
+
function acceptedOpaqueErrors(v: unknown, at: string): string[] {
|
|
135
|
+
if (!Array.isArray(v) || !v.length) return [`${at}: acceptedOpaque must be a non-empty list`];
|
|
136
|
+
return v.flatMap((a, i) => {
|
|
137
|
+
const where = `${at} acceptedOpaque ${i + 1}`;
|
|
138
|
+
if (!isObject(a)) return [`${where}: not a mapping`];
|
|
139
|
+
const e = unknownKeys(a, ACCEPTED_KEYS, where);
|
|
140
|
+
if (!(Number.isInteger(a.step) && (a.step as number) > 0)) e.push(`${where}: step must be a positive integer`);
|
|
141
|
+
if (typeof a.sha !== 'string' || !OPAQUE_SHA_RE.test(a.sha)) e.push(`${where}: sha must be 16 hex characters`);
|
|
142
|
+
return e;
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
|
|
121
146
|
/** Errors in one stored lock entry (empty = well-formed). The hash itself is checked by the reader. */
|
|
122
147
|
export function lockEntryErrors(raw: unknown, id: string): string[] {
|
|
123
148
|
const at = `scenario ${id}`;
|
|
@@ -128,6 +153,10 @@ export function lockEntryErrors(raw: unknown, id: string): string[] {
|
|
|
128
153
|
if (typeof raw.approvedAt !== 'string' || Number.isNaN(Date.parse(raw.approvedAt))) e.push(`${at}: approvedAt must be an ISO-8601 timestamp`);
|
|
129
154
|
if (typeof raw.decisionId !== 'string' || !DECISION_ID_RE.test(raw.decisionId)) e.push(`${at}: decisionId must be D-<16 hex>`);
|
|
130
155
|
if (raw.humanApproved !== undefined && raw.humanApproved !== true) e.push(`${at}: humanApproved is either true or absent`);
|
|
156
|
+
if (raw.acceptedOpaque !== undefined) {
|
|
157
|
+
e.push(...acceptedOpaqueErrors(raw.acceptedOpaque, at));
|
|
158
|
+
if (raw.humanApproved !== true) e.push(`${at}: acceptedOpaque needs humanApproved: true`);
|
|
159
|
+
}
|
|
131
160
|
if (!Array.isArray(raw.steps)) e.push(`${at}: steps must be a list`);
|
|
132
161
|
else raw.steps.forEach((s, i) => e.push(...stepErrors(s, `${at} step ${i + 1}`)));
|
|
133
162
|
return e;
|
|
@@ -80,6 +80,7 @@ export function serializeLock(lock: ContractLock): string {
|
|
|
80
80
|
scenarios[id] = {
|
|
81
81
|
hash: e.hash, implementation: e.implementation, approvedAt: e.approvedAt, decisionId: e.decisionId,
|
|
82
82
|
...(e.humanApproved === true ? { humanApproved: true as const } : {}),
|
|
83
|
+
...(e.acceptedOpaque?.length ? { acceptedOpaque: e.acceptedOpaque.map((a) => ({ step: a.step, sha: a.sha })) } : {}),
|
|
83
84
|
steps: e.steps.map(orderedStep),
|
|
84
85
|
};
|
|
85
86
|
}
|
|
@@ -4,12 +4,14 @@
|
|
|
4
4
|
* HEAD is read once (one `git ls-tree`, bounded by the caller's timeout). A unit with no lock in the
|
|
5
5
|
* working tree AND none in HEAD has no contract to break and is skipped without parsing anything.
|
|
6
6
|
* Every other unit gets its verdict rebuilt by buildContractVerdict — the same function `contract
|
|
7
|
-
* check` runs — and anything but `pass` blocks.
|
|
7
|
+
* check` runs — and anything but `pass` blocks. Its trace part gets the guard's ~3 s at most:
|
|
8
|
+
* SUNGEN_TRACE_VERDICT_BUDGET_MS can shorten that here, never lengthen it (#926). A lock deleted from the working tree still counts
|
|
8
9
|
* while HEAD holds it. If git FAILS in a checkout that has .git while a delivered unit has a lock,
|
|
9
10
|
* this throws (GitUnavailableError): the guard asks rather than calling an unverifiable lock clean.
|
|
10
11
|
*/
|
|
11
12
|
import * as fs from 'fs';
|
|
12
13
|
import { buildContractVerdict, ContractUnit, VerdictValue } from './build-contract-verdict';
|
|
14
|
+
import { guardTraceBudgetMs } from './build-trace-conformance';
|
|
13
15
|
import { CLI_GIT_TIMEOUT_MS, GitUnavailableError, headSnapshot, posixRel } from './contract-git';
|
|
14
16
|
import { HARD_CLASSES } from './contract-schema';
|
|
15
17
|
import { lockPathFor } from './contracts-lock-file';
|
|
@@ -24,7 +26,7 @@ export function contractBlockers(projectRoot: string, units: ContractUnit[], opt
|
|
|
24
26
|
const out: ContractBlocker[] = [];
|
|
25
27
|
for (const unit of unique) {
|
|
26
28
|
if (!hasWorkingLock(unit) && !snap.blobs?.has(posixRel(projectRoot, lockPathFor(unit.dir)))) continue;
|
|
27
|
-
const v = buildContractVerdict(projectRoot, unit, { snapshot: snap });
|
|
29
|
+
const v = buildContractVerdict(projectRoot, unit, { snapshot: snap, traceBudgetMs: guardTraceBudgetMs() });
|
|
28
30
|
if (v.verdict === 'pass') continue;
|
|
29
31
|
const worst = v.scenarios.find((s) => HARD_CLASSES.has(s.class)) ?? v.scenarios.find((s) => s.class === 'hash-changed');
|
|
30
32
|
const headline = worst
|
|
@@ -9,13 +9,17 @@
|
|
|
9
9
|
* The decision goes first: a crash between the two leaves a record without a lock entry, never an
|
|
10
10
|
* entry pointing at nothing. Only `sungen contract approve` calls this after its TTY confirmation;
|
|
11
11
|
* the confirmation itself lives there (approve-contract.ts).
|
|
12
|
+
*
|
|
13
|
+
* `--accept-opaque` (#923, accept-opaque-steps.ts) writes the same kind of decision for ONE scenario,
|
|
14
|
+
* listing it at its hash WITH the opaque steps QA accepted (step + sha); its lock entry is rewritten to
|
|
15
|
+
* point at that decision and carries the same list, which verify-approval-decisions.ts cross-checks.
|
|
12
16
|
*/
|
|
13
17
|
import * as path from 'path';
|
|
14
18
|
import { appendDecision } from '../decisions/append-decision';
|
|
15
19
|
import { isValidScope } from '../decisions/decision-file';
|
|
16
20
|
import type { JsonValue } from '../decisions/decision-schema';
|
|
17
21
|
import { gitUserName } from '../decisions/decision-source';
|
|
18
|
-
import { ContractLock, ContractScenario, LOCK_SCHEMA } from './contract-schema';
|
|
22
|
+
import { AcceptedOpaqueStep, ContractLock, ContractScenario, LOCK_SCHEMA } from './contract-schema';
|
|
19
23
|
import { lockEntryFor, lockPathFor, serializeLock, writeLockFile } from './contracts-lock-file';
|
|
20
24
|
import { assertNoLockSecrets } from './lock-secret-guard';
|
|
21
25
|
|
|
@@ -30,6 +34,18 @@ export interface ApprovalWrite {
|
|
|
30
34
|
}
|
|
31
35
|
|
|
32
36
|
export const DEFAULT_APPROVAL_REASON = 'QA reviewed the normalized Gherkin contract';
|
|
37
|
+
export const DEFAULT_OPAQUE_REASON = 'QA reviewed the opaque step(s) sungen cannot compare';
|
|
38
|
+
|
|
39
|
+
export interface OpaqueAcceptanceWrite {
|
|
40
|
+
projectRoot: string;
|
|
41
|
+
unitDir: string;
|
|
42
|
+
unit: string;
|
|
43
|
+
/** The scenario as it reads now — its hash equals its approved lock entry's. */
|
|
44
|
+
scenario: ContractScenario;
|
|
45
|
+
accepted: AcceptedOpaqueStep[];
|
|
46
|
+
reason?: string;
|
|
47
|
+
lock: ContractLock;
|
|
48
|
+
}
|
|
33
49
|
|
|
34
50
|
/** The decision scope of a unit directory (`screens/login`, `api/flows/pay`…). */
|
|
35
51
|
export function scopeOfUnitDir(projectRoot: string, unitDir: string): string {
|
|
@@ -68,3 +84,27 @@ export function recordContractApproval(w: ApprovalWrite): { decisionId: string;
|
|
|
68
84
|
writeLockFile(lockPathFor(w.unitDir), lock);
|
|
69
85
|
return { decisionId: decision.entry.id, lock };
|
|
70
86
|
}
|
|
87
|
+
|
|
88
|
+
function lockWithAcceptance(w: OpaqueAcceptanceWrite, decisionId: string, approvedAt: string): ContractLock {
|
|
89
|
+
const entry = { ...lockEntryFor(w.scenario, decisionId, approvedAt), acceptedOpaque: w.accepted.map((a) => ({ step: a.step, sha: a.sha })) };
|
|
90
|
+
return { schema: LOCK_SCHEMA, scenarios: { ...w.lock.scenarios, [w.scenario.id]: entry } };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function recordOpaqueAcceptance(w: OpaqueAcceptanceWrite): { decisionId: string; lock: ContractLock } {
|
|
94
|
+
const scope = scopeOfUnitDir(w.projectRoot, w.unitDir);
|
|
95
|
+
assertNoLockSecrets([w.scenario], serializeLock(lockWithAcceptance(w, 'D-0000000000000000', new Date().toISOString())));
|
|
96
|
+
const sc = w.scenario;
|
|
97
|
+
const accepted = w.accepted.map((a) => ({ step: a.step, sha: a.sha }));
|
|
98
|
+
const payload: Record<string, JsonValue> = {
|
|
99
|
+
unit: w.unit, scenarios: [{ id: sc.id, hash: sc.hash, implementation: sc.implementation, acceptedOpaque: accepted }], removed: [],
|
|
100
|
+
};
|
|
101
|
+
const decision = appendDecision(w.projectRoot, scope, {
|
|
102
|
+
kind: 'contract-approval', anchors: [], visibility: 'internal', valid_from: new Date().toISOString(), payload,
|
|
103
|
+
text: `Accepted the reviewed opaque step(s) ${[...new Set(accepted.map((a) => a.step))].join(', ')} of ${sc.id} (${w.unit})`,
|
|
104
|
+
reason: w.reason?.trim() || DEFAULT_OPAQUE_REASON,
|
|
105
|
+
source: { by: gitUserName(w.projectRoot), where: 'cli' },
|
|
106
|
+
});
|
|
107
|
+
const lock = lockWithAcceptance(w, decision.entry.id, decision.entry.valid_from);
|
|
108
|
+
writeLockFile(lockPathFor(w.unitDir), lock);
|
|
109
|
+
return { decisionId: decision.entry.id, lock };
|
|
110
|
+
}
|
|
@@ -22,6 +22,11 @@ import { isPageLevel } from './trace/trace-identity-resolution';
|
|
|
22
22
|
import type { IntentStep } from './trace/trace-steps';
|
|
23
23
|
|
|
24
24
|
export const CONTROL_CHARS = /[\u0000-\u001f\u007f\u2028\u2029]/g;
|
|
25
|
+
/** Text from a trace or report shown in one line of a finding: control characters dropped, at most `max` characters. */
|
|
26
|
+
export function clip(s: string, max = 120): string {
|
|
27
|
+
const t = s.replace(CONTROL_CHARS, '');
|
|
28
|
+
return t.length > max ? `${t.slice(0, max - 1)}\u2026` : t;
|
|
29
|
+
}
|
|
25
30
|
|
|
26
31
|
export interface ResolvedTarget { identity: TargetIdentity; facts?: TargetFacts }
|
|
27
32
|
export interface RenderedStep {
|
|
@@ -12,9 +12,12 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import * as fs from 'fs';
|
|
14
14
|
import * as path from 'path';
|
|
15
|
+
import { isIdLike } from '../../harness/parse';
|
|
16
|
+
import { clip, CONTROL_CHARS } from '../render-free-gherkin';
|
|
17
|
+
import { FileRead, readInsideProject } from './read-file-once';
|
|
15
18
|
import { bindTrace } from './trace-binding';
|
|
16
|
-
import {
|
|
17
|
-
import { TraceRejected } from './trace-zip-limits';
|
|
19
|
+
import { traceFromRead, ParsedTrace } from './trace-file-reader';
|
|
20
|
+
import { traceZipLimits, TraceRejected } from './trace-zip-limits';
|
|
18
21
|
import { finalAttempt, readPlaywrightReport, ReportResult, ReportTest, scenarioIdOfTest } from './select-final-attempt';
|
|
19
22
|
import { findRunRecords, readRunRecord, RunRecord } from './trace-run-record';
|
|
20
23
|
|
|
@@ -32,26 +35,26 @@ export interface TraceRun {
|
|
|
32
35
|
}
|
|
33
36
|
export interface ScenarioEvidence { runs: Map<string, TraceRun[]>; problems: string[]; inputs: string[] }
|
|
34
37
|
|
|
38
|
+
/** A run record or JSON report larger than this is refused before a byte is read (fail closed, #926). */
|
|
39
|
+
export const MAX_EVIDENCE_JSON_BYTES = 64 * 1024 * 1024;
|
|
40
|
+
|
|
41
|
+
/** Why an evidence file was not read — `outside the project` covers a link out, a swap and a non-file. */
|
|
42
|
+
function notRead(name: string, r: Exclude<FileRead, { status: 'read' }>): string {
|
|
43
|
+
if (r.status === 'too-large') return `${name} is larger than ${MAX_EVIDENCE_JSON_BYTES / 1024 / 1024} MiB — not read`;
|
|
44
|
+
if (r.status === 'missing') return `${name} is missing — not read`;
|
|
45
|
+
return `${name} is outside the project (a symlink?) or not a regular file — not read`;
|
|
46
|
+
}
|
|
47
|
+
|
|
35
48
|
/** The reason a run is not judged once the verdict's deadline has passed (fail closed). */
|
|
36
49
|
export const OVER_BUDGET = 'verdict budget exceeded; run `sungen contract check`';
|
|
37
50
|
const NO_RECORD = "no sungen run record under test-results-free/ (or test-results/) — run the free tests with their own config: outputDir 'test-results-free', trace: 'on' and the sungen run-record reporter";
|
|
38
51
|
|
|
39
|
-
/**
|
|
40
|
-
|
|
41
|
-
try {
|
|
42
|
-
const root = fs.realpathSync(projectRoot);
|
|
43
|
-
const real = fs.realpathSync(file);
|
|
44
|
-
return real.startsWith(root + path.sep) && fs.statSync(real).isFile() ? real : null;
|
|
45
|
-
} catch { return null; }
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function resolveAttachment(projectRoot: string, report: string, p: string): { file?: string; outside?: boolean } {
|
|
52
|
+
/** Where a report's trace attachment is: the first candidate that exists (read later, opened once). */
|
|
53
|
+
function attachmentPath(projectRoot: string, report: string, p: string): string | null {
|
|
49
54
|
for (const cand of path.isAbsolute(p) ? [p] : [path.resolve(projectRoot, p), path.resolve(path.dirname(report), p)]) {
|
|
50
|
-
if (
|
|
51
|
-
const real = insideProject(projectRoot, cand);
|
|
52
|
-
return real ? { file: real } : { outside: true };
|
|
55
|
+
if (fs.existsSync(cand)) return cand;
|
|
53
56
|
}
|
|
54
|
-
return
|
|
57
|
+
return null;
|
|
55
58
|
}
|
|
56
59
|
|
|
57
60
|
function judge(projectRoot: string, record: RunRecord, test: ReportTest, notBefore: number): TraceRun {
|
|
@@ -59,11 +62,18 @@ function judge(projectRoot: string, record: RunRecord, test: ReportTest, notBefo
|
|
|
59
62
|
const base = { scenarioId, test, record };
|
|
60
63
|
const fin = finalAttempt(test);
|
|
61
64
|
if (fin.admissibility !== 'admissible') return { ...base, result: fin.result, status: fin.admissibility, reason: fin.reason };
|
|
62
|
-
const {
|
|
63
|
-
|
|
64
|
-
if (!
|
|
65
|
+
const missing: TraceRun = { ...base, result: fin.result, status: 'unproven', reason: `no trace at ${fin.result!.tracePath} (the report points at a missing file)` };
|
|
66
|
+
const cand = attachmentPath(projectRoot, test.report, fin.result!.tracePath!);
|
|
67
|
+
if (!cand) return missing;
|
|
68
|
+
// Opened once, inside the project, and what was opened is what is parsed and hashed (#926).
|
|
69
|
+
const limits = traceZipLimits();
|
|
70
|
+
const read = readInsideProject(projectRoot, cand, limits.maxZipBytes);
|
|
71
|
+
if (read.status === 'missing') return missing;
|
|
72
|
+
if (read.status === 'outside' || read.status === 'not-a-file') {
|
|
73
|
+
return { ...base, result: fin.result, status: 'unproven', reason: `the trace ${path.basename(fin.result!.tracePath!)} is outside the project (or not a file) — not read` };
|
|
74
|
+
}
|
|
65
75
|
let trace: ParsedTrace;
|
|
66
|
-
try { trace =
|
|
76
|
+
try { trace = traceFromRead(read, cand, limits); } catch (e) {
|
|
67
77
|
const unsupported = e instanceof TraceRejected && e.code === 'trace.version-unsupported';
|
|
68
78
|
return { ...base, result: fin.result, status: unsupported ? 'unsupported' : 'refused', reason: `trace refused: ${(e as Error).message}` };
|
|
69
79
|
}
|
|
@@ -75,35 +85,62 @@ function judge(projectRoot: string, record: RunRecord, test: ReportTest, notBefo
|
|
|
75
85
|
/** Every judged run of the tests `want` accepts (derive: a unit's tests; check: by scenario id). */
|
|
76
86
|
export function collectRuns(
|
|
77
87
|
projectRoot: string, want: (id: string, test: ReportTest) => boolean, notBefore: number, deadline = Number.POSITIVE_INFINITY,
|
|
78
|
-
): { runs: TraceRun[]; problems: string[]; inputs: string[]; records: number } {
|
|
88
|
+
): { runs: TraceRun[]; problems: string[]; inputs: string[]; records: number; seen: Array<{ id: string; title: string }> } {
|
|
79
89
|
const runs: TraceRun[] = [];
|
|
80
90
|
const problems: string[] = [];
|
|
81
91
|
const inputs: string[] = [];
|
|
92
|
+
const seen: Array<{ id: string; title: string }> = [];
|
|
82
93
|
const records = findRunRecords(projectRoot);
|
|
83
94
|
for (const file of records) {
|
|
84
95
|
let record: RunRecord;
|
|
85
|
-
|
|
86
|
-
|
|
96
|
+
const recordRead = readInsideProject(projectRoot, file, MAX_EVIDENCE_JSON_BYTES);
|
|
97
|
+
if (recordRead.status !== 'read') { problems.push(notRead(path.basename(file), recordRead)); continue; }
|
|
98
|
+
try { record = readRunRecord(file, recordRead.bytes.toString('utf8')); } catch (e) { problems.push(`${path.basename(file)} is not trusted: ${(e as Error).message}`); continue; }
|
|
87
99
|
if (!record.report) { problems.push(`${path.basename(file)} names no JSON report — add the json reporter to the free-test config`); continue; }
|
|
88
|
-
const
|
|
89
|
-
if (
|
|
100
|
+
const reportRead = readInsideProject(projectRoot, path.resolve(projectRoot, record.report), MAX_EVIDENCE_JSON_BYTES);
|
|
101
|
+
if (reportRead.status !== 'read') { problems.push(`${path.basename(file)}: ${notRead(`the report ${path.basename(record.report)}`, reportRead)}`); continue; }
|
|
90
102
|
let tests: ReportTest[];
|
|
91
|
-
try { tests = readPlaywrightReport(
|
|
103
|
+
try { tests = readPlaywrightReport(reportRead.real, reportRead.bytes.toString('utf8')); } catch (e) { problems.push((e as Error).message); continue; }
|
|
92
104
|
inputs.push(`${record.runId}:${record.report}`);
|
|
93
105
|
for (const test of tests) {
|
|
94
106
|
const id = scenarioIdOfTest(test.title, test.tags);
|
|
107
|
+
seen.push({ id, title: test.title });
|
|
95
108
|
if (!want(id, test)) continue;
|
|
96
109
|
runs.push(Date.now() > deadline ? { scenarioId: id, test, record, status: 'unproven', reason: OVER_BUDGET } : judge(projectRoot, record, test, notBefore));
|
|
97
110
|
}
|
|
98
111
|
}
|
|
99
|
-
return { runs, problems, inputs, records: records.length };
|
|
112
|
+
return { runs, problems, inputs, records: records.length, seen };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** A title without its leading id, for comparing words only: `VP-X-001 Login works` → `login works`. */
|
|
116
|
+
const wordsOf = (title: string): string => {
|
|
117
|
+
const lead = title.match(/^\s*([A-Za-z][A-Za-z0-9.-]*)/)?.[1];
|
|
118
|
+
return (lead && isIdLike(lead) ? title.trim().slice(lead.length) : title).replace(CONTROL_CHARS, '').replace(/\s+/g, ' ').trim().toLowerCase();
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Why no recorded test speaks for `id` (#943): a recorded test with this scenario's title under
|
|
123
|
+
* ANOTHER id is the usual cause — the id was renamed in the .free.feature (or in the test) and the
|
|
124
|
+
* other side was not — so that mismatch is named, with the fix.
|
|
125
|
+
*/
|
|
126
|
+
function noTestFor(id: string, name: string | undefined, seen: Array<{ id: string; title: string }>): string {
|
|
127
|
+
const words = name ? wordsOf(name) : '';
|
|
128
|
+
const twin = words ? seen.find((t) => t.id !== id && wordsOf(t.title) === words) : undefined;
|
|
129
|
+
if (!twin) return `no test for ${id} in the recorded run(s)`;
|
|
130
|
+
// The title and its id come from the report — untrusted: clipped, control characters dropped.
|
|
131
|
+
const [title, other] = [clip(twin.title), clip(twin.id, 60)];
|
|
132
|
+
return `no test for ${id} in the recorded run(s) — the recorded test "${title}" (id ${other}) has this scenario's title, so the scenario id and the test no longer match: `
|
|
133
|
+
+ `give the test the id ${id} (in its title and its @tag), or the scenario the id ${other}, then run it again with trace on`;
|
|
100
134
|
}
|
|
101
135
|
|
|
102
|
-
|
|
136
|
+
/** The runs of each scenario id; `names` (id → scenario name) lets a missing test be explained. */
|
|
137
|
+
export function collectScenarioEvidence(
|
|
138
|
+
projectRoot: string, ids: ReadonlySet<string>, notBefore: number, deadline?: number, names?: ReadonlyMap<string, string>,
|
|
139
|
+
): ScenarioEvidence {
|
|
103
140
|
const found = collectRuns(projectRoot, (id) => ids.has(id), notBefore, deadline);
|
|
104
141
|
const runs = new Map<string, TraceRun[]>([...ids].map((id) => [id, found.runs.filter((r) => r.scenarioId === id)]));
|
|
105
142
|
for (const [id, list] of runs) {
|
|
106
|
-
if (!list.length) list.push({ scenarioId: id, status: 'unproven', reason: found.records ?
|
|
143
|
+
if (!list.length) list.push({ scenarioId: id, status: 'unproven', reason: found.records ? noTestFor(id, names?.get(id), found.seen) : NO_RECORD });
|
|
107
144
|
}
|
|
108
145
|
return { runs, problems: found.problems, inputs: found.inputs };
|
|
109
146
|
}
|