@sun-asterisk/sungen 3.3.0-alpha.7 → 3.3.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/case-triage-hint.d.ts +14 -0
- package/dist/cli/case-triage-hint.d.ts.map +1 -0
- package/dist/cli/case-triage-hint.js +23 -0
- package/dist/cli/case-triage-hint.js.map +1 -0
- package/dist/cli/commands/contract-derive-command.d.ts +2 -1
- package/dist/cli/commands/contract-derive-command.d.ts.map +1 -1
- package/dist/cli/commands/contract-derive-command.js +15 -3
- package/dist/cli/commands/contract-derive-command.js.map +1 -1
- package/dist/cli/commands/contract.d.ts.map +1 -1
- package/dist/cli/commands/contract.js +5 -2
- package/dist/cli/commands/contract.js.map +1 -1
- package/dist/cli/commands/delivery.d.ts.map +1 -1
- package/dist/cli/commands/delivery.js +29 -13
- package/dist/cli/commands/delivery.js.map +1 -1
- package/dist/cli/commands/gate.d.ts.map +1 -1
- package/dist/cli/commands/gate.js +2 -1
- package/dist/cli/commands/gate.js.map +1 -1
- package/dist/cli/commands/generate.d.ts.map +1 -1
- package/dist/cli/commands/generate.js +12 -2
- package/dist/cli/commands/generate.js.map +1 -1
- package/dist/cli/commands/journey.d.ts.map +1 -1
- package/dist/cli/commands/journey.js +4 -2
- package/dist/cli/commands/journey.js.map +1 -1
- package/dist/cli/commands/next.d.ts +4 -2
- package/dist/cli/commands/next.d.ts.map +1 -1
- package/dist/cli/commands/next.js +22 -6
- package/dist/cli/commands/next.js.map +1 -1
- package/dist/cli/commands/triage.d.ts.map +1 -1
- package/dist/cli/commands/triage.js +5 -3
- package/dist/cli/commands/triage.js.map +1 -1
- package/dist/cli/project-sungen-invocation.d.ts +6 -0
- package/dist/cli/project-sungen-invocation.d.ts.map +1 -1
- package/dist/cli/project-sungen-invocation.js +19 -0
- package/dist/cli/project-sungen-invocation.js.map +1 -1
- package/dist/cli/run-command-hints.d.ts +9 -0
- package/dist/cli/run-command-hints.d.ts.map +1 -0
- package/dist/cli/run-command-hints.js +84 -0
- package/dist/cli/run-command-hints.js.map +1 -0
- package/dist/cli/waiting-for-you-block.d.ts +9 -6
- package/dist/cli/waiting-for-you-block.d.ts.map +1 -1
- package/dist/cli/waiting-for-you-block.js +9 -6
- package/dist/cli/waiting-for-you-block.js.map +1 -1
- package/dist/contract/build-trace-conformance.d.ts.map +1 -1
- package/dist/contract/build-trace-conformance.js +4 -11
- package/dist/contract/build-trace-conformance.js.map +1 -1
- package/dist/contract/derive-free-feature.d.ts +9 -0
- package/dist/contract/derive-free-feature.d.ts.map +1 -1
- package/dist/contract/derive-free-feature.js +10 -3
- package/dist/contract/derive-free-feature.js.map +1 -1
- package/dist/contract/resolve-unit-trace-identities.d.ts +12 -0
- package/dist/contract/resolve-unit-trace-identities.d.ts.map +1 -1
- package/dist/contract/resolve-unit-trace-identities.js +33 -49
- package/dist/contract/resolve-unit-trace-identities.js.map +1 -1
- package/dist/contract/trace/collect-trace-evidence.d.ts +9 -3
- package/dist/contract/trace/collect-trace-evidence.d.ts.map +1 -1
- package/dist/contract/trace/collect-trace-evidence.js +45 -18
- package/dist/contract/trace/collect-trace-evidence.js.map +1 -1
- package/dist/contract/trace/scenario-id-claims.d.ts +11 -0
- package/dist/contract/trace/scenario-id-claims.d.ts.map +1 -0
- package/dist/contract/trace/scenario-id-claims.js +89 -0
- package/dist/contract/trace/scenario-id-claims.js.map +1 -0
- package/dist/contract/trace/select-final-attempt.d.ts +2 -0
- package/dist/contract/trace/select-final-attempt.d.ts.map +1 -1
- package/dist/contract/trace/select-final-attempt.js +4 -1
- package/dist/contract/trace/select-final-attempt.js.map +1 -1
- package/dist/contract/trace/trace-binding.d.ts +3 -2
- package/dist/contract/trace/trace-binding.d.ts.map +1 -1
- package/dist/contract/trace/trace-binding.js +6 -3
- package/dist/contract/trace/trace-binding.js.map +1 -1
- package/dist/contract/trace/trace-freshness.d.ts +27 -0
- package/dist/contract/trace/trace-freshness.d.ts.map +1 -0
- package/dist/contract/trace/trace-freshness.js +146 -0
- package/dist/contract/trace/trace-freshness.js.map +1 -0
- package/dist/contract/trace/trace-identity-resolution.d.ts +2 -0
- package/dist/contract/trace/trace-identity-resolution.d.ts.map +1 -1
- package/dist/contract/trace/trace-identity-resolution.js +6 -3
- package/dist/contract/trace/trace-identity-resolution.js.map +1 -1
- package/dist/contract/trace/trace-run-record.d.ts +2 -0
- package/dist/contract/trace/trace-run-record.d.ts.map +1 -1
- package/dist/contract/trace/trace-run-record.js +19 -5
- package/dist/contract/trace/trace-run-record.js.map +1 -1
- package/dist/exporters/csv-exporter.d.ts +11 -2
- package/dist/exporters/csv-exporter.d.ts.map +1 -1
- package/dist/exporters/csv-exporter.js +8 -4
- package/dist/exporters/csv-exporter.js.map +1 -1
- package/dist/exporters/matrix/export.d.ts +6 -1
- package/dist/exporters/matrix/export.d.ts.map +1 -1
- package/dist/exporters/matrix/export.js +11 -4
- package/dist/exporters/matrix/export.js.map +1 -1
- package/dist/exporters/xlsx-exporter.d.ts +3 -2
- package/dist/exporters/xlsx-exporter.d.ts.map +1 -1
- package/dist/exporters/xlsx-exporter.js +4 -4
- package/dist/exporters/xlsx-exporter.js.map +1 -1
- package/dist/graph/evaluate-graph.d.ts +3 -1
- package/dist/graph/evaluate-graph.d.ts.map +1 -1
- package/dist/graph/evaluate-graph.js +4 -2
- package/dist/graph/evaluate-graph.js.map +1 -1
- package/dist/graph/unit-artifact-probes.d.ts.map +1 -1
- package/dist/graph/unit-artifact-probes.js +3 -1
- package/dist/graph/unit-artifact-probes.js.map +1 -1
- package/dist/harness/gates/audit-report-verification.d.ts +2 -0
- package/dist/harness/gates/audit-report-verification.d.ts.map +1 -1
- package/dist/harness/gates/audit-report-verification.js +8 -3
- package/dist/harness/gates/audit-report-verification.js.map +1 -1
- package/dist/harness/next-step.d.ts +2 -0
- package/dist/harness/next-step.d.ts.map +1 -1
- package/dist/harness/next-step.js +2 -0
- package/dist/harness/next-step.js.map +1 -1
- package/dist/hooks/hand-back-core.d.ts +4 -1
- package/dist/hooks/hand-back-core.d.ts.map +1 -1
- package/dist/hooks/hand-back-core.js +7 -3
- package/dist/hooks/hand-back-core.js.map +1 -1
- package/dist/hooks/session-start-unit-state-hook.js +1 -1
- package/dist/hooks/session-start-unit-state-hook.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +4 -1
- package/package.json +3 -3
- package/src/cli/case-triage-hint.ts +31 -0
- package/src/cli/commands/contract-derive-command.ts +14 -4
- package/src/cli/commands/contract.ts +5 -2
- package/src/cli/commands/delivery.ts +29 -13
- package/src/cli/commands/gate.ts +2 -1
- package/src/cli/commands/generate.ts +11 -2
- package/src/cli/commands/journey.ts +4 -2
- package/src/cli/commands/next.ts +25 -9
- package/src/cli/commands/triage.ts +5 -3
- package/src/cli/project-sungen-invocation.ts +20 -0
- package/src/cli/run-command-hints.ts +47 -0
- package/src/cli/waiting-for-you-block.ts +14 -10
- package/src/contract/build-trace-conformance.ts +4 -6
- package/src/contract/derive-free-feature.ts +15 -3
- package/src/contract/resolve-unit-trace-identities.ts +35 -12
- package/src/contract/trace/collect-trace-evidence.ts +50 -18
- package/src/contract/trace/scenario-id-claims.ts +60 -0
- package/src/contract/trace/select-final-attempt.ts +9 -2
- package/src/contract/trace/trace-binding.ts +7 -4
- package/src/contract/trace/trace-freshness.ts +110 -0
- package/src/contract/trace/trace-identity-resolution.ts +7 -4
- package/src/contract/trace/trace-run-record.ts +20 -4
- package/src/exporters/csv-exporter.ts +12 -4
- package/src/exporters/matrix/export.ts +11 -4
- package/src/exporters/xlsx-exporter.ts +5 -4
- package/src/graph/evaluate-graph.ts +5 -3
- package/src/graph/unit-artifact-probes.ts +3 -1
- package/src/harness/gates/audit-report-verification.ts +7 -2
- package/src/harness/next-step.ts +3 -0
- package/src/hooks/hand-back-core.ts +8 -4
- package/src/hooks/session-start-unit-state-hook.ts +1 -1
- package/src/orchestrator/templates/ai-src/commands/create-test.md +4 -1
|
@@ -8,6 +8,9 @@
|
|
|
8
8
|
*
|
|
9
9
|
* sungenInvocation(root) 'npx sungen' when <root>/node_modules/@sun-asterisk/sungen exists, else 'sungen'
|
|
10
10
|
* withProjectInvocation(cmd, root) a canonical `sungen …` command as this project runs it
|
|
11
|
+
* withProjectInvocationInText(text, root)
|
|
12
|
+
* every `sungen …` command inside printed text (a refusal, a hint, a
|
|
13
|
+
* step's reason) as this project runs it (#963); prose mentions stay
|
|
11
14
|
* updateCommand(root, version?) `<invocation> update` plus `--alpha` / `--beta` (`--tag <id>` for another prerelease)
|
|
12
15
|
*
|
|
13
16
|
* Only the project root itself is looked at — where `npm i` puts a QA project's install — so a
|
|
@@ -27,6 +30,23 @@ export function withProjectInvocation(command: string, projectRoot: string): str
|
|
|
27
30
|
return inv !== 'sungen' && /^sungen\b/.test(command) ? `${inv}${command.slice('sungen'.length)}` : command;
|
|
28
31
|
}
|
|
29
32
|
|
|
33
|
+
/**
|
|
34
|
+
* A `sungen` that starts a command inside text: at the start of a line (after indentation), after a
|
|
35
|
+
* backtick or `(`, after ": ", "run " or "then " — and followed by a subcommand or an option. A prose
|
|
36
|
+
* mention ("written by sungen 3.3.0", "an older sungen", "sungen: …") is none of these.
|
|
37
|
+
*/
|
|
38
|
+
const COMMAND_IN_TEXT = /(^[ \t]*|[`(]|:[ \t]+|\b(?:[Rr]un|then)[ \t]+)sungen(?= [a-z-])/gm;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Printed text (a refusal, a hint, a step's reason) with every `sungen …` command in it as this project
|
|
42
|
+
* runs sungen (#963): the human-only commands a QA copies from a refusal must run the project's own
|
|
43
|
+
* install, not an older global one. Text for machines (`--json`) stays canonical; this is for the screen.
|
|
44
|
+
*/
|
|
45
|
+
export function withProjectInvocationInText(text: string, projectRoot: string): string {
|
|
46
|
+
const inv = sungenInvocation(projectRoot);
|
|
47
|
+
return inv === 'sungen' ? text : text.replace(COMMAND_IN_TEXT, (_m, before: string) => `${before}${inv}`);
|
|
48
|
+
}
|
|
49
|
+
|
|
30
50
|
/** The `sungen update` flag that keeps the channel of `version` (its prerelease id); '' for stable. */
|
|
31
51
|
export function updateChannelFlag(version: string = getPackageVersion()): string {
|
|
32
52
|
const id = version.match(/^\d+\.\d+\.\d+-([0-9A-Za-z-]+)/)?.[1];
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The run commands sungen suggests, each writing its results where the reader that asks for them
|
|
3
|
+
* looks (#964). A hint whose results nobody reads leaves a passing run invisible: the plain
|
|
4
|
+
* `npx playwright test` wrote `test-result.json` at the project root, which `sungen next` never reads.
|
|
5
|
+
*
|
|
6
|
+
* generatedSubdir(kind, unit) `specs/generated/<unit>` · `…/flows/<flow>` · `…/api/<area>` — where
|
|
7
|
+
* `generate` writes a unit's specs and `next` / `delivery` read its results
|
|
8
|
+
* plainRunCommand(genRel, feature) one compiled feature's spec with the JSON reporter pointed at
|
|
9
|
+
* `<genRel>/<feature>-test-result.json` (graph/unit-artifact-probes RESULT_FILE)
|
|
10
|
+
* freeRunCommand(cwd) hand-written @free / impl:code tests: the project's free-test config, which
|
|
11
|
+
* writes into its own outputDir `test-results-free/` (#941)
|
|
12
|
+
* freeRunHasResults(cwd) that run left its results (or a sungen run record) there
|
|
13
|
+
*/
|
|
14
|
+
import * as fs from 'fs';
|
|
15
|
+
import * as path from 'path';
|
|
16
|
+
import { FREE_RUN_DIR } from '../contract/trace/trace-run-record';
|
|
17
|
+
|
|
18
|
+
export type RunUnitKind = 'screen' | 'flow' | 'api';
|
|
19
|
+
|
|
20
|
+
export function generatedSubdir(kind: RunUnitKind, unit: string): string {
|
|
21
|
+
return path.posix.join('specs', 'generated', ...(kind === 'flow' ? ['flows'] : kind === 'api' ? ['api'] : []), unit);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** `[SUNGEN_ENV=<env> ]PLAYWRIGHT_JSON_OUTPUT_NAME=<genRel>/<feature>-test-result.json npx playwright test <genRel>/<feature>.spec.ts` */
|
|
25
|
+
export function plainRunCommand(genRel: string, featureBaseName: string, env?: string): string {
|
|
26
|
+
return `${env ? `SUNGEN_ENV=${env} ` : ''}PLAYWRIGHT_JSON_OUTPUT_NAME=${genRel}/${featureBaseName}-test-result.json npx playwright test ${genRel}/${featureBaseName}.spec.ts`;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const FREE_CONFIG = /^playwright\.free\.config\.(?:ts|mts|cts|js|mjs|cjs)$/;
|
|
30
|
+
|
|
31
|
+
/** The free run: `npx playwright test -c <playwright.free.config.*>` when the project has one, else what that config holds. */
|
|
32
|
+
export function freeRunCommand(cwd: string): string {
|
|
33
|
+
let config: string | undefined;
|
|
34
|
+
try { config = fs.readdirSync(cwd).find((f) => FREE_CONFIG.test(f)); } catch { /* no project dir: the generic form */ }
|
|
35
|
+
return config
|
|
36
|
+
? `npx playwright test -c ${config}`
|
|
37
|
+
: `run the hand-written tests with the project's free-test config (outputDir '${FREE_RUN_DIR}', trace: 'on', the sungen run-record reporter — "Free tests" in the sungen docs)`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** The free run left its JSON results or a sungen run record in `test-results-free/`. */
|
|
41
|
+
export function freeRunHasResults(cwd: string): boolean {
|
|
42
|
+
try {
|
|
43
|
+
return fs.readdirSync(path.join(cwd, FREE_RUN_DIR)).some((f) => f === 'results.json' || /^sungen-run-record.*\.json$/.test(f));
|
|
44
|
+
} catch {
|
|
45
|
+
return false; // never run
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -1,23 +1,27 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The "Waiting for you" block (#954) — every pending human-only action of a unit, as ready-to-copy
|
|
3
3
|
* commands with the reason under each. One renderer for the three places that show it: `sungen next`
|
|
4
|
-
* (text) and the Stop / SessionStart hook context. Commands
|
|
5
|
-
* with a local install, #939); `--json` keeps
|
|
6
|
-
*
|
|
4
|
+
* (text) and the Stop / SessionStart hook context. Commands — and those inside a reason (#963) — as the
|
|
5
|
+
* project runs sungen (`npx sungen` with a local install, #939); `--json` keeps `command` canonical in
|
|
6
|
+
* `humanActions` and adds `rendered`. An instruction (a commit through review, a `git checkout --`) is
|
|
7
|
+
* marked `✎ … (by hand, not a command)`, never shown as one to copy.
|
|
7
8
|
*/
|
|
8
9
|
import type { HumanAction } from '../graph/human-action';
|
|
9
|
-
import { withProjectInvocation } from './project-sungen-invocation';
|
|
10
|
+
import { withProjectInvocation, withProjectInvocationInText } from './project-sungen-invocation';
|
|
10
11
|
|
|
11
12
|
export const WAITING_FOR_YOU_HEAD = '⏳ Waiting for you — a human runs these in their own terminal (in Claude Code: ! <command>); an agent never runs them:';
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
|
-
* The block's lines (indented two spaces), or none when nothing waits for a human
|
|
15
|
-
* of the blocking step that cut the frontier — the block is not exhaustive then,
|
|
15
|
+
* The block's lines (indented two spaces), or none when nothing waits for a human and nothing holds the
|
|
16
|
+
* rest. `heldBy`: the title of the blocking step that cut the frontier — the block is not exhaustive then,
|
|
17
|
+
* and says what holds the rest, with that step's command (`heldByCommand`) when given (#966). A held
|
|
18
|
+
* block with no item of its own still says so, rather than vanishing (a QA read "nothing is waiting").
|
|
16
19
|
*/
|
|
17
|
-
export function waitingForYouLines(actions: readonly HumanAction[] | undefined, projectRoot: string, heldBy?: string): string[] {
|
|
18
|
-
|
|
20
|
+
export function waitingForYouLines(actions: readonly HumanAction[] | undefined, projectRoot: string, heldBy?: string, heldByCommand?: string): string[] {
|
|
21
|
+
const holds = heldBy ? `"${heldBy}" clears${heldByCommand ? ` (the step above: ${withProjectInvocationInText(heldByCommand, projectRoot)})` : ''}` : '';
|
|
22
|
+
if (!actions?.length) return heldBy ? [` ⏳ Waiting for you: nothing yet — more appear once ${holds}.`] : [];
|
|
19
23
|
return [` ${WAITING_FOR_YOU_HEAD}`, ...actions.flatMap((a) => [
|
|
20
24
|
a.instruction ? ` ✎ ${a.command} (by hand, not a command)` : ` ${withProjectInvocation(a.command, projectRoot)}`,
|
|
21
|
-
` ${a.reason}`,
|
|
22
|
-
]), ...(heldBy ? [` More appear once
|
|
25
|
+
` ${withProjectInvocationInText(a.reason, projectRoot)}`,
|
|
26
|
+
]), ...(heldBy ? [` More appear once ${holds}.`] : [])];
|
|
23
27
|
}
|
|
@@ -12,17 +12,17 @@
|
|
|
12
12
|
* Bounded (review H4, M2): a deadline (`budgetMs`; the guard's default is ~3 s) — a scenario not
|
|
13
13
|
* judged in time is `unproven` when approved, a warning while proposed: fail CLOSED. An input that
|
|
14
14
|
* cannot be read while judging (test-data, a trace) makes that scenario unproven with its reason,
|
|
15
|
-
* never a thrown verdict. A trace older than the unit's
|
|
15
|
+
* never a thrown verdict. A trace older than the unit's `.feature` / lock is refused (stale) — judged
|
|
16
|
+
* by their content, so a touch or a fresh clone stales nothing (#967, trace/trace-freshness.ts).
|
|
16
17
|
*/
|
|
17
18
|
import { createHash } from 'crypto';
|
|
18
19
|
import * as fs from 'fs';
|
|
19
20
|
import { readCapabilities } from '../harness/capability';
|
|
20
21
|
import { classifyTraceRuns, notJudged, TraceOutcome } from './classify-trace-conformance';
|
|
21
22
|
import { ContractLock, ContractScenario, isHumanApproved, LockEntry } from './contract-schema';
|
|
22
|
-
import { lockPathFor } from './contracts-lock-file';
|
|
23
23
|
import { loadTestDataIndex, TestDataIndex } from './data-normalize';
|
|
24
|
-
import { contractFeatureFiles } from './normalize-feature-steps';
|
|
25
24
|
import { renderTraceSteps, RenderedStep } from './render-free-gherkin';
|
|
25
|
+
import { unitContract } from './trace/trace-freshness';
|
|
26
26
|
import { collectScenarioEvidence, OVER_BUDGET, ScenarioEvidence, TraceRun } from './trace/collect-trace-evidence';
|
|
27
27
|
|
|
28
28
|
export { OVER_BUDGET } from './trace/collect-trace-evidence';
|
|
@@ -33,7 +33,6 @@ import { traceIntentSteps } from './trace/trace-steps';
|
|
|
33
33
|
|
|
34
34
|
export interface UnitTraceConformance { outcomes: Map<string, TraceOutcome>; warnings: string[]; inputs: unknown }
|
|
35
35
|
|
|
36
|
-
const mtimeMs = (f: string): number => { try { return fs.statSync(f).mtimeMs; } catch { return 0; } };
|
|
37
36
|
const sha = (s: string | Buffer): string => createHash('sha256').update(s).digest('hex');
|
|
38
37
|
const why = (e: unknown): string => (e instanceof Error ? e.message : String(e)).split('\n')[0].slice(0, 300);
|
|
39
38
|
|
|
@@ -65,8 +64,7 @@ export function traceConformanceForUnit(
|
|
|
65
64
|
let index: TestDataIndex = { byValue: new Map() };
|
|
66
65
|
let setupError = '';
|
|
67
66
|
try {
|
|
68
|
-
|
|
69
|
-
evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), notBefore, deadline, new Map(traced.map((s) => [s.id, s.source.name])));
|
|
67
|
+
evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), unitContract(projectRoot, unit), deadline, new Map(traced.map((s) => [s.id, s.source.name])));
|
|
70
68
|
index = loadTestDataIndex(unit.dir, unit.name);
|
|
71
69
|
} catch (e) { setupError = `trace conformance could not run: ${why(e)}`; }
|
|
72
70
|
const { cache, problem } = readIdentityCache(projectRoot);
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
* identities asked of the registered `traceIdentity` resolver (driver-ui) when one is given,
|
|
9
9
|
* otherwise the cache, otherwise core's heuristic — `# identity low` marks those steps
|
|
10
10
|
* approved a scenario QA already approved (no `@proposed`) is never rewritten
|
|
11
|
+
* one id, one two recorded tests claiming one id (#962) refuse the derive — both named with
|
|
12
|
+
* test file:line, NOTHING written for the unit: keeping one would drop the other's contract
|
|
11
13
|
* safety redaction by field meaning, machine paths scrubbed, then the secret scanner and the
|
|
12
14
|
* semantic credential check over the exact text: any hit → NOTHING is written
|
|
13
15
|
*/
|
|
@@ -25,6 +27,7 @@ import { scenariosOfFeatureFile } from './normalize-feature-steps';
|
|
|
25
27
|
import { featureFilesFor } from '../harness/unit-paths';
|
|
26
28
|
import { CONTROL_CHARS, freeFeatureText, renderTraceSteps, scenarioBlockText } from './render-free-gherkin';
|
|
27
29
|
import { collectRuns, TraceRun } from './trace/collect-trace-evidence';
|
|
30
|
+
import { idsClaimedByTwoTests } from './trace/scenario-id-claims';
|
|
28
31
|
import { readIdentityCache } from './trace/identity-cache';
|
|
29
32
|
import type { TraceIdentityResolver } from './trace/target-identity';
|
|
30
33
|
import { identitiesFor, resolveWithDriver } from './trace/trace-identity-resolution';
|
|
@@ -39,6 +42,11 @@ export interface DeriveResult {
|
|
|
39
42
|
/** `approved: true` — skipped because QA already approved it (derive never rewrites one), not for want of a trace. */
|
|
40
43
|
skipped: Array<{ id: string; reason: string; approved?: true }>;
|
|
41
44
|
refused?: SecretHit[];
|
|
45
|
+
/**
|
|
46
|
+
* Ids two or more recorded tests claim (#962), each test as `file:line "title" (run record <path>)` —
|
|
47
|
+
* nothing was written. `staleRecords`: older test-results/ records that are a claim's only source.
|
|
48
|
+
*/
|
|
49
|
+
duplicateIds?: Array<{ id: string; tests: string[]; staleRecords?: string[] }>;
|
|
42
50
|
note?: string;
|
|
43
51
|
/** A derived id another `.feature` of the unit already uses (#939) — `contract check` and approve would trip on it later. */
|
|
44
52
|
warnings?: string[];
|
|
@@ -75,7 +83,7 @@ function reusedIdWarnings(unit: { name: string; dir: string }, freeFile: string,
|
|
|
75
83
|
} catch { /* an unparseable sibling is `contract check`'s to report */ }
|
|
76
84
|
}
|
|
77
85
|
return ids.filter((id) => owners.has(id)).map((id) =>
|
|
78
|
-
`${id} is already the ID of a scenario in ${owners.get(id)} — the derived scenario reuses it; give the free test (its title's leading ID) or that scenario its own ID, or \`sungen contract check\` reports it twice and approval refuses it`);
|
|
86
|
+
`${id} is already the ID of a scenario in ${owners.get(id)} — the derived scenario reuses it; give the free test (its title's leading ID, or its @tag) or that scenario its own ID, or \`sungen contract check\` reports it twice and approval refuses it`);
|
|
79
87
|
}
|
|
80
88
|
|
|
81
89
|
/**
|
|
@@ -84,7 +92,7 @@ function reusedIdWarnings(unit: { name: string; dir: string }, freeFile: string,
|
|
|
84
92
|
* refused, or the project is mobile (`note`).
|
|
85
93
|
*/
|
|
86
94
|
export function nothingDerived(r: DeriveResult): boolean {
|
|
87
|
-
return !r.note && !r.refused?.length && r.scenarios.length === 0 && !r.skipped.some((s) => s.approved);
|
|
95
|
+
return !r.note && !r.refused?.length && !r.duplicateIds?.length && r.scenarios.length === 0 && !r.skipped.some((s) => s.approved);
|
|
88
96
|
}
|
|
89
97
|
|
|
90
98
|
export async function deriveFreeFeature(i: DeriveInput): Promise<DeriveResult> {
|
|
@@ -93,7 +101,11 @@ export async function deriveFreeFeature(i: DeriveInput): Promise<DeriveResult> {
|
|
|
93
101
|
const result: DeriveResult = { file, written: false, scenarios: [], skipped: [] };
|
|
94
102
|
if ((readCapabilities(i.projectRoot).platform ?? '').toLowerCase() === 'mobile') return { ...result, note: MOBILE_NOT_SUPPORTED };
|
|
95
103
|
const grep = i.grep ? new RegExp(i.grep, 'i') : null;
|
|
96
|
-
const found = collectRuns(i.projectRoot, (id, t) => (grep ? grep.test(`${id} ${t.title} ${t.file}`) : belongsToUnit(t.file, unitBase.toLowerCase())),
|
|
104
|
+
const found = collectRuns(i.projectRoot, (id, t) => (grep ? grep.test(`${id} ${t.title} ${t.file}`) : belongsToUnit(t.file, unitBase.toLowerCase())), null);
|
|
105
|
+
const claimed = idsClaimedByTwoTests(i.projectRoot, found.runs);
|
|
106
|
+
if (claimed.size) {
|
|
107
|
+
return { ...result, duplicateIds: [...claimed].sort(([a], [b]) => a.localeCompare(b, 'en')).map(([id, c]) => ({ id, tests: c.tests, ...(c.staleRecords.length ? { staleRecords: c.staleRecords } : {}) })) };
|
|
108
|
+
}
|
|
97
109
|
const byId = new Map<string, TraceRun[]>();
|
|
98
110
|
for (const r of found.runs) byId.set(r.scenarioId, [...(byId.get(r.scenarioId) ?? []), r]);
|
|
99
111
|
const admissible = (runs: TraceRun[]): TraceRun[] => runs.filter((r) => r.status === 'admissible' && r.trace)
|
|
@@ -3,33 +3,56 @@
|
|
|
3
3
|
* resolver (driver-ui — a real browser) about every target of the unit's admissible traces and
|
|
4
4
|
* cache the answers (DOS-14.5). The verdict itself never calls a browser; it reads the cache, so
|
|
5
5
|
* the CLI and the Claude Code guard keep building the same verdict from the same files.
|
|
6
|
+
*
|
|
7
|
+
* What it reports is split for QA (#966): a WARNING only when something needs attention — no
|
|
8
|
+
* resolver, a mobile project, a target the browser could not name (its steps stay opaque). "Every
|
|
9
|
+
* target was already known" and "N new targets identified" are information, in plain words.
|
|
6
10
|
*/
|
|
7
|
-
import * as fs from 'fs';
|
|
8
11
|
import { capabilityRegistry } from '../capabilities/registry';
|
|
9
12
|
import { readCapabilities } from '../harness/capability';
|
|
10
13
|
import { MOBILE_NOT_SUPPORTED } from './build-trace-conformance';
|
|
11
|
-
import {
|
|
12
|
-
import { contractFeatureFiles, loadContractScenarios } from './normalize-feature-steps';
|
|
14
|
+
import { loadContractScenarios } from './normalize-feature-steps';
|
|
13
15
|
import { collectScenarioEvidence } from './trace/collect-trace-evidence';
|
|
16
|
+
import { readIdentityCache } from './trace/identity-cache';
|
|
17
|
+
import { unitContract } from './trace/trace-freshness';
|
|
14
18
|
import type { TraceIdentityResolver } from './trace/target-identity';
|
|
15
19
|
import { resolveWithDriver } from './trace/trace-identity-resolution';
|
|
16
20
|
|
|
17
|
-
|
|
21
|
+
export interface TraceIdentityOutcome { warnings: string[]; info: string[] }
|
|
18
22
|
|
|
19
23
|
/** The first registered capability that can resolve trace identities (UI: driver-ui), if any. */
|
|
20
24
|
export function registeredTraceResolver(): TraceIdentityResolver | undefined {
|
|
21
25
|
return capabilityRegistry.all().find((c) => typeof c.traceIdentity === 'function')?.traceIdentity;
|
|
22
26
|
}
|
|
23
27
|
|
|
24
|
-
export async function
|
|
28
|
+
export async function traceIdentityOutcome(projectRoot: string, unit: { name: string; dir: string }, resolver = registeredTraceResolver()): Promise<TraceIdentityOutcome> {
|
|
29
|
+
const none: TraceIdentityOutcome = { warnings: [], info: [] };
|
|
25
30
|
const traced = loadContractScenarios(unit.dir, unit.name).filter((s) => s.implementation !== 'compiled');
|
|
26
|
-
if (!traced.length) return
|
|
27
|
-
if ((readCapabilities(projectRoot).platform ?? '').toLowerCase() === 'mobile') return [MOBILE_NOT_SUPPORTED];
|
|
28
|
-
if (!resolver) return ['no trace identity resolver is registered (install @sungen/driver-ui) — every target identity stays low, so nothing can match'];
|
|
29
|
-
const
|
|
30
|
-
const evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), notBefore);
|
|
31
|
+
if (!traced.length) return none;
|
|
32
|
+
if ((readCapabilities(projectRoot).platform ?? '').toLowerCase() === 'mobile') return { warnings: [MOBILE_NOT_SUPPORTED], info: [] };
|
|
33
|
+
if (!resolver) return { warnings: ['no trace identity resolver is registered (install @sungen/driver-ui) — every target identity stays low, so nothing can match'], info: [] };
|
|
34
|
+
const evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), unitContract(projectRoot, unit));
|
|
31
35
|
const traces = [...evidence.runs.values()].flat().filter((r) => r.status === 'admissible' && r.trace).map((r) => r.trace!);
|
|
32
|
-
if (!traces.length) return
|
|
36
|
+
if (!traces.length) return none;
|
|
33
37
|
const r = await resolveWithDriver(projectRoot, traces, resolver);
|
|
34
|
-
|
|
38
|
+
const { cache } = readIdentityCache(projectRoot);
|
|
39
|
+
const total = r.keys.length;
|
|
40
|
+
const unnamed = r.keys.filter((k) => cache.entries.get(k)?.identityConfidence !== 'high').length;
|
|
41
|
+
const blocked = r.blockedRequests ? `; the replayed pages tried ${r.blockedRequests} request(s), all blocked` : '';
|
|
42
|
+
const known = total - r.asked;
|
|
43
|
+
const info = r.asked
|
|
44
|
+
? `trace identity: ${r.asked} new target(s) identified in a real browser (${r.resolver})${known ? `, ${known} already known` : ''}${blocked}`
|
|
45
|
+
: `trace identity: all ${total} target(s) were already identified (identity cache) — nothing new to resolve`;
|
|
46
|
+
const warnings = unnamed
|
|
47
|
+
? [`trace identity: ${unnamed} of ${total} target(s) could not be identified (no unique accessible name) — the steps on them stay opaque, for QA to review, until the test targets a unique, named element`]
|
|
48
|
+
: [];
|
|
49
|
+
return { warnings, info: [info] };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* What `contract check --trace` adds to the verdict's warnings: only what needs attention (#966).
|
|
54
|
+
* The plain-words information is `traceIdentityOutcome(...).info`.
|
|
55
|
+
*/
|
|
56
|
+
export async function resolveUnitTraceIdentities(projectRoot: string, unit: { name: string; dir: string }, resolver = registeredTraceResolver()): Promise<string[]> {
|
|
57
|
+
return (await traceIdentityOutcome(projectRoot, unit, resolver)).warnings;
|
|
35
58
|
}
|
|
@@ -8,6 +8,9 @@
|
|
|
8
8
|
* refused the trace exists but may not speak (swapped, stale, rejected zip…)
|
|
9
9
|
* unsupported the trace's format version is unknown (exit ≠ 0)
|
|
10
10
|
*
|
|
11
|
+
* Two recorded TESTS claiming one scenario id (#962) make its evidence ambiguous: neither speaks —
|
|
12
|
+
* `unproven`, both named (scenario-id-claims.ts) — instead of judging two tests' runs as one.
|
|
13
|
+
*
|
|
11
14
|
* Only traces of the scenario ids asked for are opened, so a unit pays for its own tests only.
|
|
12
15
|
*/
|
|
13
16
|
import * as fs from 'fs';
|
|
@@ -15,7 +18,9 @@ import * as path from 'path';
|
|
|
15
18
|
import { isIdLike } from '../../harness/parse';
|
|
16
19
|
import { clip, CONTROL_CHARS } from '../render-free-gherkin';
|
|
17
20
|
import { FileRead, readInsideProject } from './read-file-once';
|
|
21
|
+
import { idsClaimedByTwoTests, twoTestsClaim } from './scenario-id-claims';
|
|
18
22
|
import { bindTrace } from './trace-binding';
|
|
23
|
+
import type { UnitContract } from './trace-freshness';
|
|
19
24
|
import { traceFromRead, ParsedTrace } from './trace-file-reader';
|
|
20
25
|
import { traceZipLimits, TraceRejected } from './trace-zip-limits';
|
|
21
26
|
import { finalAttempt, readPlaywrightReport, ReportResult, ReportTest, scenarioIdOfTest } from './select-final-attempt';
|
|
@@ -49,21 +54,33 @@ function notRead(name: string, r: Exclude<FileRead, { status: 'read' }>): string
|
|
|
49
54
|
export const OVER_BUDGET = 'verdict budget exceeded; run `sungen contract check`';
|
|
50
55
|
const NO_RECORD = "no sungen run record under test-results-free/ (or test-results/) — run the free tests with their own config: outputDir 'test-results-free', trace: 'on' and the sungen run-record reporter";
|
|
51
56
|
|
|
52
|
-
/**
|
|
53
|
-
function
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
57
|
+
/** `p` relative to `root` when it lies under it, else null. */
|
|
58
|
+
function under(root: string, p: string): string | null {
|
|
59
|
+
const rel = path.relative(root, p);
|
|
60
|
+
return rel && !rel.startsWith('..') && !path.isAbsolute(rel) ? rel : null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Where a report's trace attachment is: the first candidate that exists (read later, opened once, and
|
|
65
|
+
* only from inside the project). Playwright writes attachment paths ABSOLUTE, in the checkout that ran;
|
|
66
|
+
* after a clone or a move that checkout is elsewhere, so a path under the report's own config dir is
|
|
67
|
+
* first tried at the same place under this project (#967) — the run record's sha256 still binds the bytes.
|
|
68
|
+
*/
|
|
69
|
+
function attachmentPath(projectRoot: string, test: ReportTest, p: string): string | null {
|
|
70
|
+
const moved = path.isAbsolute(p) && test.configRoot && !under(path.resolve(projectRoot), p) ? under(test.configRoot, p) : null;
|
|
71
|
+
const candidates = path.isAbsolute(p)
|
|
72
|
+
? [...(moved ? [path.resolve(projectRoot, moved)] : []), p]
|
|
73
|
+
: [path.resolve(projectRoot, p), path.resolve(path.dirname(test.report), p)];
|
|
74
|
+
return candidates.find((c) => fs.existsSync(c)) ?? null;
|
|
58
75
|
}
|
|
59
76
|
|
|
60
|
-
function judge(projectRoot: string, record: RunRecord, test: ReportTest,
|
|
77
|
+
function judge(projectRoot: string, record: RunRecord, test: ReportTest, contract: UnitContract | null): TraceRun {
|
|
61
78
|
const scenarioId = scenarioIdOfTest(test.title, test.tags);
|
|
62
79
|
const base = { scenarioId, test, record };
|
|
63
80
|
const fin = finalAttempt(test);
|
|
64
81
|
if (fin.admissibility !== 'admissible') return { ...base, result: fin.result, status: fin.admissibility, reason: fin.reason };
|
|
65
82
|
const missing: TraceRun = { ...base, result: fin.result, status: 'unproven', reason: `no trace at ${fin.result!.tracePath} (the report points at a missing file)` };
|
|
66
|
-
const cand = attachmentPath(projectRoot, test
|
|
83
|
+
const cand = attachmentPath(projectRoot, test, fin.result!.tracePath!);
|
|
67
84
|
if (!cand) return missing;
|
|
68
85
|
// Opened once, inside the project, and what was opened is what is parsed and hashed (#926).
|
|
69
86
|
const limits = traceZipLimits();
|
|
@@ -77,17 +94,22 @@ function judge(projectRoot: string, record: RunRecord, test: ReportTest, notBefo
|
|
|
77
94
|
const unsupported = e instanceof TraceRejected && e.code === 'trace.version-unsupported';
|
|
78
95
|
return { ...base, result: fin.result, status: unsupported ? 'unsupported' : 'refused', reason: `trace refused: ${(e as Error).message}` };
|
|
79
96
|
}
|
|
80
|
-
const bound = bindTrace({ projectRoot, test, result: fin.result!, record, trace,
|
|
97
|
+
const bound = bindTrace({ projectRoot, test, result: fin.result!, record, trace, contract });
|
|
81
98
|
if (bound.ok === false) return { ...base, result: fin.result, trace, status: 'refused', reason: `trace refused: ${bound.reason}` };
|
|
82
99
|
return { ...base, result: fin.result, trace, status: 'admissible', specFile: bound.specFile };
|
|
83
100
|
}
|
|
84
101
|
|
|
85
|
-
/**
|
|
102
|
+
/**
|
|
103
|
+
* Every judged run of the tests `want` accepts (derive: a unit's tests; check: by scenario id). A trace
|
|
104
|
+
* older than `contract` — the unit's .feature files and lock, by content (#967) — is refused; derive,
|
|
105
|
+
* which writes the contract from the traces, passes null.
|
|
106
|
+
*/
|
|
86
107
|
export function collectRuns(
|
|
87
|
-
projectRoot: string, want: (id: string, test: ReportTest) => boolean,
|
|
88
|
-
): { runs: TraceRun[]; problems: string[]; inputs: string[]; records: number; seen: Array<{ id: string; title: string }> } {
|
|
108
|
+
projectRoot: string, want: (id: string, test: ReportTest) => boolean, contract: UnitContract | null, deadline = Number.POSITIVE_INFINITY,
|
|
109
|
+
): { runs: TraceRun[]; problems: string[]; untrusted: string[]; inputs: string[]; records: number; seen: Array<{ id: string; title: string }> } {
|
|
89
110
|
const runs: TraceRun[] = [];
|
|
90
111
|
const problems: string[] = [];
|
|
112
|
+
const untrusted: string[] = [];
|
|
91
113
|
const inputs: string[] = [];
|
|
92
114
|
const seen: Array<{ id: string; title: string }> = [];
|
|
93
115
|
const records = findRunRecords(projectRoot);
|
|
@@ -95,7 +117,11 @@ export function collectRuns(
|
|
|
95
117
|
let record: RunRecord;
|
|
96
118
|
const recordRead = readInsideProject(projectRoot, file, MAX_EVIDENCE_JSON_BYTES);
|
|
97
119
|
if (recordRead.status !== 'read') { problems.push(notRead(path.basename(file), recordRead)); continue; }
|
|
98
|
-
try { record = readRunRecord(file, recordRead.bytes.toString('utf8')); } catch (e) {
|
|
120
|
+
try { record = readRunRecord(file, recordRead.bytes.toString('utf8')); } catch (e) {
|
|
121
|
+
untrusted.push(`${path.basename(file)} is not trusted: ${(e as Error).message}`);
|
|
122
|
+
problems.push(untrusted[untrusted.length - 1]);
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
99
125
|
if (!record.report) { problems.push(`${path.basename(file)} names no JSON report — add the json reporter to the free-test config`); continue; }
|
|
100
126
|
const reportRead = readInsideProject(projectRoot, path.resolve(projectRoot, record.report), MAX_EVIDENCE_JSON_BYTES);
|
|
101
127
|
if (reportRead.status !== 'read') { problems.push(`${path.basename(file)}: ${notRead(`the report ${path.basename(record.report)}`, reportRead)}`); continue; }
|
|
@@ -106,10 +132,10 @@ export function collectRuns(
|
|
|
106
132
|
const id = scenarioIdOfTest(test.title, test.tags);
|
|
107
133
|
seen.push({ id, title: test.title });
|
|
108
134
|
if (!want(id, test)) continue;
|
|
109
|
-
runs.push(Date.now() > deadline ? { scenarioId: id, test, record, status: 'unproven', reason: OVER_BUDGET } : judge(projectRoot, record, test,
|
|
135
|
+
runs.push(Date.now() > deadline ? { scenarioId: id, test, record, status: 'unproven', reason: OVER_BUDGET } : judge(projectRoot, record, test, contract));
|
|
110
136
|
}
|
|
111
137
|
}
|
|
112
|
-
return { runs, problems, inputs, records: records.length, seen };
|
|
138
|
+
return { runs, problems, untrusted, inputs, records: records.length, seen };
|
|
113
139
|
}
|
|
114
140
|
|
|
115
141
|
/** A title without its leading id, for comparing words only: `VP-X-001 Login works` → `login works`. */
|
|
@@ -135,12 +161,18 @@ function noTestFor(id: string, name: string | undefined, seen: Array<{ id: strin
|
|
|
135
161
|
|
|
136
162
|
/** The runs of each scenario id; `names` (id → scenario name) lets a missing test be explained. */
|
|
137
163
|
export function collectScenarioEvidence(
|
|
138
|
-
projectRoot: string, ids: ReadonlySet<string>,
|
|
164
|
+
projectRoot: string, ids: ReadonlySet<string>, contract: UnitContract | null, deadline?: number, names?: ReadonlyMap<string, string>,
|
|
139
165
|
): ScenarioEvidence {
|
|
140
|
-
const found = collectRuns(projectRoot, (id) => ids.has(id),
|
|
166
|
+
const found = collectRuns(projectRoot, (id) => ids.has(id), contract, deadline);
|
|
141
167
|
const runs = new Map<string, TraceRun[]>([...ids].map((id) => [id, found.runs.filter((r) => r.scenarioId === id)]));
|
|
168
|
+
// Ambiguous evidence: no run of either test may speak for the id — never one picked arbitrarily.
|
|
169
|
+
for (const [id, claim] of idsClaimedByTwoTests(projectRoot, found.runs)) {
|
|
170
|
+
if (runs.has(id)) runs.set(id, [{ scenarioId: id, status: 'unproven', reason: twoTestsClaim(id, claim) }]);
|
|
171
|
+
}
|
|
142
172
|
for (const [id, list] of runs) {
|
|
143
|
-
|
|
173
|
+
// A record that was not trusted may be why (a newer reporter, a hand edit): the finding says so.
|
|
174
|
+
const why = found.untrusted.length ? ` — ${found.untrusted.join('; ')}` : '';
|
|
175
|
+
if (!list.length) list.push({ scenarioId: id, status: 'unproven', reason: found.records ? `${noTestFor(id, names?.get(id), found.seen)}${why}` : NO_RECORD });
|
|
144
176
|
}
|
|
145
177
|
return { runs, problems: found.problems, inputs: found.inputs };
|
|
146
178
|
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One scenario id, one test (#962): which ids more than one recorded TEST claims, said the same way
|
|
3
|
+
* by `contract derive` (it refuses) and `contract check` (the id is unproven). One test recorded
|
|
4
|
+
* again — another run or shard, a retry, a second Playwright project — is still one test: the same
|
|
5
|
+
* file and the same Playwright test id.
|
|
6
|
+
*
|
|
7
|
+
* Each claiming test names the run record(s) it came from. A test that only an older run's record
|
|
8
|
+
* under `test-results/` still mentions — while another claim comes from the free config's own
|
|
9
|
+
* `test-results-free/` — is most likely gone: that record is offered for deletion.
|
|
10
|
+
*/
|
|
11
|
+
import * as path from 'path';
|
|
12
|
+
import { clip } from '../render-free-gherkin';
|
|
13
|
+
import type { TraceRun } from './collect-trace-evidence';
|
|
14
|
+
import { FREE_RUN_DIR } from './trace-run-record';
|
|
15
|
+
|
|
16
|
+
export interface IdClaim {
|
|
17
|
+
/** `file:line "title" (run record <path>)` per claiming test, sorted. */
|
|
18
|
+
tests: string[];
|
|
19
|
+
/** Project-relative records under test-results/ that are a claim's only source, next to a test-results-free/ claim. */
|
|
20
|
+
staleRecords: string[];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const posix = (p: string): string => p.split(path.sep).join('/');
|
|
24
|
+
/** A run's project-relative test file: the run record's (what the binding checks), else the report's. */
|
|
25
|
+
const testFileOf = (r: TraceRun): string => r.record?.attempts.find((a) => a.testId === r.test?.testId)?.file ?? r.test?.file ?? '?';
|
|
26
|
+
const label = (r: TraceRun, records: string[]): string =>
|
|
27
|
+
`${clip(testFileOf(r), 200)}:${r.test?.line ?? 0} "${clip(r.test?.title ?? '')}" (run record ${records.map((f) => clip(f, 200)).join(', ')})`;
|
|
28
|
+
|
|
29
|
+
export function idsClaimedByTwoTests(projectRoot: string, runs: readonly TraceRun[]): Map<string, IdClaim> {
|
|
30
|
+
const byId = new Map<string, Map<string, { run: TraceRun; records: Set<string> }>>();
|
|
31
|
+
for (const r of runs) {
|
|
32
|
+
if (!r.test) continue;
|
|
33
|
+
const tests = byId.get(r.scenarioId) ?? new Map<string, { run: TraceRun; records: Set<string> }>();
|
|
34
|
+
const key = `${testFileOf(r)}\0${r.test.testId}`;
|
|
35
|
+
const entry = tests.get(key) ?? { run: r, records: new Set<string>() };
|
|
36
|
+
if (r.record?.file) entry.records.add(posix(path.relative(projectRoot, r.record.file)));
|
|
37
|
+
tests.set(key, entry);
|
|
38
|
+
byId.set(r.scenarioId, tests);
|
|
39
|
+
}
|
|
40
|
+
const claims = new Map<string, IdClaim>();
|
|
41
|
+
for (const [id, tests] of byId) {
|
|
42
|
+
if (tests.size < 2) continue;
|
|
43
|
+
const entries = [...tests.values()].map((t) => ({ ...t, records: [...t.records].sort() }));
|
|
44
|
+
const inFreeDir = entries.some((t) => t.records.some((f) => f.startsWith(`${FREE_RUN_DIR}/`)));
|
|
45
|
+
const legacyOnly = entries.filter((t) => t.records.length > 0 && t.records.every((f) => f.startsWith('test-results/')));
|
|
46
|
+
claims.set(id, {
|
|
47
|
+
tests: entries.map((t) => label(t.run, t.records)).sort(),
|
|
48
|
+
staleRecords: inFreeDir ? [...new Set(legacyOnly.flatMap((t) => t.records))].sort() : [],
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
return claims;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** What to tell the user when several tests claim one id — check and derive say it the same way. */
|
|
55
|
+
export function twoTestsClaim(id: string, claim: IdClaim): string {
|
|
56
|
+
const stale = claim.staleRecords.length
|
|
57
|
+
? `; if that test is gone, delete ${claim.staleRecords.join(', ')} — an older run's record (free tests now record under ${FREE_RUN_DIR}/)`
|
|
58
|
+
: '';
|
|
59
|
+
return `${claim.tests.length} recorded tests claim ${id}: ${claim.tests.join(', ')} — give each test its own ID (its title's leading ID, or its @tag), then run them again with trace on${stale}`;
|
|
60
|
+
}
|
|
@@ -11,10 +11,15 @@
|
|
|
11
11
|
* leading id, else the whole title — the same rule the contract uses for a scenario name.
|
|
12
12
|
*/
|
|
13
13
|
import * as fs from 'fs';
|
|
14
|
+
import * as path from 'path';
|
|
14
15
|
import { isIdLike } from '../../harness/parse';
|
|
15
16
|
|
|
16
17
|
export interface ReportResult { retry: number; status: string; startTime: string; duration: number; tracePath: string | null }
|
|
17
|
-
export interface ReportTest {
|
|
18
|
+
export interface ReportTest {
|
|
19
|
+
testId: string; title: string; file: string; line: number; tags: string[]; project: string; results: ReportResult[]; report: string;
|
|
20
|
+
/** The directory of the config that ran (`config.configFile`, absolute as Playwright writes it) — where absolute attachment paths start. */
|
|
21
|
+
configRoot?: string;
|
|
22
|
+
}
|
|
18
23
|
export type Admissibility = 'admissible' | 'inconclusive' | 'unproven';
|
|
19
24
|
|
|
20
25
|
const isObj = (v: unknown): v is Record<string, unknown> => !!v && typeof v === 'object' && !Array.isArray(v);
|
|
@@ -38,6 +43,8 @@ export function readPlaywrightReport(file: string, text?: string): ReportTest[]
|
|
|
38
43
|
try { doc = JSON.parse(text ?? fs.readFileSync(file, 'utf8')); } catch (e) { throw new Error(`the JSON report ${file} is not readable (${(e as Error).message.split('\n')[0]})`); }
|
|
39
44
|
if (!isObj(doc) || !Array.isArray(doc.suites)) throw new Error(`${file} is not a Playwright JSON report (no suites)`);
|
|
40
45
|
const out: ReportTest[] = [];
|
|
46
|
+
const configFile = isObj(doc.config) && typeof doc.config.configFile === 'string' && path.isAbsolute(doc.config.configFile) ? doc.config.configFile : null;
|
|
47
|
+
const configRoot = configFile ? { configRoot: path.dirname(configFile) } : {};
|
|
41
48
|
for (const top of doc.suites) {
|
|
42
49
|
for (const spec of specsOf(top)) {
|
|
43
50
|
if (typeof spec.id !== 'string' || typeof spec.title !== 'string' || typeof spec.file !== 'string') continue;
|
|
@@ -46,7 +53,7 @@ export function readPlaywrightReport(file: string, text?: string): ReportTest[]
|
|
|
46
53
|
const results = arr(t.results).map(resultOf).filter((r): r is ReportResult => !!r).sort((a, b) => a.retry - b.retry);
|
|
47
54
|
out.push({
|
|
48
55
|
testId: spec.id, title: spec.title, file: spec.file, line: Number(spec.line) || 0,
|
|
49
|
-
tags: arr(spec.tags).map(String), project: String(t.projectName ?? ''), results, report: file,
|
|
56
|
+
tags: arr(spec.tags).map(String), project: String(t.projectName ?? ''), results, report: file, ...configRoot,
|
|
50
57
|
});
|
|
51
58
|
}
|
|
52
59
|
}
|
|
@@ -8,12 +8,14 @@
|
|
|
8
8
|
* code sha256 of the test file NOW equals the hash the record took at run start
|
|
9
9
|
* title every library part says `<file>:<line> › … › <title>` of exactly this test
|
|
10
10
|
* clock the trace started inside the attempt's window (±2 s)
|
|
11
|
-
* fresh the trace is
|
|
11
|
+
* fresh the trace is not older than the .feature files and the lock it is asked to prove —
|
|
12
|
+
* judged by their content, not their file time (#967, trace-freshness.ts)
|
|
12
13
|
*/
|
|
13
14
|
import { createHash } from 'crypto';
|
|
14
15
|
import * as fs from 'fs';
|
|
15
16
|
import * as path from 'path';
|
|
16
17
|
import type { ParsedTrace } from './trace-file-reader';
|
|
18
|
+
import { staleReason, UnitContract } from './trace-freshness';
|
|
17
19
|
import type { RunRecord } from './trace-run-record';
|
|
18
20
|
import type { ReportResult, ReportTest } from './select-final-attempt';
|
|
19
21
|
|
|
@@ -23,8 +25,8 @@ export interface BindingInput {
|
|
|
23
25
|
result: ReportResult;
|
|
24
26
|
record: RunRecord;
|
|
25
27
|
trace: ParsedTrace;
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
+
/** The unit's contract as it is now; null when no contract is proven (derive reads traces to write one). */
|
|
29
|
+
contract: UnitContract | null;
|
|
28
30
|
}
|
|
29
31
|
export type Binding = { ok: true; specFile: string } | { ok: false; reason: string };
|
|
30
32
|
|
|
@@ -51,6 +53,7 @@ export function bindTrace(i: BindingInput): Binding {
|
|
|
51
53
|
if (typeof wall !== 'number' || Number.isNaN(start) || wall < start - CLOCK_SLACK_MS || wall > start + i.result.duration + CLOCK_SLACK_MS) {
|
|
52
54
|
return fail('the trace was not recorded during this attempt (clock window)');
|
|
53
55
|
}
|
|
54
|
-
|
|
56
|
+
const stale = i.contract ? staleReason(i.contract, i.record) : null;
|
|
57
|
+
if (stale) return fail(stale);
|
|
55
58
|
return { ok: true, specFile: attempt.file };
|
|
56
59
|
}
|