@sun-asterisk/sungen 3.3.0-alpha.8 → 3.3.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/contract-derive-command.d.ts +2 -1
- package/dist/cli/commands/contract-derive-command.d.ts.map +1 -1
- package/dist/cli/commands/contract-derive-command.js +13 -2
- package/dist/cli/commands/contract-derive-command.js.map +1 -1
- package/dist/contract/build-trace-conformance.d.ts.map +1 -1
- package/dist/contract/build-trace-conformance.js +4 -11
- package/dist/contract/build-trace-conformance.js.map +1 -1
- package/dist/contract/derive-free-feature.d.ts +9 -0
- package/dist/contract/derive-free-feature.d.ts.map +1 -1
- package/dist/contract/derive-free-feature.js +10 -3
- package/dist/contract/derive-free-feature.js.map +1 -1
- package/dist/contract/resolve-unit-trace-identities.d.ts +12 -0
- package/dist/contract/resolve-unit-trace-identities.d.ts.map +1 -1
- package/dist/contract/resolve-unit-trace-identities.js +33 -49
- package/dist/contract/resolve-unit-trace-identities.js.map +1 -1
- package/dist/contract/trace/collect-trace-evidence.d.ts +9 -3
- package/dist/contract/trace/collect-trace-evidence.d.ts.map +1 -1
- package/dist/contract/trace/collect-trace-evidence.js +45 -18
- package/dist/contract/trace/collect-trace-evidence.js.map +1 -1
- package/dist/contract/trace/scenario-id-claims.d.ts +11 -0
- package/dist/contract/trace/scenario-id-claims.d.ts.map +1 -0
- package/dist/contract/trace/scenario-id-claims.js +89 -0
- package/dist/contract/trace/scenario-id-claims.js.map +1 -0
- package/dist/contract/trace/select-final-attempt.d.ts +2 -0
- package/dist/contract/trace/select-final-attempt.d.ts.map +1 -1
- package/dist/contract/trace/select-final-attempt.js +4 -1
- package/dist/contract/trace/select-final-attempt.js.map +1 -1
- package/dist/contract/trace/trace-binding.d.ts +3 -2
- package/dist/contract/trace/trace-binding.d.ts.map +1 -1
- package/dist/contract/trace/trace-binding.js +6 -3
- package/dist/contract/trace/trace-binding.js.map +1 -1
- package/dist/contract/trace/trace-freshness.d.ts +27 -0
- package/dist/contract/trace/trace-freshness.d.ts.map +1 -0
- package/dist/contract/trace/trace-freshness.js +146 -0
- package/dist/contract/trace/trace-freshness.js.map +1 -0
- package/dist/contract/trace/trace-identity-resolution.d.ts +2 -0
- package/dist/contract/trace/trace-identity-resolution.d.ts.map +1 -1
- package/dist/contract/trace/trace-identity-resolution.js +6 -3
- package/dist/contract/trace/trace-identity-resolution.js.map +1 -1
- package/dist/contract/trace/trace-run-record.d.ts +2 -0
- package/dist/contract/trace/trace-run-record.d.ts.map +1 -1
- package/dist/contract/trace/trace-run-record.js +19 -5
- package/dist/contract/trace/trace-run-record.js.map +1 -1
- package/dist/orchestrator/templates/ai-src/commands/create-test.md +4 -1
- package/package.json +3 -3
- package/src/cli/commands/contract-derive-command.ts +12 -3
- package/src/contract/build-trace-conformance.ts +4 -6
- package/src/contract/derive-free-feature.ts +15 -3
- package/src/contract/resolve-unit-trace-identities.ts +35 -12
- package/src/contract/trace/collect-trace-evidence.ts +50 -18
- package/src/contract/trace/scenario-id-claims.ts +60 -0
- package/src/contract/trace/select-final-attempt.ts +9 -2
- package/src/contract/trace/trace-binding.ts +7 -4
- package/src/contract/trace/trace-freshness.ts +110 -0
- package/src/contract/trace/trace-identity-resolution.ts +7 -4
- package/src/contract/trace/trace-run-record.ts +20 -4
- package/src/orchestrator/templates/ai-src/commands/create-test.md +4 -1
|
@@ -46,10 +46,15 @@ exports.readRunRecord = readRunRecord;
|
|
|
46
46
|
* own outputDir (#941) — or `<project>/test-results/sungen-run-record*.json` (one per run or shard).
|
|
47
47
|
* Playwright wipes its outputDir at the start of every run, so a record and its traces share a
|
|
48
48
|
* lifetime; the free run has its own so the compiled suite's next run cannot delete its traces.
|
|
49
|
+
*
|
|
50
|
+
* Schema 2 (#967) adds `contract`: the sha256 of every `.feature` and `contracts.lock.yaml` under `qa/`
|
|
51
|
+
* when the run started — what trace freshness compares by content. Schema 1 (an older reporter) has
|
|
52
|
+
* none; its traces are dated instead (trace-freshness.ts).
|
|
49
53
|
*/
|
|
50
54
|
const fs = __importStar(require("fs"));
|
|
51
55
|
const path = __importStar(require("path"));
|
|
52
|
-
const
|
|
56
|
+
const TOP_V1 = new Set(['schema', 'tool', 'runId', 'startedAt', 'playwrightVersion', 'shard', 'report', 'specs', 'attempts']);
|
|
57
|
+
const TOP_V2 = new Set([...TOP_V1, 'contract']);
|
|
53
58
|
const ATTEMPT = new Set(['testId', 'file', 'title', 'retry', 'status', 'startTime', 'duration', 'trace']);
|
|
54
59
|
const SHA = /^[0-9a-f]{64}$/;
|
|
55
60
|
const isObj = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
|
|
@@ -98,11 +103,16 @@ function readRunRecord(file, text) {
|
|
|
98
103
|
}
|
|
99
104
|
if (!isObj(raw))
|
|
100
105
|
throw new Error('not an object');
|
|
101
|
-
|
|
106
|
+
if (raw.tool === 'sungen-trace-run-record' && typeof raw.schema === 'number' && raw.schema > 2) {
|
|
107
|
+
// A newer driver-ui wrote it (mixed versions): say what to upgrade, not just "not a record".
|
|
108
|
+
throw new Error(`record schema ${raw.schema} is newer than this sungen — upgrade @sun-asterisk/sungen (core before or together with driver-ui)`);
|
|
109
|
+
}
|
|
110
|
+
const keys = raw.schema === 2 ? TOP_V2 : raw.schema === 1 ? TOP_V1 : null;
|
|
111
|
+
if (!keys || raw.tool !== 'sungen-trace-run-record')
|
|
112
|
+
throw new Error('not a sungen run record (schema 1 or 2)');
|
|
113
|
+
const extra = Object.keys(raw).find((k) => !keys.has(k));
|
|
102
114
|
if (extra)
|
|
103
115
|
throw new Error(`unknown key "${extra}"`);
|
|
104
|
-
if (raw.schema !== 1 || raw.tool !== 'sungen-trace-run-record')
|
|
105
|
-
throw new Error('not a sungen run record (schema 1)');
|
|
106
116
|
if (typeof raw.runId !== 'string' || !raw.runId || typeof raw.startedAt !== 'string' || typeof raw.playwrightVersion !== 'string')
|
|
107
117
|
throw new Error('runId, startedAt and playwrightVersion must be strings');
|
|
108
118
|
if (raw.shard !== null && typeof raw.shard !== 'string')
|
|
@@ -111,11 +121,15 @@ function readRunRecord(file, text) {
|
|
|
111
121
|
throw new Error('report must be a string or null');
|
|
112
122
|
if (!isObj(raw.specs) || !Object.values(raw.specs).every((v) => typeof v === 'string' && (v === '' || SHA.test(v))))
|
|
113
123
|
throw new Error('specs must map files to sha256');
|
|
124
|
+
if (raw.schema === 2 && !(isObj(raw.contract) && Object.values(raw.contract).every((v) => typeof v === 'string' && SHA.test(v)))) {
|
|
125
|
+
throw new Error('contract must map contract files to sha256 (schema 2)');
|
|
126
|
+
}
|
|
114
127
|
if (!Array.isArray(raw.attempts))
|
|
115
128
|
throw new Error('attempts must be a list');
|
|
116
129
|
return {
|
|
117
130
|
file, runId: raw.runId, startedAt: raw.startedAt, playwrightVersion: raw.playwrightVersion, shard: raw.shard,
|
|
118
|
-
report: raw.report, specs: raw.specs,
|
|
131
|
+
report: raw.report, specs: raw.specs,
|
|
132
|
+
contract: raw.schema === 2 ? raw.contract : null, attempts: raw.attempts.map(attemptOf),
|
|
119
133
|
};
|
|
120
134
|
}
|
|
121
135
|
//# sourceMappingURL=trace-run-record.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trace-run-record.js","sourceRoot":"","sources":["../../../src/contract/trace/trace-run-record.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"trace-run-record.js","sourceRoot":"","sources":["../../../src/contract/trace/trace-run-record.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8CA,wCAOC;AAiBD,sCAyBC;AA/FD;;;;;;;;;;;;;;GAcG;AACH,uCAAyB;AACzB,2CAA6B;AAmB7B,MAAM,MAAM,GAAwB,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;AACnJ,MAAM,MAAM,GAAwB,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;AACrE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;AAC1G,MAAM,GAAG,GAAG,gBAAgB,CAAC;AAC7B,MAAM,KAAK,GAAG,CAAC,CAAU,EAAgC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAE9G,uHAAuH;AAC1G,QAAA,YAAY,GAAG,mBAAmB,CAAC;AAChD,gHAAgH;AACnG,QAAA,eAAe,GAAsB,CAAC,oBAAY,EAAE,cAAc,CAAC,CAAC;AAEjF,SAAgB,cAAc,CAAC,WAAmB;IAChD,OAAO,uBAAe,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACxC,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YAAC,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,EAAE,CAAC;QAAC,CAAC;QACzD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iDAAiD,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7H,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,GAAY,EAAE,CAAS;IACxC,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,IAAI,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,EAAE,wBAAwB,KAAK,GAAG,CAAC,CAAC;IAClE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC;IAC/E,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,EAAE,kDAAkD,CAAC,CAAC;IAChM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAK,KAAgB,GAAG,CAAC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC9J,MAAM,IAAI,KAAK,CAAC,GAAG,EAAE,+CAA+C,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,EAAE,kCAAkC,CAAC,CAAC;IACjM,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAe,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAkC,EAAE,CAAC;AACjI,CAAC;AAED,4HAA4H;AAC5H,SAAgB,aAAa,CAAC,IAAY,EAAE,IAAa;IACvD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QAAC,MAAM,IAAI,KAAK,CAAC,sBAAuB,CAAW,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC;IAC7J,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;IAClD,IAAI,GAAG,CAAC,IAAI,KAAK,yBAAyB,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/F,6FAA6F;QAC7F,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,CAAC,MAAM,oGAAoG,CAAC,CAAC;IACnJ,CAAC;IACD,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1E,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,yBAAyB;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAChH,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,IAAI,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,KAAK,GAAG,CAAC,CAAC;IACrD,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,iBAAiB,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC7M,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC3G,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAC9G,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACvK,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACjI,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7E,OAAO;QACL,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,EAAE,KAAK,EAAE,GAAG,CAAC,KAAsB;QAC7H,MAAM,EAAE,GAAG,CAAC,MAAuB,EAAE,KAAK,EAAE,GAAG,CAAC,KAA+B;QAC/E,QAAQ,EAAE,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAkC,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC;KAClH,CAAC;AACJ,CAAC"}
|
|
@@ -111,7 +111,10 @@ terminal (`script`, `expect`, a pty wrapper), and never edit `contracts.lock.yam
|
|
|
111
111
|
traces, write their Gherkin with `sungen contract derive --screen <name>` — deterministic, it writes
|
|
112
112
|
`features/<name>.free.feature` with every scenario tagged `@free @proposed`. Do not author those
|
|
113
113
|
steps yourself; you may name the scenarios and suggest VP-ids. A `@proposed` scenario is not
|
|
114
|
-
counted by the audit until a QA approves it (`sungen contract approve --screen <name>`).
|
|
114
|
+
counted by the audit until a QA approves it (`sungen contract approve --screen <name>`). When
|
|
115
|
+
two recorded tests claim one ID, `sungen contract derive` exits `1`, names both tests and writes
|
|
116
|
+
nothing: stop and ask the QA to give each test its own ID (its title's leading ID, or its `@tag`).
|
|
117
|
+
Never rename a test's ID — or a scenario's — yourself to get past it.
|
|
115
118
|
|
|
116
119
|
## Platform detection (do this FIRST)
|
|
117
120
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sun-asterisk/sungen",
|
|
3
|
-
"version": "3.3.0-alpha.
|
|
3
|
+
"version": "3.3.0-alpha.9",
|
|
4
4
|
"description": "Deterministic E2E Test Compiler - Gherkin + Selectors → Playwright tests",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
@@ -40,8 +40,8 @@
|
|
|
40
40
|
"@babel/types": "^7.28.5",
|
|
41
41
|
"@cucumber/gherkin": "^37.0.0",
|
|
42
42
|
"@cucumber/messages": "^31.0.0",
|
|
43
|
-
"@sungen/driver-data-factory": "3.3.0-alpha.
|
|
44
|
-
"@sungen/driver-ui": "3.3.0-alpha.
|
|
43
|
+
"@sungen/driver-data-factory": "3.3.0-alpha.9",
|
|
44
|
+
"@sungen/driver-ui": "3.3.0-alpha.9",
|
|
45
45
|
"chalk": "^5.6.2",
|
|
46
46
|
"commander": "^14.0.2",
|
|
47
47
|
"dotenv": "^17.2.3",
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
*
|
|
5
5
|
* exit 0 written, unchanged, every scenario already approved, or a mobile project (not supported yet)
|
|
6
6
|
* exit 1 nothing to derive (no admissible trace — each skipped test says why, #932), refused (a
|
|
7
|
-
* secret or a machine path would have been written — nothing was),
|
|
7
|
+
* secret or a machine path would have been written — nothing was), refused because two
|
|
8
|
+
* recorded tests claim one id (#962 — both named, nothing written), or an error
|
|
8
9
|
*/
|
|
9
10
|
import type { Command } from 'commander';
|
|
10
11
|
import * as path from 'path';
|
|
@@ -23,7 +24,15 @@ function print(r: DeriveResult, unit: string): void {
|
|
|
23
24
|
for (const w of r.warnings ?? []) L(` ⚠ ${w}`);
|
|
24
25
|
for (const s of r.scenarios) L(` + ${s.title} (${s.steps} steps${s.low ? `, ${s.low} low identity` : ''}${s.opaque ? `, ${s.opaque} opaque` : ''})`);
|
|
25
26
|
for (const s of r.skipped) L(` - ${s.id} skipped: ${s.reason}`);
|
|
26
|
-
if (r.
|
|
27
|
+
if (r.duplicateIds?.length) {
|
|
28
|
+
L(" ✗ Refused — nothing was written. One ID names one test: keeping one would silently drop the other's contract.");
|
|
29
|
+
for (const d of r.duplicateIds) {
|
|
30
|
+
L(` ${d.tests.length} recorded tests claim ${d.id}:`);
|
|
31
|
+
for (const t of d.tests) L(` ${t}`);
|
|
32
|
+
if (d.staleRecords?.length) L(` If that test is gone, delete ${d.staleRecords.join(', ')} — an older run's record (free tests now record under test-results-free/).`);
|
|
33
|
+
}
|
|
34
|
+
L(" Give each test its own ID (its title's leading ID, or its @tag), run the free tests again with trace on, then derive.");
|
|
35
|
+
} else if (r.refused?.length) {
|
|
27
36
|
L(` ✗ Refused — nothing was written. ${r.refused.length} likely secret(s) / machine path(s):`);
|
|
28
37
|
for (const h of r.refused) L(` ${h.where} — ${h.reason}`);
|
|
29
38
|
L(' Move the value into test-data (or an env overlay) and use it from there, then derive again.');
|
|
@@ -54,7 +63,7 @@ export function registerDeriveCommand(contract: Command): void {
|
|
|
54
63
|
projectRoot: process.cwd(), unit, grep: options.grep ? String(options.grep) : undefined, resolver: registeredTraceResolver() ?? null,
|
|
55
64
|
});
|
|
56
65
|
if (options.json) console.log(JSON.stringify(r, null, 2)); else print(r, unit.name);
|
|
57
|
-
process.exit(r.refused?.length || nothingDerived(r) ? 1 : 0);
|
|
66
|
+
process.exit(r.refused?.length || r.duplicateIds?.length || nothingDerived(r) ? 1 : 0);
|
|
58
67
|
} catch (error) {
|
|
59
68
|
console.error('Error:', error instanceof Error ? error.message : error);
|
|
60
69
|
process.exit(1);
|
|
@@ -12,17 +12,17 @@
|
|
|
12
12
|
* Bounded (review H4, M2): a deadline (`budgetMs`; the guard's default is ~3 s) — a scenario not
|
|
13
13
|
* judged in time is `unproven` when approved, a warning while proposed: fail CLOSED. An input that
|
|
14
14
|
* cannot be read while judging (test-data, a trace) makes that scenario unproven with its reason,
|
|
15
|
-
* never a thrown verdict. A trace older than the unit's
|
|
15
|
+
* never a thrown verdict. A trace older than the unit's `.feature` / lock is refused (stale) — judged
|
|
16
|
+
* by their content, so a touch or a fresh clone stales nothing (#967, trace/trace-freshness.ts).
|
|
16
17
|
*/
|
|
17
18
|
import { createHash } from 'crypto';
|
|
18
19
|
import * as fs from 'fs';
|
|
19
20
|
import { readCapabilities } from '../harness/capability';
|
|
20
21
|
import { classifyTraceRuns, notJudged, TraceOutcome } from './classify-trace-conformance';
|
|
21
22
|
import { ContractLock, ContractScenario, isHumanApproved, LockEntry } from './contract-schema';
|
|
22
|
-
import { lockPathFor } from './contracts-lock-file';
|
|
23
23
|
import { loadTestDataIndex, TestDataIndex } from './data-normalize';
|
|
24
|
-
import { contractFeatureFiles } from './normalize-feature-steps';
|
|
25
24
|
import { renderTraceSteps, RenderedStep } from './render-free-gherkin';
|
|
25
|
+
import { unitContract } from './trace/trace-freshness';
|
|
26
26
|
import { collectScenarioEvidence, OVER_BUDGET, ScenarioEvidence, TraceRun } from './trace/collect-trace-evidence';
|
|
27
27
|
|
|
28
28
|
export { OVER_BUDGET } from './trace/collect-trace-evidence';
|
|
@@ -33,7 +33,6 @@ import { traceIntentSteps } from './trace/trace-steps';
|
|
|
33
33
|
|
|
34
34
|
export interface UnitTraceConformance { outcomes: Map<string, TraceOutcome>; warnings: string[]; inputs: unknown }
|
|
35
35
|
|
|
36
|
-
const mtimeMs = (f: string): number => { try { return fs.statSync(f).mtimeMs; } catch { return 0; } };
|
|
37
36
|
const sha = (s: string | Buffer): string => createHash('sha256').update(s).digest('hex');
|
|
38
37
|
const why = (e: unknown): string => (e instanceof Error ? e.message : String(e)).split('\n')[0].slice(0, 300);
|
|
39
38
|
|
|
@@ -65,8 +64,7 @@ export function traceConformanceForUnit(
|
|
|
65
64
|
let index: TestDataIndex = { byValue: new Map() };
|
|
66
65
|
let setupError = '';
|
|
67
66
|
try {
|
|
68
|
-
|
|
69
|
-
evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), notBefore, deadline, new Map(traced.map((s) => [s.id, s.source.name])));
|
|
67
|
+
evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), unitContract(projectRoot, unit), deadline, new Map(traced.map((s) => [s.id, s.source.name])));
|
|
70
68
|
index = loadTestDataIndex(unit.dir, unit.name);
|
|
71
69
|
} catch (e) { setupError = `trace conformance could not run: ${why(e)}`; }
|
|
72
70
|
const { cache, problem } = readIdentityCache(projectRoot);
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
* identities asked of the registered `traceIdentity` resolver (driver-ui) when one is given,
|
|
9
9
|
* otherwise the cache, otherwise core's heuristic — `# identity low` marks those steps
|
|
10
10
|
* approved a scenario QA already approved (no `@proposed`) is never rewritten
|
|
11
|
+
* one id, one two recorded tests claiming one id (#962) refuse the derive — both named with
|
|
12
|
+
* test file:line, NOTHING written for the unit: keeping one would drop the other's contract
|
|
11
13
|
* safety redaction by field meaning, machine paths scrubbed, then the secret scanner and the
|
|
12
14
|
* semantic credential check over the exact text: any hit → NOTHING is written
|
|
13
15
|
*/
|
|
@@ -25,6 +27,7 @@ import { scenariosOfFeatureFile } from './normalize-feature-steps';
|
|
|
25
27
|
import { featureFilesFor } from '../harness/unit-paths';
|
|
26
28
|
import { CONTROL_CHARS, freeFeatureText, renderTraceSteps, scenarioBlockText } from './render-free-gherkin';
|
|
27
29
|
import { collectRuns, TraceRun } from './trace/collect-trace-evidence';
|
|
30
|
+
import { idsClaimedByTwoTests } from './trace/scenario-id-claims';
|
|
28
31
|
import { readIdentityCache } from './trace/identity-cache';
|
|
29
32
|
import type { TraceIdentityResolver } from './trace/target-identity';
|
|
30
33
|
import { identitiesFor, resolveWithDriver } from './trace/trace-identity-resolution';
|
|
@@ -39,6 +42,11 @@ export interface DeriveResult {
|
|
|
39
42
|
/** `approved: true` — skipped because QA already approved it (derive never rewrites one), not for want of a trace. */
|
|
40
43
|
skipped: Array<{ id: string; reason: string; approved?: true }>;
|
|
41
44
|
refused?: SecretHit[];
|
|
45
|
+
/**
|
|
46
|
+
* Ids two or more recorded tests claim (#962), each test as `file:line "title" (run record <path>)` —
|
|
47
|
+
* nothing was written. `staleRecords`: older test-results/ records that are a claim's only source.
|
|
48
|
+
*/
|
|
49
|
+
duplicateIds?: Array<{ id: string; tests: string[]; staleRecords?: string[] }>;
|
|
42
50
|
note?: string;
|
|
43
51
|
/** A derived id another `.feature` of the unit already uses (#939) — `contract check` and approve would trip on it later. */
|
|
44
52
|
warnings?: string[];
|
|
@@ -75,7 +83,7 @@ function reusedIdWarnings(unit: { name: string; dir: string }, freeFile: string,
|
|
|
75
83
|
} catch { /* an unparseable sibling is `contract check`'s to report */ }
|
|
76
84
|
}
|
|
77
85
|
return ids.filter((id) => owners.has(id)).map((id) =>
|
|
78
|
-
`${id} is already the ID of a scenario in ${owners.get(id)} — the derived scenario reuses it; give the free test (its title's leading ID) or that scenario its own ID, or \`sungen contract check\` reports it twice and approval refuses it`);
|
|
86
|
+
`${id} is already the ID of a scenario in ${owners.get(id)} — the derived scenario reuses it; give the free test (its title's leading ID, or its @tag) or that scenario its own ID, or \`sungen contract check\` reports it twice and approval refuses it`);
|
|
79
87
|
}
|
|
80
88
|
|
|
81
89
|
/**
|
|
@@ -84,7 +92,7 @@ function reusedIdWarnings(unit: { name: string; dir: string }, freeFile: string,
|
|
|
84
92
|
* refused, or the project is mobile (`note`).
|
|
85
93
|
*/
|
|
86
94
|
export function nothingDerived(r: DeriveResult): boolean {
|
|
87
|
-
return !r.note && !r.refused?.length && r.scenarios.length === 0 && !r.skipped.some((s) => s.approved);
|
|
95
|
+
return !r.note && !r.refused?.length && !r.duplicateIds?.length && r.scenarios.length === 0 && !r.skipped.some((s) => s.approved);
|
|
88
96
|
}
|
|
89
97
|
|
|
90
98
|
export async function deriveFreeFeature(i: DeriveInput): Promise<DeriveResult> {
|
|
@@ -93,7 +101,11 @@ export async function deriveFreeFeature(i: DeriveInput): Promise<DeriveResult> {
|
|
|
93
101
|
const result: DeriveResult = { file, written: false, scenarios: [], skipped: [] };
|
|
94
102
|
if ((readCapabilities(i.projectRoot).platform ?? '').toLowerCase() === 'mobile') return { ...result, note: MOBILE_NOT_SUPPORTED };
|
|
95
103
|
const grep = i.grep ? new RegExp(i.grep, 'i') : null;
|
|
96
|
-
const found = collectRuns(i.projectRoot, (id, t) => (grep ? grep.test(`${id} ${t.title} ${t.file}`) : belongsToUnit(t.file, unitBase.toLowerCase())),
|
|
104
|
+
const found = collectRuns(i.projectRoot, (id, t) => (grep ? grep.test(`${id} ${t.title} ${t.file}`) : belongsToUnit(t.file, unitBase.toLowerCase())), null);
|
|
105
|
+
const claimed = idsClaimedByTwoTests(i.projectRoot, found.runs);
|
|
106
|
+
if (claimed.size) {
|
|
107
|
+
return { ...result, duplicateIds: [...claimed].sort(([a], [b]) => a.localeCompare(b, 'en')).map(([id, c]) => ({ id, tests: c.tests, ...(c.staleRecords.length ? { staleRecords: c.staleRecords } : {}) })) };
|
|
108
|
+
}
|
|
97
109
|
const byId = new Map<string, TraceRun[]>();
|
|
98
110
|
for (const r of found.runs) byId.set(r.scenarioId, [...(byId.get(r.scenarioId) ?? []), r]);
|
|
99
111
|
const admissible = (runs: TraceRun[]): TraceRun[] => runs.filter((r) => r.status === 'admissible' && r.trace)
|
|
@@ -3,33 +3,56 @@
|
|
|
3
3
|
* resolver (driver-ui — a real browser) about every target of the unit's admissible traces and
|
|
4
4
|
* cache the answers (DOS-14.5). The verdict itself never calls a browser; it reads the cache, so
|
|
5
5
|
* the CLI and the Claude Code guard keep building the same verdict from the same files.
|
|
6
|
+
*
|
|
7
|
+
* What it reports is split for QA (#966): a WARNING only when something needs attention — no
|
|
8
|
+
* resolver, a mobile project, a target the browser could not name (its steps stay opaque). "Every
|
|
9
|
+
* target was already known" and "N new targets identified" are information, in plain words.
|
|
6
10
|
*/
|
|
7
|
-
import * as fs from 'fs';
|
|
8
11
|
import { capabilityRegistry } from '../capabilities/registry';
|
|
9
12
|
import { readCapabilities } from '../harness/capability';
|
|
10
13
|
import { MOBILE_NOT_SUPPORTED } from './build-trace-conformance';
|
|
11
|
-
import {
|
|
12
|
-
import { contractFeatureFiles, loadContractScenarios } from './normalize-feature-steps';
|
|
14
|
+
import { loadContractScenarios } from './normalize-feature-steps';
|
|
13
15
|
import { collectScenarioEvidence } from './trace/collect-trace-evidence';
|
|
16
|
+
import { readIdentityCache } from './trace/identity-cache';
|
|
17
|
+
import { unitContract } from './trace/trace-freshness';
|
|
14
18
|
import type { TraceIdentityResolver } from './trace/target-identity';
|
|
15
19
|
import { resolveWithDriver } from './trace/trace-identity-resolution';
|
|
16
20
|
|
|
17
|
-
|
|
21
|
+
export interface TraceIdentityOutcome { warnings: string[]; info: string[] }
|
|
18
22
|
|
|
19
23
|
/** The first registered capability that can resolve trace identities (UI: driver-ui), if any. */
|
|
20
24
|
export function registeredTraceResolver(): TraceIdentityResolver | undefined {
|
|
21
25
|
return capabilityRegistry.all().find((c) => typeof c.traceIdentity === 'function')?.traceIdentity;
|
|
22
26
|
}
|
|
23
27
|
|
|
24
|
-
export async function
|
|
28
|
+
export async function traceIdentityOutcome(projectRoot: string, unit: { name: string; dir: string }, resolver = registeredTraceResolver()): Promise<TraceIdentityOutcome> {
|
|
29
|
+
const none: TraceIdentityOutcome = { warnings: [], info: [] };
|
|
25
30
|
const traced = loadContractScenarios(unit.dir, unit.name).filter((s) => s.implementation !== 'compiled');
|
|
26
|
-
if (!traced.length) return
|
|
27
|
-
if ((readCapabilities(projectRoot).platform ?? '').toLowerCase() === 'mobile') return [MOBILE_NOT_SUPPORTED];
|
|
28
|
-
if (!resolver) return ['no trace identity resolver is registered (install @sungen/driver-ui) — every target identity stays low, so nothing can match'];
|
|
29
|
-
const
|
|
30
|
-
const evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), notBefore);
|
|
31
|
+
if (!traced.length) return none;
|
|
32
|
+
if ((readCapabilities(projectRoot).platform ?? '').toLowerCase() === 'mobile') return { warnings: [MOBILE_NOT_SUPPORTED], info: [] };
|
|
33
|
+
if (!resolver) return { warnings: ['no trace identity resolver is registered (install @sungen/driver-ui) — every target identity stays low, so nothing can match'], info: [] };
|
|
34
|
+
const evidence = collectScenarioEvidence(projectRoot, new Set(traced.map((s) => s.id)), unitContract(projectRoot, unit));
|
|
31
35
|
const traces = [...evidence.runs.values()].flat().filter((r) => r.status === 'admissible' && r.trace).map((r) => r.trace!);
|
|
32
|
-
if (!traces.length) return
|
|
36
|
+
if (!traces.length) return none;
|
|
33
37
|
const r = await resolveWithDriver(projectRoot, traces, resolver);
|
|
34
|
-
|
|
38
|
+
const { cache } = readIdentityCache(projectRoot);
|
|
39
|
+
const total = r.keys.length;
|
|
40
|
+
const unnamed = r.keys.filter((k) => cache.entries.get(k)?.identityConfidence !== 'high').length;
|
|
41
|
+
const blocked = r.blockedRequests ? `; the replayed pages tried ${r.blockedRequests} request(s), all blocked` : '';
|
|
42
|
+
const known = total - r.asked;
|
|
43
|
+
const info = r.asked
|
|
44
|
+
? `trace identity: ${r.asked} new target(s) identified in a real browser (${r.resolver})${known ? `, ${known} already known` : ''}${blocked}`
|
|
45
|
+
: `trace identity: all ${total} target(s) were already identified (identity cache) — nothing new to resolve`;
|
|
46
|
+
const warnings = unnamed
|
|
47
|
+
? [`trace identity: ${unnamed} of ${total} target(s) could not be identified (no unique accessible name) — the steps on them stay opaque, for QA to review, until the test targets a unique, named element`]
|
|
48
|
+
: [];
|
|
49
|
+
return { warnings, info: [info] };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* What `contract check --trace` adds to the verdict's warnings: only what needs attention (#966).
|
|
54
|
+
* The plain-words information is `traceIdentityOutcome(...).info`.
|
|
55
|
+
*/
|
|
56
|
+
export async function resolveUnitTraceIdentities(projectRoot: string, unit: { name: string; dir: string }, resolver = registeredTraceResolver()): Promise<string[]> {
|
|
57
|
+
return (await traceIdentityOutcome(projectRoot, unit, resolver)).warnings;
|
|
35
58
|
}
|
|
@@ -8,6 +8,9 @@
|
|
|
8
8
|
* refused the trace exists but may not speak (swapped, stale, rejected zip…)
|
|
9
9
|
* unsupported the trace's format version is unknown (exit ≠ 0)
|
|
10
10
|
*
|
|
11
|
+
* Two recorded TESTS claiming one scenario id (#962) make its evidence ambiguous: neither speaks —
|
|
12
|
+
* `unproven`, both named (scenario-id-claims.ts) — instead of judging two tests' runs as one.
|
|
13
|
+
*
|
|
11
14
|
* Only traces of the scenario ids asked for are opened, so a unit pays for its own tests only.
|
|
12
15
|
*/
|
|
13
16
|
import * as fs from 'fs';
|
|
@@ -15,7 +18,9 @@ import * as path from 'path';
|
|
|
15
18
|
import { isIdLike } from '../../harness/parse';
|
|
16
19
|
import { clip, CONTROL_CHARS } from '../render-free-gherkin';
|
|
17
20
|
import { FileRead, readInsideProject } from './read-file-once';
|
|
21
|
+
import { idsClaimedByTwoTests, twoTestsClaim } from './scenario-id-claims';
|
|
18
22
|
import { bindTrace } from './trace-binding';
|
|
23
|
+
import type { UnitContract } from './trace-freshness';
|
|
19
24
|
import { traceFromRead, ParsedTrace } from './trace-file-reader';
|
|
20
25
|
import { traceZipLimits, TraceRejected } from './trace-zip-limits';
|
|
21
26
|
import { finalAttempt, readPlaywrightReport, ReportResult, ReportTest, scenarioIdOfTest } from './select-final-attempt';
|
|
@@ -49,21 +54,33 @@ function notRead(name: string, r: Exclude<FileRead, { status: 'read' }>): string
|
|
|
49
54
|
export const OVER_BUDGET = 'verdict budget exceeded; run `sungen contract check`';
|
|
50
55
|
const NO_RECORD = "no sungen run record under test-results-free/ (or test-results/) — run the free tests with their own config: outputDir 'test-results-free', trace: 'on' and the sungen run-record reporter";
|
|
51
56
|
|
|
52
|
-
/**
|
|
53
|
-
function
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
57
|
+
/** `p` relative to `root` when it lies under it, else null. */
|
|
58
|
+
function under(root: string, p: string): string | null {
|
|
59
|
+
const rel = path.relative(root, p);
|
|
60
|
+
return rel && !rel.startsWith('..') && !path.isAbsolute(rel) ? rel : null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Where a report's trace attachment is: the first candidate that exists (read later, opened once, and
|
|
65
|
+
* only from inside the project). Playwright writes attachment paths ABSOLUTE, in the checkout that ran;
|
|
66
|
+
* after a clone or a move that checkout is elsewhere, so a path under the report's own config dir is
|
|
67
|
+
* first tried at the same place under this project (#967) — the run record's sha256 still binds the bytes.
|
|
68
|
+
*/
|
|
69
|
+
function attachmentPath(projectRoot: string, test: ReportTest, p: string): string | null {
|
|
70
|
+
const moved = path.isAbsolute(p) && test.configRoot && !under(path.resolve(projectRoot), p) ? under(test.configRoot, p) : null;
|
|
71
|
+
const candidates = path.isAbsolute(p)
|
|
72
|
+
? [...(moved ? [path.resolve(projectRoot, moved)] : []), p]
|
|
73
|
+
: [path.resolve(projectRoot, p), path.resolve(path.dirname(test.report), p)];
|
|
74
|
+
return candidates.find((c) => fs.existsSync(c)) ?? null;
|
|
58
75
|
}
|
|
59
76
|
|
|
60
|
-
function judge(projectRoot: string, record: RunRecord, test: ReportTest,
|
|
77
|
+
function judge(projectRoot: string, record: RunRecord, test: ReportTest, contract: UnitContract | null): TraceRun {
|
|
61
78
|
const scenarioId = scenarioIdOfTest(test.title, test.tags);
|
|
62
79
|
const base = { scenarioId, test, record };
|
|
63
80
|
const fin = finalAttempt(test);
|
|
64
81
|
if (fin.admissibility !== 'admissible') return { ...base, result: fin.result, status: fin.admissibility, reason: fin.reason };
|
|
65
82
|
const missing: TraceRun = { ...base, result: fin.result, status: 'unproven', reason: `no trace at ${fin.result!.tracePath} (the report points at a missing file)` };
|
|
66
|
-
const cand = attachmentPath(projectRoot, test
|
|
83
|
+
const cand = attachmentPath(projectRoot, test, fin.result!.tracePath!);
|
|
67
84
|
if (!cand) return missing;
|
|
68
85
|
// Opened once, inside the project, and what was opened is what is parsed and hashed (#926).
|
|
69
86
|
const limits = traceZipLimits();
|
|
@@ -77,17 +94,22 @@ function judge(projectRoot: string, record: RunRecord, test: ReportTest, notBefo
|
|
|
77
94
|
const unsupported = e instanceof TraceRejected && e.code === 'trace.version-unsupported';
|
|
78
95
|
return { ...base, result: fin.result, status: unsupported ? 'unsupported' : 'refused', reason: `trace refused: ${(e as Error).message}` };
|
|
79
96
|
}
|
|
80
|
-
const bound = bindTrace({ projectRoot, test, result: fin.result!, record, trace,
|
|
97
|
+
const bound = bindTrace({ projectRoot, test, result: fin.result!, record, trace, contract });
|
|
81
98
|
if (bound.ok === false) return { ...base, result: fin.result, trace, status: 'refused', reason: `trace refused: ${bound.reason}` };
|
|
82
99
|
return { ...base, result: fin.result, trace, status: 'admissible', specFile: bound.specFile };
|
|
83
100
|
}
|
|
84
101
|
|
|
85
|
-
/**
|
|
102
|
+
/**
|
|
103
|
+
* Every judged run of the tests `want` accepts (derive: a unit's tests; check: by scenario id). A trace
|
|
104
|
+
* older than `contract` — the unit's .feature files and lock, by content (#967) — is refused; derive,
|
|
105
|
+
* which writes the contract from the traces, passes null.
|
|
106
|
+
*/
|
|
86
107
|
export function collectRuns(
|
|
87
|
-
projectRoot: string, want: (id: string, test: ReportTest) => boolean,
|
|
88
|
-
): { runs: TraceRun[]; problems: string[]; inputs: string[]; records: number; seen: Array<{ id: string; title: string }> } {
|
|
108
|
+
projectRoot: string, want: (id: string, test: ReportTest) => boolean, contract: UnitContract | null, deadline = Number.POSITIVE_INFINITY,
|
|
109
|
+
): { runs: TraceRun[]; problems: string[]; untrusted: string[]; inputs: string[]; records: number; seen: Array<{ id: string; title: string }> } {
|
|
89
110
|
const runs: TraceRun[] = [];
|
|
90
111
|
const problems: string[] = [];
|
|
112
|
+
const untrusted: string[] = [];
|
|
91
113
|
const inputs: string[] = [];
|
|
92
114
|
const seen: Array<{ id: string; title: string }> = [];
|
|
93
115
|
const records = findRunRecords(projectRoot);
|
|
@@ -95,7 +117,11 @@ export function collectRuns(
|
|
|
95
117
|
let record: RunRecord;
|
|
96
118
|
const recordRead = readInsideProject(projectRoot, file, MAX_EVIDENCE_JSON_BYTES);
|
|
97
119
|
if (recordRead.status !== 'read') { problems.push(notRead(path.basename(file), recordRead)); continue; }
|
|
98
|
-
try { record = readRunRecord(file, recordRead.bytes.toString('utf8')); } catch (e) {
|
|
120
|
+
try { record = readRunRecord(file, recordRead.bytes.toString('utf8')); } catch (e) {
|
|
121
|
+
untrusted.push(`${path.basename(file)} is not trusted: ${(e as Error).message}`);
|
|
122
|
+
problems.push(untrusted[untrusted.length - 1]);
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
99
125
|
if (!record.report) { problems.push(`${path.basename(file)} names no JSON report — add the json reporter to the free-test config`); continue; }
|
|
100
126
|
const reportRead = readInsideProject(projectRoot, path.resolve(projectRoot, record.report), MAX_EVIDENCE_JSON_BYTES);
|
|
101
127
|
if (reportRead.status !== 'read') { problems.push(`${path.basename(file)}: ${notRead(`the report ${path.basename(record.report)}`, reportRead)}`); continue; }
|
|
@@ -106,10 +132,10 @@ export function collectRuns(
|
|
|
106
132
|
const id = scenarioIdOfTest(test.title, test.tags);
|
|
107
133
|
seen.push({ id, title: test.title });
|
|
108
134
|
if (!want(id, test)) continue;
|
|
109
|
-
runs.push(Date.now() > deadline ? { scenarioId: id, test, record, status: 'unproven', reason: OVER_BUDGET } : judge(projectRoot, record, test,
|
|
135
|
+
runs.push(Date.now() > deadline ? { scenarioId: id, test, record, status: 'unproven', reason: OVER_BUDGET } : judge(projectRoot, record, test, contract));
|
|
110
136
|
}
|
|
111
137
|
}
|
|
112
|
-
return { runs, problems, inputs, records: records.length, seen };
|
|
138
|
+
return { runs, problems, untrusted, inputs, records: records.length, seen };
|
|
113
139
|
}
|
|
114
140
|
|
|
115
141
|
/** A title without its leading id, for comparing words only: `VP-X-001 Login works` → `login works`. */
|
|
@@ -135,12 +161,18 @@ function noTestFor(id: string, name: string | undefined, seen: Array<{ id: strin
|
|
|
135
161
|
|
|
136
162
|
/** The runs of each scenario id; `names` (id → scenario name) lets a missing test be explained. */
|
|
137
163
|
export function collectScenarioEvidence(
|
|
138
|
-
projectRoot: string, ids: ReadonlySet<string>,
|
|
164
|
+
projectRoot: string, ids: ReadonlySet<string>, contract: UnitContract | null, deadline?: number, names?: ReadonlyMap<string, string>,
|
|
139
165
|
): ScenarioEvidence {
|
|
140
|
-
const found = collectRuns(projectRoot, (id) => ids.has(id),
|
|
166
|
+
const found = collectRuns(projectRoot, (id) => ids.has(id), contract, deadline);
|
|
141
167
|
const runs = new Map<string, TraceRun[]>([...ids].map((id) => [id, found.runs.filter((r) => r.scenarioId === id)]));
|
|
168
|
+
// Ambiguous evidence: no run of either test may speak for the id — never one picked arbitrarily.
|
|
169
|
+
for (const [id, claim] of idsClaimedByTwoTests(projectRoot, found.runs)) {
|
|
170
|
+
if (runs.has(id)) runs.set(id, [{ scenarioId: id, status: 'unproven', reason: twoTestsClaim(id, claim) }]);
|
|
171
|
+
}
|
|
142
172
|
for (const [id, list] of runs) {
|
|
143
|
-
|
|
173
|
+
// A record that was not trusted may be why (a newer reporter, a hand edit): the finding says so.
|
|
174
|
+
const why = found.untrusted.length ? ` — ${found.untrusted.join('; ')}` : '';
|
|
175
|
+
if (!list.length) list.push({ scenarioId: id, status: 'unproven', reason: found.records ? `${noTestFor(id, names?.get(id), found.seen)}${why}` : NO_RECORD });
|
|
144
176
|
}
|
|
145
177
|
return { runs, problems: found.problems, inputs: found.inputs };
|
|
146
178
|
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One scenario id, one test (#962): which ids more than one recorded TEST claims, said the same way
|
|
3
|
+
* by `contract derive` (it refuses) and `contract check` (the id is unproven). One test recorded
|
|
4
|
+
* again — another run or shard, a retry, a second Playwright project — is still one test: the same
|
|
5
|
+
* file and the same Playwright test id.
|
|
6
|
+
*
|
|
7
|
+
* Each claiming test names the run record(s) it came from. A test that only an older run's record
|
|
8
|
+
* under `test-results/` still mentions — while another claim comes from the free config's own
|
|
9
|
+
* `test-results-free/` — is most likely gone: that record is offered for deletion.
|
|
10
|
+
*/
|
|
11
|
+
import * as path from 'path';
|
|
12
|
+
import { clip } from '../render-free-gherkin';
|
|
13
|
+
import type { TraceRun } from './collect-trace-evidence';
|
|
14
|
+
import { FREE_RUN_DIR } from './trace-run-record';
|
|
15
|
+
|
|
16
|
+
export interface IdClaim {
|
|
17
|
+
/** `file:line "title" (run record <path>)` per claiming test, sorted. */
|
|
18
|
+
tests: string[];
|
|
19
|
+
/** Project-relative records under test-results/ that are a claim's only source, next to a test-results-free/ claim. */
|
|
20
|
+
staleRecords: string[];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const posix = (p: string): string => p.split(path.sep).join('/');
|
|
24
|
+
/** A run's project-relative test file: the run record's (what the binding checks), else the report's. */
|
|
25
|
+
const testFileOf = (r: TraceRun): string => r.record?.attempts.find((a) => a.testId === r.test?.testId)?.file ?? r.test?.file ?? '?';
|
|
26
|
+
const label = (r: TraceRun, records: string[]): string =>
|
|
27
|
+
`${clip(testFileOf(r), 200)}:${r.test?.line ?? 0} "${clip(r.test?.title ?? '')}" (run record ${records.map((f) => clip(f, 200)).join(', ')})`;
|
|
28
|
+
|
|
29
|
+
export function idsClaimedByTwoTests(projectRoot: string, runs: readonly TraceRun[]): Map<string, IdClaim> {
|
|
30
|
+
const byId = new Map<string, Map<string, { run: TraceRun; records: Set<string> }>>();
|
|
31
|
+
for (const r of runs) {
|
|
32
|
+
if (!r.test) continue;
|
|
33
|
+
const tests = byId.get(r.scenarioId) ?? new Map<string, { run: TraceRun; records: Set<string> }>();
|
|
34
|
+
const key = `${testFileOf(r)}\0${r.test.testId}`;
|
|
35
|
+
const entry = tests.get(key) ?? { run: r, records: new Set<string>() };
|
|
36
|
+
if (r.record?.file) entry.records.add(posix(path.relative(projectRoot, r.record.file)));
|
|
37
|
+
tests.set(key, entry);
|
|
38
|
+
byId.set(r.scenarioId, tests);
|
|
39
|
+
}
|
|
40
|
+
const claims = new Map<string, IdClaim>();
|
|
41
|
+
for (const [id, tests] of byId) {
|
|
42
|
+
if (tests.size < 2) continue;
|
|
43
|
+
const entries = [...tests.values()].map((t) => ({ ...t, records: [...t.records].sort() }));
|
|
44
|
+
const inFreeDir = entries.some((t) => t.records.some((f) => f.startsWith(`${FREE_RUN_DIR}/`)));
|
|
45
|
+
const legacyOnly = entries.filter((t) => t.records.length > 0 && t.records.every((f) => f.startsWith('test-results/')));
|
|
46
|
+
claims.set(id, {
|
|
47
|
+
tests: entries.map((t) => label(t.run, t.records)).sort(),
|
|
48
|
+
staleRecords: inFreeDir ? [...new Set(legacyOnly.flatMap((t) => t.records))].sort() : [],
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
return claims;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** What to tell the user when several tests claim one id — check and derive say it the same way. */
|
|
55
|
+
export function twoTestsClaim(id: string, claim: IdClaim): string {
|
|
56
|
+
const stale = claim.staleRecords.length
|
|
57
|
+
? `; if that test is gone, delete ${claim.staleRecords.join(', ')} — an older run's record (free tests now record under ${FREE_RUN_DIR}/)`
|
|
58
|
+
: '';
|
|
59
|
+
return `${claim.tests.length} recorded tests claim ${id}: ${claim.tests.join(', ')} — give each test its own ID (its title's leading ID, or its @tag), then run them again with trace on${stale}`;
|
|
60
|
+
}
|
|
@@ -11,10 +11,15 @@
|
|
|
11
11
|
* leading id, else the whole title — the same rule the contract uses for a scenario name.
|
|
12
12
|
*/
|
|
13
13
|
import * as fs from 'fs';
|
|
14
|
+
import * as path from 'path';
|
|
14
15
|
import { isIdLike } from '../../harness/parse';
|
|
15
16
|
|
|
16
17
|
export interface ReportResult { retry: number; status: string; startTime: string; duration: number; tracePath: string | null }
|
|
17
|
-
export interface ReportTest {
|
|
18
|
+
export interface ReportTest {
|
|
19
|
+
testId: string; title: string; file: string; line: number; tags: string[]; project: string; results: ReportResult[]; report: string;
|
|
20
|
+
/** The directory of the config that ran (`config.configFile`, absolute as Playwright writes it) — where absolute attachment paths start. */
|
|
21
|
+
configRoot?: string;
|
|
22
|
+
}
|
|
18
23
|
export type Admissibility = 'admissible' | 'inconclusive' | 'unproven';
|
|
19
24
|
|
|
20
25
|
const isObj = (v: unknown): v is Record<string, unknown> => !!v && typeof v === 'object' && !Array.isArray(v);
|
|
@@ -38,6 +43,8 @@ export function readPlaywrightReport(file: string, text?: string): ReportTest[]
|
|
|
38
43
|
try { doc = JSON.parse(text ?? fs.readFileSync(file, 'utf8')); } catch (e) { throw new Error(`the JSON report ${file} is not readable (${(e as Error).message.split('\n')[0]})`); }
|
|
39
44
|
if (!isObj(doc) || !Array.isArray(doc.suites)) throw new Error(`${file} is not a Playwright JSON report (no suites)`);
|
|
40
45
|
const out: ReportTest[] = [];
|
|
46
|
+
const configFile = isObj(doc.config) && typeof doc.config.configFile === 'string' && path.isAbsolute(doc.config.configFile) ? doc.config.configFile : null;
|
|
47
|
+
const configRoot = configFile ? { configRoot: path.dirname(configFile) } : {};
|
|
41
48
|
for (const top of doc.suites) {
|
|
42
49
|
for (const spec of specsOf(top)) {
|
|
43
50
|
if (typeof spec.id !== 'string' || typeof spec.title !== 'string' || typeof spec.file !== 'string') continue;
|
|
@@ -46,7 +53,7 @@ export function readPlaywrightReport(file: string, text?: string): ReportTest[]
|
|
|
46
53
|
const results = arr(t.results).map(resultOf).filter((r): r is ReportResult => !!r).sort((a, b) => a.retry - b.retry);
|
|
47
54
|
out.push({
|
|
48
55
|
testId: spec.id, title: spec.title, file: spec.file, line: Number(spec.line) || 0,
|
|
49
|
-
tags: arr(spec.tags).map(String), project: String(t.projectName ?? ''), results, report: file,
|
|
56
|
+
tags: arr(spec.tags).map(String), project: String(t.projectName ?? ''), results, report: file, ...configRoot,
|
|
50
57
|
});
|
|
51
58
|
}
|
|
52
59
|
}
|
|
@@ -8,12 +8,14 @@
|
|
|
8
8
|
* code sha256 of the test file NOW equals the hash the record took at run start
|
|
9
9
|
* title every library part says `<file>:<line> › … › <title>` of exactly this test
|
|
10
10
|
* clock the trace started inside the attempt's window (±2 s)
|
|
11
|
-
* fresh the trace is
|
|
11
|
+
* fresh the trace is not older than the .feature files and the lock it is asked to prove —
|
|
12
|
+
* judged by their content, not their file time (#967, trace-freshness.ts)
|
|
12
13
|
*/
|
|
13
14
|
import { createHash } from 'crypto';
|
|
14
15
|
import * as fs from 'fs';
|
|
15
16
|
import * as path from 'path';
|
|
16
17
|
import type { ParsedTrace } from './trace-file-reader';
|
|
18
|
+
import { staleReason, UnitContract } from './trace-freshness';
|
|
17
19
|
import type { RunRecord } from './trace-run-record';
|
|
18
20
|
import type { ReportResult, ReportTest } from './select-final-attempt';
|
|
19
21
|
|
|
@@ -23,8 +25,8 @@ export interface BindingInput {
|
|
|
23
25
|
result: ReportResult;
|
|
24
26
|
record: RunRecord;
|
|
25
27
|
trace: ParsedTrace;
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
+
/** The unit's contract as it is now; null when no contract is proven (derive reads traces to write one). */
|
|
29
|
+
contract: UnitContract | null;
|
|
28
30
|
}
|
|
29
31
|
export type Binding = { ok: true; specFile: string } | { ok: false; reason: string };
|
|
30
32
|
|
|
@@ -51,6 +53,7 @@ export function bindTrace(i: BindingInput): Binding {
|
|
|
51
53
|
if (typeof wall !== 'number' || Number.isNaN(start) || wall < start - CLOCK_SLACK_MS || wall > start + i.result.duration + CLOCK_SLACK_MS) {
|
|
52
54
|
return fail('the trace was not recorded during this attempt (clock window)');
|
|
53
55
|
}
|
|
54
|
-
|
|
56
|
+
const stale = i.contract ? staleReason(i.contract, i.record) : null;
|
|
57
|
+
if (stale) return fail(stale);
|
|
55
58
|
return { ok: true, specFile: attempt.file };
|
|
56
59
|
}
|