@sanqianx/project-knowledge 5.0.25 → 5.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/_site/_test/helpers/qa-r44-env.js +418 -0
- package/_site/_test/helpers/qa-r44-report.js +656 -0
- package/_site/_test/helpers/qa-r44-ui.js +347 -0
- package/_site/_test/helpers/qa-r49-wb-host.js +58 -0
- package/_site/_test/qa-r44-permission-question.js +756 -0
- package/_site/_test/qa-r49-sys003-real-blank-probe.js +374 -0
- package/module-runtime-manifest.json +30 -28
- package/package.json +2 -2
- package/plugins/project-knowledge/.claude-plugin/plugin.json +1 -1
- package/plugins/project-knowledge/.codex-plugin/plugin.json +1 -1
- package/plugins/project-knowledge/.mcp.json +1 -1
|
@@ -0,0 +1,656 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// _site/_test/helpers/qa-r44-report.js
|
|
3
|
+
//
|
|
4
|
+
// The report builder of the R44 lane.
|
|
5
|
+
//
|
|
6
|
+
// R44D: this builder is GENERIC. It reports what the run it is handed
|
|
7
|
+
// actually produced, and nothing else:
|
|
8
|
+
//
|
|
9
|
+
// * it never narrates a past run. The historical facts of the accepted R44
|
|
10
|
+
// acceptance live in the frozen R44 report; for --report-only they are
|
|
11
|
+
// quoted in the frozenAcceptance block of the portability report, and
|
|
12
|
+
// nowhere else;
|
|
13
|
+
// * it never manufactures a PASS. The verdict is the verdict of the summary
|
|
14
|
+
// it was given, and a summary that lacks a required case yields a
|
|
15
|
+
// non-PASS verdict;
|
|
16
|
+
// * it never names a dependency it did not resolve: the shared files are
|
|
17
|
+
// read from the ACTUAL require-closure of the lane files.
|
|
18
|
+
//
|
|
19
|
+
// Every section is either read from the run, read from the repository, or
|
|
20
|
+
// read from the run history of that run. A missing fact is reported as
|
|
21
|
+
// missing.
|
|
22
|
+
const fs = require('node:fs');
|
|
23
|
+
const path = require('node:path');
|
|
24
|
+
const crypto = require('node:crypto');
|
|
25
|
+
|
|
26
|
+
const LANE = 'P-WB-PERMISSION-QUESTION-R44';
|
|
27
|
+
const LANE_OWNER = 'P-WB-QA-CANONICAL-R44D';
|
|
28
|
+
|
|
29
|
+
// The files of the lane and what they are for. The list is a description of
|
|
30
|
+
// the package, not a record of a run.
|
|
31
|
+
const LANE_FILES = [
|
|
32
|
+
{ path: '_site/_test/helpers/qa-r44-env.js', role: 'driver-level isolation (env capture, hardening, fail-closed assertion)' },
|
|
33
|
+
{ path: '_site/_test/helpers/qa-r44-ui.js', role: "real UI actions on the product's own permission / question cards" },
|
|
34
|
+
{ path: '_site/_test/helpers/qa-r44-report.js', role: 'this report builder' },
|
|
35
|
+
{ path: '_site/_test/qa-r44-permission-question.js', role: 'the WB-006 / WB-007 scenarios' },
|
|
36
|
+
{ path: 'qa-autopilot/tools/product-probes/qa-r44-permission-question.js', role: 'the portable canonical entry point' },
|
|
37
|
+
{ path: 'qa-autopilot/tools/product-probes/qa-r44-report-regression.js', role: 'the explicit zero-model regression probe' },
|
|
38
|
+
];
|
|
39
|
+
|
|
40
|
+
function sha256File(file) {
|
|
41
|
+
try { return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); } catch { return null; }
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function findRepoRoot(start) {
|
|
45
|
+
let dir = path.resolve(start);
|
|
46
|
+
for (;;) {
|
|
47
|
+
if (fs.existsSync(path.join(dir, 'package.json')) && fs.existsSync(path.join(dir, 'qa-autopilot'))) return dir;
|
|
48
|
+
const parent = path.dirname(dir);
|
|
49
|
+
if (parent === dir) throw new Error(`no repository root above ${start}`);
|
|
50
|
+
dir = parent;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// The expected contract of the cases of this lane, verbatim.
|
|
55
|
+
function readExpectedCases(repoRoot, ids) {
|
|
56
|
+
const file = path.join(repoRoot, 'qa-autopilot', 'config', 'test-cases.json');
|
|
57
|
+
const raw = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
58
|
+
const list = Array.isArray(raw) ? raw : (raw.cases || []);
|
|
59
|
+
const out = {};
|
|
60
|
+
for (const id of ids) {
|
|
61
|
+
const found = list.find(entry => entry && entry.id === id);
|
|
62
|
+
out[id] = found
|
|
63
|
+
? { title: found.title, mode: found.mode, steps: found.steps || [], expected: found.expected || [], evidence: found.evidence || [], preconditions: found.preconditions || [], source: found.source || null, statusInConfig: found.status || null }
|
|
64
|
+
: { missing: true };
|
|
65
|
+
}
|
|
66
|
+
return { file: path.relative(repoRoot, file).replace(/\\/g, '/'), sha256: sha256File(file), cases: out };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Every file of this lane, with its digest and its owner.
|
|
70
|
+
function laneArtifacts(repoRoot, files) {
|
|
71
|
+
const list = (files && files.length ? files : LANE_FILES).map(entry => (typeof entry === 'string' ? { path: entry } : entry));
|
|
72
|
+
return list.map(entry => {
|
|
73
|
+
const full = path.join(repoRoot, ...entry.path.split('/'));
|
|
74
|
+
const known = LANE_FILES.find(knownEntry => knownEntry.path === entry.path);
|
|
75
|
+
return {
|
|
76
|
+
path: entry.path,
|
|
77
|
+
owner: LANE_OWNER,
|
|
78
|
+
originalLane: LANE,
|
|
79
|
+
role: entry.role || (known ? known.role : 'lane file'),
|
|
80
|
+
sha256: sha256File(full),
|
|
81
|
+
bytes: fs.existsSync(full) ? fs.statSync(full).size : null,
|
|
82
|
+
trackedByGit: trackedByGit(repoRoot, entry.path),
|
|
83
|
+
};
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function trackedByGit(repoRoot, relative) {
|
|
88
|
+
try {
|
|
89
|
+
const result = require('node:child_process').spawnSync('git', ['ls-files', '--error-unmatch', relative], { cwd: repoRoot, encoding: 'utf8', windowsHide: true, timeout: 30000 });
|
|
90
|
+
return result.status === 0;
|
|
91
|
+
} catch { return null; }
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// The relative require specifiers of a source text.
|
|
95
|
+
function requireSpecifiers(source) {
|
|
96
|
+
const requireRe = /require\(\s*['"](\.[^'"]+)['"]\s*\)/g;
|
|
97
|
+
const out = [];
|
|
98
|
+
let match;
|
|
99
|
+
requireRe.lastIndex = 0;
|
|
100
|
+
while ((match = requireRe.exec(source)) !== null) out.push(match[1]);
|
|
101
|
+
return out;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// The require-closure of the given roots, resolved like node resolves it.
|
|
105
|
+
function dependencyClosure(repoRoot, roots) {
|
|
106
|
+
const seen = new Map();
|
|
107
|
+
const unresolved = [];
|
|
108
|
+
const walk = absolute => {
|
|
109
|
+
const relative = path.relative(repoRoot, absolute).split(path.sep).join('/');
|
|
110
|
+
if (seen.has(relative)) return;
|
|
111
|
+
if (!fs.existsSync(absolute)) { seen.set(relative, { exists: false }); return; }
|
|
112
|
+
seen.set(relative, { exists: true, sha256: sha256File(absolute) });
|
|
113
|
+
for (const specifier of requireSpecifiers(fs.readFileSync(absolute, 'utf8'))) {
|
|
114
|
+
const base = path.resolve(path.dirname(absolute), specifier);
|
|
115
|
+
const hit = [base, `${base}.js`, path.join(base, 'index.js')]
|
|
116
|
+
.find(candidate => fs.existsSync(candidate) && fs.statSync(candidate).isFile());
|
|
117
|
+
if (!hit) { unresolved.push(`${relative} -> ${specifier}`); continue; }
|
|
118
|
+
walk(hit);
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
for (const root of roots) walk(path.resolve(repoRoot, root));
|
|
122
|
+
return { files: seen, unresolved };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// The product files the acceptance depends on. Their digests freeze the
|
|
126
|
+
// behaviour that was proved.
|
|
127
|
+
function productFiles(repoRoot, summary) {
|
|
128
|
+
const root = summary && summary.pinned && summary.pinned.resolvedRoot;
|
|
129
|
+
const relative = [
|
|
130
|
+
'apps/agent-terminal/app.js',
|
|
131
|
+
'apps/agent-terminal/user-question-card.js',
|
|
132
|
+
'packages/client/index.js',
|
|
133
|
+
'packages/server/lib/user-questions.js',
|
|
134
|
+
'packages/server/lib/workbench-service.js',
|
|
135
|
+
'packages/server/lib/http-handler.js',
|
|
136
|
+
'src/backend/lib/claude-cli-runner.js',
|
|
137
|
+
];
|
|
138
|
+
if (!root) return { root: null, files: [] };
|
|
139
|
+
return {
|
|
140
|
+
root,
|
|
141
|
+
version: summary.pinned.version || null,
|
|
142
|
+
files: relative.map(rel => ({ path: rel, sha256: sha256File(path.join(root, ...rel.split('/'))) })),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function caseDetail(summary, caseId) {
|
|
147
|
+
const entry = (summary.cases_detail || []).find(item => item.caseId === caseId);
|
|
148
|
+
return entry || null;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// The digests of the lane source, as they were DURING the run that is being
|
|
152
|
+
// reported. They are read from the run history of that run; a report without
|
|
153
|
+
// a run history carries none.
|
|
154
|
+
function sourceDigestsDuringRun(runHistory) {
|
|
155
|
+
if (!runHistory || !runHistory.sourceDigestsDuringRun) return null;
|
|
156
|
+
return runHistory.sourceDigestsDuringRun;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// The build of the report of ONE run. The verdict is the verdict of the run.
|
|
160
|
+
function buildReport(options) {
|
|
161
|
+
const { repoRoot, evidenceDir, summary, runHistory, proposalFiles } = options;
|
|
162
|
+
const cases = ['WB-006', 'WB-007'];
|
|
163
|
+
const expected = readExpectedCases(repoRoot, cases);
|
|
164
|
+
const artifacts = laneArtifacts(repoRoot, proposalFiles);
|
|
165
|
+
const lanePaths = new Set(artifacts.map(entry => entry.path));
|
|
166
|
+
|
|
167
|
+
const wb006 = summary.wb006 || {};
|
|
168
|
+
const wb007 = summary.wb007 || {};
|
|
169
|
+
const wb006Case = caseDetail(summary, 'R44.c-wb006-real-permission-card-raised');
|
|
170
|
+
const wb006Deny = caseDetail(summary, 'R44.d-wb006-deny-executed-by-the-real-ui');
|
|
171
|
+
const wb007Card = caseDetail(summary, 'R44.e-wb007-real-question-card-and-pause');
|
|
172
|
+
const wb007Refusals = caseDetail(summary, 'R44.f-wb007-bypass-cannot-substitute-empty-answer-or-let-it-through');
|
|
173
|
+
const wb007Bypass = caseDetail(summary, 'R44.g-wb007-bypass-mode-does-not-release-the-question');
|
|
174
|
+
const wb007Submit = caseDetail(summary, 'R44.h-wb007-ui-enables-the-submit-only-after-a-complete-answer');
|
|
175
|
+
const wb007Receipt = caseDetail(summary, 'R44.i-wb007-real-ui-selection-submitted-and-the-receipt-matches');
|
|
176
|
+
const wb007BypassTurn = caseDetail(summary, 'R44.j-wb007-the-bypass-mode-still-asks-the-question');
|
|
177
|
+
const realProfileCase = caseDetail(summary, 'R44.k-real-profiles-byte-identical');
|
|
178
|
+
|
|
179
|
+
// The shared files the lane really depends on, read from the closure.
|
|
180
|
+
const closure = dependencyClosure(repoRoot, [...lanePaths]);
|
|
181
|
+
const sharedDependencies = [...closure.files.entries()]
|
|
182
|
+
.filter(([relative]) => !lanePaths.has(relative))
|
|
183
|
+
.map(([relative, info]) => ({
|
|
184
|
+
path: relative,
|
|
185
|
+
exists: info.exists,
|
|
186
|
+
sha256: info.sha256,
|
|
187
|
+
trackedByGit: trackedByGit(repoRoot, relative),
|
|
188
|
+
kind: 'shared, read-only',
|
|
189
|
+
owner: 'the root QA harness, not this lane',
|
|
190
|
+
}));
|
|
191
|
+
|
|
192
|
+
// What THIS run proved and what it did not. It is read from the summary of
|
|
193
|
+
// the run; a run that did not record a case has not proved it.
|
|
194
|
+
const evaluation = summary.caseEvaluation || null;
|
|
195
|
+
const notProvedCases = evaluation
|
|
196
|
+
? evaluation.notPassed
|
|
197
|
+
: (summary.cases_detail || []).filter(entry => entry.status !== 'PASS').map(entry => ({ caseId: entry.caseId, status: entry.status }));
|
|
198
|
+
const bypassCases = ['R44.g-wb007-bypass-mode-does-not-release-the-question', 'R44.j-wb007-the-bypass-mode-still-asks-the-question'];
|
|
199
|
+
const bypassProven = bypassCases.map(caseId => {
|
|
200
|
+
const found = (summary.cases_detail || []).find(entry => entry.caseId === caseId);
|
|
201
|
+
return { caseId, status: found ? found.status : 'MISSING', proven: Boolean(found) && found.status === 'PASS' };
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
const report = {
|
|
205
|
+
schema: 'qa-autopilot/wb-permission-question-r44/report/v1',
|
|
206
|
+
lane: LANE,
|
|
207
|
+
generatedAt: new Date().toISOString(),
|
|
208
|
+
verdict: summary.verdict,
|
|
209
|
+
cases,
|
|
210
|
+
evidenceDir: evidenceDir || null,
|
|
211
|
+
|
|
212
|
+
// What the run was and what it produced. This is the run's own summary.
|
|
213
|
+
summary,
|
|
214
|
+
|
|
215
|
+
contract: {
|
|
216
|
+
source: 'qa-autopilot/config/test-cases.json',
|
|
217
|
+
file: expected.file,
|
|
218
|
+
sha256: expected.sha256,
|
|
219
|
+
expected: expected.cases,
|
|
220
|
+
note: 'the expected items are quoted verbatim from the shared case file; the shared file itself was NOT modified by this lane',
|
|
221
|
+
},
|
|
222
|
+
|
|
223
|
+
identity: {
|
|
224
|
+
product: summary.module || null,
|
|
225
|
+
pinned: summary.pinned || null,
|
|
226
|
+
workbenchSource: summary.source || null,
|
|
227
|
+
productFiles: productFiles(repoRoot, summary),
|
|
228
|
+
artifacts,
|
|
229
|
+
},
|
|
230
|
+
|
|
231
|
+
isolation: summary.isolation || null,
|
|
232
|
+
profile: summary.profile || null,
|
|
233
|
+
paidTurnsSent: summary.paidTurnsSent,
|
|
234
|
+
networkUsed: summary.networkUsed,
|
|
235
|
+
|
|
236
|
+
// What THIS run observed. The method notes describe the lane; the
|
|
237
|
+
// observed values are read from the run.
|
|
238
|
+
evidence: {
|
|
239
|
+
'WB-006': {
|
|
240
|
+
expectedVerbatim: expected.cases['WB-006'] ? expected.cases['WB-006'].expected : null,
|
|
241
|
+
method: 'the card is raised by the product and decided through the product\'s own 拒绝 button in the real browser; no API call performs the decision',
|
|
242
|
+
ui: {
|
|
243
|
+
card: wb006Case && wb006Case.detail ? wb006Case.detail.card : null,
|
|
244
|
+
settledState: wb006.uiSettledState || null,
|
|
245
|
+
settledCard: wb006.uiSettledCard || null,
|
|
246
|
+
},
|
|
247
|
+
api: {
|
|
248
|
+
permissionRequest: wb006Case && wb006Case.detail ? wb006Case.detail.requestEvent : null,
|
|
249
|
+
permissionResolved: wb006.permissionResolved || null,
|
|
250
|
+
effectivePermissionMode: wb006.effectivePermissionMode || null,
|
|
251
|
+
pendingPermissionAfterTheDecision: wb006.pendingPermissionAfterTheDecision || null,
|
|
252
|
+
toolResult: wb006.toolResult || null,
|
|
253
|
+
turnResult: wb006.turnResult || null,
|
|
254
|
+
},
|
|
255
|
+
disk: {
|
|
256
|
+
markerPath: wb006.markerPath || null,
|
|
257
|
+
markerExistsAfterTheDenial: wb006.markerExistsAfterTheDenial,
|
|
258
|
+
filesCarryingTheMarkerText: wb006.filesCarryingTheMarkerText || null,
|
|
259
|
+
note: 'the marker lives in the lane\'s own fixture project below the isolated run root; it is not inside the repository and not a real user path',
|
|
260
|
+
},
|
|
261
|
+
verdict: wb006Deny ? wb006Deny.status : null,
|
|
262
|
+
},
|
|
263
|
+
'WB-007': {
|
|
264
|
+
expectedVerbatim: expected.cases['WB-007'] ? expected.cases['WB-007'].expected : null,
|
|
265
|
+
method: 'the selection is made by driving the product\'s own radio / checkbox / custom input and its own 提交答案 button in the real browser; the resolve endpoint is used only as a negative control',
|
|
266
|
+
ui: {
|
|
267
|
+
card: wb007Card && wb007Card.detail ? wb007Card.detail.card : null,
|
|
268
|
+
pausedWithoutSubmission: wb007Card && wb007Card.detail ? wb007Card.detail.pausedWithoutSubmission : null,
|
|
269
|
+
submitDisabledBeforeAnyAnswer: wb007Submit && wb007Submit.detail ? wb007Submit.detail.submitDisabledBeforeAnyAnswer : null,
|
|
270
|
+
submitDisabledAfterTheCompleteSelection: wb007Submit && wb007Submit.detail ? wb007Submit.detail.submitDisabledAfterTheCompleteSelection : null,
|
|
271
|
+
settledTitle: wb007.uiSettledTitle || null,
|
|
272
|
+
settledCard: wb007.uiSettledCard || null,
|
|
273
|
+
selection: wb007.selection || null,
|
|
274
|
+
},
|
|
275
|
+
api: {
|
|
276
|
+
questionRequest: wb007Card && wb007Card.detail ? wb007Card.detail.requestEvent : null,
|
|
277
|
+
refusedResolutions: wb007Refusals && wb007Refusals.detail ? wb007Refusals.detail.attempts : null,
|
|
278
|
+
permissionResolved: wb007.permissionResolved || null,
|
|
279
|
+
productPendingPermissionWhilePending: wb007Card && wb007Card.detail ? wb007Card.detail.productPendingPermission : null,
|
|
280
|
+
answeredToolCall: wb007.answeredToolCall || null,
|
|
281
|
+
toolResult: wb007.toolResult || null,
|
|
282
|
+
finalAnswerEchoesEverySelection: wb007.finalAnswerEchoesEverySelection,
|
|
283
|
+
finalAnswerPreview: wb007.finalAnswerPreview || null,
|
|
284
|
+
},
|
|
285
|
+
disk: {
|
|
286
|
+
note: 'WB-007 writes nothing outside the isolated run root; the marker scan of the fixture project is reported under WB-006',
|
|
287
|
+
filesCarryingTheMarkerText: wb006.filesCarryingTheMarkerText || null,
|
|
288
|
+
},
|
|
289
|
+
bypassMode: {
|
|
290
|
+
menuSwitch: wb007Bypass && wb007Bypass.detail ? wb007Bypass.detail : null,
|
|
291
|
+
boundedBypassTurn: wb007BypassTurn && wb007BypassTurn.detail ? wb007BypassTurn.detail : null,
|
|
292
|
+
required: true,
|
|
293
|
+
proven: bypassProven,
|
|
294
|
+
},
|
|
295
|
+
verdict: wb007Receipt ? wb007Receipt.status : null,
|
|
296
|
+
},
|
|
297
|
+
},
|
|
298
|
+
|
|
299
|
+
// The run history of THIS lane's runs, as it was recorded.
|
|
300
|
+
runs: runHistory ? runHistory.runs || [] : null,
|
|
301
|
+
firstFailure: runHistory ? runHistory.firstFailure || null : null,
|
|
302
|
+
retest: runHistory ? runHistory.retest || null : null,
|
|
303
|
+
commands: runHistory ? runHistory.commands || null : null,
|
|
304
|
+
sourceDigestsDuringRun: sourceDigestsDuringRun(runHistory),
|
|
305
|
+
|
|
306
|
+
tooling: {
|
|
307
|
+
dependencies: [
|
|
308
|
+
{ name: 'node', kind: 'the runtime that drives the lane', version: process.version },
|
|
309
|
+
{ name: 'the pinned Workbench module', kind: 'the published module, hosted as an isolated child process', version: summary.module && summary.module.version, root: summary.module && summary.module.root },
|
|
310
|
+
{ name: 'Claude Code CLI', kind: 'the executable the module spawns for the real turn', resolvedBy: 'the lane isolation helper (CLAUDE_CODE_EXECPATH, then the npm global package, then the user-local bin, then PATH, then where.exe)' },
|
|
311
|
+
{ name: 'Google Chrome', kind: 'the real browser driven over CDP', resolvedBy: 'the shared find-chrome helper, with the path captured BEFORE the isolation' },
|
|
312
|
+
...sharedDependencies,
|
|
313
|
+
],
|
|
314
|
+
ownership: {
|
|
315
|
+
laneOwned: artifacts.map(entry => ({ path: entry.path, owner: entry.owner, sha256: entry.sha256 })),
|
|
316
|
+
reusedReadOnly: sharedDependencies.map(entry => entry.path),
|
|
317
|
+
neverTouched: [
|
|
318
|
+
'the Workbench source checkout (read-only)',
|
|
319
|
+
'the installed module (never patched, never reinstalled)',
|
|
320
|
+
'the shared case file qa-autopilot/config/test-cases.json',
|
|
321
|
+
'the shared bindings and the files of the other lanes',
|
|
322
|
+
],
|
|
323
|
+
},
|
|
324
|
+
},
|
|
325
|
+
|
|
326
|
+
// What THIS run proved, stated from the run's own results.
|
|
327
|
+
runFacts: {
|
|
328
|
+
verdict: summary.verdict,
|
|
329
|
+
paidTurnsSent: summary.paidTurnsSent,
|
|
330
|
+
requiredCases: evaluation ? evaluation.essential : null,
|
|
331
|
+
requiredCasesNotProven: notProvedCases,
|
|
332
|
+
bypassCasesProven: bypassProven,
|
|
333
|
+
realProfileStoreUnchanged: realProfileCase ? realProfileCase.status === 'PASS' : null,
|
|
334
|
+
note: 'every entry is read from the results of the run this report is about',
|
|
335
|
+
},
|
|
336
|
+
|
|
337
|
+
// The scope of the lane. These statements are about the lane, not about
|
|
338
|
+
// a particular run.
|
|
339
|
+
notProved: [
|
|
340
|
+
...notProvedCases.map(entry => `The required case ${entry.caseId} was not proven by this run: its recorded status is ${entry.status}.`),
|
|
341
|
+
'Only the claude-code transport of the agent-terminal UI was exercised; the other surfaces of the product were not exercised by this run.',
|
|
342
|
+
'No quota, balance or PAYG endpoint was queried, and no credential was written to a fixture, a log, a URL, the browser or this report.',
|
|
343
|
+
],
|
|
344
|
+
|
|
345
|
+
cleanup: {
|
|
346
|
+
ownedProcesses: 'the isolated host child process and the Chrome process tree of this run are terminated by the lane itself',
|
|
347
|
+
retainedEvidence: 'the logs of this run are kept below the lane temp root and the evidence directory of the run',
|
|
348
|
+
realProfileStore: realProfileCase ? `the original subscription profile store was opened read-only; its byte-level digest is ${realProfileCase.status === 'PASS' ? 'unchanged' : 'NOT proven unchanged'} (case R44.k)` : 'the profile store case was not recorded by this run',
|
|
349
|
+
},
|
|
350
|
+
};
|
|
351
|
+
return report;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
function writeReport(options) {
|
|
355
|
+
const report = buildReport(options);
|
|
356
|
+
fs.mkdirSync(options.evidenceDir, { recursive: true });
|
|
357
|
+
const file = path.join(options.evidenceDir, 'report.json');
|
|
358
|
+
fs.writeFileSync(file, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
|
359
|
+
return { file, report };
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// The freshness of a recorded check. A check is fresh only when the bytes of
|
|
363
|
+
// the files it checked are still the current bytes. A stale check must not be
|
|
364
|
+
// presented as a PASS.
|
|
365
|
+
function checkFreshness(check, repoRoot, proposalFiles) {
|
|
366
|
+
const files = proposalFiles || LANE_FILES.map(entry => entry.path);
|
|
367
|
+
const recorded = new Map();
|
|
368
|
+
const results = (check && check.results) || [];
|
|
369
|
+
// The recorded digests are the ones the check wrote for the files it
|
|
370
|
+
// checked. They are found by their shape, not by the name of the check.
|
|
371
|
+
for (const entry of results) {
|
|
372
|
+
const list = entry && entry.detail && Array.isArray(entry.detail.files) ? entry.detail.files : null;
|
|
373
|
+
if (!list) continue;
|
|
374
|
+
for (const file of list) {
|
|
375
|
+
if (file && file.path && file.sha256) recorded.set(file.path, file.sha256);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
const mismatches = [];
|
|
379
|
+
const unrecorded = [];
|
|
380
|
+
for (const relative of files) {
|
|
381
|
+
const current = sha256File(path.join(repoRoot, ...relative.split('/')));
|
|
382
|
+
if (!recorded.has(relative)) { unrecorded.push(relative); continue; }
|
|
383
|
+
if (recorded.get(relative) !== current) {
|
|
384
|
+
mismatches.push({ path: relative, recordedSha256: recorded.get(relative), currentSha256: current });
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
return {
|
|
388
|
+
fresh: mismatches.length === 0 && unrecorded.length === 0 && recorded.size > 0,
|
|
389
|
+
mismatches,
|
|
390
|
+
unrecorded,
|
|
391
|
+
recordedFiles: [...recorded.keys()],
|
|
392
|
+
note: 'a check is fresh only when the bytes of every proposed file are still the bytes the check ran against',
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// The portability report of the canonical lane.
|
|
397
|
+
//
|
|
398
|
+
// It never re-runs anything and it never presents the accepted R44 model
|
|
399
|
+
// result as a model retest of this packaging. The digests of the source that
|
|
400
|
+
// produced the R44 acceptance and the digests of the packaging delivered here
|
|
401
|
+
// are two separate blocks and are never merged.
|
|
402
|
+
//
|
|
403
|
+
// R44D: it states a clean-checkout run or a negative control ONLY when the
|
|
404
|
+
// corresponding file was actually read. Everything it claims about the
|
|
405
|
+
// evidence is derived from the files it read.
|
|
406
|
+
function buildPortabilityReport(options) {
|
|
407
|
+
const { repoRoot, evidenceDir, check, frozenR44, proposalFiles } = options;
|
|
408
|
+
const cases = ['WB-006', 'WB-007'];
|
|
409
|
+
|
|
410
|
+
const checkResults = (check && check.results) || [];
|
|
411
|
+
const closureCheck = checkResults.find(entry => String(entry.id || '').startsWith('R44C.c'));
|
|
412
|
+
const trackedSharedFiles = closureCheck && closureCheck.detail ? (closureCheck.detail.trackedSharedFiles || []) : [];
|
|
413
|
+
|
|
414
|
+
// The source that actually produced the accepted R44 evidence.
|
|
415
|
+
const frozenReport = frozenR44 && frozenR44.available ? frozenR44.report : null;
|
|
416
|
+
const frozenHistoryFile = frozenR44 && frozenR44.available
|
|
417
|
+
? path.join(path.dirname(frozenR44.file), 'run-history.json')
|
|
418
|
+
: null;
|
|
419
|
+
const frozenHistory = frozenHistoryFile && fs.existsSync(frozenHistoryFile)
|
|
420
|
+
? JSON.parse(fs.readFileSync(frozenHistoryFile, 'utf8'))
|
|
421
|
+
: null;
|
|
422
|
+
const acceptanceTimeDigests = frozenHistory && frozenHistory.acceptanceArtifacts && frozenHistory.acceptanceArtifacts.files
|
|
423
|
+
? frozenHistory.acceptanceArtifacts.files
|
|
424
|
+
: [];
|
|
425
|
+
|
|
426
|
+
// The packaging delivered here.
|
|
427
|
+
const currentPackaging = (proposalFiles || []).map(relative => {
|
|
428
|
+
const full = path.join(repoRoot, ...String(relative).split('/'));
|
|
429
|
+
return {
|
|
430
|
+
path: relative,
|
|
431
|
+
sha256: sha256File(full),
|
|
432
|
+
bytes: fs.existsSync(full) ? fs.statSync(full).size : null,
|
|
433
|
+
trackedByGit: trackedByGit(repoRoot, relative),
|
|
434
|
+
};
|
|
435
|
+
});
|
|
436
|
+
|
|
437
|
+
// The commands of this lane, as they were actually executed. They are read
|
|
438
|
+
// from the lane's own progress file, so the report carries real command
|
|
439
|
+
// lines and real exit codes and nothing is written from memory. A report
|
|
440
|
+
// that lives in a subdirectory of the lane reads the progress file of the
|
|
441
|
+
// lane root.
|
|
442
|
+
const progressCandidates = evidenceDir
|
|
443
|
+
? [path.join(evidenceDir, 'progress.json'), path.join(path.dirname(evidenceDir), 'progress.json')]
|
|
444
|
+
: [];
|
|
445
|
+
const progressFile = progressCandidates.find(candidate => fs.existsSync(candidate)) || null;
|
|
446
|
+
const progress = progressFile ? JSON.parse(fs.readFileSync(progressFile, 'utf8')) : null;
|
|
447
|
+
|
|
448
|
+
// The clean-checkout run and the negative control are claimed only when the
|
|
449
|
+
// corresponding file was read.
|
|
450
|
+
const cleanCheckoutFile = evidenceDir ? path.join(evidenceDir, 'clean-checkout', 'check.json') : null;
|
|
451
|
+
const cleanCheckout = cleanCheckoutFile && fs.existsSync(cleanCheckoutFile)
|
|
452
|
+
? JSON.parse(fs.readFileSync(cleanCheckoutFile, 'utf8'))
|
|
453
|
+
: null;
|
|
454
|
+
const noDepsFile = evidenceDir ? path.join(evidenceDir, 'check-clean-checkout-without-npm-deps.json') : null;
|
|
455
|
+
const noDeps = noDepsFile && fs.existsSync(noDepsFile)
|
|
456
|
+
? JSON.parse(fs.readFileSync(noDepsFile, 'utf8'))
|
|
457
|
+
: null;
|
|
458
|
+
|
|
459
|
+
// A stale check must not be reported as a PASS.
|
|
460
|
+
const freshness = check ? checkFreshness(check, repoRoot, proposalFiles) : null;
|
|
461
|
+
|
|
462
|
+
const checkPassed = Boolean(check && check.verdict === 'PASS');
|
|
463
|
+
const cleanCheckoutPassed = Boolean(cleanCheckout && cleanCheckout.verdict === 'PASS');
|
|
464
|
+
const frozenPassed = Boolean(frozenReport && frozenReport.verdict === 'PASS');
|
|
465
|
+
const fresh = Boolean(freshness && freshness.fresh);
|
|
466
|
+
|
|
467
|
+
// The verdict has three tiers. A missing piece of evidence is a BLOCKED,
|
|
468
|
+
// not a FAIL: nothing was disproved, nothing was proven either. A FAIL is
|
|
469
|
+
// for evidence that is present and does not hold.
|
|
470
|
+
let verdict;
|
|
471
|
+
let verdictReason;
|
|
472
|
+
if (!check) {
|
|
473
|
+
verdict = 'BLOCKED';
|
|
474
|
+
verdictReason = 'there is no check result to report on, so nothing can be reported as proven';
|
|
475
|
+
} else if (!fresh) {
|
|
476
|
+
verdict = 'BLOCKED';
|
|
477
|
+
verdictReason = 'the recorded check does not match the current bytes of the proposed files, so its result does not apply to this packaging';
|
|
478
|
+
} else if (!cleanCheckout) {
|
|
479
|
+
verdict = 'BLOCKED';
|
|
480
|
+
verdictReason = 'there is no clean-checkout result to report on, so the portability claim is not established';
|
|
481
|
+
} else if (!frozenReport) {
|
|
482
|
+
verdict = 'BLOCKED';
|
|
483
|
+
verdictReason = 'the frozen R44 acceptance is not readable, so the accepted result cannot be quoted';
|
|
484
|
+
} else if (checkPassed && cleanCheckoutPassed && frozenPassed) {
|
|
485
|
+
verdict = 'PASS';
|
|
486
|
+
verdictReason = 'the working-tree check, the clean-checkout check and the frozen acceptance are all present and hold';
|
|
487
|
+
} else {
|
|
488
|
+
verdict = 'FAIL';
|
|
489
|
+
verdictReason = 'a check that is present does not hold';
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
const observations = [];
|
|
493
|
+
if (check) {
|
|
494
|
+
observations.push('The zero-model check was run in this working tree: it parses the proposed files, resolves their require-closure, proves that every file of the closure outside the proposal is tracked by git, loads the shared modules and proves that the check starts no service, no browser and no model request.');
|
|
495
|
+
}
|
|
496
|
+
if (cleanCheckout) {
|
|
497
|
+
observations.push('The zero-model check was also run inside a clean tracked checkout of HEAD with only the proposed files overlaid; the tracked list was handed in, because the clean checkout has no .git of its own.');
|
|
498
|
+
}
|
|
499
|
+
if (noDeps) {
|
|
500
|
+
observations.push('A negative control was run in the same clean checkout without the npm dependencies; it failed on the missing "ws" package, which is a missing dependency of the checkout and not a portability defect of the lane.');
|
|
501
|
+
}
|
|
502
|
+
if (frozenReport) {
|
|
503
|
+
observations.push('The accepted R44 acceptance is quoted from the frozen report and is not a retest of this packaging: the digests of the source that produced it are kept separate from the digests of the packaging delivered here.');
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
const report = {
|
|
507
|
+
schema: 'qa-autopilot/wb-qa-canonical-r44c/report/v1',
|
|
508
|
+
lane: 'P-WB-QA-CANONICAL-R44C',
|
|
509
|
+
generatedAt: new Date().toISOString(),
|
|
510
|
+
verdict,
|
|
511
|
+
verdictReason,
|
|
512
|
+
mode: 'portability report only: no model request was sent by this lane and nothing was re-run',
|
|
513
|
+
cases,
|
|
514
|
+
evidenceDir: evidenceDir || null,
|
|
515
|
+
|
|
516
|
+
scope: {
|
|
517
|
+
question: 'can the R44 lane be re-used from a clean tracked checkout, with only its own files added?',
|
|
518
|
+
answered: verdict === 'PASS'
|
|
519
|
+
? 'yes for the dependency closure, the syntax and the zero-model bootstrap, in the working tree AND in a clean tracked checkout of HEAD with the proposed-file overlay'
|
|
520
|
+
: `not established: ${verdictReason}`,
|
|
521
|
+
boundary: [
|
|
522
|
+
'no model request was sent by this lane',
|
|
523
|
+
'no service, no browser and no real profile was opened by the check',
|
|
524
|
+
'the R44 report, run-history and logs stay frozen and are only read',
|
|
525
|
+
'the shared case file and the shared bindings were not modified',
|
|
526
|
+
'no other lane\'s files were modified',
|
|
527
|
+
],
|
|
528
|
+
},
|
|
529
|
+
|
|
530
|
+
// The accepted R44 result. It is quoted as the ACCEPTED result of the R44
|
|
531
|
+
// lane, produced by the source digests below. It is explicitly NOT a
|
|
532
|
+
// model retest of this packaging.
|
|
533
|
+
frozenAcceptance: {
|
|
534
|
+
available: Boolean(frozenReport),
|
|
535
|
+
reportFile: frozenR44 && frozenR44.file ? frozenR44.file : null,
|
|
536
|
+
reportSha256: frozenR44 && frozenR44.file ? sha256File(frozenR44.file) : null,
|
|
537
|
+
runHistoryFile: frozenHistoryFile,
|
|
538
|
+
runHistorySha256: frozenHistoryFile ? sha256File(frozenHistoryFile) : null,
|
|
539
|
+
verdict: frozenReport ? frozenReport.verdict : null,
|
|
540
|
+
generatedAt: frozenReport ? frozenReport.generatedAt : null,
|
|
541
|
+
paidTurnsSent: frozenReport ? frozenReport.paidTurnsSent : null,
|
|
542
|
+
runs: frozenHistory ? (frozenHistory.runs || []) : null,
|
|
543
|
+
firstFailure: frozenHistory ? (frozenHistory.firstFailure || null) : null,
|
|
544
|
+
retest: frozenHistory ? (frozenHistory.retest || null) : null,
|
|
545
|
+
commands: frozenHistory ? (frozenHistory.commands || null) : null,
|
|
546
|
+
executedWith: {
|
|
547
|
+
note: 'the digests of the lane source as it was DURING the accepted R44 acceptance',
|
|
548
|
+
files: acceptanceTimeDigests,
|
|
549
|
+
},
|
|
550
|
+
isAModelRetestOfThisPackaging: false,
|
|
551
|
+
modelRetestNote: 'the accepted model result belongs to the digests above. The packaging delivered here differs from them and has NOT been exercised against a model by this lane, so it carries no model result at all.',
|
|
552
|
+
},
|
|
553
|
+
|
|
554
|
+
// The packaging delivered here. Its digests are the ones the zero-model
|
|
555
|
+
// check actually ran against.
|
|
556
|
+
currentPackaging: {
|
|
557
|
+
note: 'the files this lane proposes, as they are delivered; the zero-model check ran against exactly these bytes',
|
|
558
|
+
files: currentPackaging,
|
|
559
|
+
},
|
|
560
|
+
|
|
561
|
+
// The zero-model check.
|
|
562
|
+
check: {
|
|
563
|
+
schema: check && check.schema ? check.schema : null,
|
|
564
|
+
startedAt: check ? check.startedAt : null,
|
|
565
|
+
finishedAt: check ? check.finishedAt : null,
|
|
566
|
+
verdict: check ? check.verdict : null,
|
|
567
|
+
freshness,
|
|
568
|
+
results: checkResults,
|
|
569
|
+
modelRequestsSent: 0,
|
|
570
|
+
paidTurnsSent: 0,
|
|
571
|
+
},
|
|
572
|
+
|
|
573
|
+
// The clean-checkout run, claimed only when its result file was read.
|
|
574
|
+
cleanCheckoutCheck: {
|
|
575
|
+
ran: Boolean(cleanCheckout),
|
|
576
|
+
file: cleanCheckoutFile,
|
|
577
|
+
repoRoot: cleanCheckout ? cleanCheckout.repoRoot : null,
|
|
578
|
+
verdict: cleanCheckout ? cleanCheckout.verdict : null,
|
|
579
|
+
results: cleanCheckout ? cleanCheckout.results : null,
|
|
580
|
+
trackedListSource: cleanCheckout && cleanCheckout.results
|
|
581
|
+
? ((cleanCheckout.results.find(entry => String(entry.id || '').startsWith('R44C.c')) || {}).detail || {}).trackedListSource || null
|
|
582
|
+
: null,
|
|
583
|
+
npmDependenciesNeeded: 'the tracked browser helper requires the "ws" package, so the clean checkout needs its npm dependencies (npm ci) or NODE_PATH pointing at an installed dependency tree',
|
|
584
|
+
negativeControlWithoutNpmDependencies: noDeps ? {
|
|
585
|
+
file: noDepsFile,
|
|
586
|
+
verdict: noDeps.verdict,
|
|
587
|
+
failedCheck: (noDeps.results || []).filter(entry => entry.status === 'FAIL').map(entry => ({ id: entry.id, error: entry.detail && entry.detail.error ? entry.detail.error : null })),
|
|
588
|
+
} : null,
|
|
589
|
+
},
|
|
590
|
+
|
|
591
|
+
// The dependency closure, as the check measured it.
|
|
592
|
+
dependencyClosure: {
|
|
593
|
+
note: 'every file of the require-closure of the proposed files; only the proposed files may be untracked, everything else must be tracked by git',
|
|
594
|
+
trackedSharedFiles,
|
|
595
|
+
retiredHarness: '_site/_test/helpers/parallel-workbench-runtime-harness.js',
|
|
596
|
+
retiredHarnessUsed: false,
|
|
597
|
+
},
|
|
598
|
+
|
|
599
|
+
tooling: {
|
|
600
|
+
dependencies: [
|
|
601
|
+
{ name: 'node', kind: 'the runtime that parses, resolves and loads the lane', version: process.version },
|
|
602
|
+
{ name: 'git', kind: 'the source of truth for "tracked"', resolvedBy: 'git ls-files (or QA_R44_TRACKED_LIST in a checkout without .git)' },
|
|
603
|
+
],
|
|
604
|
+
ownership: {
|
|
605
|
+
laneOwned: currentPackaging.map(entry => ({ path: entry.path, owner: 'P-WB-QA-CANONICAL-R44C', sha256: entry.sha256 })),
|
|
606
|
+
reusedReadOnly: trackedSharedFiles,
|
|
607
|
+
neverTouched: [
|
|
608
|
+
'the R44 report, run-history and logs (frozen, read-only)',
|
|
609
|
+
'the shared case file qa-autopilot/config/test-cases.json',
|
|
610
|
+
'the shared bindings',
|
|
611
|
+
'the files of the other lanes',
|
|
612
|
+
'the Workbench source checkout (read-only)',
|
|
613
|
+
'the installed module (never patched, never reinstalled)',
|
|
614
|
+
'the published packages (no publish, no tag, no version change)',
|
|
615
|
+
],
|
|
616
|
+
},
|
|
617
|
+
},
|
|
618
|
+
|
|
619
|
+
// The real commands of this lane, with their real exit codes.
|
|
620
|
+
commands: progress && progress.commands ? progress.commands : null,
|
|
621
|
+
stages: progress && progress.stages ? progress.stages : null,
|
|
622
|
+
|
|
623
|
+
observations,
|
|
624
|
+
|
|
625
|
+
notProved: [
|
|
626
|
+
'This lane sent NO model request, so the delivered packaging has NOT been exercised against a real model turn. The accepted WB-006 / WB-007 result belongs to the frozen acceptance-time digests and is not a retest of this packaging.',
|
|
627
|
+
'The zero-model check proves the dependency closure, the syntax, the module loading and the API surface. It does NOT prove that the browser-driven acceptance still passes on the delivered packaging; that needs an explicit --run with PK_WB_ALLOW_PAID_TURNS=1, which this lane did not perform.',
|
|
628
|
+
'The check proves that the module loading starts no service, no browser and no model request. It does not prove the behaviour of the isolated host, because no host was started.',
|
|
629
|
+
'The exit-lingering remedy (a lane-owned event reader with a cleared timer) is verified by construction only. No paid run was performed, so the lingering was not re-measured after the change.',
|
|
630
|
+
'No quota, balance or PAYG endpoint was queried, and no credential was written to a fixture, a log, a URL, the browser or this report.',
|
|
631
|
+
],
|
|
632
|
+
|
|
633
|
+
cleanup: {
|
|
634
|
+
ownedProcesses: 'the zero-model check starts no process; the lane started nothing that had to be killed',
|
|
635
|
+
retainedEvidence: 'the check result, this report, the progress file and the pre-change snapshot of the lane files are kept below the lane evidence directory',
|
|
636
|
+
frozenEvidence: 'the R44 report, run-history and logs are unchanged',
|
|
637
|
+
},
|
|
638
|
+
};
|
|
639
|
+
return report;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
module.exports = {
|
|
643
|
+
LANE,
|
|
644
|
+
LANE_OWNER,
|
|
645
|
+
LANE_FILES,
|
|
646
|
+
buildReport,
|
|
647
|
+
writeReport,
|
|
648
|
+
buildPortabilityReport,
|
|
649
|
+
checkFreshness,
|
|
650
|
+
readExpectedCases,
|
|
651
|
+
laneArtifacts,
|
|
652
|
+
dependencyClosure,
|
|
653
|
+
requireSpecifiers,
|
|
654
|
+
sha256File,
|
|
655
|
+
findRepoRoot,
|
|
656
|
+
};
|