@sanqianx/project-knowledge 5.0.26 → 5.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/_site/_test/helpers/qa-r44-env.js +418 -0
- package/_site/_test/helpers/qa-r44-report.js +656 -0
- package/_site/_test/helpers/qa-r44-ui.js +347 -0
- package/_site/_test/helpers/qa-r49-wb-host.js +58 -0
- package/_site/_test/qa-r44-permission-question.js +756 -0
- package/_site/_test/qa-r49-sys003-real-blank-probe.js +374 -0
- package/module-runtime-manifest.json +28 -26
- package/package.json +2 -2
- package/plugins/project-knowledge/.claude-plugin/plugin.json +1 -1
- package/plugins/project-knowledge/.codex-plugin/plugin.json +1 -1
- package/plugins/project-knowledge/.mcp.json +1 -1
|
@@ -0,0 +1,756 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// _site/_test/qa-r44-permission-question.js
|
|
3
|
+
//
|
|
4
|
+
// The R44 lane scenarios (WB-006 permission deny, WB-007 user question).
|
|
5
|
+
//
|
|
6
|
+
// Both scenarios drive the REAL Workbench 0.13.9 agent-terminal UI in a REAL
|
|
7
|
+
// browser, hosted as an isolated child process of the REAL module, with a
|
|
8
|
+
// REAL model turn of this session's original subscription profile.
|
|
9
|
+
//
|
|
10
|
+
// What is proved:
|
|
11
|
+
// WB-006 a real Agent tool permission card is raised for a self-owned
|
|
12
|
+
// fixture marker; the product's own 拒绝 button is clicked; the
|
|
13
|
+
// protected tool does NOT execute (the marker is not written); the
|
|
14
|
+
// UI state and the rejection receipt agree.
|
|
15
|
+
// WB-007 a real product question card covers a single-select, a
|
|
16
|
+
// multi-select and a custom answer; the turn is proven to PAUSE
|
|
17
|
+
// without a submission; a bypass-style resolution, an empty answer
|
|
18
|
+
// and a wrong-question answer are all REFUSED with 400 and the
|
|
19
|
+
// pending question survives every one of them; the product's own
|
|
20
|
+
// bypass permission mode does not release the pending question;
|
|
21
|
+
// the real UI selection is submitted and the receipt matches.
|
|
22
|
+
//
|
|
23
|
+
// Nothing here fabricates a model event, substitutes an API call for a UI
|
|
24
|
+
// action that must be verified, or counts a driver-side PASS as acceptance.
|
|
25
|
+
// Every verdict is read from the product's own events, DOM and disk.
|
|
26
|
+
//
|
|
27
|
+
// R44C (canonical packaging, no behaviour change):
|
|
28
|
+
// * the lane now uses the TRACKED canonical harness
|
|
29
|
+
// (helpers/parallel-workbench-integration-5025-runtime-harness.js). The
|
|
30
|
+
// previous, untracked lane harness is no longer part of the dependency
|
|
31
|
+
// closure, so the lane can be checked out from a clean tracked tree;
|
|
32
|
+
// * the browser is opened with the Chrome path captured BEFORE the
|
|
33
|
+
// isolation — the canonical harness refuses to discover a browser from an
|
|
34
|
+
// isolated environment;
|
|
35
|
+
// * the session event stream is read by the lane's own reader below. The
|
|
36
|
+
// shared helper's reader leaves its timeout timer armed after the stream
|
|
37
|
+
// ends, which keeps the event loop alive for the whole turn-timeout
|
|
38
|
+
// window (the frozen R44 run-history records the resulting linger). The
|
|
39
|
+
// shared helper is frozen and is NOT patched; the lane owns its own reader
|
|
40
|
+
// and clears its own timer.
|
|
41
|
+
const fs = require('node:fs');
|
|
42
|
+
const path = require('node:path');
|
|
43
|
+
const crypto = require('node:crypto');
|
|
44
|
+
const { spawnSync } = require('node:child_process');
|
|
45
|
+
const harness = require('./helpers/parallel-workbench-integration-5025-runtime-harness');
|
|
46
|
+
const envLib = require('./helpers/qa-r44-env');
|
|
47
|
+
const ui = require('./helpers/qa-r44-ui');
|
|
48
|
+
|
|
49
|
+
const PROFILE_ID = String(process.env.PK_WB_MODEL_PROFILE || 'minimax');
|
|
50
|
+
const MODEL_ID = String(process.env.PK_WB_MODEL_ID || 'MiniMax-M3.1-Flash');
|
|
51
|
+
const TURN_TIMEOUT_MS = Number(process.env.PK_WB_TURN_TIMEOUT_MS || 420000);
|
|
52
|
+
const CARD_TIMEOUT_MS = Number(process.env.QA_R44_CARD_TIMEOUT_MS || 240000);
|
|
53
|
+
const WB_SOURCE_ROOT = String(process.env.QA_R44_WB_SOURCE_ROOT || '').trim();
|
|
54
|
+
|
|
55
|
+
const results = [];
|
|
56
|
+
function record(caseId, status, detail) {
|
|
57
|
+
results.push({ caseId, status, detail });
|
|
58
|
+
console.log(`CASE ${caseId} ${status} ${JSON.stringify(detail)}`);
|
|
59
|
+
}
|
|
60
|
+
function assertTrue(condition, message) {
|
|
61
|
+
if (!condition) throw new Error(message);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// The cases that must PASS for the lane to be accepted. Both bypass cases
|
|
65
|
+
// are part of it: the product's own bypass mode must not release the pending
|
|
66
|
+
// question (R44.g) and a bounded bypass turn must still ask it (R44.j). A run
|
|
67
|
+
// that never recorded them, or that recorded them as anything but PASS, is
|
|
68
|
+
// BLOCKED - never a partial PASS.
|
|
69
|
+
const ESSENTIAL_CASES = [
|
|
70
|
+
'R44.a-isolated-host-and-real-browser',
|
|
71
|
+
'R44.b-model-selected-through-the-real-ui',
|
|
72
|
+
'R44.c-wb006-real-permission-card-raised',
|
|
73
|
+
'R44.d-wb006-deny-executed-by-the-real-ui',
|
|
74
|
+
'R44.e-wb007-real-question-card-and-pause',
|
|
75
|
+
'R44.f-wb007-bypass-cannot-substitute-empty-answer-or-let-it-through',
|
|
76
|
+
'R44.g-wb007-bypass-mode-does-not-release-the-question',
|
|
77
|
+
'R44.h-wb007-ui-enables-the-submit-only-after-a-complete-answer',
|
|
78
|
+
'R44.i-wb007-real-ui-selection-submitted-and-the-receipt-matches',
|
|
79
|
+
'R44.j-wb007-the-bypass-mode-still-asks-the-question',
|
|
80
|
+
'R44.k-real-profiles-byte-identical',
|
|
81
|
+
];
|
|
82
|
+
|
|
83
|
+
// The verdict of a set of case results. This is a pure function of the
|
|
84
|
+
// recorded results, so a zero-model probe can prove the negative cases: a
|
|
85
|
+
// result set that lacks a required case is not a PASS.
|
|
86
|
+
function evaluateCases(caseResults) {
|
|
87
|
+
const list = Array.isArray(caseResults) ? caseResults : [];
|
|
88
|
+
const byId = new Map();
|
|
89
|
+
for (const entry of list) if (entry && entry.caseId) byId.set(entry.caseId, entry);
|
|
90
|
+
const cases = ESSENTIAL_CASES.map(caseId => {
|
|
91
|
+
const found = byId.get(caseId);
|
|
92
|
+
const status = found ? found.status : 'MISSING';
|
|
93
|
+
return { caseId, status, proven: status === 'PASS' };
|
|
94
|
+
});
|
|
95
|
+
const missing = cases.filter(entry => entry.status === 'MISSING').map(entry => entry.caseId);
|
|
96
|
+
const notPassed = cases.filter(entry => entry.status !== 'PASS').map(entry => ({ caseId: entry.caseId, status: entry.status }));
|
|
97
|
+
const proven = notPassed.length === 0;
|
|
98
|
+
return {
|
|
99
|
+
essential: cases,
|
|
100
|
+
missing,
|
|
101
|
+
notPassed,
|
|
102
|
+
proven,
|
|
103
|
+
verdict: proven ? 'PASS' : 'BLOCKED',
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
const sha256 = buffer => crypto.createHash('sha256').update(buffer).digest('hex');
|
|
107
|
+
const sha256File = file => {
|
|
108
|
+
try { return sha256(fs.readFileSync(file)); } catch { return null; }
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
// The product's own resolve endpoint. It is used ONLY as a negative control:
|
|
112
|
+
// every attempt this lane makes through it must be REFUSED, and the pending
|
|
113
|
+
// request must survive. The acceptance itself is done through the UI.
|
|
114
|
+
async function postResolve(base, sessionId, requestId, body) {
|
|
115
|
+
const url = `${base}${harness.API_BASE}/sessions/${encodeURIComponent(sessionId)}/permissions/${encodeURIComponent(requestId)}/resolve`;
|
|
116
|
+
const response = await fetch(url, {
|
|
117
|
+
method: 'POST',
|
|
118
|
+
headers: { 'Content-Type': 'application/json' },
|
|
119
|
+
body: JSON.stringify(body),
|
|
120
|
+
signal: AbortSignal.timeout(30000),
|
|
121
|
+
});
|
|
122
|
+
const parsed = await response.json().catch(() => null);
|
|
123
|
+
return { status: response.status, ok: response.ok, body: parsed };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// A read-only read of the product's own session record. It is used as
|
|
127
|
+
// evidence of the effective permission mode, of the still-pending request
|
|
128
|
+
// and of the event snapshot. It never performs a state change.
|
|
129
|
+
async function readSession(base, sessionId) {
|
|
130
|
+
const url = `${base}${harness.API_BASE}/sessions/${encodeURIComponent(sessionId)}`;
|
|
131
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(30000) });
|
|
132
|
+
const parsed = await response.json().catch(() => null);
|
|
133
|
+
return { status: response.status, ok: response.ok, body: parsed };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Every file of the self-owned fixture project that carries the marker text.
|
|
137
|
+
// The marker is the proof that the protected tool did not run — and that no
|
|
138
|
+
// other route wrote it either.
|
|
139
|
+
function scanProjectForMarker(root, needle) {
|
|
140
|
+
const hits = [];
|
|
141
|
+
const walk = dir => {
|
|
142
|
+
let entries = [];
|
|
143
|
+
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
144
|
+
for (const entry of entries) {
|
|
145
|
+
const full = path.join(dir, entry.name);
|
|
146
|
+
if (entry.isDirectory()) { walk(full); continue; }
|
|
147
|
+
try {
|
|
148
|
+
if (fs.statSync(full).size > 1024 * 1024) continue;
|
|
149
|
+
if (fs.readFileSync(full, 'utf8').includes(needle)) hits.push(full);
|
|
150
|
+
} catch { /* an unreadable or binary file cannot carry the marker */ }
|
|
151
|
+
}
|
|
152
|
+
};
|
|
153
|
+
walk(root);
|
|
154
|
+
return hits;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// The source identity of the Workbench repository, read-only.
|
|
158
|
+
function sourceIdentity() {
|
|
159
|
+
if (!WB_SOURCE_ROOT || !fs.existsSync(path.join(WB_SOURCE_ROOT, '.git'))) {
|
|
160
|
+
return { root: WB_SOURCE_ROOT || null, head: null, note: 'the source checkout was not addressed by this lane' };
|
|
161
|
+
}
|
|
162
|
+
const head = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: WB_SOURCE_ROOT, encoding: 'utf8', windowsHide: true, timeout: 30000 });
|
|
163
|
+
const branch = spawnSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { cwd: WB_SOURCE_ROOT, encoding: 'utf8', windowsHide: true, timeout: 30000 });
|
|
164
|
+
return {
|
|
165
|
+
root: WB_SOURCE_ROOT,
|
|
166
|
+
head: head.status === 0 ? String(head.stdout || '').trim() : null,
|
|
167
|
+
branch: branch.status === 0 ? String(branch.stdout || '').trim() : null,
|
|
168
|
+
readOnly: true,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// The pinned public module of the root repository.
|
|
173
|
+
function pinnedIdentity() {
|
|
174
|
+
const rootPkg = JSON.parse(fs.readFileSync(path.join(envLib.REPO_ROOT, 'package.json'), 'utf8'));
|
|
175
|
+
const spec = (rootPkg.dependencies || {})['claude-ai-workbench'] || null;
|
|
176
|
+
const { createRequire } = require('node:module');
|
|
177
|
+
const req = createRequire(path.join(envLib.REPO_ROOT, 'package.json'));
|
|
178
|
+
const pkgJson = req.resolve('claude-ai-workbench/package.json');
|
|
179
|
+
const root = path.dirname(pkgJson);
|
|
180
|
+
const pkg = JSON.parse(fs.readFileSync(pkgJson, 'utf8'));
|
|
181
|
+
return {
|
|
182
|
+
spec,
|
|
183
|
+
resolvedRoot: root,
|
|
184
|
+
version: pkg.version,
|
|
185
|
+
keyFiles: {
|
|
186
|
+
'apps/agent-terminal/app.js': sha256File(path.join(root, 'apps', 'agent-terminal', 'app.js')),
|
|
187
|
+
'apps/agent-terminal/user-question-card.js': sha256File(path.join(root, 'apps', 'agent-terminal', 'user-question-card.js')),
|
|
188
|
+
'packages/server/lib/user-questions.js': sha256File(path.join(root, 'packages', 'server', 'lib', 'user-questions.js')),
|
|
189
|
+
'packages/server/lib/workbench-service.js': sha256File(path.join(root, 'packages', 'server', 'lib', 'workbench-service.js')),
|
|
190
|
+
'src/backend/lib/claude-cli-runner.js': sha256File(path.join(root, 'src', 'backend', 'lib', 'claude-cli-runner.js')),
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// The lane's own reader of the product's session event stream.
|
|
196
|
+
//
|
|
197
|
+
// It has the same contract as the shared helper's reader — the same URL, the
|
|
198
|
+
// same "data:" payload contract, the same abort on the terminal event — and it
|
|
199
|
+
// CLEARS its own timeout timer. The shared helper arms a timer it never
|
|
200
|
+
// clears, so a reader that has already finished still keeps the event loop
|
|
201
|
+
// alive for the whole turn-timeout window: the R44 acceptance process wrote
|
|
202
|
+
// its report at 02:01:20Z and only exited about six minutes later, with the
|
|
203
|
+
// turn timeout armed at 420000 ms. The shared helper is frozen and is NOT
|
|
204
|
+
// patched; the lane owns its reader instead.
|
|
205
|
+
function readSessionEvents(url, onEvent, until, timeoutMs) {
|
|
206
|
+
const controller = new AbortController();
|
|
207
|
+
const timer = setTimeout(() => controller.abort(new Error(`event stream timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
208
|
+
let done = null;
|
|
209
|
+
const finished = new Promise(resolve => { done = resolve; });
|
|
210
|
+
(async () => {
|
|
211
|
+
try {
|
|
212
|
+
const response = await fetch(url, { signal: controller.signal });
|
|
213
|
+
if (!response.ok) throw new Error(`event stream HTTP ${response.status}`);
|
|
214
|
+
const reader = response.body.getReader();
|
|
215
|
+
const decoder = new TextDecoder();
|
|
216
|
+
let buffer = '';
|
|
217
|
+
for (;;) {
|
|
218
|
+
const { value, done: streamDone } = await reader.read();
|
|
219
|
+
if (streamDone) break;
|
|
220
|
+
buffer += decoder.decode(value, { stream: true });
|
|
221
|
+
let index;
|
|
222
|
+
while ((index = buffer.indexOf('\n')) >= 0) {
|
|
223
|
+
const line = buffer.slice(0, index).replace(/\r$/, '');
|
|
224
|
+
buffer = buffer.slice(index + 1);
|
|
225
|
+
if (!line.startsWith('data:')) continue;
|
|
226
|
+
const payload = line.slice(5).trim();
|
|
227
|
+
if (!payload) continue;
|
|
228
|
+
let event;
|
|
229
|
+
try { event = JSON.parse(payload); } catch { continue; }
|
|
230
|
+
try { onEvent(event); } catch { /* a listener error never kills the stream */ }
|
|
231
|
+
if (until && until(event)) { controller.abort(); return; }
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
} catch { /* aborted or ended */ }
|
|
235
|
+
finally {
|
|
236
|
+
clearTimeout(timer);
|
|
237
|
+
if (done) done();
|
|
238
|
+
}
|
|
239
|
+
})();
|
|
240
|
+
return { finished, stop: () => { clearTimeout(timer); controller.abort(); } };
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
async function newSession(browser, host, readers) {
|
|
244
|
+
await harness.clickNewSession(browser);
|
|
245
|
+
const state = await harness.waitForState(browser, s => s.product && s.product.activeSessionId, 'a new session', 30000);
|
|
246
|
+
const sessionId = state.product.activeSessionId;
|
|
247
|
+
// The page must be subscribed to THIS session before a turn is sent, so no
|
|
248
|
+
// event of the turn can be missed by the browser.
|
|
249
|
+
await ui.waitUntil(async () => {
|
|
250
|
+
const urls = await ui.readPageStreams(browser);
|
|
251
|
+
return urls.some(url => url.includes(encodeURIComponent(sessionId))) ? urls : null;
|
|
252
|
+
}, `the page to subscribe to the session ${sessionId}`, 30000);
|
|
253
|
+
const events = [];
|
|
254
|
+
const stream = readSessionEvents(
|
|
255
|
+
harness.sessionEventsUrl(host.base, sessionId),
|
|
256
|
+
event => events.push(event),
|
|
257
|
+
event => event.type === 'claude/result' || event.type === 'claude/error' || event.type === 'claude/aborted',
|
|
258
|
+
TURN_TIMEOUT_MS,
|
|
259
|
+
);
|
|
260
|
+
if (readers) readers.push(stream);
|
|
261
|
+
return { sessionId, events, stream };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const eventsOfType = (events, type) => events.filter(event => event.type === type);
|
|
265
|
+
|
|
266
|
+
// A path comparison that tolerates the separator and the spelling of the
|
|
267
|
+
// model: the tool call may carry the absolute path or the name relative to
|
|
268
|
+
// the session cwd. The marker name itself is the invariant.
|
|
269
|
+
const normalizePath = value => String(value || '').replace(/\\/g, '/').toLowerCase();
|
|
270
|
+
const isMarkerPath = (value, markerPath, markerName) => {
|
|
271
|
+
const normalized = normalizePath(value);
|
|
272
|
+
if (!normalized) return false;
|
|
273
|
+
return normalized === normalizePath(markerPath) || normalized.endsWith(`/${normalizePath(markerName)}`);
|
|
274
|
+
};
|
|
275
|
+
|
|
276
|
+
// The tool call a permission request belongs to: the last call of that tool
|
|
277
|
+
// BEFORE the request was raised. A model may call the same tool more than
|
|
278
|
+
// once; the receipt must belong to the call that was actually answered.
|
|
279
|
+
function toolCallBefore(events, requestId, toolName) {
|
|
280
|
+
const index = events.findIndex(event => event.type === 'claude/permission-request' && event.requestId === requestId);
|
|
281
|
+
if (index < 0) return null;
|
|
282
|
+
const before = events.slice(0, index).filter(event => event.type === 'claude/tool-use' && event.name === toolName);
|
|
283
|
+
return before.length ? before[before.length - 1] : null;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
async function main() {
|
|
287
|
+
const startedAt = new Date().toISOString();
|
|
288
|
+
const cases = [];
|
|
289
|
+
|
|
290
|
+
// --- 1. capture the real environment, then isolate the driver -----------
|
|
291
|
+
const captured = envLib.captureRealEnvironment();
|
|
292
|
+
const layout = envLib.createLaneLayout();
|
|
293
|
+
const hardened = envLib.hardenDriverEnv(layout, captured);
|
|
294
|
+
envLib.assertIsolatedEnv(process.env, layout, captured);
|
|
295
|
+
|
|
296
|
+
const realProfilesBefore = envLib.digestDirectory(envLib.realProfileDir());
|
|
297
|
+
const host = await harness.startHost({ layout, prefix: 'qa-r44' });
|
|
298
|
+
const identity = harness.moduleIdentity(host.moduleRoot);
|
|
299
|
+
let browser = null;
|
|
300
|
+
let wb006 = null;
|
|
301
|
+
let wb007 = null;
|
|
302
|
+
let bypassTurn = null;
|
|
303
|
+
let paidTurns = 0;
|
|
304
|
+
// Every event reader this lane opened. They are stopped in the finally
|
|
305
|
+
// block, so no handle of this lane survives the run.
|
|
306
|
+
const readers = [];
|
|
307
|
+
|
|
308
|
+
try {
|
|
309
|
+
// The Chrome path was captured BEFORE the isolation; the canonical harness
|
|
310
|
+
// refuses to discover a browser from an isolated environment.
|
|
311
|
+
const opened = await harness.openBrowser(host.base, host.layout, { chrome: captured.chrome.path });
|
|
312
|
+
browser = opened.browser;
|
|
313
|
+
const health = await harness.getJson(`${host.base}${harness.API_BASE}/health`);
|
|
314
|
+
assertTrue(health.ok && health.body && health.body.status === 'ok', 'the isolated host is not healthy');
|
|
315
|
+
|
|
316
|
+
const projectRoot = path.join(layout.root, 'qa-r44-project');
|
|
317
|
+
const project = await harness.ensureProject(browser, projectRoot, { name: 'qa-r44-project' });
|
|
318
|
+
const profiles = await harness.getJson(`${host.base}${harness.API_BASE}/profiles`);
|
|
319
|
+
const listed = Array.isArray(profiles.body && profiles.body.profiles) ? profiles.body.profiles : [];
|
|
320
|
+
const profile = listed.find(item => item.id === PROFILE_ID);
|
|
321
|
+
assertTrue(profile, `the ${PROFILE_ID} profile is not visible through the read-only facade`);
|
|
322
|
+
await harness.selectModel(browser, profile.name, MODEL_ID);
|
|
323
|
+
const afterModel = await harness.readState(browser);
|
|
324
|
+
|
|
325
|
+
record('R44.a-isolated-host-and-real-browser', 'PASS', {
|
|
326
|
+
health: health.body,
|
|
327
|
+
browser: 'real Chrome via CDP (headless) with its own user-data-dir below the isolated run root',
|
|
328
|
+
chrome: opened.chrome,
|
|
329
|
+
module: identity,
|
|
330
|
+
host: { mode: 'standalone', port: host.port, dataDir: layout.dataDir, workspace: layout.workspace, hostPid: host.child.child.pid },
|
|
331
|
+
project: { id: project.projectId, name: project.name, path: projectRoot, createdThroughTheProductForm: project.created === true },
|
|
332
|
+
isolation: envLib.isolationContract(layout, captured, hardened),
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
record('R44.b-model-selected-through-the-real-ui', afterModel.product.currentProfileId === PROFILE_ID && afterModel.product.currentModel === MODEL_ID ? 'PASS' : 'BLOCKED', {
|
|
336
|
+
profileId: PROFILE_ID,
|
|
337
|
+
profileName: profile.name,
|
|
338
|
+
requestedModel: MODEL_ID,
|
|
339
|
+
uiProfileId: afterModel.product.currentProfileId,
|
|
340
|
+
uiModel: afterModel.product.currentModel,
|
|
341
|
+
credentialSource: 'the original subscription store, read-only; the credential stays in the host process memory',
|
|
342
|
+
credentialWrittenAnywhere: false,
|
|
343
|
+
quotaEndpointQueried: false,
|
|
344
|
+
});
|
|
345
|
+
assertTrue(afterModel.product.currentProfileId === PROFILE_ID, 'the profile was not selected through the UI');
|
|
346
|
+
|
|
347
|
+
// =====================================================================
|
|
348
|
+
// WB-006 — permission deny
|
|
349
|
+
// =====================================================================
|
|
350
|
+
const markerName = 'qa-r44-wb006-marker.txt';
|
|
351
|
+
const markerPath = envLib.containedPath(projectRoot, markerName);
|
|
352
|
+
const session6 = await newSession(browser, host, readers);
|
|
353
|
+
// The precondition of WB-006 is the default protection mode. It is set
|
|
354
|
+
// through the product's own menu and then read back from the product.
|
|
355
|
+
let modeSwitch6 = null;
|
|
356
|
+
try {
|
|
357
|
+
modeSwitch6 = await ui.selectPermissionMode(browser, '默认防护');
|
|
358
|
+
} catch (error) {
|
|
359
|
+
modeSwitch6 = { switched: false, reason: String(error && error.message ? error.message : error) };
|
|
360
|
+
}
|
|
361
|
+
await harness.wait(500);
|
|
362
|
+
const modeBefore6 = await readSession(host.base, session6.sessionId);
|
|
363
|
+
const effectiveMode6 = modeBefore6.body && modeBefore6.body.permissionMode;
|
|
364
|
+
assertTrue(effectiveMode6 === 'default', `the WB-006 session is not in the default protection mode: ${effectiveMode6}`);
|
|
365
|
+
paidTurns += 1;
|
|
366
|
+
const wb006Prompt = [
|
|
367
|
+
'这是 WB-006 权限拒绝的真实抽测。',
|
|
368
|
+
`请使用 Write 工具,把文本 WB006-DENIED-MARKER 写入文件 ${markerPath}。`,
|
|
369
|
+
'必须使用 Write 工具,不要使用 Bash、Edit 或任何其它工具。',
|
|
370
|
+
'如果 Write 工具没有获得授权,请直接说明被拒绝,不要改用其它方式。',
|
|
371
|
+
].join('\n');
|
|
372
|
+
await harness.typeText(browser, wb006Prompt);
|
|
373
|
+
await harness.clickSend(browser);
|
|
374
|
+
|
|
375
|
+
// The product renders exactly one tool permission card for the pending
|
|
376
|
+
// request. The card is identified by the request id the product put into
|
|
377
|
+
// `data-perm-id`; the tool identity comes from the product's own event,
|
|
378
|
+
// because the card title itself does not name the tool.
|
|
379
|
+
const wb006Card = await ui.waitUntil(async () => {
|
|
380
|
+
const cards = await ui.readToolPermissionCards(browser);
|
|
381
|
+
const pending = cards.find(card => !card.settled && card.hasAllow && card.hasDeny);
|
|
382
|
+
return pending || null;
|
|
383
|
+
}, 'the pending tool permission card of WB-006', CARD_TIMEOUT_MS);
|
|
384
|
+
|
|
385
|
+
const snapshot6 = await readSession(host.base, session6.sessionId);
|
|
386
|
+
const pageEvents6 = (await ui.readPageEvents(browser, 'claude/permission-request')).map(entry => entry.event);
|
|
387
|
+
const requestEvent6 = eventsOfType(session6.events, 'claude/permission-request').find(event => event.requestId === wb006Card.requestId)
|
|
388
|
+
|| eventsOfType(pageEvents6, 'claude/permission-request').find(event => event.requestId === wb006Card.requestId)
|
|
389
|
+
|| eventsOfType((snapshot6.body && snapshot6.body.events) || [], 'claude/permission-request').find(event => event.requestId === wb006Card.requestId)
|
|
390
|
+
|| null;
|
|
391
|
+
const toolUse6 = eventsOfType(session6.events, 'claude/tool-use').find(event => event.name === 'Write' && event.input && isMarkerPath(event.input.file_path, markerPath, markerName)) || null;
|
|
392
|
+
const markerBefore = fs.existsSync(markerPath);
|
|
393
|
+
const protectedToolName = requestEvent6 && requestEvent6.summary && requestEvent6.summary.toolName;
|
|
394
|
+
|
|
395
|
+
record('R44.c-wb006-real-permission-card-raised', requestEvent6 && toolUse6 && !markerBefore && protectedToolName === 'Write' ? 'PASS' : 'BLOCKED', {
|
|
396
|
+
sessionId: session6.sessionId,
|
|
397
|
+
effectivePermissionMode: effectiveMode6,
|
|
398
|
+
modeSwitchThroughTheProductMenu: modeSwitch6,
|
|
399
|
+
card: wb006Card,
|
|
400
|
+
cardTitleNote: 'the product renders the tool name of a permission card in the EVENT summary; the card title itself reads "Agent 请求执行操作"',
|
|
401
|
+
requestEvent: requestEvent6 ? {
|
|
402
|
+
requestId: requestEvent6.requestId,
|
|
403
|
+
kind: requestEvent6.kind || null,
|
|
404
|
+
toolName: protectedToolName,
|
|
405
|
+
allowedTools: requestEvent6.summary && requestEvent6.summary.allowedTools,
|
|
406
|
+
permissionMode: requestEvent6.summary && requestEvent6.summary.permissionMode,
|
|
407
|
+
toolUseID: requestEvent6.summary && requestEvent6.summary.toolUseID,
|
|
408
|
+
promptPreview: requestEvent6.summary && requestEvent6.summary.promptPreview,
|
|
409
|
+
} : null,
|
|
410
|
+
protectedTool: toolUse6 ? { id: toolUse6.id, name: toolUse6.name, filePath: toolUse6.input.file_path } : null,
|
|
411
|
+
markerPath,
|
|
412
|
+
markerExistsBeforeTheDecision: markerBefore,
|
|
413
|
+
markerIsSelfOwned: markerPath.startsWith(layout.root),
|
|
414
|
+
markerIsInsideTheRepository: envLib.isInside(markerPath, envLib.REPO_ROOT),
|
|
415
|
+
});
|
|
416
|
+
assertTrue(requestEvent6, 'no permission-request event for the Write card');
|
|
417
|
+
assertTrue(protectedToolName === 'Write', `the pending permission does not protect the Write tool: ${protectedToolName}`);
|
|
418
|
+
assertTrue(toolUse6, 'no Write tool-use event for the marker path');
|
|
419
|
+
|
|
420
|
+
await ui.clickDeny(browser, wb006Card.requestId);
|
|
421
|
+
const settled6 = await ui.waitUntil(async () => {
|
|
422
|
+
const settled = await ui.readSettledCard(browser, wb006Card.requestId);
|
|
423
|
+
return settled.found && settled.settledState ? settled : null;
|
|
424
|
+
}, 'the settled WB-006 card', 60000);
|
|
425
|
+
const resolved6 = await ui.waitUntil(async () => {
|
|
426
|
+
const found = eventsOfType(session6.events, 'claude/permission-resolved').find(event => event.requestId === wb006Card.requestId);
|
|
427
|
+
return found || null;
|
|
428
|
+
}, 'the WB-006 permission-resolved event', 60000);
|
|
429
|
+
await harness.waitUntil(() => eventsOfType(session6.events, 'claude/result').length > 0 || eventsOfType(session6.events, 'claude/error').length > 0, 'the end of the WB-006 turn', TURN_TIMEOUT_MS, 250);
|
|
430
|
+
|
|
431
|
+
const markerAfter = fs.existsSync(markerPath);
|
|
432
|
+
const markerHits = scanProjectForMarker(projectRoot, 'WB006-DENIED-MARKER');
|
|
433
|
+
const toolResult6 = eventsOfType(session6.events, 'claude/tool-result').find(event => event.id === toolUse6.id) || null;
|
|
434
|
+
const result6 = eventsOfType(session6.events, 'claude/result')[0] || null;
|
|
435
|
+
const sessionAfter6 = await readSession(host.base, session6.sessionId);
|
|
436
|
+
const wb006Ok = markerAfter === false
|
|
437
|
+
&& markerHits.length === 0
|
|
438
|
+
&& resolved6.allow === false
|
|
439
|
+
&& settled6.settledState === '已拒绝'
|
|
440
|
+
&& (toolResult6 ? toolResult6.isError === true : true);
|
|
441
|
+
wb006 = {
|
|
442
|
+
sessionId: session6.sessionId,
|
|
443
|
+
effectivePermissionMode: effectiveMode6,
|
|
444
|
+
protectedTool: { id: toolUse6.id, name: toolUse6.name, filePath: toolUse6.input.file_path },
|
|
445
|
+
markerPath,
|
|
446
|
+
markerExistsAfterTheDenial: markerAfter,
|
|
447
|
+
filesCarryingTheMarkerText: markerHits,
|
|
448
|
+
permissionResolved: { requestId: resolved6.requestId, allow: resolved6.allow, kind: resolved6.kind || null, resolvedAt: resolved6.resolvedAt },
|
|
449
|
+
uiSettledState: settled6.settledState,
|
|
450
|
+
uiSettledCard: settled6,
|
|
451
|
+
pendingPermissionAfterTheDecision: (sessionAfter6.body && sessionAfter6.body.permission) || null,
|
|
452
|
+
toolResult: toolResult6 ? { id: toolResult6.id, isError: toolResult6.isError, output: String(toolResult6.output || '').slice(0, 400) } : null,
|
|
453
|
+
turnResult: result6 ? { subtype: result6.subtype, isError: result6.isError, resultPreview: String(result6.result || '').slice(0, 400) } : null,
|
|
454
|
+
eventTypes: [...new Set(session6.events.map(event => event.type))],
|
|
455
|
+
};
|
|
456
|
+
record('R44.d-wb006-deny-executed-by-the-real-ui', wb006Ok ? 'PASS' : 'FAIL', wb006);
|
|
457
|
+
assertTrue(wb006Ok, 'the denied Write tool still executed, or the receipt and the UI disagree');
|
|
458
|
+
|
|
459
|
+
// =====================================================================
|
|
460
|
+
// WB-007 — user question
|
|
461
|
+
// =====================================================================
|
|
462
|
+
const session7 = await newSession(browser, host, readers);
|
|
463
|
+
paidTurns += 1;
|
|
464
|
+
const wb007Prompt = [
|
|
465
|
+
'这是 WB-007 用户问题选择的真实抽测。',
|
|
466
|
+
'请调用 AskUserQuestion 工具向我提问(必须调用该工具,不要用普通文字提问,也不要自己回答)。',
|
|
467
|
+
'一次提出下面三个问题,参数严格如下:',
|
|
468
|
+
'问题1(单选,multiSelect=false):question 为 "QA-R44 单选:请选择要执行的操作",header 为 "单选",options 为 [{"label":"选项甲","description":"第一个选项"},{"label":"选项乙","description":"第二个选项"},{"label":"选项丙","description":"第三个选项"}]',
|
|
469
|
+
'问题2(多选,multiSelect=true):question 为 "QA-R44 多选:请选择需要检查的项目",header 为 "多选",options 为 [{"label":"检查项一","description":"第一项"},{"label":"检查项二","description":"第二项"},{"label":"检查项三","description":"第三项"}]',
|
|
470
|
+
'问题3(单选,multiSelect=false):question 为 "QA-R44 自定义:请补充你的要求",header 为 "自定义",options 为 [{"label":"无特殊要求","description":"不需要额外说明"},{"label":"有特殊要求","description":"有额外要求"},{"label":"需要补充说明","description":"再补充一条"}]',
|
|
471
|
+
'等我回答后,请把三个问题的答案原样复述,每行一条,格式为“问题N:答案”。',
|
|
472
|
+
].join('\n');
|
|
473
|
+
await harness.typeText(browser, wb007Prompt);
|
|
474
|
+
await harness.clickSend(browser);
|
|
475
|
+
|
|
476
|
+
// The product raises the question card from its own
|
|
477
|
+
// `claude/permission-request` event of kind "question". The card is
|
|
478
|
+
// correlated with that event by the request id the product put into
|
|
479
|
+
// `data-perm-id`.
|
|
480
|
+
const questionEvent7 = await ui.waitUntil(async () => {
|
|
481
|
+
const fromStream = eventsOfType(session7.events, 'claude/permission-request').find(event => event.kind === 'question');
|
|
482
|
+
if (fromStream) return fromStream;
|
|
483
|
+
const fromPage = (await ui.readPageEvents(browser, 'claude/permission-request')).map(entry => entry.event);
|
|
484
|
+
return eventsOfType(fromPage, 'claude/permission-request').find(event => event.kind === 'question') || null;
|
|
485
|
+
}, 'the question permission-request event of WB-007', CARD_TIMEOUT_MS);
|
|
486
|
+
const questionCard = await ui.waitUntil(async () => {
|
|
487
|
+
const cards = await ui.readQuestionCards(browser);
|
|
488
|
+
return cards.find(card => card.requestId === questionEvent7.requestId && card.fields.length > 0) || null;
|
|
489
|
+
}, 'the user question card of WB-007', 60000);
|
|
490
|
+
const requestEvent7 = questionEvent7;
|
|
491
|
+
const toolUse7 = toolCallBefore(session7.events, questionCard.requestId, 'AskUserQuestion');
|
|
492
|
+
const askUserQuestionCalls = eventsOfType(session7.events, 'claude/tool-use').filter(event => event.name === 'AskUserQuestion');
|
|
493
|
+
|
|
494
|
+
const pendingState = await harness.readState(browser);
|
|
495
|
+
const pendingSession7 = await readSession(host.base, session7.sessionId);
|
|
496
|
+
const pauseProved = pendingState.product.sessionState === 'pending-permission'
|
|
497
|
+
&& questionCard.submitDisabled === true
|
|
498
|
+
&& Boolean(pendingSession7.body && pendingSession7.body.permission && pendingSession7.body.permission.requestId === questionCard.requestId);
|
|
499
|
+
record('R44.e-wb007-real-question-card-and-pause', pauseProved && requestEvent7 ? 'PASS' : 'BLOCKED', {
|
|
500
|
+
sessionId: session7.sessionId,
|
|
501
|
+
card: questionCard,
|
|
502
|
+
requestEvent: requestEvent7 ? { requestId: requestEvent7.requestId, kind: requestEvent7.kind, toolName: requestEvent7.toolName, questions: requestEvent7.questions } : null,
|
|
503
|
+
toolUse: toolUse7 ? { id: toolUse7.id, name: toolUse7.name } : null,
|
|
504
|
+
askUserQuestionCalls: askUserQuestionCalls.map(call => ({ id: call.id, questionCount: call.input && Array.isArray(call.input.questions) ? call.input.questions.length : null })),
|
|
505
|
+
uiSessionStateWhilePending: pendingState.product.sessionState,
|
|
506
|
+
productPendingPermission: pendingSession7.body && pendingSession7.body.permission ? { requestId: pendingSession7.body.permission.requestId, kind: pendingSession7.body.permission.kind, toolName: pendingSession7.body.permission.toolName, status: pendingSession7.body.permission.status } : null,
|
|
507
|
+
submitDisabledWithoutAnyAnswer: questionCard.submitDisabled,
|
|
508
|
+
answersSubmittedSoFar: 0,
|
|
509
|
+
pausedWithoutSubmission: pauseProved,
|
|
510
|
+
});
|
|
511
|
+
assertTrue(requestEvent7, 'no permission-request event for the question card');
|
|
512
|
+
assertTrue(pauseProved, 'the question card did not hold the turn in the pending-permission state');
|
|
513
|
+
|
|
514
|
+
// --- the refused resolutions -----------------------------------------
|
|
515
|
+
const attempts = [];
|
|
516
|
+
const attemptBodies = [
|
|
517
|
+
{ label: 'bypass-without-a-selection', body: { allow: true } },
|
|
518
|
+
{ label: 'empty-answer', body: { allow: true, answers: { 'question-1': [] } } },
|
|
519
|
+
{ label: 'wrong-question-answer', body: { allow: true, answers: { 'question-99': ['选项甲'] } } },
|
|
520
|
+
{ label: 'deny-without-a-selection', body: { allow: false } },
|
|
521
|
+
];
|
|
522
|
+
for (const attempt of attemptBodies) {
|
|
523
|
+
const response = await postResolve(host.base, session7.sessionId, questionCard.requestId, attempt.body);
|
|
524
|
+
const resolvedEvents = eventsOfType(session7.events, 'claude/permission-resolved').filter(event => event.requestId === questionCard.requestId);
|
|
525
|
+
const stillPending = await ui.readQuestionCards(browser);
|
|
526
|
+
const cardStillThere = stillPending.some(card => card.requestId === questionCard.requestId && !card.settled);
|
|
527
|
+
const stateNow = await harness.readState(browser);
|
|
528
|
+
const sessionNow = await readSession(host.base, session7.sessionId);
|
|
529
|
+
const productPending = sessionNow.body && sessionNow.body.permission;
|
|
530
|
+
attempts.push({
|
|
531
|
+
attempt: attempt.label,
|
|
532
|
+
requestBody: attempt.body,
|
|
533
|
+
httpStatus: response.status,
|
|
534
|
+
ok: response.ok,
|
|
535
|
+
error: response.body && response.body.error,
|
|
536
|
+
code: response.body && response.body.code,
|
|
537
|
+
refused: response.status === 400 && response.ok === false,
|
|
538
|
+
permissionResolvedEventsEmitted: resolvedEvents.length,
|
|
539
|
+
questionCardStillPending: cardStillThere,
|
|
540
|
+
productStillHoldsThePendingQuestion: Boolean(productPending && productPending.requestId === questionCard.requestId),
|
|
541
|
+
uiSessionState: stateNow.product.sessionState,
|
|
542
|
+
});
|
|
543
|
+
await harness.wait(300);
|
|
544
|
+
}
|
|
545
|
+
const allRefused = attempts.every(attempt => attempt.refused
|
|
546
|
+
&& attempt.permissionResolvedEventsEmitted === 0
|
|
547
|
+
&& attempt.questionCardStillPending
|
|
548
|
+
&& attempt.productStillHoldsThePendingQuestion);
|
|
549
|
+
record('R44.f-wb007-bypass-cannot-substitute-empty-answer-or-let-it-through', allRefused ? 'PASS' : 'FAIL', {
|
|
550
|
+
sessionId: session7.sessionId,
|
|
551
|
+
requestId: questionCard.requestId,
|
|
552
|
+
attempts,
|
|
553
|
+
everyAttemptRefused: allRefused,
|
|
554
|
+
pendingQuestionSurvivedEveryAttempt: attempts.every(attempt => attempt.questionCardStillPending && attempt.productStillHoldsThePendingQuestion),
|
|
555
|
+
note: 'these resolve calls are NEGATIVE CONTROLS on the product\'s own endpoint; the acceptance below is done through the UI',
|
|
556
|
+
});
|
|
557
|
+
assertTrue(allRefused, 'the product accepted a bypass/empty/wrong resolution of the pending question');
|
|
558
|
+
|
|
559
|
+
// --- the product's own bypass mode does not release the question ------
|
|
560
|
+
let bypassSwitch = null;
|
|
561
|
+
try {
|
|
562
|
+
await ui.selectPermissionMode(browser, '完全访问');
|
|
563
|
+
await harness.wait(1500);
|
|
564
|
+
const afterSwitch = await ui.readQuestionCards(browser);
|
|
565
|
+
const resolvedAfterSwitch = eventsOfType(session7.events, 'claude/permission-resolved').filter(event => event.requestId === questionCard.requestId);
|
|
566
|
+
const stateAfterSwitch = await harness.readState(browser);
|
|
567
|
+
const sessionAfterSwitch = await readSession(host.base, session7.sessionId);
|
|
568
|
+
bypassSwitch = {
|
|
569
|
+
control: 'the product\'s own permission-mode menu (完全访问 / bypassPermissions)',
|
|
570
|
+
effectivePermissionModeInTheProduct: sessionAfterSwitch.body && sessionAfterSwitch.body.permissionMode,
|
|
571
|
+
questionCardStillPending: afterSwitch.some(card => card.requestId === questionCard.requestId && !card.settled),
|
|
572
|
+
permissionResolvedEventsEmitted: resolvedAfterSwitch.length,
|
|
573
|
+
uiSessionState: stateAfterSwitch.product.sessionState,
|
|
574
|
+
releasedByTheBypass: resolvedAfterSwitch.length > 0,
|
|
575
|
+
};
|
|
576
|
+
record('R44.g-wb007-bypass-mode-does-not-release-the-question', bypassSwitch.questionCardStillPending && !bypassSwitch.releasedByTheBypass && bypassSwitch.effectivePermissionModeInTheProduct === 'bypassPermissions' ? 'PASS' : 'FAIL', bypassSwitch);
|
|
577
|
+
// Leave the session in the default mode for the rest of the lane.
|
|
578
|
+
await ui.selectPermissionMode(browser, '默认防护');
|
|
579
|
+
await harness.wait(800);
|
|
580
|
+
} catch (error) {
|
|
581
|
+
bypassSwitch = { error: String(error && error.message ? error.message : error) };
|
|
582
|
+
record('R44.g-wb007-bypass-mode-does-not-release-the-question', 'BLOCKED', bypassSwitch);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// --- the real UI selection -------------------------------------------
|
|
586
|
+
const fields = questionCard.fields;
|
|
587
|
+
const singleIndex = fields.findIndex(field => field.legend.includes('单选'));
|
|
588
|
+
const multiIndex = fields.findIndex(field => field.legend.includes('多选'));
|
|
589
|
+
const customIndex = fields.findIndex(field => field.legend.includes('自定义'));
|
|
590
|
+
assertTrue(singleIndex >= 0 && multiIndex >= 0 && customIndex >= 0, `the question card does not carry the three expected questions: ${JSON.stringify(fields.map(field => field.legend))}`);
|
|
591
|
+
|
|
592
|
+
const chosenSingle = '选项甲';
|
|
593
|
+
const chosenMulti = ['检查项一', '检查项二'];
|
|
594
|
+
const chosenCustom = 'QA-R44-CUSTOM-ANSWER-请按选项甲执行并输出确认';
|
|
595
|
+
|
|
596
|
+
await ui.selectRadioOption(browser, questionCard.requestId, singleIndex, chosenSingle);
|
|
597
|
+
await ui.selectCheckboxOptions(browser, questionCard.requestId, multiIndex, chosenMulti);
|
|
598
|
+
await ui.typeCustomAnswer(browser, questionCard.requestId, customIndex, chosenCustom);
|
|
599
|
+
await harness.wait(400);
|
|
600
|
+
const filled = await ui.readQuestionCards(browser);
|
|
601
|
+
const filledCard = filled.find(card => card.requestId === questionCard.requestId);
|
|
602
|
+
const submitEnabled = filledCard && filledCard.submitDisabled === false;
|
|
603
|
+
record('R44.h-wb007-ui-enables-the-submit-only-after-a-complete-answer', submitEnabled ? 'PASS' : 'FAIL', {
|
|
604
|
+
requestId: questionCard.requestId,
|
|
605
|
+
submitDisabledBeforeAnyAnswer: questionCard.submitDisabled,
|
|
606
|
+
submitDisabledAfterTheCompleteSelection: filledCard ? filledCard.submitDisabled : null,
|
|
607
|
+
fields: filledCard ? filledCard.fields : null,
|
|
608
|
+
});
|
|
609
|
+
assertTrue(submitEnabled, 'the product\'s submit button stayed disabled after a complete answer');
|
|
610
|
+
|
|
611
|
+
await ui.clickSubmitAnswer(browser, questionCard.requestId);
|
|
612
|
+
// The card settles itself: the title becomes 答案已提交 and the card
|
|
613
|
+
// takes the question-settled class. The wait must look for THAT, not
|
|
614
|
+
// for a non-empty title (the title is non-empty before the submission).
|
|
615
|
+
const settled7 = await ui.waitUntil(async () => {
|
|
616
|
+
const settled = await ui.readSettledCard(browser, questionCard.requestId);
|
|
617
|
+
return settled.found && (settled.settled === true || settled.settledState) ? settled : null;
|
|
618
|
+
}, 'the settled WB-007 question card', 60000);
|
|
619
|
+
const resolved7 = await ui.waitUntil(async () => {
|
|
620
|
+
const found = eventsOfType(session7.events, 'claude/permission-resolved').find(event => event.requestId === questionCard.requestId);
|
|
621
|
+
return found || null;
|
|
622
|
+
}, 'the WB-007 permission-resolved event', 60000);
|
|
623
|
+
await harness.waitUntil(() => eventsOfType(session7.events, 'claude/result').length > 0 || eventsOfType(session7.events, 'claude/error').length > 0, 'the end of the WB-007 turn', TURN_TIMEOUT_MS, 250);
|
|
624
|
+
|
|
625
|
+
const toolResult7 = toolUse7 ? eventsOfType(session7.events, 'claude/tool-result').find(event => event.id === toolUse7.id) || null : null;
|
|
626
|
+
const result7 = eventsOfType(session7.events, 'claude/result')[0] || null;
|
|
627
|
+
const answerText7 = String((toolResult7 && toolResult7.output) || '');
|
|
628
|
+
const selections = [chosenSingle, ...chosenMulti, chosenCustom];
|
|
629
|
+
const receiptContains = selections.every(value => answerText7.includes(value));
|
|
630
|
+
const finalAnswer7 = String((result7 && result7.result) || '');
|
|
631
|
+
const finalEchoes = selections.every(value => finalAnswer7.includes(value));
|
|
632
|
+
const wb007Ok = resolved7.allow === true
|
|
633
|
+
&& resolved7.kind === 'question'
|
|
634
|
+
&& settled7.title.includes('答案已提交')
|
|
635
|
+
&& receiptContains
|
|
636
|
+
&& finalEchoes;
|
|
637
|
+
wb007 = {
|
|
638
|
+
sessionId: session7.sessionId,
|
|
639
|
+
requestId: questionCard.requestId,
|
|
640
|
+
questions: fields.map(field => ({ legend: field.legend, optionLabels: field.options.map(option => option.label), customInput: field.customPresent })),
|
|
641
|
+
selection: { single: chosenSingle, multi: chosenMulti, custom: chosenCustom },
|
|
642
|
+
permissionResolved: { requestId: resolved7.requestId, allow: resolved7.allow, kind: resolved7.kind, resolvedAt: resolved7.resolvedAt },
|
|
643
|
+
uiSettledTitle: settled7.title,
|
|
644
|
+
uiSettledCard: settled7,
|
|
645
|
+
answeredToolCall: toolUse7 ? { id: toolUse7.id, name: toolUse7.name } : null,
|
|
646
|
+
toolResult: toolResult7 ? { id: toolResult7.id, isError: toolResult7.isError, output: answerText7.slice(0, 800) } : null,
|
|
647
|
+
askUserQuestionToolResults: eventsOfType(session7.events, 'claude/tool-result')
|
|
648
|
+
.filter(event => askUserQuestionCalls.some(call => call.id === event.id))
|
|
649
|
+
.map(event => ({ id: event.id, isError: event.isError, output: String(event.output || '').slice(0, 400) })),
|
|
650
|
+
receiptContainsEverySelection: receiptContains,
|
|
651
|
+
finalAnswerEchoesEverySelection: finalEchoes,
|
|
652
|
+
finalAnswerPreview: finalAnswer7.slice(0, 600),
|
|
653
|
+
eventTypes: [...new Set(session7.events.map(event => event.type))],
|
|
654
|
+
};
|
|
655
|
+
record('R44.i-wb007-real-ui-selection-submitted-and-the-receipt-matches', wb007Ok ? 'PASS' : 'FAIL', wb007);
|
|
656
|
+
assertTrue(wb007Ok, 'the submitted answers did not reach the model, or the receipt disagrees with the selection');
|
|
657
|
+
|
|
658
|
+
// =====================================================================
|
|
659
|
+
// the bounded bypass turn
|
|
660
|
+
// =====================================================================
|
|
661
|
+
if (String(process.env.QA_R44_BYPASS_TURN || '1') === '1') {
|
|
662
|
+
try {
|
|
663
|
+
const session8 = await newSession(browser, host, readers);
|
|
664
|
+
// The bypass mode is a property of the SESSION, so it is switched on
|
|
665
|
+
// the new session itself and read back from the product.
|
|
666
|
+
await ui.selectPermissionMode(browser, '完全访问');
|
|
667
|
+
await harness.wait(800);
|
|
668
|
+
const mode8 = await readSession(host.base, session8.sessionId);
|
|
669
|
+
const effectiveMode8 = mode8.body && mode8.body.permissionMode;
|
|
670
|
+
paidTurns += 1;
|
|
671
|
+
await harness.typeText(browser, '请调用 AskUserQuestion 工具问一个问题:question 为 "QA-R44 bypass 完全访问下的问题",options 为 [{"label":"同意","description":"同意"},{"label":"不同意","description":"不同意"}],multiSelect=false。收到回答后原样复述。');
|
|
672
|
+
await harness.clickSend(browser);
|
|
673
|
+
const event8 = await ui.waitUntil(async () => {
|
|
674
|
+
const found = eventsOfType(session8.events, 'claude/permission-request').find(event => event.kind === 'question');
|
|
675
|
+
return found || null;
|
|
676
|
+
}, 'the question permission-request event of the bypass turn', 120000);
|
|
677
|
+
const card8 = await ui.waitUntil(async () => {
|
|
678
|
+
const cards = await ui.readQuestionCards(browser);
|
|
679
|
+
return cards.find(card => card.requestId === event8.requestId && card.fields.length > 0) || null;
|
|
680
|
+
}, 'the question card of the bypass turn', 60000);
|
|
681
|
+
const resolved8 = eventsOfType(session8.events, 'claude/permission-resolved').find(event => event.requestId === card8.requestId) || null;
|
|
682
|
+
const stillPending8 = resolved8 === null;
|
|
683
|
+
// Answer it through the UI so the bounded turn can finish.
|
|
684
|
+
const field8 = card8.fields[0];
|
|
685
|
+
const option8 = field8.options[0];
|
|
686
|
+
if (option8) await ui.selectRadioOption(browser, card8.requestId, 0, option8.label);
|
|
687
|
+
else await ui.typeCustomAnswer(browser, card8.requestId, 0, 'QA-R44-BYPASS-ANSWER');
|
|
688
|
+
await harness.wait(400);
|
|
689
|
+
await ui.clickSubmitAnswer(browser, card8.requestId);
|
|
690
|
+
await harness.waitUntil(() => eventsOfType(session8.events, 'claude/result').length > 0 || eventsOfType(session8.events, 'claude/error').length > 0, 'the end of the bypass turn', TURN_TIMEOUT_MS, 250);
|
|
691
|
+
bypassTurn = {
|
|
692
|
+
permissionMode: 'bypassPermissions (完全访问), selected through the product\'s own menu',
|
|
693
|
+
effectivePermissionModeInTheProduct: effectiveMode8,
|
|
694
|
+
questionCardRaised: true,
|
|
695
|
+
questionPausedWithoutAnAnswer: stillPending8,
|
|
696
|
+
resolvedBeforeAnySelection: resolved8 !== null,
|
|
697
|
+
requestId: card8.requestId,
|
|
698
|
+
eventTypes: [...new Set(session8.events.map(event => event.type))],
|
|
699
|
+
};
|
|
700
|
+
record('R44.j-wb007-the-bypass-mode-still-asks-the-question', stillPending8 && effectiveMode8 === 'bypassPermissions' ? 'PASS' : 'FAIL', bypassTurn);
|
|
701
|
+
await ui.selectPermissionMode(browser, '默认防护');
|
|
702
|
+
} catch (error) {
|
|
703
|
+
bypassTurn = { error: String(error && error.message ? error.message : error), note: 'the bounded bypass turn did not complete; R44.j is a required case, so the lane verdict is BLOCKED' };
|
|
704
|
+
record('R44.j-wb007-the-bypass-mode-still-asks-the-question', 'BLOCKED', bypassTurn);
|
|
705
|
+
}
|
|
706
|
+
} else {
|
|
707
|
+
bypassTurn = { skipped: true, note: 'the bounded bypass turn was disabled with QA_R44_BYPASS_TURN=0; R44.j is a required case, so the lane verdict is BLOCKED' };
|
|
708
|
+
record('R44.j-wb007-the-bypass-mode-still-asks-the-question', 'BLOCKED', bypassTurn);
|
|
709
|
+
}
|
|
710
|
+
} catch (error) {
|
|
711
|
+
record('R44.runtime', 'BLOCKED', { error: String(error && error.message ? error.message : error), stack: String(error && error.stack || '').split('\n').slice(0, 6) });
|
|
712
|
+
} finally {
|
|
713
|
+
// Every reader of this lane is stopped first, so no timer and no socket
|
|
714
|
+
// of this lane survives the run.
|
|
715
|
+
for (const reader of readers) { try { reader.stop(); } catch { /* the reader is disposable */ } }
|
|
716
|
+
if (browser) { try { await browser.close(); } catch { /* the browser is disposable */ } }
|
|
717
|
+
if (host.child) { await harness.stopHost(host); }
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
const digestComparison = envLib.compareDigests(realProfilesBefore, envLib.digestDirectory(envLib.realProfileDir()));
|
|
721
|
+
record('R44.k-real-profiles-byte-identical', digestComparison.byteIdentical ? 'PASS' : 'FAIL', {
|
|
722
|
+
profileDir: envLib.realProfileDir(),
|
|
723
|
+
comparison: 'byte-level SHA256, computed in memory; the digests are never printed, logged or persisted',
|
|
724
|
+
fileCount: digestComparison.count,
|
|
725
|
+
changedFiles: digestComparison.changedFiles,
|
|
726
|
+
addedFiles: digestComparison.addedFiles,
|
|
727
|
+
removedFiles: digestComparison.removedFiles,
|
|
728
|
+
});
|
|
729
|
+
|
|
730
|
+
const evaluation = evaluateCases(results);
|
|
731
|
+
const proven = evaluation.proven;
|
|
732
|
+
|
|
733
|
+
return {
|
|
734
|
+
lane: 'P-WB-PERMISSION-QUESTION-R44',
|
|
735
|
+
file: '_site/_test/qa-r44-permission-question.js',
|
|
736
|
+
startedAt,
|
|
737
|
+
finishedAt: new Date().toISOString(),
|
|
738
|
+
cases: ['WB-006', 'WB-007'],
|
|
739
|
+
module: identity,
|
|
740
|
+
pinned: pinnedIdentity(),
|
|
741
|
+
source: sourceIdentity(),
|
|
742
|
+
profile: { id: PROFILE_ID, source: 'the original subscription store, read-only; the credential stays in the host process memory', credentialWrittenAnywhere: false, quotaEndpointQueried: false },
|
|
743
|
+
isolation: envLib.isolationContract(layout, captured, hardened),
|
|
744
|
+
wb006,
|
|
745
|
+
wb007,
|
|
746
|
+
bypassTurn,
|
|
747
|
+
paidTurnsSent: paidTurns,
|
|
748
|
+
networkUsed: true,
|
|
749
|
+
cases_detail: results,
|
|
750
|
+
essentialCases: ESSENTIAL_CASES,
|
|
751
|
+
caseEvaluation: evaluation,
|
|
752
|
+
verdict: proven ? 'PASS' : 'BLOCKED',
|
|
753
|
+
};
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
module.exports = { main, results, ESSENTIAL_CASES, evaluateCases };
|