agent-device-lab 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +10 -0
- package/README.md +150 -2
- package/assets/dashboard/app.css +238 -0
- package/assets/dashboard/app.js +976 -0
- package/assets/dashboard/index.html +120 -0
- package/bin/agentlab.js +8 -0
- package/dist/cli/ci.js +443 -0
- package/dist/cli/flow.js +127 -0
- package/dist/cli/main.js +855 -0
- package/dist/cli/signals.js +38 -0
- package/dist/core/action-log.js +125 -0
- package/dist/core/auth.js +175 -0
- package/dist/core/bundle.js +432 -0
- package/dist/core/checks.js +137 -0
- package/dist/core/ci.js +389 -0
- package/dist/core/commands.js +366 -0
- package/dist/core/control.js +217 -0
- package/dist/core/detectors.js +392 -0
- package/dist/core/devices.js +83 -0
- package/dist/core/doctor.js +203 -0
- package/dist/core/explore-safety.js +110 -0
- package/dist/core/extract.js +1445 -0
- package/dist/core/feed.js +493 -0
- package/dist/core/findings.js +179 -0
- package/dist/core/format.js +311 -0
- package/dist/core/init.js +607 -0
- package/dist/core/lab.js +1979 -0
- package/dist/core/migrate.js +35 -0
- package/dist/core/observation.js +161 -0
- package/dist/core/ownership.js +142 -0
- package/dist/core/process-group.js +206 -0
- package/dist/core/process-identity.js +56 -0
- package/dist/core/profile.js +816 -0
- package/dist/core/project-runner.js +359 -0
- package/dist/core/replay.js +303 -0
- package/dist/core/reproduction.js +36 -0
- package/dist/core/scan-policy.js +156 -0
- package/dist/core/scan-report.js +145 -0
- package/dist/core/scan.js +455 -0
- package/dist/core/schema.js +39 -0
- package/dist/core/services.js +134 -0
- package/dist/core/steps.js +80 -0
- package/dist/core/sweep.js +138 -0
- package/dist/core/trace-sanitize.js +641 -0
- package/dist/core/url-redact.js +25 -0
- package/dist/core/versions.js +29 -0
- package/dist/core/zip.js +264 -0
- package/dist/daemon/main.js +160 -0
- package/dist/daemon/state.js +93 -0
- package/dist/dashboard/open.js +25 -0
- package/dist/dashboard/server.js +366 -0
- package/dist/mcp/server.js +89 -0
- package/package.json +63 -4
|
@@ -0,0 +1,641 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto';
|
|
2
|
+
import { readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { basename, dirname, join, resolve } from 'node:path';
|
|
4
|
+
import { LabError } from './schema.js';
|
|
5
|
+
import { readZip, writeZip, ZipError } from './zip.js';
|
|
6
|
+
/**
|
|
7
|
+
* Fail-closed sanitizer for Playwright trace zips, so a trace can leave the machine as a CI artifact or failure
|
|
8
|
+
* bundle. Product rule: it must never contain authentication state, request bodies, password values, sensitive
|
|
9
|
+
* headers or uploaded-file contents.
|
|
10
|
+
*
|
|
11
|
+
* What Playwright 1.63 records (verified by test/trace-sanitize.test.mjs, which is also the format-compatibility
|
|
12
|
+
* test: a Playwright upgrade that changes the format fails there, and unknown entries fail here):
|
|
13
|
+
* - `trace.trace` (JSON lines): `context-options` (extraHTTPHeaders, httpCredentials, storageState with cookie
|
|
14
|
+
* values and localStorage), `before`/`after` calls with `params` (fill `value`, type `text`, evaluate
|
|
15
|
+
* `expression`, addCookies `cookies`), `log` lines that repeat those values, `frame-snapshot` DOM arrays where
|
|
16
|
+
* every input carries `__playwright_value_` (including `<input type=password>`), and `screencast-frame`s.
|
|
17
|
+
* - `trace.network` (JSON lines): `resource-snapshot` HAR-like entries with request/response headers, cookies,
|
|
18
|
+
* `queryString`, `postData` and a `content._file` blob for every body.
|
|
19
|
+
* - `trace.stacks` (JSON): source locations. `resources/<sha1>.<ext>` blobs and `screencast/*.jpeg` frames.
|
|
20
|
+
*
|
|
21
|
+
* Structural removals come first; then every known secret (the caller's, plus values learned while sanitizing)
|
|
22
|
+
* is searched for in raw, JSON-escaped, URL-encoded, HTML-escaped and base64 forms across every entry. Text
|
|
23
|
+
* entries get the occurrences replaced, binary blobs holding one are dropped, and a residual is an error.
|
|
24
|
+
*
|
|
25
|
+
* Deliberate limits, all fail-closed except where noted: `evaluate*` expressions, arguments and results are
|
|
26
|
+
* masked wholesale (code cannot be sanitized structurally); an entry this module does not know is an error;
|
|
27
|
+
* screenshots are kept (they show what the page showed, and password fields render as dots); console text and
|
|
28
|
+
* page-owned DOM text are only covered by the known-secret pass; keys pressed one at a time are masked only
|
|
29
|
+
* when they are a single character.
|
|
30
|
+
*/
|
|
31
|
+
const REDACTED = '\u2039redacted\u203A';
|
|
32
|
+
const MIN_SECRET = 4;
|
|
33
|
+
/** Values learned from the trace itself are replaced everywhere only above this length, to avoid mangling text. */
|
|
34
|
+
const MIN_LEARNED = 6;
|
|
35
|
+
const UNSANITIZABLE = 'trace_unsanitizable';
|
|
36
|
+
function unsanitizable(message) {
|
|
37
|
+
throw new LabError(UNSANITIZABLE, message, { hint: 'The trace was not written. Do not attach the raw trace to an artifact.' });
|
|
38
|
+
}
|
|
39
|
+
const isRec = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
40
|
+
const SECRET_NAME = /token|secret|session|auth|key|cookie|signature|credential|csrf|xsrf|passw/i;
|
|
41
|
+
const SENSITIVE_HEADERS = new Set([
|
|
42
|
+
'authorization', 'proxy-authorization', 'cookie', 'set-cookie', 'x-api-key', 'x-auth-token', 'x-csrf-token', 'x-xsrf-token',
|
|
43
|
+
]);
|
|
44
|
+
const isSensitiveName = (name) => SENSITIVE_HEADERS.has(name.toLowerCase()) || SECRET_NAME.test(name);
|
|
45
|
+
/** Response bodies are kept only for these resource types; everything else (xhr, fetch, websocket, ...) is dropped. */
|
|
46
|
+
const KEEP_BODY_TYPES = new Set(['document', 'stylesheet', 'script', 'image', 'font', 'media']);
|
|
47
|
+
const CODE_METHOD = /evaluate|waitForFunction|addInitScript|addScriptTag|addStyleTag|exposeBinding|exposeFunction/i;
|
|
48
|
+
const KEY_DELETE = new Set(['storageState', 'httpCredentials', 'clientCertificates']);
|
|
49
|
+
const KEY_MASK = /^(password|passwd|pwd|secret|authorization|postData|jsonData|formData|multipartData|accessToken|refreshToken|idToken|apiKey)$/i;
|
|
50
|
+
const QUERY_RE = /([?&#;])([^=&#;\s"'\\<>]*(?:token|secret|session|auth|key|cookie|signature|credential|csrf|xsrf|password)[^=&#;\s"'\\<>]*)=([^&#;\s"'\\<>]+)/gi;
|
|
51
|
+
class State {
|
|
52
|
+
blobNames;
|
|
53
|
+
maskedValues = 0;
|
|
54
|
+
droppedBodies = 0;
|
|
55
|
+
strippedHeaders = 0;
|
|
56
|
+
/** Values learned while sanitizing (cookie values, header values, sensitive input values). */
|
|
57
|
+
learned = new Set();
|
|
58
|
+
/** Blob entry names whose reference was removed; deleted if nothing else refers to them. */
|
|
59
|
+
droppedBlobs = new Set();
|
|
60
|
+
/** Kept response bodies, re-checked against the full secret set before serializing. */
|
|
61
|
+
keptBodies = [];
|
|
62
|
+
contextOptions = 0;
|
|
63
|
+
constructor(blobNames) {
|
|
64
|
+
this.blobNames = blobNames;
|
|
65
|
+
}
|
|
66
|
+
note(value, min = MIN_LEARNED) {
|
|
67
|
+
if (typeof value === 'string' && value.length >= min)
|
|
68
|
+
this.learned.add(value);
|
|
69
|
+
}
|
|
70
|
+
noteHeader(value) {
|
|
71
|
+
if (typeof value !== 'string')
|
|
72
|
+
return;
|
|
73
|
+
this.note(value);
|
|
74
|
+
const scheme = /^(?:bearer|basic|token|digest)\s+(.+)$/i.exec(value);
|
|
75
|
+
if (scheme)
|
|
76
|
+
this.note(scheme[1]);
|
|
77
|
+
if (value.includes('='))
|
|
78
|
+
for (const part of value.split(';'))
|
|
79
|
+
this.note(part.slice(part.indexOf('=') + 1).trim());
|
|
80
|
+
}
|
|
81
|
+
/** Resolves a `_file` or `_sha1` reference to the zip entry name it points at. */
|
|
82
|
+
blobFor(ref) {
|
|
83
|
+
if (typeof ref !== 'string' || ref === '')
|
|
84
|
+
return undefined;
|
|
85
|
+
return this.blobNames.get(ref) ?? this.blobNames.get(`resources/${ref}`) ?? this.blobNames.get(basename(ref).split('.')[0] ?? '');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
function stripHeaders(list, st) {
|
|
89
|
+
if (Array.isArray(list)) {
|
|
90
|
+
return list.filter((h) => {
|
|
91
|
+
if (isRec(h) && typeof h.name === 'string' && isSensitiveName(h.name)) {
|
|
92
|
+
st.noteHeader(h.value);
|
|
93
|
+
st.strippedHeaders++;
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
return true;
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
if (isRec(list)) {
|
|
100
|
+
for (const k of Object.keys(list)) {
|
|
101
|
+
if (!isSensitiveName(k))
|
|
102
|
+
continue;
|
|
103
|
+
st.noteHeader(list[k]);
|
|
104
|
+
delete list[k];
|
|
105
|
+
st.strippedHeaders++;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return list;
|
|
109
|
+
}
|
|
110
|
+
function maskCookies(list, st) {
|
|
111
|
+
if (!Array.isArray(list))
|
|
112
|
+
return;
|
|
113
|
+
for (const c of list) {
|
|
114
|
+
if (isRec(c) && typeof c.value === 'string' && c.value !== REDACTED) {
|
|
115
|
+
st.note(c.value);
|
|
116
|
+
c.value = REDACTED;
|
|
117
|
+
st.maskedValues++;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
/** Generic structural pass over any parsed trace JSON: name/value pairs, headers, cookies and secret-named keys. */
|
|
122
|
+
function deep(v, st) {
|
|
123
|
+
if (Array.isArray(v)) {
|
|
124
|
+
for (const x of v)
|
|
125
|
+
deep(x, st);
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
if (!isRec(v))
|
|
129
|
+
return;
|
|
130
|
+
if (typeof v.name === 'string' && typeof v.value === 'string' && v.value !== REDACTED && SECRET_NAME.test(v.name)) {
|
|
131
|
+
st.note(v.value);
|
|
132
|
+
v.value = REDACTED;
|
|
133
|
+
st.maskedValues++;
|
|
134
|
+
}
|
|
135
|
+
for (const k of Object.keys(v)) {
|
|
136
|
+
const val = v[k];
|
|
137
|
+
if (KEY_DELETE.has(k)) {
|
|
138
|
+
delete v[k];
|
|
139
|
+
st.maskedValues++;
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
if (KEY_MASK.test(k) && val !== undefined && val !== null && val !== '' && val !== REDACTED) {
|
|
143
|
+
st.note(val);
|
|
144
|
+
v[k] = REDACTED;
|
|
145
|
+
st.maskedValues++;
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
if (k === 'headers' || k === 'extraHTTPHeaders')
|
|
149
|
+
v[k] = stripHeaders(val, st);
|
|
150
|
+
else if (k === 'cookies')
|
|
151
|
+
maskCookies(val, st);
|
|
152
|
+
deep(v[k], st);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function collectSecrets(v, st) {
|
|
156
|
+
if (Array.isArray(v))
|
|
157
|
+
return void v.forEach((x) => collectSecrets(x, st));
|
|
158
|
+
if (!isRec(v))
|
|
159
|
+
return;
|
|
160
|
+
for (const [k, val] of Object.entries(v)) {
|
|
161
|
+
if (typeof val === 'string' && /^(value|password|token)$/i.test(k))
|
|
162
|
+
st.note(val, MIN_SECRET);
|
|
163
|
+
else
|
|
164
|
+
collectSecrets(val, st);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
function sanitizeContextOptions(ev, st) {
|
|
168
|
+
st.contextOptions++;
|
|
169
|
+
const opts = ev.options;
|
|
170
|
+
if (!isRec(opts))
|
|
171
|
+
return;
|
|
172
|
+
for (const key of KEY_DELETE) {
|
|
173
|
+
if (key in opts) {
|
|
174
|
+
collectSecrets(opts[key], st);
|
|
175
|
+
delete opts[key];
|
|
176
|
+
st.maskedValues++;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
if (isRec(opts.proxy)) {
|
|
180
|
+
st.note(opts.proxy.password, MIN_SECRET);
|
|
181
|
+
delete opts.proxy.username;
|
|
182
|
+
delete opts.proxy.password;
|
|
183
|
+
}
|
|
184
|
+
if ('extraHTTPHeaders' in opts)
|
|
185
|
+
opts.extraHTTPHeaders = stripHeaders(opts.extraHTTPHeaders, st);
|
|
186
|
+
}
|
|
187
|
+
function sanitizeBefore(ev, st, callValues, codeCalls) {
|
|
188
|
+
const params = ev.params;
|
|
189
|
+
if (!isRec(params))
|
|
190
|
+
return;
|
|
191
|
+
const method = typeof ev.method === 'string' ? ev.method : '';
|
|
192
|
+
const values = [];
|
|
193
|
+
for (const k of ['value', 'text']) {
|
|
194
|
+
const v = params[k];
|
|
195
|
+
if (typeof v === 'string' && v !== '' && v !== REDACTED) {
|
|
196
|
+
values.push(v);
|
|
197
|
+
params[k] = REDACTED;
|
|
198
|
+
st.maskedValues++;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
if (typeof params.key === 'string' && [...params.key].length === 1) {
|
|
202
|
+
values.push(params.key);
|
|
203
|
+
params.key = REDACTED;
|
|
204
|
+
st.maskedValues++;
|
|
205
|
+
}
|
|
206
|
+
if (CODE_METHOD.test(method)) {
|
|
207
|
+
if (typeof ev.callId === 'string')
|
|
208
|
+
codeCalls.add(ev.callId);
|
|
209
|
+
for (const k of ['expression', 'arg', 'source', 'script', 'content']) {
|
|
210
|
+
if (params[k] !== undefined && params[k] !== REDACTED) {
|
|
211
|
+
params[k] = REDACTED;
|
|
212
|
+
st.maskedValues++;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
if (typeof params.expression === 'string' && /value/i.test(params.expression)) {
|
|
217
|
+
for (const k of ['expectedText', 'expectedValue']) {
|
|
218
|
+
if (params[k] !== undefined && params[k] !== REDACTED) {
|
|
219
|
+
params[k] = REDACTED;
|
|
220
|
+
st.maskedValues++;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if (Array.isArray(params.payloads)) {
|
|
225
|
+
for (const p of params.payloads) {
|
|
226
|
+
if (isRec(p) && p.buffer !== undefined && p.buffer !== '<Buffer>') {
|
|
227
|
+
p.buffer = '<Buffer>';
|
|
228
|
+
st.maskedValues++;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
if (values.length && typeof ev.callId === 'string')
|
|
233
|
+
callValues.set(ev.callId, values);
|
|
234
|
+
}
|
|
235
|
+
function isSensitiveField(attrs) {
|
|
236
|
+
const type = typeof attrs.type === 'string' ? attrs.type.toLowerCase() : '';
|
|
237
|
+
if (type === 'password' || type === 'hidden')
|
|
238
|
+
return true;
|
|
239
|
+
if (typeof attrs.autocomplete === 'string' && /password|one-time-code|cc-number|cc-csc|cc-exp/i.test(attrs.autocomplete))
|
|
240
|
+
return true;
|
|
241
|
+
for (const k of ['name', 'id']) {
|
|
242
|
+
const v = attrs[k];
|
|
243
|
+
if (typeof v === 'string' && /pass(word|wd)?|pwd|otp|cvv|cvc|secret|token|api[-_]?key|csrf|xsrf|credential/i.test(v))
|
|
244
|
+
return true;
|
|
245
|
+
}
|
|
246
|
+
return false;
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* A snapshot node is a string (text), `[tag, attrs?, ...children]`, or `[[snapshotsAgo, index]]` (a reference to
|
|
250
|
+
* a node of an earlier snapshot, which was sanitized when that snapshot was processed).
|
|
251
|
+
*/
|
|
252
|
+
function scrubSnapshot(node, st, onSensitiveTarget) {
|
|
253
|
+
if (!Array.isArray(node) || typeof node[0] !== 'string')
|
|
254
|
+
return;
|
|
255
|
+
const tag = node[0].toLowerCase();
|
|
256
|
+
const attrs = isRec(node[1]) ? node[1] : undefined;
|
|
257
|
+
if (attrs && (tag === 'input' || tag === 'textarea') && isSensitiveField(attrs)) {
|
|
258
|
+
for (const k of ['__playwright_value_', 'value']) {
|
|
259
|
+
const v = attrs[k];
|
|
260
|
+
if (typeof v === 'string' && v !== '' && v !== REDACTED) {
|
|
261
|
+
st.note(v, MIN_SECRET);
|
|
262
|
+
attrs[k] = REDACTED;
|
|
263
|
+
st.maskedValues++;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if ('__playwright_target__' in attrs)
|
|
267
|
+
onSensitiveTarget();
|
|
268
|
+
}
|
|
269
|
+
for (let i = attrs ? 2 : 1; i < node.length; i++)
|
|
270
|
+
scrubSnapshot(node[i], st, onSensitiveTarget);
|
|
271
|
+
}
|
|
272
|
+
function dropContent(content, ref, st) {
|
|
273
|
+
delete content._file;
|
|
274
|
+
delete content._sha1;
|
|
275
|
+
delete content.text;
|
|
276
|
+
const blob = st.blobFor(ref);
|
|
277
|
+
if (blob)
|
|
278
|
+
st.droppedBlobs.add(blob);
|
|
279
|
+
st.droppedBodies++;
|
|
280
|
+
}
|
|
281
|
+
function sanitizeResource(snap, st) {
|
|
282
|
+
const req = snap.request;
|
|
283
|
+
if (isRec(req)) {
|
|
284
|
+
req.headers = stripHeaders(req.headers, st);
|
|
285
|
+
if (Array.isArray(req.cookies)) {
|
|
286
|
+
maskCookies(req.cookies, st);
|
|
287
|
+
req.cookies = [];
|
|
288
|
+
}
|
|
289
|
+
if (isRec(req.postData)) {
|
|
290
|
+
const ref = req.postData._file ?? req.postData._sha1;
|
|
291
|
+
const blob = st.blobFor(ref);
|
|
292
|
+
if (blob)
|
|
293
|
+
st.droppedBlobs.add(blob);
|
|
294
|
+
delete req.postData;
|
|
295
|
+
st.droppedBodies++;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
const res = snap.response;
|
|
299
|
+
if (isRec(res)) {
|
|
300
|
+
res.headers = stripHeaders(res.headers, st);
|
|
301
|
+
if (Array.isArray(res.cookies)) {
|
|
302
|
+
maskCookies(res.cookies, st);
|
|
303
|
+
res.cookies = [];
|
|
304
|
+
}
|
|
305
|
+
const content = res.content;
|
|
306
|
+
if (isRec(content)) {
|
|
307
|
+
const ref = (content._file ?? content._sha1);
|
|
308
|
+
const type = typeof snap._resourceType === 'string' ? snap._resourceType : 'other';
|
|
309
|
+
if (!KEEP_BODY_TYPES.has(type)) {
|
|
310
|
+
if (ref !== undefined || content.text !== undefined)
|
|
311
|
+
dropContent(content, ref, st);
|
|
312
|
+
}
|
|
313
|
+
else if (ref !== undefined || content.text !== undefined)
|
|
314
|
+
st.keptBodies.push({ content, ref });
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
/** Replaces `value` in every string of an event (used for the log lines and titles that repeat a typed value). */
|
|
319
|
+
function replaceInStrings(v, values) {
|
|
320
|
+
if (typeof v === 'string') {
|
|
321
|
+
let s = v;
|
|
322
|
+
for (const value of values) {
|
|
323
|
+
s = s.split(JSON.stringify(value)).join(JSON.stringify(REDACTED));
|
|
324
|
+
if (value.length >= 3)
|
|
325
|
+
s = s.split(value).join(REDACTED);
|
|
326
|
+
}
|
|
327
|
+
return s;
|
|
328
|
+
}
|
|
329
|
+
if (Array.isArray(v))
|
|
330
|
+
return v.map((x) => replaceInStrings(x, values));
|
|
331
|
+
if (isRec(v)) {
|
|
332
|
+
for (const k of Object.keys(v))
|
|
333
|
+
v[k] = replaceInStrings(v[k], values);
|
|
334
|
+
}
|
|
335
|
+
return v;
|
|
336
|
+
}
|
|
337
|
+
function parseLines(name, data) {
|
|
338
|
+
const events = [];
|
|
339
|
+
for (const line of data.toString('utf8').split('\n')) {
|
|
340
|
+
if (line.trim() === '')
|
|
341
|
+
continue;
|
|
342
|
+
let parsed;
|
|
343
|
+
try {
|
|
344
|
+
parsed = JSON.parse(line);
|
|
345
|
+
}
|
|
346
|
+
catch {
|
|
347
|
+
return unsanitizable(`${name} has a line that is not valid JSON (unexpected trace format)`);
|
|
348
|
+
}
|
|
349
|
+
if (!isRec(parsed) || typeof parsed.type !== 'string')
|
|
350
|
+
return unsanitizable(`${name} has an unexpected event shape`);
|
|
351
|
+
events.push(parsed);
|
|
352
|
+
}
|
|
353
|
+
return events;
|
|
354
|
+
}
|
|
355
|
+
function processEvents(events, st) {
|
|
356
|
+
const callValues = new Map();
|
|
357
|
+
const codeCalls = new Set();
|
|
358
|
+
const sensitiveCalls = new Set();
|
|
359
|
+
for (const ev of events) {
|
|
360
|
+
switch (ev.type) {
|
|
361
|
+
case 'context-options':
|
|
362
|
+
sanitizeContextOptions(ev, st);
|
|
363
|
+
break;
|
|
364
|
+
case 'before':
|
|
365
|
+
sanitizeBefore(ev, st, callValues, codeCalls);
|
|
366
|
+
break;
|
|
367
|
+
case 'after':
|
|
368
|
+
if (typeof ev.callId === 'string' && codeCalls.has(ev.callId) && ev.result !== undefined && ev.result !== REDACTED) {
|
|
369
|
+
ev.result = REDACTED;
|
|
370
|
+
st.maskedValues++;
|
|
371
|
+
}
|
|
372
|
+
break;
|
|
373
|
+
case 'frame-snapshot': {
|
|
374
|
+
const snap = ev.snapshot;
|
|
375
|
+
if (isRec(snap)) {
|
|
376
|
+
const callId = typeof snap.callId === 'string' ? snap.callId : undefined;
|
|
377
|
+
scrubSnapshot(snap.html, st, () => callId && sensitiveCalls.add(callId));
|
|
378
|
+
}
|
|
379
|
+
break;
|
|
380
|
+
}
|
|
381
|
+
case 'resource-snapshot':
|
|
382
|
+
if (isRec(ev.snapshot))
|
|
383
|
+
sanitizeResource(ev.snapshot, st);
|
|
384
|
+
break;
|
|
385
|
+
default:
|
|
386
|
+
break;
|
|
387
|
+
}
|
|
388
|
+
deep(ev, st);
|
|
389
|
+
}
|
|
390
|
+
// A value typed into a sensitive field is learned so it is replaced everywhere, not just in its own call.
|
|
391
|
+
for (const callId of sensitiveCalls)
|
|
392
|
+
for (const v of callValues.get(callId) ?? [])
|
|
393
|
+
st.note(v, MIN_SECRET);
|
|
394
|
+
for (let i = 0; i < events.length; i++) {
|
|
395
|
+
const ev = events[i];
|
|
396
|
+
const callId = typeof ev.callId === 'string' ? ev.callId : undefined;
|
|
397
|
+
const values = callId ? callValues.get(callId) : undefined;
|
|
398
|
+
if (values && ev.type !== 'frame-snapshot')
|
|
399
|
+
events[i] = replaceInStrings(ev, values);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
function base64Variants(s) {
|
|
403
|
+
const bytes = Buffer.from(s, 'utf8');
|
|
404
|
+
const out = [];
|
|
405
|
+
for (let k = 0; k < 3; k++) {
|
|
406
|
+
const padded = Buffer.concat([Buffer.alloc(k), bytes]);
|
|
407
|
+
let e = padded.toString('base64').replace(/=+$/, '');
|
|
408
|
+
if (padded.length % 3 !== 0)
|
|
409
|
+
e = e.slice(0, -1); // the last character also depends on the following byte
|
|
410
|
+
e = e.slice([0, 2, 3][k]); // leading characters that also depend on the preceding bytes
|
|
411
|
+
if (e.length >= 4)
|
|
412
|
+
out.push(e, e.replace(/\+/g, '-').replace(/\//g, '_'));
|
|
413
|
+
}
|
|
414
|
+
return out;
|
|
415
|
+
}
|
|
416
|
+
function variantsOf(secret) {
|
|
417
|
+
const forms = new Set([
|
|
418
|
+
secret,
|
|
419
|
+
JSON.stringify(secret).slice(1, -1),
|
|
420
|
+
encodeURIComponent(secret),
|
|
421
|
+
encodeURIComponent(secret).replace(/%20/g, '+'),
|
|
422
|
+
secret.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, '''),
|
|
423
|
+
...base64Variants(secret),
|
|
424
|
+
]);
|
|
425
|
+
return [...forms].filter((f) => f.length > 0);
|
|
426
|
+
}
|
|
427
|
+
function buildVariants(secrets) {
|
|
428
|
+
const all = new Set();
|
|
429
|
+
for (const s of secrets)
|
|
430
|
+
for (const v of variantsOf(s))
|
|
431
|
+
all.add(v);
|
|
432
|
+
return { strings: [...all].sort((a, b) => b.length - a.length) };
|
|
433
|
+
}
|
|
434
|
+
function replaceVariants(text, variants) {
|
|
435
|
+
let hits = 0;
|
|
436
|
+
let out = text;
|
|
437
|
+
for (const v of variants.strings) {
|
|
438
|
+
if (!out.includes(v))
|
|
439
|
+
continue;
|
|
440
|
+
const parts = out.split(v);
|
|
441
|
+
hits += parts.length - 1;
|
|
442
|
+
out = parts.join(REDACTED);
|
|
443
|
+
}
|
|
444
|
+
return { text: out, hits };
|
|
445
|
+
}
|
|
446
|
+
function bytesContain(data, variants) {
|
|
447
|
+
return variants.strings.some((v) => data.includes(Buffer.from(v, 'utf8')));
|
|
448
|
+
}
|
|
449
|
+
function asText(data) {
|
|
450
|
+
if (data.includes(0))
|
|
451
|
+
return undefined;
|
|
452
|
+
try {
|
|
453
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(data);
|
|
454
|
+
}
|
|
455
|
+
catch {
|
|
456
|
+
return undefined;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
function assertNoSecrets(entries, variants) {
|
|
460
|
+
for (const e of entries) {
|
|
461
|
+
if (bytesContain(e.data, variants))
|
|
462
|
+
unsanitizable(`a known secret is still present in ${e.name}`);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
function classify(name) {
|
|
466
|
+
if (/^resources\/src@/.test(name))
|
|
467
|
+
return 'source';
|
|
468
|
+
if (name.endsWith('.trace') || name.endsWith('.network'))
|
|
469
|
+
return 'lines';
|
|
470
|
+
if (name.endsWith('.stacks'))
|
|
471
|
+
return 'stacks';
|
|
472
|
+
if (name.startsWith('resources/') || name.startsWith('screencast/'))
|
|
473
|
+
return 'blob';
|
|
474
|
+
return unsanitizable(`unexpected entry "${name}" in the trace (unknown trace format)`);
|
|
475
|
+
}
|
|
476
|
+
function sanitizeEntries(entries, secrets) {
|
|
477
|
+
const kinds = new Map(entries.map((e) => [e.name, classify(e.name)]));
|
|
478
|
+
const blobNames = new Map();
|
|
479
|
+
for (const e of entries) {
|
|
480
|
+
if (kinds.get(e.name) !== 'blob')
|
|
481
|
+
continue;
|
|
482
|
+
blobNames.set(e.name, e.name);
|
|
483
|
+
const base = basename(e.name);
|
|
484
|
+
blobNames.set(base, e.name);
|
|
485
|
+
blobNames.set(base.split('.')[0] ?? base, e.name);
|
|
486
|
+
}
|
|
487
|
+
const st = new State(blobNames);
|
|
488
|
+
const known = new Set(secrets.filter((s) => s.length >= MIN_SECRET));
|
|
489
|
+
const lineFiles = new Map();
|
|
490
|
+
const stacks = new Map();
|
|
491
|
+
for (const e of entries) {
|
|
492
|
+
const kind = kinds.get(e.name);
|
|
493
|
+
if (kind === 'lines') {
|
|
494
|
+
const events = parseLines(e.name, e.data);
|
|
495
|
+
processEvents(events, st);
|
|
496
|
+
lineFiles.set(e.name, events);
|
|
497
|
+
}
|
|
498
|
+
else if (kind === 'stacks') {
|
|
499
|
+
let parsed;
|
|
500
|
+
try {
|
|
501
|
+
parsed = JSON.parse(e.data.toString('utf8'));
|
|
502
|
+
}
|
|
503
|
+
catch {
|
|
504
|
+
return unsanitizable(`${e.name} is not valid JSON (unexpected trace format)`);
|
|
505
|
+
}
|
|
506
|
+
deep(parsed, st);
|
|
507
|
+
stacks.set(e.name, parsed);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
if (st.contextOptions === 0)
|
|
511
|
+
unsanitizable('the archive has no Playwright trace context (unexpected trace format)');
|
|
512
|
+
const removed = new Set();
|
|
513
|
+
for (const e of entries)
|
|
514
|
+
if (kinds.get(e.name) === 'source')
|
|
515
|
+
removed.add(e.name);
|
|
516
|
+
// Documents and other kept bodies that hold a secret are dropped rather than edited.
|
|
517
|
+
const bodyVariants = buildVariants([...known, ...st.learned]);
|
|
518
|
+
const byName = new Map(entries.map((e) => [e.name, e]));
|
|
519
|
+
for (const body of st.keptBodies) {
|
|
520
|
+
const blob = st.blobFor(body.ref);
|
|
521
|
+
const data = blob ? byName.get(blob)?.data : undefined;
|
|
522
|
+
if (data && bytesContain(data, bodyVariants))
|
|
523
|
+
dropContent(body.content, body.ref, st);
|
|
524
|
+
}
|
|
525
|
+
// Serialize the structured entries, then redact query parameters and known secrets in the text.
|
|
526
|
+
const variants = buildVariants([...known, ...st.learned]);
|
|
527
|
+
const texts = new Map();
|
|
528
|
+
for (const [name, events] of lineFiles)
|
|
529
|
+
texts.set(name, events.map((ev) => JSON.stringify(ev)).join('\n') + '\n');
|
|
530
|
+
for (const [name, value] of stacks)
|
|
531
|
+
texts.set(name, JSON.stringify(value));
|
|
532
|
+
for (const [name, text] of texts) {
|
|
533
|
+
let next = text.replace(QUERY_RE, (m, sep, param, value) => {
|
|
534
|
+
if (value === 'redacted')
|
|
535
|
+
return m;
|
|
536
|
+
st.maskedValues++;
|
|
537
|
+
return `${sep}${param}=redacted`;
|
|
538
|
+
});
|
|
539
|
+
next = replaceVariants(next, variants).text;
|
|
540
|
+
for (const line of next.split('\n')) {
|
|
541
|
+
if (line === '')
|
|
542
|
+
continue;
|
|
543
|
+
try {
|
|
544
|
+
JSON.parse(line);
|
|
545
|
+
}
|
|
546
|
+
catch {
|
|
547
|
+
unsanitizable(`redaction produced invalid JSON in ${name}`);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
texts.set(name, next);
|
|
551
|
+
}
|
|
552
|
+
// Blobs: drop the ones whose reference was removed (unless something else still refers to them), then apply the
|
|
553
|
+
// known-secret pass, editing text blobs and dropping binary ones.
|
|
554
|
+
const referenced = (blob) => {
|
|
555
|
+
const base = basename(blob);
|
|
556
|
+
for (const t of texts.values())
|
|
557
|
+
if (t.includes(base))
|
|
558
|
+
return true;
|
|
559
|
+
return false;
|
|
560
|
+
};
|
|
561
|
+
for (const blob of st.droppedBlobs)
|
|
562
|
+
if (!referenced(blob))
|
|
563
|
+
removed.add(blob);
|
|
564
|
+
const out = [];
|
|
565
|
+
for (const e of entries) {
|
|
566
|
+
if (removed.has(e.name))
|
|
567
|
+
continue;
|
|
568
|
+
const text = texts.get(e.name);
|
|
569
|
+
if (text !== undefined) {
|
|
570
|
+
out.push({ name: e.name, data: Buffer.from(text, 'utf8') });
|
|
571
|
+
continue;
|
|
572
|
+
}
|
|
573
|
+
if (kinds.get(e.name) === 'blob' && bytesContain(e.data, variants)) {
|
|
574
|
+
const asStr = asText(e.data);
|
|
575
|
+
if (asStr === undefined) {
|
|
576
|
+
removed.add(e.name);
|
|
577
|
+
continue;
|
|
578
|
+
}
|
|
579
|
+
out.push({ name: e.name, data: Buffer.from(replaceVariants(asStr, variants).text, 'utf8') });
|
|
580
|
+
continue;
|
|
581
|
+
}
|
|
582
|
+
out.push(e);
|
|
583
|
+
}
|
|
584
|
+
assertNoSecrets(out, variants);
|
|
585
|
+
return {
|
|
586
|
+
out,
|
|
587
|
+
report: {
|
|
588
|
+
entries: entries.length,
|
|
589
|
+
removedEntries: removed.size,
|
|
590
|
+
maskedValues: st.maskedValues,
|
|
591
|
+
droppedBodies: st.droppedBodies,
|
|
592
|
+
strippedHeaders: st.strippedHeaders,
|
|
593
|
+
},
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
/**
|
|
597
|
+
* Reads the Playwright trace at `input` and writes a sanitized copy to `output`. Any failure (unreadable zip,
|
|
598
|
+
* unexpected format, a secret that could not be removed) throws `trace_unsanitizable` and leaves nothing at
|
|
599
|
+
* `output`. The report has counts only; no message or report ever contains a secret value.
|
|
600
|
+
*/
|
|
601
|
+
export async function sanitizeTrace(input, output, opts = {}) {
|
|
602
|
+
const inPath = resolve(input);
|
|
603
|
+
const outPath = resolve(output);
|
|
604
|
+
if (inPath === outPath)
|
|
605
|
+
throw new LabError('invalid_request', 'output must differ from the input trace', { hint: 'Write the sanitized trace to a new path.' });
|
|
606
|
+
const tmp = join(dirname(outPath), `.${basename(outPath)}.${randomBytes(6).toString('hex')}.tmp`);
|
|
607
|
+
try {
|
|
608
|
+
let raw;
|
|
609
|
+
try {
|
|
610
|
+
raw = await readFile(inPath);
|
|
611
|
+
}
|
|
612
|
+
catch {
|
|
613
|
+
return unsanitizable('the trace file cannot be read');
|
|
614
|
+
}
|
|
615
|
+
let entries;
|
|
616
|
+
try {
|
|
617
|
+
entries = readZip(raw);
|
|
618
|
+
}
|
|
619
|
+
catch (err) {
|
|
620
|
+
return unsanitizable(err instanceof ZipError ? `the trace is not a usable zip: ${err.message}` : 'the trace is not a usable zip');
|
|
621
|
+
}
|
|
622
|
+
const secrets = (opts.secrets ?? []).filter((s) => typeof s === 'string');
|
|
623
|
+
const { out, report } = sanitizeEntries(entries, secrets);
|
|
624
|
+
const zip = writeZip(out);
|
|
625
|
+
await writeFile(tmp, zip, { mode: 0o600 });
|
|
626
|
+
await rename(tmp, outPath);
|
|
627
|
+
// Verify what is on disk, not what was intended.
|
|
628
|
+
const back = readZip(await readFile(outPath));
|
|
629
|
+
assertNoSecrets(back, buildVariants(secrets.filter((s) => s.length >= MIN_SECRET)));
|
|
630
|
+
return report;
|
|
631
|
+
}
|
|
632
|
+
catch (err) {
|
|
633
|
+
await rm(tmp, { force: true }).catch(() => undefined);
|
|
634
|
+
await rm(outPath, { force: true }).catch(() => undefined);
|
|
635
|
+
if (err instanceof LabError)
|
|
636
|
+
throw err;
|
|
637
|
+
// Never forward the original message: parser errors can quote the content that failed to parse.
|
|
638
|
+
return unsanitizable(`the trace could not be sanitized (${err instanceof Error ? err.name : 'unknown error'})`);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
//# sourceMappingURL=trace-sanitize.js.map
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Secrets carried in a URL (a sign-in token in the fragment, an API key in the query, a password in the
|
|
2
|
+
// userinfo) must not reach results, events, logs or reports. Pure: the value of a secret-looking
|
|
3
|
+
// parameter is replaced, the rest of the URL is kept so a route still reads as a route.
|
|
4
|
+
// A parameter name is secret-looking when it contains one of these words anywhere (accessToken,
|
|
5
|
+
// client_secret, X-Amz-Signature, refresh_token[0]) ...
|
|
6
|
+
const ANYWHERE = 'token|secret|passw(?:or)?d|pwd|api[_.-]?key|signature|jwt|credential|bearer|session';
|
|
7
|
+
// ... or is one of these short words on its own or as a delimited part (sig, auth, otp, key, code would
|
|
8
|
+
// otherwise match "design", "author", "hotpink"). A camelCase tail (userAuth, idSig) counts too.
|
|
9
|
+
const WHOLE = 'sig|auth|otp|pin|pass|sid';
|
|
10
|
+
const PART = '[\\w.\\-\\[\\]%]*';
|
|
11
|
+
const NAME_I = `(?:${PART}(?:${ANYWHERE})${PART}|(?:${PART}[_.\\-\\]])?(?:${WHOLE})(?:[_.\\-\\[]${PART})?)`;
|
|
12
|
+
const NAME_CAMEL = `${PART}[a-z0-9](?:Sig|Auth|Otp|Pin|Pass)`;
|
|
13
|
+
const param = (name, flags) => new RegExp(`([?&#/;!]|%3[fF]|%2[36fF])(${name})(=|%3[dD])((?:(?!%2[36])[^&#;\\s"'<>])*)`, flags);
|
|
14
|
+
// A URL nested in a parameter arrives percent-encoded (%3F ? · %26 & · %23 # · %2F / · %3D =).
|
|
15
|
+
const SECRET_PARAM_I = param(NAME_I, 'gi');
|
|
16
|
+
// The camelCase tail is matched with its capitals, so "compass" and "spin" stay as they are.
|
|
17
|
+
const SECRET_PARAM = param(NAME_CAMEL, 'g');
|
|
18
|
+
const USERINFO = /(\b[a-z][a-z0-9+.-]*:\/\/)[^/?#\s@]*@/gi;
|
|
19
|
+
export const REDACTED_PARAM = '‹redacted›';
|
|
20
|
+
/** A URL, route or message with the values of secret-looking parameters and any URL userinfo replaced. */
|
|
21
|
+
export function redactUrlSecrets(text) {
|
|
22
|
+
const swap = (_m, sep, key, eq) => `${sep}${key}${eq}${REDACTED_PARAM}`;
|
|
23
|
+
return text.replace(USERINFO, `$1${REDACTED_PARAM}@`).replace(SECRET_PARAM_I, swap).replace(SECRET_PARAM, swap);
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=url-redact.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { SCHEMA_VERSION } from './schema.js';
|
|
3
|
+
/**
|
|
4
|
+
* Every versioned contract agentlab exposes, in one place. A number changes only when its shape changes
|
|
5
|
+
* incompatibly; `agentlab version` prints them so a project or an agent can check compatibility.
|
|
6
|
+
*/
|
|
7
|
+
export const CONTRACT_VERSIONS = {
|
|
8
|
+
/** The `schemaVersion` written into agentlab.json by `init` and `migrate`. */
|
|
9
|
+
profile: 2,
|
|
10
|
+
/** The `schemaVersion`s of agentlab.json this version of agentlab reads. */
|
|
11
|
+
profileSupported: [1, 2],
|
|
12
|
+
/** The `schemaVersion` on CLI and MCP JSON results (`SCHEMA_VERSION` in schema.ts). */
|
|
13
|
+
results: SCHEMA_VERSION,
|
|
14
|
+
/** MCP tool names and their input schemas. */
|
|
15
|
+
mcpTools: 1,
|
|
16
|
+
/** Scan and CI report JSON. */
|
|
17
|
+
report: 1,
|
|
18
|
+
/** The failure bundle. */
|
|
19
|
+
bundle: 1,
|
|
20
|
+
/** Dashboard feed messages. */
|
|
21
|
+
events: 1,
|
|
22
|
+
};
|
|
23
|
+
let cached;
|
|
24
|
+
/** The package.json version of this installation. Read once. */
|
|
25
|
+
export function productVersion() {
|
|
26
|
+
cached ??= JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')).version;
|
|
27
|
+
return cached;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=versions.js.map
|