@broberg/lens-daemon 0.1.15 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/auth/index.ts +28 -13
- package/src/browser.ts +33 -12
- package/src/extension-flow.ts +70 -9
- package/src/flow-store.ts +15 -1
- package/src/flow.ts +157 -8
- package/src/mcp.ts +1 -0
- package/src/service.ts +18 -4
- package/src/storage.ts +3 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@broberg/lens-daemon",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.20",
|
|
4
4
|
"description": "The Lens module of the cardmem daemon (flows, critic, coverage, capture sessions, the /lens/* routes and the cardmem-lens MCP) as a package. The daemon mounts createLensRouter(deps) + createLensMcpRouter(deps). Bun runtime.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/src/auth/index.ts
CHANGED
|
@@ -196,6 +196,26 @@ async function applyMintEndpoint(
|
|
|
196
196
|
cfg: MintEndpointAuth,
|
|
197
197
|
env: AuthEnv,
|
|
198
198
|
): Promise<void> {
|
|
199
|
+
const json = (await fetchMintJson(cfg, env, 'mintEndpoint auth')) as StorageStateShape;
|
|
200
|
+
if (!json.cookies?.length && !json.origins?.length) {
|
|
201
|
+
throw new Error(`mintEndpoint auth: ${cfg.url} returned a storageState with no cookies/origins`);
|
|
202
|
+
}
|
|
203
|
+
await applyStorageStateJson(context, json);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** The narrow mint call shared by auth.mintEndpoint and F009.129's
|
|
207
|
+
* extension.storage_local_mint: resolve the secret (secretPath wins over
|
|
208
|
+
* secretEnv), POST/GET with `Authorization: Bearer`, require 2xx + JSON. Error
|
|
209
|
+
* messages name the url and status only — never the secret, never the body. */
|
|
210
|
+
export interface MintCall {
|
|
211
|
+
url: string;
|
|
212
|
+
method?: 'POST' | 'GET';
|
|
213
|
+
secretEnv?: string;
|
|
214
|
+
secretPath?: string;
|
|
215
|
+
body?: unknown;
|
|
216
|
+
headers?: Record<string, string>;
|
|
217
|
+
}
|
|
218
|
+
export async function fetchMintJson(cfg: MintCall, env: AuthEnv, label: string): Promise<unknown> {
|
|
199
219
|
// Resolve the narrow mint secret — secretPath (gitignored file, no restart)
|
|
200
220
|
// wins over secretEnv. The secret is NOT in the manifest/call, only its source.
|
|
201
221
|
let secret: string | undefined;
|
|
@@ -204,18 +224,18 @@ async function applyMintEndpoint(
|
|
|
204
224
|
// some other repo's secret, which would then be sent to the caller's url.
|
|
205
225
|
// Callers resolve against their project root first (resolveAuthPaths).
|
|
206
226
|
if (!isAbsolute(cfg.secretPath)) {
|
|
207
|
-
throw new Error(
|
|
227
|
+
throw new Error(`${label}: secretPath must be absolute by the time it is read (got "${cfg.secretPath}") — register the project or pass an absolute path`);
|
|
208
228
|
}
|
|
209
229
|
try {
|
|
210
230
|
secret = readFileSync(cfg.secretPath, 'utf-8').trim();
|
|
211
231
|
} catch (err) {
|
|
212
|
-
throw new Error(
|
|
232
|
+
throw new Error(`${label}: could not read secretPath ${cfg.secretPath}: ${err instanceof Error ? err.message : err}`);
|
|
213
233
|
}
|
|
214
234
|
} else if (cfg.secretEnv) {
|
|
215
235
|
secret = env[cfg.secretEnv];
|
|
216
|
-
if (!secret) throw new Error(
|
|
236
|
+
if (!secret) throw new Error(`${label}: env ${cfg.secretEnv} is not set`);
|
|
217
237
|
} else {
|
|
218
|
-
throw new Error(
|
|
238
|
+
throw new Error(`${label}: set secretPath or secretEnv`);
|
|
219
239
|
}
|
|
220
240
|
|
|
221
241
|
// Optional JSON body (POST only) — lets the endpoint mint a session for a
|
|
@@ -235,21 +255,16 @@ async function applyMintEndpoint(
|
|
|
235
255
|
...(sendBody ? { body: JSON.stringify(cfg.body) } : {}),
|
|
236
256
|
});
|
|
237
257
|
} catch (err) {
|
|
238
|
-
throw new Error(
|
|
258
|
+
throw new Error(`${label}: request to ${cfg.url} failed: ${err instanceof Error ? err.message : err}`);
|
|
239
259
|
}
|
|
240
260
|
if (!res.ok) {
|
|
241
|
-
throw new Error(
|
|
261
|
+
throw new Error(`${label}: ${cfg.url} returned ${res.status} ${res.statusText}`);
|
|
242
262
|
}
|
|
243
|
-
let json: StorageStateShape;
|
|
244
263
|
try {
|
|
245
|
-
|
|
264
|
+
return await res.json();
|
|
246
265
|
} catch (err) {
|
|
247
|
-
throw new Error(
|
|
266
|
+
throw new Error(`${label}: ${cfg.url} did not return JSON: ${err instanceof Error ? err.message : err}`);
|
|
248
267
|
}
|
|
249
|
-
if (!json.cookies?.length && !json.origins?.length) {
|
|
250
|
-
throw new Error(`mintEndpoint auth: ${cfg.url} returned a storageState with no cookies/origins`);
|
|
251
|
-
}
|
|
252
|
-
await applyStorageStateJson(context, json);
|
|
253
268
|
}
|
|
254
269
|
|
|
255
270
|
async function applyCookieJwt(
|
package/src/browser.ts
CHANGED
|
@@ -320,23 +320,44 @@ export interface LoadedExtension {
|
|
|
320
320
|
* reason this can run on a headless host (and therefore on Cloud Lens) instead
|
|
321
321
|
* of needing a screen. */
|
|
322
322
|
const EXTENSION_CHANNEL = 'chromium';
|
|
323
|
+
const EXTENSION_LAUNCH_TIMEOUT_MS = 30_000;
|
|
324
|
+
const EXTENSION_LAUNCH_ATTEMPTS = 2;
|
|
323
325
|
|
|
324
326
|
async function launchWithExtension(
|
|
325
327
|
opts: WithPageOptions,
|
|
326
328
|
extPath: string,
|
|
327
329
|
): Promise<{ context: BrowserContext; extension: LoadedExtension; userDataDir: string }> {
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
330
|
+
// Bounded and retried once: Playwright's default launch timeout is 180 s, so a
|
|
331
|
+
// launch that stalls (seen on loaded Linux CI) used to hang the whole run past
|
|
332
|
+
// any test or caller deadline with no word on WHERE. Now it names the stage.
|
|
333
|
+
let userDataDir = '';
|
|
334
|
+
let context: BrowserContext | null = null;
|
|
335
|
+
let lastLaunchError = '';
|
|
336
|
+
for (let attempt = 1; attempt <= EXTENSION_LAUNCH_ATTEMPTS && !context; attempt++) {
|
|
337
|
+
userDataDir = await mkdtemp(join(tmpdir(), 'lens-ext-'));
|
|
338
|
+
try {
|
|
339
|
+
context = await chromium.launchPersistentContext(userDataDir, {
|
|
340
|
+
...contextOptions(opts),
|
|
341
|
+
headless: true,
|
|
342
|
+
channel: EXTENSION_CHANNEL,
|
|
343
|
+
timeout: EXTENSION_LAUNCH_TIMEOUT_MS,
|
|
344
|
+
args: [
|
|
345
|
+
`--disable-extensions-except=${extPath}`,
|
|
346
|
+
`--load-extension=${extPath}`,
|
|
347
|
+
'--disable-dev-shm-usage',
|
|
348
|
+
'--no-sandbox',
|
|
349
|
+
],
|
|
350
|
+
});
|
|
351
|
+
} catch (err) {
|
|
352
|
+
lastLaunchError = err instanceof Error ? err.message.split('\n')[0]! : String(err);
|
|
353
|
+
await rm(userDataDir, { recursive: true, force: true }).catch(() => {});
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
if (!context) {
|
|
357
|
+
throw new Error(
|
|
358
|
+
`extension browser did not start (${EXTENSION_LAUNCH_ATTEMPTS} attempts × ${EXTENSION_LAUNCH_TIMEOUT_MS} ms): ${lastLaunchError}`,
|
|
359
|
+
);
|
|
360
|
+
}
|
|
340
361
|
// MV3 registers a service worker; MV2 a background page. Wait for whichever
|
|
341
362
|
// appears — the id lives in its URL host either way.
|
|
342
363
|
let worker = context.serviceWorkers()[0];
|
package/src/extension-flow.ts
CHANGED
|
@@ -88,16 +88,77 @@ export const SIMULATED_ACCESS_NOTE =
|
|
|
88
88
|
* Read back and compared, so a seed that did not land fails the run here instead
|
|
89
89
|
* of showing up later as a popup that is mysteriously logged out. */
|
|
90
90
|
export async function seedExtensionStorage(context: BrowserContext, values: Record<string, unknown>): Promise<void> {
|
|
91
|
-
const target = context.serviceWorkers()[0] ?? context.backgroundPages()[0];
|
|
92
|
-
if (!target) throw new Error('storage_local: the extension has no service worker or background page to write chrome.storage.local from');
|
|
93
91
|
const keys = Object.keys(values);
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
92
|
+
// An MV3 service worker can be stopped and restarted by Chrome at any moment, and an
|
|
93
|
+
// evaluate on the stopped one never settles — observed as a 90 s hang under a loaded
|
|
94
|
+
// test suite, never in isolation. So every attempt has a deadline, and each retry asks
|
|
95
|
+
// the context for the worker that is live NOW instead of reusing the old handle.
|
|
96
|
+
let back: Record<string, unknown> | null | undefined;
|
|
97
|
+
let lastError = 'no attempt made';
|
|
98
|
+
for (let attempt = 1; attempt <= SEED_ATTEMPTS && back === undefined; attempt++) {
|
|
99
|
+
const target =
|
|
100
|
+
context.serviceWorkers()[0] ??
|
|
101
|
+
context.backgroundPages()[0] ??
|
|
102
|
+
(await context.waitForEvent('serviceworker', { timeout: SEED_DEADLINE_MS }).catch(() => undefined));
|
|
103
|
+
if (!target) {
|
|
104
|
+
lastError = 'no service worker or background page to write chrome.storage.local from';
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
try {
|
|
108
|
+
back = await deadline(
|
|
109
|
+
target.evaluate(async (v: Record<string, unknown>) => {
|
|
110
|
+
const c = (globalThis as unknown as { chrome: { storage?: { local: { set(x: object): Promise<void>; get(k: string[]): Promise<Record<string, unknown>> } } } }).chrome;
|
|
111
|
+
if (!c.storage) return null;
|
|
112
|
+
await c.storage.local.set(v);
|
|
113
|
+
return c.storage.local.get(Object.keys(v));
|
|
114
|
+
}, values),
|
|
115
|
+
SEED_DEADLINE_MS,
|
|
116
|
+
);
|
|
117
|
+
} catch (err) {
|
|
118
|
+
lastError = err instanceof Error ? err.message : String(err);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
if (back === undefined) throw new Error(`storage_local: could not write chrome.storage.local after ${SEED_ATTEMPTS} attempts — ${lastError}`);
|
|
100
122
|
if (!back) throw new Error('storage_local: the extension has no chrome.storage — its manifest does not request the "storage" permission, so there is nowhere to seed');
|
|
101
|
-
const missing = keys.filter((k) =>
|
|
123
|
+
const missing = keys.filter((k) => !sameJson(back![k], values[k]));
|
|
102
124
|
if (missing.length) throw new Error(`storage_local: chrome.storage.local did not keep ${missing.join(', ')} — read back differs from what was written`);
|
|
103
125
|
}
|
|
126
|
+
|
|
127
|
+
const SEED_ATTEMPTS = 3;
|
|
128
|
+
const SEED_DEADLINE_MS = 5_000;
|
|
129
|
+
|
|
130
|
+
/** Reject after `ms` instead of waiting forever on a call that will never settle. */
|
|
131
|
+
export function deadline<T>(p: Promise<T>, ms: number): Promise<T> {
|
|
132
|
+
return new Promise<T>((resolve, reject) => {
|
|
133
|
+
const t = setTimeout(() => reject(new Error(`no answer within ${ms} ms`)), ms);
|
|
134
|
+
t.unref?.();
|
|
135
|
+
p.then(
|
|
136
|
+
(v) => {
|
|
137
|
+
clearTimeout(t);
|
|
138
|
+
resolve(v);
|
|
139
|
+
},
|
|
140
|
+
(e) => {
|
|
141
|
+
clearTimeout(t);
|
|
142
|
+
reject(e);
|
|
143
|
+
},
|
|
144
|
+
);
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** F009.127 — structural equality for JSON values, blind to object key ORDER.
|
|
149
|
+
* chrome.storage hands an object back with its keys reordered, so comparing
|
|
150
|
+
* JSON text refused a login object that had landed perfectly. Array order and
|
|
151
|
+
* every value still count. Exported for test. */
|
|
152
|
+
export function sameJson(a: unknown, b: unknown): boolean {
|
|
153
|
+
if (a === b) return true;
|
|
154
|
+
if (typeof a !== typeof b || a === null || b === null || typeof a !== 'object') return false;
|
|
155
|
+
if (Array.isArray(a) !== Array.isArray(b)) return false;
|
|
156
|
+
if (Array.isArray(a)) {
|
|
157
|
+
const bb = b as unknown[];
|
|
158
|
+
return a.length === bb.length && a.every((v, i) => sameJson(v, bb[i]));
|
|
159
|
+
}
|
|
160
|
+
const ao = a as Record<string, unknown>;
|
|
161
|
+
const bo = b as Record<string, unknown>;
|
|
162
|
+
const ak = Object.keys(ao);
|
|
163
|
+
return ak.length === Object.keys(bo).length && ak.every((k) => Object.hasOwn(bo, k) && sameJson(ao[k], bo[k]));
|
|
164
|
+
}
|
package/src/flow-store.ts
CHANGED
|
@@ -9,7 +9,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
9
9
|
import type { LensFlowRunRow } from './schema.ts';
|
|
10
10
|
import { artifactKeys, type LensStorage } from './storage.ts';
|
|
11
11
|
import { enqueueArtifacts } from './artifact-uploads.ts';
|
|
12
|
-
import type { FlowResult, FailedResponse } from './flow.ts';
|
|
12
|
+
import type { FlowDownload, FlowResult, FailedResponse } from './flow.ts';
|
|
13
13
|
|
|
14
14
|
/** A per-step entry as persisted (screenshot bytes already offloaded to an
|
|
15
15
|
* artifact key — null when the step took no shot). */
|
|
@@ -110,6 +110,18 @@ export async function recordFlowRun(
|
|
|
110
110
|
await storage.put(videoKey, result.video);
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
+
// F009.128 — downloaded files go where the stills go; the row keeps only the key.
|
|
114
|
+
const storedDownloads: Array<Omit<FlowDownload, 'bytes'> & { key?: string }> = [];
|
|
115
|
+
for (const [n, d] of (result.downloads ?? []).entries()) {
|
|
116
|
+
const { bytes, ...meta } = d;
|
|
117
|
+
let key: string | undefined;
|
|
118
|
+
if (bytes) {
|
|
119
|
+
key = artifactKeys.flowDownload(projectSlug, id, n, d.name);
|
|
120
|
+
await storage.put(key, bytes);
|
|
121
|
+
}
|
|
122
|
+
storedDownloads.push({ ...meta, ...(key ? { key } : {}) });
|
|
123
|
+
}
|
|
124
|
+
|
|
113
125
|
const row: LensFlowRunRow = {
|
|
114
126
|
id,
|
|
115
127
|
project_slug: projectSlug,
|
|
@@ -157,6 +169,8 @@ export async function recordFlowRun(
|
|
|
157
169
|
...(result.extension ? { extension: result.extension } : {}),
|
|
158
170
|
...(result.password_in_url ? { password_in_url: result.password_in_url } : {}),
|
|
159
171
|
...(result.not_listening ? { not_listening: result.not_listening } : {}),
|
|
172
|
+
// F009.128 — always written when the run reported it, empty included.
|
|
173
|
+
...(result.downloads ? { downloads: storedDownloads } : {}),
|
|
160
174
|
}),
|
|
161
175
|
};
|
|
162
176
|
|
package/src/flow.ts
CHANGED
|
@@ -24,8 +24,8 @@ import {
|
|
|
24
24
|
type Target,
|
|
25
25
|
} from '@broberg/lens-engine';
|
|
26
26
|
import { withPage, type LensEngine } from './browser.ts';
|
|
27
|
-
import { defaultPopupOf, prepareExtension, seedExtensionStorage, SIMULATED_ACCESS_NOTE } from './extension-flow.ts';
|
|
28
|
-
import { authPrepare, type AuthConfig, type AuthEnv } from './auth/index.ts';
|
|
27
|
+
import { deadline, defaultPopupOf, prepareExtension, seedExtensionStorage, SIMULATED_ACCESS_NOTE } from './extension-flow.ts';
|
|
28
|
+
import { authPrepare, fetchMintJson, type AuthConfig, type AuthEnv } from './auth/index.ts';
|
|
29
29
|
import { takeShot, settleForShot } from './capture.ts';
|
|
30
30
|
import { runCritic, topFindingMessage, type CriticMode, type CriticReport } from './critic.ts';
|
|
31
31
|
import { waitForBuild, observeBuild } from './build-marker.ts';
|
|
@@ -194,6 +194,19 @@ export interface FlowExtension {
|
|
|
194
194
|
/** F009.126 — written to the extension's chrome.storage.local before step 0, so a
|
|
195
195
|
* popup that keeps its login there starts signed in (OAuth cannot run headless). */
|
|
196
196
|
storage_local?: Record<string, unknown>;
|
|
197
|
+
/** F009.129 — the daemon MINTS the value itself (Bearer secret from secretPath /
|
|
198
|
+
* secretEnv) and writes the JSON answer to chrome.storage.local[key], so a real
|
|
199
|
+
* token never passes through the calling agent. Wins over storage_local on the
|
|
200
|
+
* same key. */
|
|
201
|
+
storage_local_mint?: StorageLocalMint;
|
|
202
|
+
}
|
|
203
|
+
export interface StorageLocalMint {
|
|
204
|
+
key: string;
|
|
205
|
+
url: string;
|
|
206
|
+
method?: 'POST' | 'GET';
|
|
207
|
+
secretPath?: string;
|
|
208
|
+
secretEnv?: string;
|
|
209
|
+
body?: Record<string, unknown>;
|
|
197
210
|
}
|
|
198
211
|
|
|
199
212
|
/**
|
|
@@ -359,7 +372,36 @@ export class InstrumentError extends Error {
|
|
|
359
372
|
}
|
|
360
373
|
}
|
|
361
374
|
|
|
375
|
+
/** F009.128 — one file the run's browser downloaded. `bytes` is dropped once the
|
|
376
|
+
* store has written it; a reader gets `url` instead. */
|
|
377
|
+
export interface FlowDownload {
|
|
378
|
+
name: string;
|
|
379
|
+
size: number;
|
|
380
|
+
/** Which page triggered it: the page under test, the extension popup, or another. */
|
|
381
|
+
page: 'page' | 'popup' | 'other';
|
|
382
|
+
/** Index of the step that was running when it started; null before the first. */
|
|
383
|
+
step: number | null;
|
|
384
|
+
bytes?: Buffer;
|
|
385
|
+
error?: string;
|
|
386
|
+
/** Not kept because a limit was hit (MAX_DOWNLOADS / MAX_DOWNLOAD_BYTES). */
|
|
387
|
+
skipped?: 'cap';
|
|
388
|
+
}
|
|
389
|
+
export const MAX_DOWNLOADS = 10;
|
|
390
|
+
export const MAX_DOWNLOAD_BYTES = 25 * 1024 * 1024;
|
|
391
|
+
const DOWNLOAD_SETTLE_MS = 5_000;
|
|
392
|
+
const DOWNLOAD_FINISH_MS = 30_000;
|
|
393
|
+
|
|
394
|
+
/** F009.128 — wait for the run's downloads, bounded. Whatever has not finished by
|
|
395
|
+
* then is marked as such, never dropped and never waited on forever. Exported for test. */
|
|
396
|
+
export async function finishDownloads(pending: Promise<void>[], downloads: FlowDownload[], ms: number): Promise<void> {
|
|
397
|
+
await deadline(Promise.allSettled(pending), ms).catch(() => {
|
|
398
|
+
for (const d of downloads) if (!d.bytes && !d.error && !d.skipped) d.error = `did not finish within ${ms} ms`;
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
|
|
362
402
|
export interface FlowResult {
|
|
403
|
+
/** F009.128 — files the run downloaded. Always present on a finished run. */
|
|
404
|
+
downloads?: FlowDownload[];
|
|
363
405
|
/** F009.125 — the extension this run loaded: its id (read at load time), the
|
|
364
406
|
* build path the caller named, and whether host access was SIMULATED on a copy. */
|
|
365
407
|
extension?: { id: string; path: string; host_access: 'as-built' | 'simulated'; note?: string } | null;
|
|
@@ -467,8 +509,8 @@ export function parseFlow(input: unknown): Flow {
|
|
|
467
509
|
if (f.extension !== undefined) {
|
|
468
510
|
const e = f.extension as unknown as Record<string, unknown>;
|
|
469
511
|
const keys = e && typeof e === 'object' && !Array.isArray(e) ? Object.keys(e) : null;
|
|
470
|
-
if (!keys || typeof e.path !== 'string' || !e.path || keys.some((k) =>
|
|
471
|
-
throw new Error('flow.extension must be {path: "<unpacked build dir>", grant_host_access?: boolean, storage_local?: {…}}');
|
|
512
|
+
if (!keys || typeof e.path !== 'string' || !e.path || keys.some((k) => !['path', 'grant_host_access', 'storage_local', 'storage_local_mint'].includes(k))) {
|
|
513
|
+
throw new Error('flow.extension must be {path: "<unpacked build dir>", grant_host_access?: boolean, storage_local?: {…}, storage_local_mint?: {…}}');
|
|
472
514
|
}
|
|
473
515
|
if (e.grant_host_access !== undefined && typeof e.grant_host_access !== 'boolean') {
|
|
474
516
|
throw new Error('flow.extension.grant_host_access must be a boolean');
|
|
@@ -477,10 +519,12 @@ export function parseFlow(input: unknown): Flow {
|
|
|
477
519
|
if (sl !== undefined && (sl === null || typeof sl !== 'object' || Array.isArray(sl))) {
|
|
478
520
|
throw new Error('flow.extension.storage_local must be an object of {key: JSON value} — it is passed to chrome.storage.local.set as-is');
|
|
479
521
|
}
|
|
522
|
+
const mint = e.storage_local_mint === undefined ? undefined : parseStorageLocalMint(e.storage_local_mint);
|
|
480
523
|
extension = {
|
|
481
524
|
path: e.path,
|
|
482
525
|
...(e.grant_host_access === true ? { grant_host_access: true } : {}),
|
|
483
526
|
...(sl !== undefined ? { storage_local: sl as Record<string, unknown> } : {}),
|
|
527
|
+
...(mint ? { storage_local_mint: mint } : {}),
|
|
484
528
|
};
|
|
485
529
|
}
|
|
486
530
|
validateAndNormalizeSteps(f.steps);
|
|
@@ -511,6 +555,8 @@ function requestRegex(u: string): RegExp {
|
|
|
511
555
|
export interface SeenRequest {
|
|
512
556
|
method: string;
|
|
513
557
|
url: string;
|
|
558
|
+
/** Playwright's resource type; 'document' is a navigation (and what a link download is). */
|
|
559
|
+
type?: string;
|
|
514
560
|
/** null while in flight, or when it failed before a response. */
|
|
515
561
|
status: number | null;
|
|
516
562
|
failed?: string;
|
|
@@ -531,6 +577,30 @@ function redactQuery(u: string): string {
|
|
|
531
577
|
return q === -1 ? u : `${u.slice(0, q)}?…`;
|
|
532
578
|
}
|
|
533
579
|
|
|
580
|
+
/** F009.129 — refused when malformed, the same way the rest of flow.extension is. */
|
|
581
|
+
function parseStorageLocalMint(raw: unknown): StorageLocalMint {
|
|
582
|
+
const m = raw as Record<string, unknown>;
|
|
583
|
+
const shape = 'flow.extension.storage_local_mint must be {key, url, method?, secretPath | secretEnv, body?}';
|
|
584
|
+
if (!m || typeof m !== 'object' || Array.isArray(m)) throw new Error(shape);
|
|
585
|
+
const allowed = ['key', 'url', 'method', 'secretPath', 'secretEnv', 'body'];
|
|
586
|
+
const unknownKey = Object.keys(m).find((k) => !allowed.includes(k));
|
|
587
|
+
if (unknownKey) throw new Error(`${shape} — unknown key "${unknownKey}"`);
|
|
588
|
+
if (typeof m.key !== 'string' || !m.key) throw new Error(`${shape} — key is required`);
|
|
589
|
+
if (typeof m.url !== 'string' || !/^https?:\/\//.test(m.url)) throw new Error(`${shape} — url must be http(s)`);
|
|
590
|
+
if (m.method !== undefined && m.method !== 'POST' && m.method !== 'GET') throw new Error(`${shape} — method must be POST or GET`);
|
|
591
|
+
const hasPath = typeof m.secretPath === 'string' && m.secretPath !== '';
|
|
592
|
+
const hasEnv = typeof m.secretEnv === 'string' && m.secretEnv !== '';
|
|
593
|
+
if (hasPath === hasEnv) throw new Error(`${shape} — give exactly one of secretPath or secretEnv`);
|
|
594
|
+
if (m.body !== undefined && (m.body === null || typeof m.body !== 'object' || Array.isArray(m.body))) throw new Error(`${shape} — body must be an object`);
|
|
595
|
+
return {
|
|
596
|
+
key: m.key,
|
|
597
|
+
url: m.url,
|
|
598
|
+
...(m.method ? { method: m.method as 'POST' | 'GET' } : {}),
|
|
599
|
+
...(hasPath ? { secretPath: m.secretPath as string } : { secretEnv: m.secretEnv as string }),
|
|
600
|
+
...(m.body ? { body: m.body as Record<string, unknown> } : {}),
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
|
|
534
604
|
function usesPopup(steps: FlowStep[]): boolean {
|
|
535
605
|
return steps.some(
|
|
536
606
|
(st) =>
|
|
@@ -1562,6 +1632,14 @@ export async function clickAndActivate(
|
|
|
1562
1632
|
if (t === 'xhr' || t === 'fetch' || t === 'document') requests += 1;
|
|
1563
1633
|
};
|
|
1564
1634
|
page.on('request', onRequest);
|
|
1635
|
+
// F009.128 — a download IS an effect. A button whose only job is a download
|
|
1636
|
+
// (a blob handed to <a download>) touches no DOM and makes no request, so it
|
|
1637
|
+
// read as a silent no-op and got pressed twice: two files for one click.
|
|
1638
|
+
let downloaded = false;
|
|
1639
|
+
const onDownload = () => {
|
|
1640
|
+
downloaded = true;
|
|
1641
|
+
};
|
|
1642
|
+
page.on('download', onDownload);
|
|
1565
1643
|
const urlBefore = page.url();
|
|
1566
1644
|
await page
|
|
1567
1645
|
.evaluate(() => {
|
|
@@ -1606,7 +1684,7 @@ export async function clickAndActivate(
|
|
|
1606
1684
|
const mutations = await page
|
|
1607
1685
|
.evaluate(() => (window as unknown as { __lensMut?: number }).__lensMut ?? 0)
|
|
1608
1686
|
.catch(() => 1); // page navigated away → that IS an effect
|
|
1609
|
-
if (mutations > 0 || requests > 0 || page.url() !== urlBefore) return false;
|
|
1687
|
+
if (mutations > 0 || requests > 0 || downloaded || page.url() !== urlBefore) return false;
|
|
1610
1688
|
const focusMoved = await page
|
|
1611
1689
|
.evaluate(
|
|
1612
1690
|
([before, clicked]) => {
|
|
@@ -1629,6 +1707,7 @@ export async function clickAndActivate(
|
|
|
1629
1707
|
return true;
|
|
1630
1708
|
} finally {
|
|
1631
1709
|
page.off('request', onRequest);
|
|
1710
|
+
page.off('download', onDownload);
|
|
1632
1711
|
await focusBefore.dispose().catch(() => {});
|
|
1633
1712
|
await target?.dispose().catch(() => {});
|
|
1634
1713
|
await page
|
|
@@ -2741,6 +2820,7 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
|
|
|
2741
2820
|
// F213.11 — read BEFORE the step runs, so the number is "how long after the
|
|
2742
2821
|
// page loaded did this start", never "how long did it take".
|
|
2743
2822
|
const stepStartedAt = Date.now();
|
|
2823
|
+
currentStep = i;
|
|
2744
2824
|
const msAfterNav = stepStartedAt - lastNavAt;
|
|
2745
2825
|
// F213.13 — a goto's reading is measured from the PREVIOUS navigation, so it
|
|
2746
2826
|
// is not the same quantity as its siblings and must not sit in their column.
|
|
@@ -2928,12 +3008,58 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
|
|
|
2928
3008
|
extCtx.popup = popup;
|
|
2929
3009
|
return null;
|
|
2930
3010
|
};
|
|
3011
|
+
// F009.128 — files the run downloads. A download is awaited before the context
|
|
3012
|
+
// closes; otherwise the bytes die with the browser and the report says nothing.
|
|
3013
|
+
const downloads: FlowDownload[] = [];
|
|
3014
|
+
const pendingDownloads: Promise<void>[] = [];
|
|
3015
|
+
let currentStep: number | null = null;
|
|
3016
|
+
const watchDownloads = (context: BrowserContext): void => {
|
|
3017
|
+
const hooked = new WeakSet<Page>();
|
|
3018
|
+
const hook = (p: Page) => {
|
|
3019
|
+
if (hooked.has(p)) return;
|
|
3020
|
+
hooked.add(p);
|
|
3021
|
+
p.on('download', (d) => {
|
|
3022
|
+
const entry: FlowDownload = {
|
|
3023
|
+
name: d.suggestedFilename(),
|
|
3024
|
+
size: 0,
|
|
3025
|
+
page: p === extCtx?.popup ? 'popup' : p === extCtx?.main || p === videoPage ? 'page' : 'other',
|
|
3026
|
+
step: currentStep,
|
|
3027
|
+
};
|
|
3028
|
+
downloads.push(entry);
|
|
3029
|
+
if (downloads.length > MAX_DOWNLOADS) {
|
|
3030
|
+
entry.skipped = 'cap';
|
|
3031
|
+
pendingDownloads.push(d.cancel().catch(() => {}));
|
|
3032
|
+
return;
|
|
3033
|
+
}
|
|
3034
|
+
pendingDownloads.push(
|
|
3035
|
+
(async () => {
|
|
3036
|
+
const failure = await d.failure();
|
|
3037
|
+
// `??=`: once the run gave up waiting, the context closing cancels the
|
|
3038
|
+
// download — and 'canceled' would hide that it simply never finished.
|
|
3039
|
+
if (failure) {
|
|
3040
|
+
entry.error ??= failure;
|
|
3041
|
+
return;
|
|
3042
|
+
}
|
|
3043
|
+
const path = await d.path();
|
|
3044
|
+
const bytes = await readFile(path);
|
|
3045
|
+
entry.size = bytes.length;
|
|
3046
|
+
if (bytes.length > MAX_DOWNLOAD_BYTES) entry.skipped = 'cap';
|
|
3047
|
+
else entry.bytes = bytes;
|
|
3048
|
+
})().catch((err) => {
|
|
3049
|
+
entry.error ??= err instanceof Error ? err.message : String(err);
|
|
3050
|
+
}),
|
|
3051
|
+
);
|
|
3052
|
+
});
|
|
3053
|
+
};
|
|
3054
|
+
context.pages().forEach(hook);
|
|
3055
|
+
context.on('page', hook);
|
|
3056
|
+
};
|
|
2931
3057
|
// F009.126 — every request the run's main browser makes, from the first step on.
|
|
2932
3058
|
const seenRequests: SeenRequest[] = [];
|
|
2933
3059
|
const watchRequests = (context: BrowserContext): void => {
|
|
2934
3060
|
const byReq = new Map<object, SeenRequest>();
|
|
2935
3061
|
context.on('request', (req) => {
|
|
2936
|
-
const r: SeenRequest = { method: req.method(), url: req.url(), status: null };
|
|
3062
|
+
const r: SeenRequest = { method: req.method(), url: req.url(), status: null, type: req.resourceType() };
|
|
2937
3063
|
byReq.set(req, r);
|
|
2938
3064
|
seenRequests.push(r);
|
|
2939
3065
|
});
|
|
@@ -3017,9 +3143,15 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
|
|
|
3017
3143
|
videoPage = page;
|
|
3018
3144
|
collectFailures(page);
|
|
3019
3145
|
watchRequests(context);
|
|
3146
|
+
watchDownloads(context);
|
|
3020
3147
|
if (ext && flow.extension && prepared) {
|
|
3021
3148
|
extCtx = { context, id: ext.id, main: page, popup: null };
|
|
3022
|
-
|
|
3149
|
+
const mint = flow.extension.storage_local_mint;
|
|
3150
|
+
const seed = {
|
|
3151
|
+
...(flow.extension.storage_local ?? {}),
|
|
3152
|
+
...(mint ? { [mint.key]: await fetchMintJson(mint, env, 'storage_local_mint') } : {}),
|
|
3153
|
+
};
|
|
3154
|
+
if (Object.keys(seed).length) await seedExtensionStorage(context, seed);
|
|
3023
3155
|
extensionInfo = {
|
|
3024
3156
|
id: ext.id,
|
|
3025
3157
|
path: flow.extension.path,
|
|
@@ -3118,6 +3250,22 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
|
|
|
3118
3250
|
} catch {
|
|
3119
3251
|
// page already closed — final_url stays null (not measured)
|
|
3120
3252
|
}
|
|
3253
|
+
// F009.128 — before the context (and the files) go away. A link download is a
|
|
3254
|
+
// navigation whose 'download' event fires only once its headers arrive, which can
|
|
3255
|
+
// be AFTER the last step returned; so wait for navigations still in flight (bounded)
|
|
3256
|
+
// and only then for the downloads they turned into. Runs with nothing in flight
|
|
3257
|
+
// pay nothing.
|
|
3258
|
+
// A navigation Chromium turns into a download is ABORTED as a request
|
|
3259
|
+
// (net::ERR_ABORTED) and only then raises 'download' — so count those too.
|
|
3260
|
+
const settleUntil = Date.now() + DOWNLOAD_SETTLE_MS;
|
|
3261
|
+
const navInFlight = () => seenRequests.some((r) => r.type === 'document' && r.status === null && !r.failed);
|
|
3262
|
+
const becameDownload = () => seenRequests.filter((r) => r.type === 'document' && (r.failed ?? '').includes('ERR_ABORTED')).length;
|
|
3263
|
+
while (Date.now() < settleUntil && (navInFlight() || downloads.length < becameDownload())) {
|
|
3264
|
+
await new Promise((res) => setTimeout(res, 100));
|
|
3265
|
+
}
|
|
3266
|
+
// Bounded: a download that never finishes must not hang the run. Whatever is
|
|
3267
|
+
// still unfinished is reported as such, never dropped.
|
|
3268
|
+
await finishDownloads(pendingDownloads, downloads, DOWNLOAD_FINISH_MS);
|
|
3121
3269
|
},
|
|
3122
3270
|
);
|
|
3123
3271
|
} catch (err) {
|
|
@@ -3125,7 +3273,7 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
|
|
|
3125
3273
|
// Keep the recording even on a crash — watching HOW it broke is the whole point.
|
|
3126
3274
|
const video = await readVideo();
|
|
3127
3275
|
await prepared?.cleanup();
|
|
3128
|
-
return { ...(extensionInfo ? { extension: extensionInfo } : {}), name: flow.name, status: 'error', steps: results.sort((a, b) => a.index - b.index), failed_at: firstFailedAt, reason: leadWithRedirect(reason, firstRedirect), duration_ms: Date.now() - startedAt, video, build: observedBuild, final_url: finalUrl, redirected: firstRedirect, password_in_url: passwordInUrl(finalUrl, filledPasswords), not_listening: notListening[0] ?? null, reduced_motion: opts.reducedMotion ?? 'reduce' };
|
|
3276
|
+
return { downloads, ...(extensionInfo ? { extension: extensionInfo } : {}), name: flow.name, status: 'error', steps: results.sort((a, b) => a.index - b.index), failed_at: firstFailedAt, reason: leadWithRedirect(reason, firstRedirect), duration_ms: Date.now() - startedAt, video, build: observedBuild, final_url: finalUrl, redirected: firstRedirect, password_in_url: passwordInUrl(finalUrl, filledPasswords), not_listening: notListening[0] ?? null, reduced_motion: opts.reducedMotion ?? 'reduce' };
|
|
3129
3277
|
}
|
|
3130
3278
|
|
|
3131
3279
|
const video = await readVideo();
|
|
@@ -3179,6 +3327,7 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
|
|
|
3179
3327
|
// context can never disagree about it.
|
|
3180
3328
|
reduced_motion: opts.reducedMotion ?? 'reduce',
|
|
3181
3329
|
...(extensionInfo ? { extension: extensionInfo } : {}),
|
|
3330
|
+
downloads,
|
|
3182
3331
|
};
|
|
3183
3332
|
}
|
|
3184
3333
|
|
package/src/mcp.ts
CHANGED
|
@@ -369,6 +369,7 @@ export const TOOLS = [
|
|
|
369
369
|
path: { type: 'string' },
|
|
370
370
|
grant_host_access: { type: 'boolean' },
|
|
371
371
|
storage_local: { type: 'object', description: 'F009.126 — {key: JSON value} for chrome.storage.local, set before step 0.' },
|
|
372
|
+
storage_local_mint: { type: 'object', description: 'F009.129 — {key, url, method?, secretPath | secretEnv, body?}: the DAEMON calls the mint url (Bearer secret; secretPath relative to local_path, sandboxed) and writes the JSON answer to chrome.storage.local[key] before step 0, so a real token never passes through you. Wins over storage_local on the same key. Value and secret never appear in the report.' },
|
|
372
373
|
},
|
|
373
374
|
},
|
|
374
375
|
steps: {
|
package/src/service.ts
CHANGED
|
@@ -666,11 +666,16 @@ function parseAuthJson(raw: string | null | undefined): { source: string; adapte
|
|
|
666
666
|
/** NULL = the row predates the column, so we have no answer; `{}` = the run
|
|
667
667
|
* looked and found nothing. Malformed JSON is reported as unknown rather than
|
|
668
668
|
* as "clean": a parse failure must never read as an all-clear. */
|
|
669
|
-
function parseFindingsJson(raw: string | null | undefined): Record<string, unknown> | null {
|
|
669
|
+
function parseFindingsJson(raw: string | null | undefined, port?: number): Record<string, unknown> | null {
|
|
670
670
|
if (raw == null) return null;
|
|
671
671
|
try {
|
|
672
672
|
const v = JSON.parse(raw);
|
|
673
|
-
|
|
673
|
+
if (!v || typeof v !== 'object') return null;
|
|
674
|
+
// F009.128 — a stored download carries its storage key; a reader gets a url.
|
|
675
|
+
if (port !== undefined && Array.isArray(v.downloads)) {
|
|
676
|
+
v.downloads = v.downloads.map(({ key, ...d }: { key?: string }) => (key ? { ...d, url: artifactUrl(port, key) } : d));
|
|
677
|
+
}
|
|
678
|
+
return v as Record<string, unknown>;
|
|
674
679
|
} catch {
|
|
675
680
|
return null;
|
|
676
681
|
}
|
|
@@ -748,7 +753,7 @@ export function flowRunView(row: LensFlowRunRow, port: number) {
|
|
|
748
753
|
// and lens_list_flow_runs for the string and found neither — "the instrument
|
|
749
754
|
// measures right and nobody is told", which is the same shape as the probe
|
|
750
755
|
// that sat unwired to `fill` for a month.
|
|
751
|
-
findings: parseFindingsJson(row.findings_json),
|
|
756
|
+
findings: parseFindingsJson(row.findings_json, port),
|
|
752
757
|
steps,
|
|
753
758
|
// F255.5 — present only when the run was recorded. Same artifact mechanism as
|
|
754
759
|
// the step screenshots, so a consumer fetches it exactly the same way.
|
|
@@ -869,7 +874,7 @@ export function toCloudFlowResult(result: EngineFlowResult, row: LensFlowRunRow,
|
|
|
869
874
|
// (a CI runner with no daemon) is the consumer LEAST able to go and look
|
|
870
875
|
// somewhere else, so leaving them off this shape would be the gap at its
|
|
871
876
|
// worst.
|
|
872
|
-
findings: parseFindingsJson(row.findings_json),
|
|
877
|
+
findings: parseFindingsJson(row.findings_json, port),
|
|
873
878
|
steps: result.steps.map((s) => ({
|
|
874
879
|
index: s.index,
|
|
875
880
|
action: s.action,
|
|
@@ -938,6 +943,15 @@ export async function lensRunFlow(deps: LensDeps, input: RunFlowInput) {
|
|
|
938
943
|
const abs = raw.startsWith('/') ? raw : input.local_path ? `${input.local_path.replace(/\/$/, '')}/${raw}` : null;
|
|
939
944
|
if (!abs) throw new Error('flow.extension.path is relative — pass local_path, or give an absolute path');
|
|
940
945
|
flow.extension = { ...flow.extension, path: resolveSandboxedPath(abs, deps.sandboxRoots) };
|
|
946
|
+
// F009.129 — the mint secret file follows auth.mintEndpoint's rule: relative to
|
|
947
|
+
// the project, and inside the sandbox, so a flow can never point the daemon at
|
|
948
|
+
// another repo's secret and send it to its own url.
|
|
949
|
+
const mint = flow.extension.storage_local_mint;
|
|
950
|
+
if (mint?.secretPath) {
|
|
951
|
+
const sp = mint.secretPath.startsWith('/') ? mint.secretPath : input.local_path ? `${input.local_path.replace(/\/$/, '')}/${mint.secretPath}` : null;
|
|
952
|
+
if (!sp) throw new Error('flow.extension.storage_local_mint.secretPath is relative — pass local_path, or give an absolute path');
|
|
953
|
+
flow.extension = { ...flow.extension, storage_local_mint: { ...mint, secretPath: resolveSandboxedPath(sp, deps.sandboxRoots) } };
|
|
954
|
+
}
|
|
941
955
|
}
|
|
942
956
|
// F082.6 — auth reaches flows: own auth (relative paths resolved) or inherit
|
|
943
957
|
// the manifest's. Fixes the blocker where authed routes redirected to login.
|
package/src/storage.ts
CHANGED
|
@@ -118,6 +118,9 @@ export const artifactKeys = {
|
|
|
118
118
|
`${project}/flows/${flowRunId}/step-${stepIndex}.png`,
|
|
119
119
|
// F255.5 — the whole driven run as a video, alongside its step stills.
|
|
120
120
|
flowVideo: (project: string, flowRunId: string) => `${project}/flows/${flowRunId}/run.webm`,
|
|
121
|
+
// F009.128 — a file the run downloaded. Numbered so two files with one name both survive.
|
|
122
|
+
flowDownload: (project: string, flowRunId: string, n: number, name: string) =>
|
|
123
|
+
`${project}/flows/${flowRunId}/download-${n}-${name.replace(/[^\w.-]+/g, '_').slice(0, 120)}`,
|
|
121
124
|
// F074.63 — a vector export (PDF). Its own prefix, not runs/: an export is not
|
|
122
125
|
// a capture, carries no baseline and must never be diffed against one.
|
|
123
126
|
exportDoc: (project: string, exportId: string, ext: string) =>
|