@lincoln504/pi-research 1.7.8 → 1.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +5 -1
  2. package/agent-skill/pi-research/README.md +3 -3
  3. package/agent-skill/pi-research/SKILL.md +1 -1
  4. package/dist/cli-bin.mjs +3 -0
  5. package/dist/cli.mjs +673 -383
  6. package/dist/pdf-extract-worker.mjs +1 -1
  7. package/dist/thread-worker.mjs +6 -6
  8. package/docs/AGENT-SKILL.md +6 -4
  9. package/docs/ARCHITECTURE.md +64 -50
  10. package/docs/CHANGELOG.md +41 -0
  11. package/docs/CONFIGURATION.md +2 -2
  12. package/docs/PI-EXTENSION.md +4 -0
  13. package/docs/SDK.md +10 -7
  14. package/docs/THIRD-PARTY-NOTICES.md +11 -7
  15. package/package.json +10 -8
  16. package/scripts/camoufox-layout.cjs +455 -0
  17. package/scripts/cleanup.cjs +72 -29
  18. package/scripts/setup.cjs +73 -111
  19. package/src/cli-bin.mjs +3 -0
  20. package/src/cli.ts +12 -13
  21. package/src/core/pi-version.ts +9 -1
  22. package/src/healthcheck/index.ts +1 -1
  23. package/src/infrastructure/browser/browser-cleanup.ts +1 -1
  24. package/src/infrastructure/browser/browser-error-utils.ts +5 -4
  25. package/src/infrastructure/browser/camoufox-layout.ts +450 -0
  26. package/src/infrastructure/browser/config.ts +46 -103
  27. package/src/infrastructure/browser/ensure-browser.ts +90 -52
  28. package/src/infrastructure/browser/thread-worker-browser.ts +38 -42
  29. package/src/infrastructure/browser/thread-worker.mjs +6 -6
  30. package/src/infrastructure/knowledge-store-service.ts +6 -6
  31. package/src/orchestration/researcher-executor.ts +47 -1
  32. package/src/tui/research-health.ts +1 -1
  33. package/src/utils/log-utils.ts +1 -1
  34. package/src/web-research/browser-search.ts +8 -10
  35. package/src/web-research/pdf-extract-worker.mjs +1 -1
  36. package/src/web-research/scraper-utils.ts +11 -2
  37. package/src/web-research/web-scraper.ts +1 -1
@@ -0,0 +1,450 @@
1
+ /**
2
+ * Camoufox install-layout helpers for the runtime (ensure-browser, config, cli).
3
+ *
4
+ * TWIN of scripts/camoufox-layout.cjs (the install/uninstall half). The two
5
+ * are kept equivalent on purpose and test/unit/infrastructure/
6
+ * camoufox-layout.test.ts runs both against the same fixtures, because a drift
7
+ * between "where setup installed it" and "where the runtime looks" is exactly the
8
+ * install/runtime mismatch this package has shipped before.
9
+ *
10
+ * Why this exists. `@camoufox/camoufox` (0.5.x) differs from the old `camoufox-js`:
11
+ * - It installs under <INSTALL_DIR>/browsers/<repo>/<version>/ (multi-version),
12
+ * with `config.json`, `repo_cache.json` and a `.0.5_FLAG` beside it. camoufox-js
13
+ * put the browser files FLAT in INSTALL_DIR with a root `version.json`.
14
+ * - INSTALL_DIR is derived ONLY from platform cache dirs (XDG_CACHE_HOME on Linux,
15
+ * LOCALAPPDATA on Windows, $HOME/Library/Caches on macOS). It does NOT read
16
+ * CAMOUFOX_INSTALL_DIR or PLAYWRIGHT_BROWSERS_PATH.
17
+ * - `camoufox fetch` does `rm -rf INSTALL_DIR` when it is non-empty and has no
18
+ * `.0.5_FLAG` ("Cleaning old data...") BEFORE it downloads anything, so a failed
19
+ * download over a legacy install leaves the user with no browser at all.
20
+ * - A newer pi-research release that pins a newer browser installs the new build
21
+ * beside the old one and never removes it; the old one is dead weight because a
22
+ * released library only ever launches its own pinned build.
23
+ */
24
+
25
+ import * as fs from 'node:fs';
26
+ import * as os from 'node:os';
27
+ import * as path from 'node:path';
28
+ import { createRequire } from 'node:module';
29
+
30
+ export const COMPAT_FLAG = '.0.5_FLAG';
31
+
32
+ /** Names whose presence, with a root version.json, proves a legacy camoufox-js install. */
33
+ const LEGACY_MARKERS = ['camoufox-bin', 'camoufox', 'camoufox.exe', 'application.ini', 'Camoufox.app', 'properties.json'];
34
+
35
+ function isDir(p: string): boolean {
36
+ try {
37
+ return fs.statSync(p).isDirectory();
38
+ } catch {
39
+ return false;
40
+ }
41
+ }
42
+
43
+ function isFile(p: string): boolean {
44
+ try {
45
+ return fs.statSync(p).isFile();
46
+ } catch {
47
+ return false;
48
+ }
49
+ }
50
+
51
+ /** The custom location the user asked for, if any (camoufox itself ignores both variables). */
52
+ export function customDir(env: NodeJS.ProcessEnv = process.env): string | null {
53
+ return env['CAMOUFOX_INSTALL_DIR'] || env['PLAYWRIGHT_BROWSERS_PATH'] || null;
54
+ }
55
+
56
+ /** Mirrors @camoufox/camoufox paths.js userCacheDir('camoufox') for a given environment. */
57
+ export function defaultInstallDir(env: NodeJS.ProcessEnv = process.env, platform: NodeJS.Platform = process.platform, home: string = os.homedir()): string {
58
+ if (platform === 'win32') {
59
+ const base = env['LOCALAPPDATA'] && path.win32.isAbsolute(env['LOCALAPPDATA'])
60
+ ? env['LOCALAPPDATA']
61
+ : path.join(home, 'AppData', 'Local');
62
+ return path.join(base, 'camoufox', 'camoufox', 'Cache');
63
+ }
64
+ if (platform === 'darwin') return path.join(home, 'Library', 'Caches', 'camoufox');
65
+ const xdg = (env['XDG_CACHE_HOME'] || '').trim();
66
+ return path.join(xdg ? xdg : path.join(home, '.cache'), 'camoufox');
67
+ }
68
+
69
+ /**
70
+ * Resolve where the browser lives and which env overrides make camoufox use it.
71
+ *
72
+ * - No custom dir: the platform default, no overrides.
73
+ * - Linux: custom is the cache HOME (XDG_CACHE_HOME=custom, browser in custom/camoufox),
74
+ * except when custom already ends in `camoufox`, where its parent becomes the
75
+ * cache home so the browser lives in custom itself (old camoufox-js semantics).
76
+ * - Windows: same idea through LOCALAPPDATA (browser in <base>\camoufox\camoufox\Cache).
77
+ * - macOS: camoufox derives the cache from $HOME only; relocating it would mean
78
+ * redirecting HOME for the browser process, which also moves Firefox's own
79
+ * state. The custom dir is ignored there (`ignoredCustom` is set so callers can say so).
80
+ */
81
+ export interface ResolvedInstall {
82
+ dir: string;
83
+ overrides: Record<string, string>;
84
+ custom: string | null;
85
+ ignoredCustom: boolean;
86
+ }
87
+
88
+ export function resolveInstall(env: NodeJS.ProcessEnv = process.env, platform: NodeJS.Platform = process.platform, home: string = os.homedir()): ResolvedInstall {
89
+ const custom = customDir(env);
90
+ if (!custom) return { dir: defaultInstallDir(env, platform, home), overrides: {}, custom: null, ignoredCustom: false };
91
+ if (platform === 'darwin') {
92
+ return { dir: defaultInstallDir(env, platform, home), overrides: {}, custom, ignoredCustom: true };
93
+ }
94
+ const resolved = (platform === 'win32' ? path.win32 : path).resolve(custom);
95
+ if (platform === 'win32') {
96
+ // <LOCALAPPDATA>\camoufox\camoufox\Cache : one fixed shape, custom = LOCALAPPDATA.
97
+ const overrides: Record<string, string> = { LOCALAPPDATA: resolved };
98
+ return { dir: defaultInstallDir({ ...env, ...overrides }, platform, home), overrides, custom, ignoredCustom: false };
99
+ }
100
+ const endsInCamoufox = path.basename(resolved).toLowerCase() === 'camoufox';
101
+ const cacheHome = endsInCamoufox ? path.dirname(resolved) : resolved;
102
+ const overrides: Record<string, string> = { XDG_CACHE_HOME: cacheHome };
103
+ return { dir: defaultInstallDir({ ...env, ...overrides }, platform, home), overrides, custom, ignoredCustom: false };
104
+ }
105
+
106
+ /** Every `browsers/<repo>/<version>/` directory that holds a parseable version.json. */
107
+ export interface InstalledBuild { repo: string; name: string; path: string; rel: string; version: string; build: string }
108
+
109
+ /** The browser build @camoufox/camoufox was released with (repoName lowercased). */
110
+ export interface BrowserPin { repoName: string; version: string; build: string }
111
+
112
+ export function listInstalls(dir: string): InstalledBuild[] {
113
+ const out: InstalledBuild[] = [];
114
+ const browsers = path.join(dir, 'browsers');
115
+ let repos: fs.Dirent[];
116
+ try {
117
+ repos = fs.readdirSync(browsers, { withFileTypes: true });
118
+ } catch {
119
+ return out;
120
+ }
121
+ for (const repo of repos) {
122
+ if (!repo.isDirectory() || repo.name.startsWith('.')) continue;
123
+ const repoDir = path.join(browsers, repo.name);
124
+ let versions: fs.Dirent[];
125
+ try {
126
+ versions = fs.readdirSync(repoDir, { withFileTypes: true });
127
+ } catch {
128
+ continue;
129
+ }
130
+ for (const v of versions) {
131
+ if (!v.isDirectory()) continue;
132
+ const vdir = path.join(repoDir, v.name);
133
+ let meta: { version?: unknown; build?: unknown } | null;
134
+ try {
135
+ meta = JSON.parse(fs.readFileSync(path.join(vdir, 'version.json'), 'utf8')) as { version?: unknown; build?: unknown } | null;
136
+ } catch {
137
+ continue; // missing/corrupt version.json: the library skips it too
138
+ }
139
+ out.push({
140
+ repo: repo.name,
141
+ name: v.name,
142
+ path: vdir,
143
+ rel: `browsers/${repo.name}/${v.name}`,
144
+ version: meta && typeof meta.version === 'string' ? meta.version : '',
145
+ build: meta && typeof meta.build === 'string' ? meta.build : '',
146
+ });
147
+ }
148
+ }
149
+ return out;
150
+ }
151
+
152
+ /** True when the NEW layout holds at least one usable browser version. */
153
+ export function hasInstall(dir: string): boolean {
154
+ return listInstalls(dir).length > 0;
155
+ }
156
+
157
+ /**
158
+ * Proof that `dir` is a legacy flat camoufox-js install: a root version.json carrying
159
+ * `version` + `release`, no `.0.5_FLAG`, and at least one file only a camoufox build has.
160
+ * Anything short of that is NOT claimed, so it is never removed or moved.
161
+ */
162
+ export function isLegacyFlat(dir: string): boolean {
163
+ if (!isDir(dir) || isFile(path.join(dir, COMPAT_FLAG))) return false;
164
+ let meta: { version?: unknown; release?: unknown } | null;
165
+ try {
166
+ meta = JSON.parse(fs.readFileSync(path.join(dir, 'version.json'), 'utf8')) as typeof meta;
167
+ } catch {
168
+ return false;
169
+ }
170
+ if (!meta || typeof meta.version !== 'string' || typeof meta.release !== 'string') return false;
171
+ return LEGACY_MARKERS.some((m) => fs.existsSync(path.join(dir, m)));
172
+ }
173
+
174
+ /**
175
+ * Whether `dir` is provably camoufox's own install dir (new or legacy layout, or the
176
+ * empty shell camoufox leaves behind). Used as the guard before any recursive delete.
177
+ */
178
+ export function isCamoufoxDir(dir: string): boolean {
179
+ if (!isDir(dir)) return false;
180
+ if (isFile(path.join(dir, COMPAT_FLAG))) return true;
181
+ if (hasInstall(dir)) return true;
182
+ if (isLegacyFlat(dir)) return true;
183
+ return false;
184
+ }
185
+
186
+ /** True when the user chose a channel/pin themselves (camoufox then honours it, so keep their versions). */
187
+ export function hasExplicitChoice(dir: string): boolean {
188
+ try {
189
+ const cfg = JSON.parse(fs.readFileSync(path.join(dir, 'config.json'), 'utf8')) as { pinned?: string; channel?: string } | null;
190
+ return Boolean(cfg && (cfg.pinned || cfg.channel));
191
+ } catch {
192
+ return false;
193
+ }
194
+ }
195
+
196
+ /**
197
+ * The browser build @camoufox/camoufox was released with, from its browser-pin.json,
198
+ * or null when unpinned/unreadable. `packageRoot` is the @camoufox/camoufox directory.
199
+ */
200
+ export function loadPin(packageRoot: string): BrowserPin | null {
201
+ try {
202
+ const data = JSON.parse(fs.readFileSync(path.join(packageRoot, 'dist', 'data-files', 'browser-pin.json'), 'utf8')) as { tag?: string; repo_name?: unknown; version?: unknown; build?: unknown } | null;
203
+ if (!data || !data.tag) return null;
204
+ return { repoName: String(data.repo_name).toLowerCase(), version: String(data.version), build: String(data.build) };
205
+ } catch {
206
+ return null;
207
+ }
208
+ }
209
+
210
+ /** The @camoufox/camoufox package directory as seen from `fromUrl`, or null. */
211
+ export function findPackageRoot(fromUrl: string): string | null {
212
+ try {
213
+ const require = createRequire(fromUrl);
214
+ return path.dirname(require.resolve('@camoufox/camoufox/package.json'));
215
+ } catch {
216
+ return null;
217
+ }
218
+ }
219
+
220
+ /** Whether `build` is the one `pin` names. */
221
+ export function matchesPin(build: InstalledBuild, pin: BrowserPin): boolean {
222
+ return build.repo.toLowerCase() === pin.repoName && build.version === pin.version && build.build === pin.build;
223
+ }
224
+
225
+ /**
226
+ * Remove every installed version except the one the library will launch: the paired
227
+ * build when `pin` is given and the user made no explicit choice, otherwise the active one. A released camoufox launches
228
+ * only the build it was released with, so superseded builds are pure disk weight.
229
+ * Does nothing when the user made an explicit `camoufox set` choice, when there is no
230
+ * identifiable active version, or when the active version's folder is missing.
231
+ * Returns the removed relative paths.
232
+ */
233
+ export function pruneSuperseded(dir: string, log: (m: string) => void = () => {}, pin: BrowserPin | null = null): string[] {
234
+ if (hasExplicitChoice(dir)) return [];
235
+ const installs = listInstalls(dir);
236
+ let keep: string | null = null;
237
+ if (pin) {
238
+ const paired = installs.find((i) => matchesPin(i, pin));
239
+ keep = paired ? paired.rel : null;
240
+ } else {
241
+ try {
242
+ keep = (JSON.parse(fs.readFileSync(path.join(dir, 'config.json'), 'utf8')) as { active_version?: string }).active_version || null;
243
+ } catch {
244
+ return [];
245
+ }
246
+ }
247
+ if (!keep || !installs.some((i) => i.rel === keep)) return [];
248
+ const removed: string[] = [];
249
+ for (const i of installs) {
250
+ if (i.rel === keep) continue;
251
+ try {
252
+ fs.rmSync(i.path, { recursive: true, force: true });
253
+ removed.push(i.rel);
254
+ log(`pi-research: removed superseded Camoufox build ${i.rel}`);
255
+ } catch (e) {
256
+ log(`pi-research: could not remove superseded Camoufox build ${i.rel}: ${e instanceof Error ? e.message : String(e)}`);
257
+ }
258
+ }
259
+ // Drop now-empty repo folders so listings stay clean.
260
+ try {
261
+ for (const r of fs.readdirSync(path.join(dir, 'browsers'))) {
262
+ const rd = path.join(dir, 'browsers', r);
263
+ if (isDir(rd) && fs.readdirSync(rd).length === 0) fs.rmdirSync(rd);
264
+ }
265
+ } catch { /* best effort */ }
266
+ return removed;
267
+ }
268
+
269
+ /**
270
+ * Move a PROVEN legacy flat install out of the way so `camoufox fetch` (which would
271
+ * rm -rf it before downloading) cannot destroy a working browser if the download fails.
272
+ * Returns the stash path, or null when `dir` is not a provable legacy install.
273
+ */
274
+ export function stashLegacy(dir: string): string | null {
275
+ if (!isLegacyFlat(dir)) return null;
276
+ const stash = `${dir}.legacy-${process.pid}-${Date.now()}`;
277
+ fs.renameSync(dir, stash);
278
+ return stash;
279
+ }
280
+
281
+ /** Put a stashed legacy install back (download failed). */
282
+ export function restoreLegacy(dir: string, stash: string | null): void {
283
+ if (!stash || !fs.existsSync(stash)) return;
284
+ try {
285
+ fs.rmSync(dir, { recursive: true, force: true });
286
+ } catch { /* partial new dir; best effort */ }
287
+ fs.renameSync(stash, dir);
288
+ }
289
+
290
+ /** Delete a stashed legacy install (new layout verified usable). */
291
+ export function discardLegacy(stash: string | null): void {
292
+ if (!stash) return;
293
+ fs.rmSync(stash, { recursive: true, force: true });
294
+ }
295
+
296
+ /** Leftover legacy stash dirs from a crashed run next to `dir`; deleted only when the new layout is usable. */
297
+ export function sweepStaleStashes(dir: string, log: (m: string) => void = () => {}): string[] {
298
+ if (!hasInstall(dir)) return [];
299
+ const parent = path.dirname(dir);
300
+ const prefix = `${path.basename(dir)}.legacy-`;
301
+ const removed: string[] = [];
302
+ let entries: string[];
303
+ try {
304
+ entries = fs.readdirSync(parent);
305
+ } catch {
306
+ return removed;
307
+ }
308
+ for (const e of entries) {
309
+ if (!e.startsWith(prefix)) continue;
310
+ const p = path.join(parent, e);
311
+ if (!isLegacyFlat(p)) continue;
312
+ try {
313
+ fs.rmSync(p, { recursive: true, force: true });
314
+ removed.push(p);
315
+ log(`pi-research: removed leftover legacy Camoufox stash ${p}`);
316
+ } catch { /* best effort */ }
317
+ }
318
+ return removed;
319
+ }
320
+
321
+ /**
322
+ * Remove the whole install dir, but only when it is provably camoufox's own.
323
+ * Returns 'removed' | 'absent' | 'refused'.
324
+ */
325
+ export function removeInstallDir(dir: string): 'removed' | 'absent' | 'refused' {
326
+ if (!fs.existsSync(dir)) return 'absent';
327
+ if (!isCamoufoxDir(dir)) return 'refused';
328
+ fs.rmSync(dir, { recursive: true, force: true });
329
+ return 'removed';
330
+ }
331
+
332
+ /** True when the NEW layout can launch: the compat flag plus the paired build (or any build when unpinned). */
333
+ export function isUsable(dir: string, pin: BrowserPin | null = null): boolean {
334
+ if (!isFile(path.join(dir, COMPAT_FLAG))) return false;
335
+ const installs = listInstalls(dir);
336
+ if (installs.length === 0) return false;
337
+ // A released library launches ONLY its paired build (unless the user chose one with
338
+ // `camoufox set`), and downloads it itself if absent. So "usable" means the paired
339
+ // build is there, which makes an upgrade to a new pin re-fetch centrally instead of in
340
+ // every worker at once.
341
+ if (pin && !hasExplicitChoice(dir)) return installs.some((i) => matchesPin(i, pin));
342
+ return true;
343
+ }
344
+
345
+ /** Stash dirs (`<dir>.legacy-<pid>-<ts>`) left beside `dir`, newest first, each proven legacy. */
346
+ export function listStashes(dir: string): string[] {
347
+ const parent = path.dirname(dir);
348
+ const prefix = `${path.basename(dir)}.legacy-`;
349
+ let entries: string[];
350
+ try {
351
+ entries = fs.readdirSync(parent);
352
+ } catch {
353
+ return [];
354
+ }
355
+ return entries
356
+ .filter((e) => e.startsWith(prefix))
357
+ .map((e) => path.join(parent, e))
358
+ .filter((p) => isLegacyFlat(p))
359
+ .sort()
360
+ .reverse();
361
+ }
362
+
363
+ /**
364
+ * Crash recovery: a previous run moved a legacy install aside and died before the
365
+ * new build landed. If `dir` is now absent/empty and a proven legacy stash exists,
366
+ * put it back so the user is not left without a browser. Returns the restored stash or null.
367
+ */
368
+ export function recoverStash(dir: string, log: (m: string) => void = () => {}): string | null {
369
+ let empty = !fs.existsSync(dir);
370
+ if (!empty) {
371
+ try {
372
+ empty = isDir(dir) && fs.readdirSync(dir).length === 0;
373
+ } catch {
374
+ return null;
375
+ }
376
+ }
377
+ if (!empty) return null;
378
+ const stash = listStashes(dir)[0];
379
+ if (!stash) return null;
380
+ try {
381
+ if (fs.existsSync(dir)) fs.rmdirSync(dir);
382
+ fs.renameSync(stash, dir);
383
+ log(`pi-research: restored the previous Camoufox install from ${stash}`);
384
+ return stash;
385
+ } catch {
386
+ return null;
387
+ }
388
+ }
389
+
390
+ export interface ProvisionResult {
391
+ status: 'present' | 'installed' | 'failed';
392
+ pruned: string[];
393
+ reason?: string;
394
+ }
395
+
396
+ /**
397
+ * Install the pinned browser without ever leaving the user with less than they had.
398
+ *
399
+ * `camoufox fetch` (and the library's own first launch) run `rm -rf <dir>` on a
400
+ * non-empty dir that has no compat flag, i.e. on every legacy flat install, BEFORE
401
+ * downloading. So a legacy install is moved aside first, the fetch runs against a clean
402
+ * dir, and the stash is deleted only after the new layout verifies usable; any failure
403
+ * restores it. `fetch` resolves true when the fetch process exited 0.
404
+ */
405
+ export async function provisionBrowser(
406
+ dir: string,
407
+ fetch: () => Promise<boolean>,
408
+ log: (m: string) => void = () => {},
409
+ pin: BrowserPin | null = null,
410
+ ): Promise<ProvisionResult> {
411
+ recoverStash(dir, log);
412
+ if (isUsable(dir, pin)) {
413
+ sweepStaleStashes(dir, log);
414
+ return { status: 'present', pruned: pruneSuperseded(dir, log, pin) };
415
+ }
416
+ let stash: string | null;
417
+ try {
418
+ stash = stashLegacy(dir);
419
+ } catch (e) {
420
+ // Could not move it (in use on Windows, permissions): fetching now would delete it.
421
+ return { status: 'failed', pruned: [], reason: `could not move the existing legacy install aside (${e instanceof Error ? e.message : String(e)}); left untouched` };
422
+ }
423
+ if (stash) log(`pi-research: legacy Camoufox install moved aside to ${stash} until the new browser is verified`);
424
+ let ok = false;
425
+ let reason: string | undefined;
426
+ try {
427
+ ok = (await fetch()) === true;
428
+ if (!ok) reason = 'camoufox fetch did not exit 0';
429
+ } catch (e) {
430
+ reason = e instanceof Error ? e.message : String(e);
431
+ }
432
+ // A non-zero exit is NOT proof the browser is unusable. The launcher extracts the
433
+ // browser and writes the compat flag BEFORE it fetches its companion geoip database
434
+ // from a separate GitHub release, so a failure there (observed: `UND_ERR_SOCKET other
435
+ // side closed` on ubuntu-latest, three runs in a row on 2026-10-05) exits non-zero with
436
+ // a fully launchable paired build already in place. Verified: a real headless launch
437
+ // succeeds with the mmdb removed. Keep the install and drop the legacy stash rather
438
+ // than reporting a failure and restoring an older browser on top of a working one.
439
+ if (isUsable(dir, pin)) {
440
+ discardLegacy(stash);
441
+ if (stash) log('pi-research: removed the legacy Camoufox install (replaced by the new build)');
442
+ if (!ok) log(`pi-research: camoufox fetch reported a failure after the browser was installed (${reason}); keeping it. Run "npx camoufox fetch" to retry the companion download (e.g. the geoip database).`);
443
+ sweepStaleStashes(dir, log);
444
+ return { status: 'installed', pruned: pruneSuperseded(dir, log, pin) };
445
+ }
446
+ if (ok) reason = 'camoufox fetch exited 0 but no usable build was found';
447
+ restoreLegacy(dir, stash);
448
+ if (stash) log('pi-research: restored the previous Camoufox install after a failed upgrade');
449
+ return { status: 'failed', pruned: [], reason };
450
+ }
@@ -9,6 +9,7 @@ import * as crypto from 'node:crypto';
9
9
  import { join } from 'node:path';
10
10
  import { existsSync, mkdirSync } from 'node:fs';
11
11
  import { platform, homedir } from 'node:os';
12
+ import { resolveInstall, isUsable, loadPin, findPackageRoot } from './camoufox-layout.ts';
12
13
  import type { Config } from '../../config.ts';
13
14
  import { getConfig } from '../../config.ts';
14
15
  import { getLogger } from '../../logger.ts';
@@ -19,63 +20,26 @@ import { createRequire } from 'node:module';
19
20
  // ============================================================================
20
21
 
21
22
  /**
22
- * The directory where camoufox-js installs the browser on Windows.
23
+ * Where the Camoufox browser lives, and which environment overrides make the
24
+ * browser launcher agree with that.
23
25
  *
24
- * camoufox-js's userCacheDir("camoufox") resolves to
25
- * os.homedir()/AppData/Local/camoufox/camoufox/Cache
26
- * — note the DOUBLED "camoufox" segment, and that it uses os.homedir() rather
27
- * than %LOCALAPPDATA%. We must mirror it exactly, otherwise isBrowserAvailable()
28
- * looks in the wrong place and every browser test silently skips on Windows
29
- * even though the binary downloaded and launches fine.
26
+ * `@camoufox/camoufox` derives its install directory ONLY from the platform cache
27
+ * location (XDG_CACHE_HOME on Linux, LOCALAPPDATA on Windows, $HOME/Library/Caches on
28
+ * macOS). Unlike the camoufox-js it replaced, it does NOT read CAMOUFOX_INSTALL_DIR
29
+ * or PLAYWRIGHT_BROWSERS_PATH. Both variables are still honoured here as the
30
+ * documented way to relocate the browser, by translating them into the platform
31
+ * override the launcher does read (see resolveInstall in camoufox-layout.ts). The
32
+ * lookup (getCamoufoxBinaryPath), the install (scripts/setup.cjs,
33
+ * ensure-browser.ts) and the worker launch (getBrowserEnv) all go through that one
34
+ * function, so they cannot disagree.
30
35
  */
31
- function getWindowsCamoufoxDir(): string {
32
- return join(homedir(), 'AppData', 'Local', 'camoufox', 'camoufox', 'Cache');
33
- }
34
-
35
- /**
36
- * The custom camoufox location, if the user set one.
37
- *
38
- * CAMOUFOX_INSTALL_DIR takes precedence because it is camoufox-js's OWN
39
- * variable and the only one that can ever move the install: camoufox-js ≥0.12.0
40
- * resolves INSTALL_DIR from it. PLAYWRIGHT_BROWSERS_PATH is kept as a documented
41
- * alias, but on its own it only ever moved where we LOOK — camoufox went on
42
- * installing to the default cache, so setting it made the browser permanently
43
- * "not found" right after a successful download. Both are exported onward to
44
- * camoufox-js (getBrowserEnv, scripts/setup.cjs) so that install and lookup
45
- * cannot disagree.
46
- *
47
- * HISTORY: camoufox-js <0.12.0 hardcoded `userCacheDir("camoufox")` and
48
- * honoured NEITHER variable, which made this plumbing deliberately inert for
49
- * two pin cycles. The 0.12.0 refresh (2026-08-30) honours CAMOUFOX_INSTALL_DIR
50
- * again, so a custom location now relocates the install itself, not just our
51
- * lookup — the two can no longer disagree via version skew, but keep the dual
52
- * export (install + lookup) so they can never drift apart in future.
53
- */
54
- function getCustomCamoufoxDir(): string | undefined {
55
- return process.env['CAMOUFOX_INSTALL_DIR'] || process.env['PLAYWRIGHT_BROWSERS_PATH'] || undefined;
56
- }
57
36
 
58
37
  /**
59
- * Get the camoufox binary cache directory.
60
- * Uses the custom location above if set, otherwise the standard user cache location
61
- * that camoufox uses by default (~/.cache/camoufox on Linux).
38
+ * Get the camoufox install directory (the directory that holds `browsers/`).
62
39
  * We do NOT override HOME — that trick was unreliable and caused install/runtime mismatches.
63
40
  */
64
41
  export function getBrowserCacheDir(): string {
65
- const custom = getCustomCamoufoxDir();
66
- if (custom) {
67
- return custom;
68
- }
69
- // Mirror getCamoufoxBinaryPath() so both functions agree on the cache location
70
- const osPlatform = platform();
71
- if (osPlatform === 'win32') {
72
- return getWindowsCamoufoxDir();
73
- } else if (osPlatform === 'darwin') {
74
- return join(homedir(), 'Library', 'Caches', 'camoufox');
75
- } else {
76
- const cacheHome = process.env['XDG_CACHE_HOME'] || join(homedir(), '.cache');
77
- return join(cacheHome, 'camoufox');
78
- }
42
+ return resolveInstall(process.env, platform(), homedir()).dir;
79
43
  }
80
44
 
81
45
  /**
@@ -134,18 +98,16 @@ export function getBrowserProfileDir(config?: Config): string {
134
98
  */
135
99
  export function getBrowserEnv(config?: Config): NodeJS.ProcessEnv {
136
100
  const env: NodeJS.ProcessEnv = { ...process.env };
137
- const customPath = getCustomCamoufoxDir();
138
- if (customPath) {
139
- env['PLAYWRIGHT_BROWSERS_PATH'] = customPath;
140
- // The one that camoufox-js actually reads. Without it the worker's
141
- // camoufox launches from the DEFAULT cache while everything on this side
142
- // resolves the custom path — the install/runtime mismatch this file's
143
- // header warns about, just arriving through a different door.
144
- env['CAMOUFOX_INSTALL_DIR'] = customPath;
145
- } else {
146
- delete env['PLAYWRIGHT_BROWSERS_PATH'];
147
- delete env['CAMOUFOX_INSTALL_DIR'];
148
- }
101
+ // Hand the worker the placement overrides the browser launcher actually reads
102
+ // (XDG_CACHE_HOME / LOCALAPPDATA), derived from the user's custom dir. Without
103
+ // them the worker launches from the DEFAULT cache while everything on this side
104
+ // resolves the custom one: the install/runtime mismatch this file's header warns
105
+ // about. The two user-facing variables are removed so a nested tool cannot
106
+ // reinterpret them.
107
+ const install = resolveInstall(process.env, platform(), homedir());
108
+ Object.assign(env, install.overrides);
109
+ delete env['PLAYWRIGHT_BROWSERS_PATH'];
110
+ delete env['CAMOUFOX_INSTALL_DIR'];
149
111
  // Tell thread-workers where to write their lifecycle/error log. Precedence:
150
112
  // 1. A user-set PI_RESEARCH_LOG_FILE (already copied from process.env above) wins —
151
113
  // never clobber an explicit override.
@@ -194,26 +156,12 @@ export function ensureBrowserCacheDir(): string {
194
156
  }
195
157
 
196
158
  /**
197
- * Get the expected path where camoufox installs its binaries.
198
- * Matches camoufox-js's own resolution logic.
159
+ * Get the path where camoufox installs its browsers (the directory holding `browsers/`).
160
+ * Same answer as getBrowserCacheDir(); kept as a separate name for the callers that
161
+ * think of it as "the binary location".
199
162
  */
200
163
  export function getCamoufoxBinaryPath(): string {
201
- const customPath = getCustomCamoufoxDir();
202
- if (customPath) {
203
- return customPath;
204
- }
205
-
206
- const osPlatform = platform();
207
-
208
- if (osPlatform === 'win32') {
209
- return getWindowsCamoufoxDir();
210
- } else if (osPlatform === 'darwin') {
211
- return join(homedir(), 'Library', 'Caches', 'camoufox');
212
- } else {
213
- // Linux and others
214
- const cacheHome = process.env['XDG_CACHE_HOME'] || join(homedir(), '.cache');
215
- return join(cacheHome, 'camoufox');
216
- }
164
+ return getBrowserCacheDir();
217
165
  }
218
166
 
219
167
  // ============================================================================
@@ -255,7 +203,7 @@ export function getSchedulerVersion(config?: Config): string {
255
203
  *
256
204
  * Returns true (true headless — no visible window) on Windows and macOS.
257
205
  * Historically Windows used headless:false because headless:true crashed Firefox
258
- * (exit 0x80000003, camoufox-js issue #614); that is fixed in camoufox-js >=0.10
206
+ * (exit 0x80000003, camoufox issue #614); that is fixed in camoufox-js >=0.10
259
207
  * (verified on Windows 11 x64 with 0.10.2: headless:true launches and navigates
260
208
  * reliably and, crucially, NO browser window pops up on the desktop). Using
261
209
  * headless:false on a real Windows desktop flashed visible, sometimes fullscreen,
@@ -302,8 +250,8 @@ export function resolveHeadlessMode(): boolean | 'virtual' {
302
250
  * Check if the browser is available for meaningful browser pool testing.
303
251
  *
304
252
  * Returns false if:
305
- * - camoufox-js package is not installed
306
- * - camoufox binary is not present at expected path
253
+ * - @camoufox/camoufox package is not installed
254
+ * - no browser build is installed under the cache dir (new `browsers/<repo>/<version>` layout)
307
255
  * - FULL_MOCK_MODE is active (both search and scrape mocked)
308
256
  *
309
257
  * In FULL_MOCK_MODE, browser pool tests are not meaningful because:
@@ -317,22 +265,16 @@ export function resolveHeadlessMode(): boolean | 'virtual' {
317
265
  * Whether the browser stack's NATIVE dependencies can actually load.
318
266
  *
319
267
  * `isBrowserAvailable()` answers a different question — is the camoufox binary on
320
- * disk — and a browser can be fully downloaded and still unable to launch. camoufox-js
321
- * requires `better-sqlite3`, whose binding is produced by a dependency INSTALL SCRIPT,
322
- * and npm 12 turns those off by default. The module then imports fine and throws only
323
- * when first used, so nothing notices until every browser worker dies mid-run with
324
- * "Could not locate the bindings file" and the run reports a network problem it does
325
- * not have.
326
- *
327
- * Measured, not assumed: of this package's native dependencies, only better-sqlite3
328
- * fails a scripts-blocked install. onnxruntime-node ships its binding inside its own
329
- * tarball, and lancedb, impit and html-to-markdown all resolve prebuilt platform
330
- * packages, so the knowledge store keeps working while search is dead — exactly the
331
- * split seen in the field.
268
+ * disk — and a browser can be fully downloaded and still unable to launch.
332
269
  *
333
- * Opening an in-memory database is the only check that settles it: resolving the
334
- * module or stat-ing a path does not, because the failure is in the binding lookup at
335
- * first use. It costs a few milliseconds and is confined to diagnostics.
270
+ * History: camoufox-js required `better-sqlite3`, whose binding came from a dependency
271
+ * INSTALL SCRIPT that npm 12 turns off, so the module imported fine and every browser
272
+ * worker died at first use. `@camoufox/camoufox` has no better-sqlite3; the one native
273
+ * module left in the launcher's path is `impit` (its IP/geo lookups), which resolves a
274
+ * prebuilt platform package. That package can still be dropped by npm's optional-
275
+ * dependency bug (npm/cli#4828), and the failure is again at first use, so the probe
276
+ * loads it: resolving the module name or stat-ing a path does not settle it. It costs a
277
+ * few milliseconds and is confined to diagnostics.
336
278
  */
337
279
  export function probeBrowserNativeDeps(
338
280
  /** Injected for tests. Production callers pass nothing. */
@@ -350,16 +292,17 @@ export function probeBrowserNativeDeps(
350
292
 
351
293
  function defaultNativeDepLoad(): void {
352
294
  const require = createRequire(import.meta.url);
353
- const Database = require('better-sqlite3') as new (p: string) => { close(): void };
354
- const db = new Database(':memory:');
355
- db.close();
295
+ // Requiring `impit` loads its native binding eagerly and throws if the platform
296
+ // package is missing.
297
+ require('impit');
356
298
  }
357
299
 
358
300
  export function isBrowserAvailable(): boolean {
359
301
  if (isFullMockMode()) return false;
360
302
  try {
361
- import.meta.resolve('camoufox-js');
362
- return existsSync(getCamoufoxBinaryPath());
303
+ import.meta.resolve('@camoufox/camoufox');
304
+ const root = findPackageRoot(import.meta.url);
305
+ return isUsable(getCamoufoxBinaryPath(), root ? loadPin(root) : null);
363
306
  } catch {
364
307
  return false;
365
308
  }