@lincoln504/pi-research 1.7.8 → 1.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent-skill/pi-research/README.md +3 -3
- package/agent-skill/pi-research/SKILL.md +1 -1
- package/dist/cli-bin.mjs +3 -0
- package/dist/cli.mjs +652 -380
- package/dist/pdf-extract-worker.mjs +1 -1
- package/dist/thread-worker.mjs +6 -6
- package/docs/AGENT-SKILL.md +3 -3
- package/docs/ARCHITECTURE.md +64 -50
- package/docs/CHANGELOG.md +21 -0
- package/docs/CONFIGURATION.md +2 -2
- package/docs/SDK.md +7 -6
- package/docs/THIRD-PARTY-NOTICES.md +11 -7
- package/package.json +10 -8
- package/scripts/camoufox-layout.cjs +455 -0
- package/scripts/cleanup.cjs +72 -29
- package/scripts/setup.cjs +72 -111
- package/src/cli-bin.mjs +3 -0
- package/src/cli.ts +12 -13
- package/src/healthcheck/index.ts +1 -1
- package/src/infrastructure/browser/browser-cleanup.ts +1 -1
- package/src/infrastructure/browser/browser-error-utils.ts +5 -4
- package/src/infrastructure/browser/camoufox-layout.ts +450 -0
- package/src/infrastructure/browser/config.ts +46 -103
- package/src/infrastructure/browser/ensure-browser.ts +63 -49
- package/src/infrastructure/browser/thread-worker-browser.ts +38 -42
- package/src/infrastructure/browser/thread-worker.mjs +6 -6
- package/src/infrastructure/knowledge-store-service.ts +6 -6
- package/src/orchestration/researcher-executor.ts +47 -1
- package/src/tui/research-health.ts +1 -1
- package/src/utils/log-utils.ts +1 -1
- package/src/web-research/browser-search.ts +8 -10
- package/src/web-research/pdf-extract-worker.mjs +1 -1
- package/src/web-research/scraper-utils.ts +11 -2
- package/src/web-research/web-scraper.ts +1 -1
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Camoufox install-layout helpers for the runtime (ensure-browser, config, cli).
|
|
3
|
+
*
|
|
4
|
+
* TWIN of scripts/camoufox-layout.cjs (the install/uninstall half). The two
|
|
5
|
+
* are kept equivalent on purpose and test/unit/infrastructure/
|
|
6
|
+
* camoufox-layout.test.ts runs both against the same fixtures, because a drift
|
|
7
|
+
* between "where setup installed it" and "where the runtime looks" is exactly the
|
|
8
|
+
* install/runtime mismatch this package has shipped before.
|
|
9
|
+
*
|
|
10
|
+
* Why this exists. `@camoufox/camoufox` (0.5.x) differs from the old `camoufox-js`:
|
|
11
|
+
* - It installs under <INSTALL_DIR>/browsers/<repo>/<version>/ (multi-version),
|
|
12
|
+
* with `config.json`, `repo_cache.json` and a `.0.5_FLAG` beside it. camoufox-js
|
|
13
|
+
* put the browser files FLAT in INSTALL_DIR with a root `version.json`.
|
|
14
|
+
* - INSTALL_DIR is derived ONLY from platform cache dirs (XDG_CACHE_HOME on Linux,
|
|
15
|
+
* LOCALAPPDATA on Windows, $HOME/Library/Caches on macOS). It does NOT read
|
|
16
|
+
* CAMOUFOX_INSTALL_DIR or PLAYWRIGHT_BROWSERS_PATH.
|
|
17
|
+
* - `camoufox fetch` does `rm -rf INSTALL_DIR` when it is non-empty and has no
|
|
18
|
+
* `.0.5_FLAG` ("Cleaning old data...") BEFORE it downloads anything, so a failed
|
|
19
|
+
* download over a legacy install leaves the user with no browser at all.
|
|
20
|
+
* - A newer pi-research release that pins a newer browser installs the new build
|
|
21
|
+
* beside the old one and never removes it; the old one is dead weight because a
|
|
22
|
+
* released library only ever launches its own pinned build.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import * as fs from 'node:fs';
|
|
26
|
+
import * as os from 'node:os';
|
|
27
|
+
import * as path from 'node:path';
|
|
28
|
+
import { createRequire } from 'node:module';
|
|
29
|
+
|
|
30
|
+
export const COMPAT_FLAG = '.0.5_FLAG';
|
|
31
|
+
|
|
32
|
+
/** Names whose presence, with a root version.json, proves a legacy camoufox-js install. */
|
|
33
|
+
const LEGACY_MARKERS = ['camoufox-bin', 'camoufox', 'camoufox.exe', 'application.ini', 'Camoufox.app', 'properties.json'];
|
|
34
|
+
|
|
35
|
+
function isDir(p: string): boolean {
|
|
36
|
+
try {
|
|
37
|
+
return fs.statSync(p).isDirectory();
|
|
38
|
+
} catch {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function isFile(p: string): boolean {
|
|
44
|
+
try {
|
|
45
|
+
return fs.statSync(p).isFile();
|
|
46
|
+
} catch {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** The custom location the user asked for, if any (camoufox itself ignores both variables). */
|
|
52
|
+
export function customDir(env: NodeJS.ProcessEnv = process.env): string | null {
|
|
53
|
+
return env['CAMOUFOX_INSTALL_DIR'] || env['PLAYWRIGHT_BROWSERS_PATH'] || null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Mirrors @camoufox/camoufox paths.js userCacheDir('camoufox') for a given environment. */
|
|
57
|
+
export function defaultInstallDir(env: NodeJS.ProcessEnv = process.env, platform: NodeJS.Platform = process.platform, home: string = os.homedir()): string {
|
|
58
|
+
if (platform === 'win32') {
|
|
59
|
+
const base = env['LOCALAPPDATA'] && path.win32.isAbsolute(env['LOCALAPPDATA'])
|
|
60
|
+
? env['LOCALAPPDATA']
|
|
61
|
+
: path.join(home, 'AppData', 'Local');
|
|
62
|
+
return path.join(base, 'camoufox', 'camoufox', 'Cache');
|
|
63
|
+
}
|
|
64
|
+
if (platform === 'darwin') return path.join(home, 'Library', 'Caches', 'camoufox');
|
|
65
|
+
const xdg = (env['XDG_CACHE_HOME'] || '').trim();
|
|
66
|
+
return path.join(xdg ? xdg : path.join(home, '.cache'), 'camoufox');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Resolve where the browser lives and which env overrides make camoufox use it.
|
|
71
|
+
*
|
|
72
|
+
* - No custom dir: the platform default, no overrides.
|
|
73
|
+
* - Linux: custom is the cache HOME (XDG_CACHE_HOME=custom, browser in custom/camoufox),
|
|
74
|
+
* except when custom already ends in `camoufox`, where its parent becomes the
|
|
75
|
+
* cache home so the browser lives in custom itself (old camoufox-js semantics).
|
|
76
|
+
* - Windows: same idea through LOCALAPPDATA (browser in <base>\camoufox\camoufox\Cache).
|
|
77
|
+
* - macOS: camoufox derives the cache from $HOME only; relocating it would mean
|
|
78
|
+
* redirecting HOME for the browser process, which also moves Firefox's own
|
|
79
|
+
* state. The custom dir is ignored there (`ignoredCustom` is set so callers can say so).
|
|
80
|
+
*/
|
|
81
|
+
export interface ResolvedInstall {
|
|
82
|
+
dir: string;
|
|
83
|
+
overrides: Record<string, string>;
|
|
84
|
+
custom: string | null;
|
|
85
|
+
ignoredCustom: boolean;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function resolveInstall(env: NodeJS.ProcessEnv = process.env, platform: NodeJS.Platform = process.platform, home: string = os.homedir()): ResolvedInstall {
|
|
89
|
+
const custom = customDir(env);
|
|
90
|
+
if (!custom) return { dir: defaultInstallDir(env, platform, home), overrides: {}, custom: null, ignoredCustom: false };
|
|
91
|
+
if (platform === 'darwin') {
|
|
92
|
+
return { dir: defaultInstallDir(env, platform, home), overrides: {}, custom, ignoredCustom: true };
|
|
93
|
+
}
|
|
94
|
+
const resolved = (platform === 'win32' ? path.win32 : path).resolve(custom);
|
|
95
|
+
if (platform === 'win32') {
|
|
96
|
+
// <LOCALAPPDATA>\camoufox\camoufox\Cache : one fixed shape, custom = LOCALAPPDATA.
|
|
97
|
+
const overrides: Record<string, string> = { LOCALAPPDATA: resolved };
|
|
98
|
+
return { dir: defaultInstallDir({ ...env, ...overrides }, platform, home), overrides, custom, ignoredCustom: false };
|
|
99
|
+
}
|
|
100
|
+
const endsInCamoufox = path.basename(resolved).toLowerCase() === 'camoufox';
|
|
101
|
+
const cacheHome = endsInCamoufox ? path.dirname(resolved) : resolved;
|
|
102
|
+
const overrides: Record<string, string> = { XDG_CACHE_HOME: cacheHome };
|
|
103
|
+
return { dir: defaultInstallDir({ ...env, ...overrides }, platform, home), overrides, custom, ignoredCustom: false };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Every `browsers/<repo>/<version>/` directory that holds a parseable version.json. */
|
|
107
|
+
export interface InstalledBuild { repo: string; name: string; path: string; rel: string; version: string; build: string }
|
|
108
|
+
|
|
109
|
+
/** The browser build @camoufox/camoufox was released with (repoName lowercased). */
|
|
110
|
+
export interface BrowserPin { repoName: string; version: string; build: string }
|
|
111
|
+
|
|
112
|
+
export function listInstalls(dir: string): InstalledBuild[] {
|
|
113
|
+
const out: InstalledBuild[] = [];
|
|
114
|
+
const browsers = path.join(dir, 'browsers');
|
|
115
|
+
let repos: fs.Dirent[];
|
|
116
|
+
try {
|
|
117
|
+
repos = fs.readdirSync(browsers, { withFileTypes: true });
|
|
118
|
+
} catch {
|
|
119
|
+
return out;
|
|
120
|
+
}
|
|
121
|
+
for (const repo of repos) {
|
|
122
|
+
if (!repo.isDirectory() || repo.name.startsWith('.')) continue;
|
|
123
|
+
const repoDir = path.join(browsers, repo.name);
|
|
124
|
+
let versions: fs.Dirent[];
|
|
125
|
+
try {
|
|
126
|
+
versions = fs.readdirSync(repoDir, { withFileTypes: true });
|
|
127
|
+
} catch {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
for (const v of versions) {
|
|
131
|
+
if (!v.isDirectory()) continue;
|
|
132
|
+
const vdir = path.join(repoDir, v.name);
|
|
133
|
+
let meta: { version?: unknown; build?: unknown } | null;
|
|
134
|
+
try {
|
|
135
|
+
meta = JSON.parse(fs.readFileSync(path.join(vdir, 'version.json'), 'utf8')) as { version?: unknown; build?: unknown } | null;
|
|
136
|
+
} catch {
|
|
137
|
+
continue; // missing/corrupt version.json: the library skips it too
|
|
138
|
+
}
|
|
139
|
+
out.push({
|
|
140
|
+
repo: repo.name,
|
|
141
|
+
name: v.name,
|
|
142
|
+
path: vdir,
|
|
143
|
+
rel: `browsers/${repo.name}/${v.name}`,
|
|
144
|
+
version: meta && typeof meta.version === 'string' ? meta.version : '',
|
|
145
|
+
build: meta && typeof meta.build === 'string' ? meta.build : '',
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return out;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** True when the NEW layout holds at least one usable browser version. */
|
|
153
|
+
export function hasInstall(dir: string): boolean {
|
|
154
|
+
return listInstalls(dir).length > 0;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Proof that `dir` is a legacy flat camoufox-js install: a root version.json carrying
|
|
159
|
+
* `version` + `release`, no `.0.5_FLAG`, and at least one file only a camoufox build has.
|
|
160
|
+
* Anything short of that is NOT claimed, so it is never removed or moved.
|
|
161
|
+
*/
|
|
162
|
+
export function isLegacyFlat(dir: string): boolean {
|
|
163
|
+
if (!isDir(dir) || isFile(path.join(dir, COMPAT_FLAG))) return false;
|
|
164
|
+
let meta: { version?: unknown; release?: unknown } | null;
|
|
165
|
+
try {
|
|
166
|
+
meta = JSON.parse(fs.readFileSync(path.join(dir, 'version.json'), 'utf8')) as typeof meta;
|
|
167
|
+
} catch {
|
|
168
|
+
return false;
|
|
169
|
+
}
|
|
170
|
+
if (!meta || typeof meta.version !== 'string' || typeof meta.release !== 'string') return false;
|
|
171
|
+
return LEGACY_MARKERS.some((m) => fs.existsSync(path.join(dir, m)));
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Whether `dir` is provably camoufox's own install dir (new or legacy layout, or the
|
|
176
|
+
* empty shell camoufox leaves behind). Used as the guard before any recursive delete.
|
|
177
|
+
*/
|
|
178
|
+
export function isCamoufoxDir(dir: string): boolean {
|
|
179
|
+
if (!isDir(dir)) return false;
|
|
180
|
+
if (isFile(path.join(dir, COMPAT_FLAG))) return true;
|
|
181
|
+
if (hasInstall(dir)) return true;
|
|
182
|
+
if (isLegacyFlat(dir)) return true;
|
|
183
|
+
return false;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** True when the user chose a channel/pin themselves (camoufox then honours it, so keep their versions). */
|
|
187
|
+
export function hasExplicitChoice(dir: string): boolean {
|
|
188
|
+
try {
|
|
189
|
+
const cfg = JSON.parse(fs.readFileSync(path.join(dir, 'config.json'), 'utf8')) as { pinned?: string; channel?: string } | null;
|
|
190
|
+
return Boolean(cfg && (cfg.pinned || cfg.channel));
|
|
191
|
+
} catch {
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* The browser build @camoufox/camoufox was released with, from its browser-pin.json,
|
|
198
|
+
* or null when unpinned/unreadable. `packageRoot` is the @camoufox/camoufox directory.
|
|
199
|
+
*/
|
|
200
|
+
export function loadPin(packageRoot: string): BrowserPin | null {
|
|
201
|
+
try {
|
|
202
|
+
const data = JSON.parse(fs.readFileSync(path.join(packageRoot, 'dist', 'data-files', 'browser-pin.json'), 'utf8')) as { tag?: string; repo_name?: unknown; version?: unknown; build?: unknown } | null;
|
|
203
|
+
if (!data || !data.tag) return null;
|
|
204
|
+
return { repoName: String(data.repo_name).toLowerCase(), version: String(data.version), build: String(data.build) };
|
|
205
|
+
} catch {
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** The @camoufox/camoufox package directory as seen from `fromUrl`, or null. */
|
|
211
|
+
export function findPackageRoot(fromUrl: string): string | null {
|
|
212
|
+
try {
|
|
213
|
+
const require = createRequire(fromUrl);
|
|
214
|
+
return path.dirname(require.resolve('@camoufox/camoufox/package.json'));
|
|
215
|
+
} catch {
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** Whether `build` is the one `pin` names. */
|
|
221
|
+
export function matchesPin(build: InstalledBuild, pin: BrowserPin): boolean {
|
|
222
|
+
return build.repo.toLowerCase() === pin.repoName && build.version === pin.version && build.build === pin.build;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Remove every installed version except the one the library will launch: the paired
|
|
227
|
+
* build when `pin` is given and the user made no explicit choice, otherwise the active one. A released camoufox launches
|
|
228
|
+
* only the build it was released with, so superseded builds are pure disk weight.
|
|
229
|
+
* Does nothing when the user made an explicit `camoufox set` choice, when there is no
|
|
230
|
+
* identifiable active version, or when the active version's folder is missing.
|
|
231
|
+
* Returns the removed relative paths.
|
|
232
|
+
*/
|
|
233
|
+
export function pruneSuperseded(dir: string, log: (m: string) => void = () => {}, pin: BrowserPin | null = null): string[] {
|
|
234
|
+
if (hasExplicitChoice(dir)) return [];
|
|
235
|
+
const installs = listInstalls(dir);
|
|
236
|
+
let keep: string | null = null;
|
|
237
|
+
if (pin) {
|
|
238
|
+
const paired = installs.find((i) => matchesPin(i, pin));
|
|
239
|
+
keep = paired ? paired.rel : null;
|
|
240
|
+
} else {
|
|
241
|
+
try {
|
|
242
|
+
keep = (JSON.parse(fs.readFileSync(path.join(dir, 'config.json'), 'utf8')) as { active_version?: string }).active_version || null;
|
|
243
|
+
} catch {
|
|
244
|
+
return [];
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
if (!keep || !installs.some((i) => i.rel === keep)) return [];
|
|
248
|
+
const removed: string[] = [];
|
|
249
|
+
for (const i of installs) {
|
|
250
|
+
if (i.rel === keep) continue;
|
|
251
|
+
try {
|
|
252
|
+
fs.rmSync(i.path, { recursive: true, force: true });
|
|
253
|
+
removed.push(i.rel);
|
|
254
|
+
log(`pi-research: removed superseded Camoufox build ${i.rel}`);
|
|
255
|
+
} catch (e) {
|
|
256
|
+
log(`pi-research: could not remove superseded Camoufox build ${i.rel}: ${e instanceof Error ? e.message : String(e)}`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
// Drop now-empty repo folders so listings stay clean.
|
|
260
|
+
try {
|
|
261
|
+
for (const r of fs.readdirSync(path.join(dir, 'browsers'))) {
|
|
262
|
+
const rd = path.join(dir, 'browsers', r);
|
|
263
|
+
if (isDir(rd) && fs.readdirSync(rd).length === 0) fs.rmdirSync(rd);
|
|
264
|
+
}
|
|
265
|
+
} catch { /* best effort */ }
|
|
266
|
+
return removed;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Move a PROVEN legacy flat install out of the way so `camoufox fetch` (which would
|
|
271
|
+
* rm -rf it before downloading) cannot destroy a working browser if the download fails.
|
|
272
|
+
* Returns the stash path, or null when `dir` is not a provable legacy install.
|
|
273
|
+
*/
|
|
274
|
+
export function stashLegacy(dir: string): string | null {
|
|
275
|
+
if (!isLegacyFlat(dir)) return null;
|
|
276
|
+
const stash = `${dir}.legacy-${process.pid}-${Date.now()}`;
|
|
277
|
+
fs.renameSync(dir, stash);
|
|
278
|
+
return stash;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Put a stashed legacy install back (download failed). */
|
|
282
|
+
export function restoreLegacy(dir: string, stash: string | null): void {
|
|
283
|
+
if (!stash || !fs.existsSync(stash)) return;
|
|
284
|
+
try {
|
|
285
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
286
|
+
} catch { /* partial new dir; best effort */ }
|
|
287
|
+
fs.renameSync(stash, dir);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** Delete a stashed legacy install (new layout verified usable). */
|
|
291
|
+
export function discardLegacy(stash: string | null): void {
|
|
292
|
+
if (!stash) return;
|
|
293
|
+
fs.rmSync(stash, { recursive: true, force: true });
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/** Leftover legacy stash dirs from a crashed run next to `dir`; deleted only when the new layout is usable. */
|
|
297
|
+
export function sweepStaleStashes(dir: string, log: (m: string) => void = () => {}): string[] {
|
|
298
|
+
if (!hasInstall(dir)) return [];
|
|
299
|
+
const parent = path.dirname(dir);
|
|
300
|
+
const prefix = `${path.basename(dir)}.legacy-`;
|
|
301
|
+
const removed: string[] = [];
|
|
302
|
+
let entries: string[];
|
|
303
|
+
try {
|
|
304
|
+
entries = fs.readdirSync(parent);
|
|
305
|
+
} catch {
|
|
306
|
+
return removed;
|
|
307
|
+
}
|
|
308
|
+
for (const e of entries) {
|
|
309
|
+
if (!e.startsWith(prefix)) continue;
|
|
310
|
+
const p = path.join(parent, e);
|
|
311
|
+
if (!isLegacyFlat(p)) continue;
|
|
312
|
+
try {
|
|
313
|
+
fs.rmSync(p, { recursive: true, force: true });
|
|
314
|
+
removed.push(p);
|
|
315
|
+
log(`pi-research: removed leftover legacy Camoufox stash ${p}`);
|
|
316
|
+
} catch { /* best effort */ }
|
|
317
|
+
}
|
|
318
|
+
return removed;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* Remove the whole install dir, but only when it is provably camoufox's own.
|
|
323
|
+
* Returns 'removed' | 'absent' | 'refused'.
|
|
324
|
+
*/
|
|
325
|
+
export function removeInstallDir(dir: string): 'removed' | 'absent' | 'refused' {
|
|
326
|
+
if (!fs.existsSync(dir)) return 'absent';
|
|
327
|
+
if (!isCamoufoxDir(dir)) return 'refused';
|
|
328
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
329
|
+
return 'removed';
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** True when the NEW layout can launch: the compat flag plus the paired build (or any build when unpinned). */
|
|
333
|
+
export function isUsable(dir: string, pin: BrowserPin | null = null): boolean {
|
|
334
|
+
if (!isFile(path.join(dir, COMPAT_FLAG))) return false;
|
|
335
|
+
const installs = listInstalls(dir);
|
|
336
|
+
if (installs.length === 0) return false;
|
|
337
|
+
// A released library launches ONLY its paired build (unless the user chose one with
|
|
338
|
+
// `camoufox set`), and downloads it itself if absent. So "usable" means the paired
|
|
339
|
+
// build is there, which makes an upgrade to a new pin re-fetch centrally instead of in
|
|
340
|
+
// every worker at once.
|
|
341
|
+
if (pin && !hasExplicitChoice(dir)) return installs.some((i) => matchesPin(i, pin));
|
|
342
|
+
return true;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/** Stash dirs (`<dir>.legacy-<pid>-<ts>`) left beside `dir`, newest first, each proven legacy. */
|
|
346
|
+
export function listStashes(dir: string): string[] {
|
|
347
|
+
const parent = path.dirname(dir);
|
|
348
|
+
const prefix = `${path.basename(dir)}.legacy-`;
|
|
349
|
+
let entries: string[];
|
|
350
|
+
try {
|
|
351
|
+
entries = fs.readdirSync(parent);
|
|
352
|
+
} catch {
|
|
353
|
+
return [];
|
|
354
|
+
}
|
|
355
|
+
return entries
|
|
356
|
+
.filter((e) => e.startsWith(prefix))
|
|
357
|
+
.map((e) => path.join(parent, e))
|
|
358
|
+
.filter((p) => isLegacyFlat(p))
|
|
359
|
+
.sort()
|
|
360
|
+
.reverse();
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Crash recovery: a previous run moved a legacy install aside and died before the
|
|
365
|
+
* new build landed. If `dir` is now absent/empty and a proven legacy stash exists,
|
|
366
|
+
* put it back so the user is not left without a browser. Returns the restored stash or null.
|
|
367
|
+
*/
|
|
368
|
+
export function recoverStash(dir: string, log: (m: string) => void = () => {}): string | null {
|
|
369
|
+
let empty = !fs.existsSync(dir);
|
|
370
|
+
if (!empty) {
|
|
371
|
+
try {
|
|
372
|
+
empty = isDir(dir) && fs.readdirSync(dir).length === 0;
|
|
373
|
+
} catch {
|
|
374
|
+
return null;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
if (!empty) return null;
|
|
378
|
+
const stash = listStashes(dir)[0];
|
|
379
|
+
if (!stash) return null;
|
|
380
|
+
try {
|
|
381
|
+
if (fs.existsSync(dir)) fs.rmdirSync(dir);
|
|
382
|
+
fs.renameSync(stash, dir);
|
|
383
|
+
log(`pi-research: restored the previous Camoufox install from ${stash}`);
|
|
384
|
+
return stash;
|
|
385
|
+
} catch {
|
|
386
|
+
return null;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
export interface ProvisionResult {
|
|
391
|
+
status: 'present' | 'installed' | 'failed';
|
|
392
|
+
pruned: string[];
|
|
393
|
+
reason?: string;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Install the pinned browser without ever leaving the user with less than they had.
|
|
398
|
+
*
|
|
399
|
+
* `camoufox fetch` (and the library's own first launch) run `rm -rf <dir>` on a
|
|
400
|
+
* non-empty dir that has no compat flag, i.e. on every legacy flat install, BEFORE
|
|
401
|
+
* downloading. So a legacy install is moved aside first, the fetch runs against a clean
|
|
402
|
+
* dir, and the stash is deleted only after the new layout verifies usable; any failure
|
|
403
|
+
* restores it. `fetch` resolves true when the fetch process exited 0.
|
|
404
|
+
*/
|
|
405
|
+
export async function provisionBrowser(
|
|
406
|
+
dir: string,
|
|
407
|
+
fetch: () => Promise<boolean>,
|
|
408
|
+
log: (m: string) => void = () => {},
|
|
409
|
+
pin: BrowserPin | null = null,
|
|
410
|
+
): Promise<ProvisionResult> {
|
|
411
|
+
recoverStash(dir, log);
|
|
412
|
+
if (isUsable(dir, pin)) {
|
|
413
|
+
sweepStaleStashes(dir, log);
|
|
414
|
+
return { status: 'present', pruned: pruneSuperseded(dir, log, pin) };
|
|
415
|
+
}
|
|
416
|
+
let stash: string | null;
|
|
417
|
+
try {
|
|
418
|
+
stash = stashLegacy(dir);
|
|
419
|
+
} catch (e) {
|
|
420
|
+
// Could not move it (in use on Windows, permissions): fetching now would delete it.
|
|
421
|
+
return { status: 'failed', pruned: [], reason: `could not move the existing legacy install aside (${e instanceof Error ? e.message : String(e)}); left untouched` };
|
|
422
|
+
}
|
|
423
|
+
if (stash) log(`pi-research: legacy Camoufox install moved aside to ${stash} until the new browser is verified`);
|
|
424
|
+
let ok = false;
|
|
425
|
+
let reason: string | undefined;
|
|
426
|
+
try {
|
|
427
|
+
ok = (await fetch()) === true;
|
|
428
|
+
if (!ok) reason = 'camoufox fetch did not exit 0';
|
|
429
|
+
} catch (e) {
|
|
430
|
+
reason = e instanceof Error ? e.message : String(e);
|
|
431
|
+
}
|
|
432
|
+
// A non-zero exit is NOT proof the browser is unusable. The launcher extracts the
|
|
433
|
+
// browser and writes the compat flag BEFORE it fetches its companion geoip database
|
|
434
|
+
// from a separate GitHub release, so a failure there (observed: `UND_ERR_SOCKET other
|
|
435
|
+
// side closed` on ubuntu-latest, three runs in a row on 2026-10-05) exits non-zero with
|
|
436
|
+
// a fully launchable paired build already in place. Verified: a real headless launch
|
|
437
|
+
// succeeds with the mmdb removed. Keep the install and drop the legacy stash rather
|
|
438
|
+
// than reporting a failure and restoring an older browser on top of a working one.
|
|
439
|
+
if (isUsable(dir, pin)) {
|
|
440
|
+
discardLegacy(stash);
|
|
441
|
+
if (stash) log('pi-research: removed the legacy Camoufox install (replaced by the new build)');
|
|
442
|
+
if (!ok) log(`pi-research: camoufox fetch reported a failure after the browser was installed (${reason}); keeping it. Run "npx camoufox fetch" to retry the companion download (e.g. the geoip database).`);
|
|
443
|
+
sweepStaleStashes(dir, log);
|
|
444
|
+
return { status: 'installed', pruned: pruneSuperseded(dir, log, pin) };
|
|
445
|
+
}
|
|
446
|
+
if (ok) reason = 'camoufox fetch exited 0 but no usable build was found';
|
|
447
|
+
restoreLegacy(dir, stash);
|
|
448
|
+
if (stash) log('pi-research: restored the previous Camoufox install after a failed upgrade');
|
|
449
|
+
return { status: 'failed', pruned: [], reason };
|
|
450
|
+
}
|
|
@@ -9,6 +9,7 @@ import * as crypto from 'node:crypto';
|
|
|
9
9
|
import { join } from 'node:path';
|
|
10
10
|
import { existsSync, mkdirSync } from 'node:fs';
|
|
11
11
|
import { platform, homedir } from 'node:os';
|
|
12
|
+
import { resolveInstall, isUsable, loadPin, findPackageRoot } from './camoufox-layout.ts';
|
|
12
13
|
import type { Config } from '../../config.ts';
|
|
13
14
|
import { getConfig } from '../../config.ts';
|
|
14
15
|
import { getLogger } from '../../logger.ts';
|
|
@@ -19,63 +20,26 @@ import { createRequire } from 'node:module';
|
|
|
19
20
|
// ============================================================================
|
|
20
21
|
|
|
21
22
|
/**
|
|
22
|
-
*
|
|
23
|
+
* Where the Camoufox browser lives, and which environment overrides make the
|
|
24
|
+
* browser launcher agree with that.
|
|
23
25
|
*
|
|
24
|
-
* camoufox
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
26
|
+
* `@camoufox/camoufox` derives its install directory ONLY from the platform cache
|
|
27
|
+
* location (XDG_CACHE_HOME on Linux, LOCALAPPDATA on Windows, $HOME/Library/Caches on
|
|
28
|
+
* macOS). Unlike the camoufox-js it replaced, it does NOT read CAMOUFOX_INSTALL_DIR
|
|
29
|
+
* or PLAYWRIGHT_BROWSERS_PATH. Both variables are still honoured here as the
|
|
30
|
+
* documented way to relocate the browser, by translating them into the platform
|
|
31
|
+
* override the launcher does read (see resolveInstall in camoufox-layout.ts). The
|
|
32
|
+
* lookup (getCamoufoxBinaryPath), the install (scripts/setup.cjs,
|
|
33
|
+
* ensure-browser.ts) and the worker launch (getBrowserEnv) all go through that one
|
|
34
|
+
* function, so they cannot disagree.
|
|
30
35
|
*/
|
|
31
|
-
function getWindowsCamoufoxDir(): string {
|
|
32
|
-
return join(homedir(), 'AppData', 'Local', 'camoufox', 'camoufox', 'Cache');
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* The custom camoufox location, if the user set one.
|
|
37
|
-
*
|
|
38
|
-
* CAMOUFOX_INSTALL_DIR takes precedence because it is camoufox-js's OWN
|
|
39
|
-
* variable and the only one that can ever move the install: camoufox-js ≥0.12.0
|
|
40
|
-
* resolves INSTALL_DIR from it. PLAYWRIGHT_BROWSERS_PATH is kept as a documented
|
|
41
|
-
* alias, but on its own it only ever moved where we LOOK — camoufox went on
|
|
42
|
-
* installing to the default cache, so setting it made the browser permanently
|
|
43
|
-
* "not found" right after a successful download. Both are exported onward to
|
|
44
|
-
* camoufox-js (getBrowserEnv, scripts/setup.cjs) so that install and lookup
|
|
45
|
-
* cannot disagree.
|
|
46
|
-
*
|
|
47
|
-
* HISTORY: camoufox-js <0.12.0 hardcoded `userCacheDir("camoufox")` and
|
|
48
|
-
* honoured NEITHER variable, which made this plumbing deliberately inert for
|
|
49
|
-
* two pin cycles. The 0.12.0 refresh (2026-08-30) honours CAMOUFOX_INSTALL_DIR
|
|
50
|
-
* again, so a custom location now relocates the install itself, not just our
|
|
51
|
-
* lookup — the two can no longer disagree via version skew, but keep the dual
|
|
52
|
-
* export (install + lookup) so they can never drift apart in future.
|
|
53
|
-
*/
|
|
54
|
-
function getCustomCamoufoxDir(): string | undefined {
|
|
55
|
-
return process.env['CAMOUFOX_INSTALL_DIR'] || process.env['PLAYWRIGHT_BROWSERS_PATH'] || undefined;
|
|
56
|
-
}
|
|
57
36
|
|
|
58
37
|
/**
|
|
59
|
-
* Get the camoufox
|
|
60
|
-
* Uses the custom location above if set, otherwise the standard user cache location
|
|
61
|
-
* that camoufox uses by default (~/.cache/camoufox on Linux).
|
|
38
|
+
* Get the camoufox install directory (the directory that holds `browsers/`).
|
|
62
39
|
* We do NOT override HOME — that trick was unreliable and caused install/runtime mismatches.
|
|
63
40
|
*/
|
|
64
41
|
export function getBrowserCacheDir(): string {
|
|
65
|
-
|
|
66
|
-
if (custom) {
|
|
67
|
-
return custom;
|
|
68
|
-
}
|
|
69
|
-
// Mirror getCamoufoxBinaryPath() so both functions agree on the cache location
|
|
70
|
-
const osPlatform = platform();
|
|
71
|
-
if (osPlatform === 'win32') {
|
|
72
|
-
return getWindowsCamoufoxDir();
|
|
73
|
-
} else if (osPlatform === 'darwin') {
|
|
74
|
-
return join(homedir(), 'Library', 'Caches', 'camoufox');
|
|
75
|
-
} else {
|
|
76
|
-
const cacheHome = process.env['XDG_CACHE_HOME'] || join(homedir(), '.cache');
|
|
77
|
-
return join(cacheHome, 'camoufox');
|
|
78
|
-
}
|
|
42
|
+
return resolveInstall(process.env, platform(), homedir()).dir;
|
|
79
43
|
}
|
|
80
44
|
|
|
81
45
|
/**
|
|
@@ -134,18 +98,16 @@ export function getBrowserProfileDir(config?: Config): string {
|
|
|
134
98
|
*/
|
|
135
99
|
export function getBrowserEnv(config?: Config): NodeJS.ProcessEnv {
|
|
136
100
|
const env: NodeJS.ProcessEnv = { ...process.env };
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
delete env['CAMOUFOX_INSTALL_DIR'];
|
|
148
|
-
}
|
|
101
|
+
// Hand the worker the placement overrides the browser launcher actually reads
|
|
102
|
+
// (XDG_CACHE_HOME / LOCALAPPDATA), derived from the user's custom dir. Without
|
|
103
|
+
// them the worker launches from the DEFAULT cache while everything on this side
|
|
104
|
+
// resolves the custom one: the install/runtime mismatch this file's header warns
|
|
105
|
+
// about. The two user-facing variables are removed so a nested tool cannot
|
|
106
|
+
// reinterpret them.
|
|
107
|
+
const install = resolveInstall(process.env, platform(), homedir());
|
|
108
|
+
Object.assign(env, install.overrides);
|
|
109
|
+
delete env['PLAYWRIGHT_BROWSERS_PATH'];
|
|
110
|
+
delete env['CAMOUFOX_INSTALL_DIR'];
|
|
149
111
|
// Tell thread-workers where to write their lifecycle/error log. Precedence:
|
|
150
112
|
// 1. A user-set PI_RESEARCH_LOG_FILE (already copied from process.env above) wins —
|
|
151
113
|
// never clobber an explicit override.
|
|
@@ -194,26 +156,12 @@ export function ensureBrowserCacheDir(): string {
|
|
|
194
156
|
}
|
|
195
157
|
|
|
196
158
|
/**
|
|
197
|
-
* Get the
|
|
198
|
-
*
|
|
159
|
+
* Get the path where camoufox installs its browsers (the directory holding `browsers/`).
|
|
160
|
+
* Same answer as getBrowserCacheDir(); kept as a separate name for the callers that
|
|
161
|
+
* think of it as "the binary location".
|
|
199
162
|
*/
|
|
200
163
|
export function getCamoufoxBinaryPath(): string {
|
|
201
|
-
|
|
202
|
-
if (customPath) {
|
|
203
|
-
return customPath;
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
const osPlatform = platform();
|
|
207
|
-
|
|
208
|
-
if (osPlatform === 'win32') {
|
|
209
|
-
return getWindowsCamoufoxDir();
|
|
210
|
-
} else if (osPlatform === 'darwin') {
|
|
211
|
-
return join(homedir(), 'Library', 'Caches', 'camoufox');
|
|
212
|
-
} else {
|
|
213
|
-
// Linux and others
|
|
214
|
-
const cacheHome = process.env['XDG_CACHE_HOME'] || join(homedir(), '.cache');
|
|
215
|
-
return join(cacheHome, 'camoufox');
|
|
216
|
-
}
|
|
164
|
+
return getBrowserCacheDir();
|
|
217
165
|
}
|
|
218
166
|
|
|
219
167
|
// ============================================================================
|
|
@@ -255,7 +203,7 @@ export function getSchedulerVersion(config?: Config): string {
|
|
|
255
203
|
*
|
|
256
204
|
* Returns true (true headless — no visible window) on Windows and macOS.
|
|
257
205
|
* Historically Windows used headless:false because headless:true crashed Firefox
|
|
258
|
-
* (exit 0x80000003, camoufox
|
|
206
|
+
* (exit 0x80000003, camoufox issue #614); that is fixed in camoufox-js >=0.10
|
|
259
207
|
* (verified on Windows 11 x64 with 0.10.2: headless:true launches and navigates
|
|
260
208
|
* reliably and, crucially, NO browser window pops up on the desktop). Using
|
|
261
209
|
* headless:false on a real Windows desktop flashed visible, sometimes fullscreen,
|
|
@@ -302,8 +250,8 @@ export function resolveHeadlessMode(): boolean | 'virtual' {
|
|
|
302
250
|
* Check if the browser is available for meaningful browser pool testing.
|
|
303
251
|
*
|
|
304
252
|
* Returns false if:
|
|
305
|
-
* - camoufox
|
|
306
|
-
* -
|
|
253
|
+
* - @camoufox/camoufox package is not installed
|
|
254
|
+
* - no browser build is installed under the cache dir (new `browsers/<repo>/<version>` layout)
|
|
307
255
|
* - FULL_MOCK_MODE is active (both search and scrape mocked)
|
|
308
256
|
*
|
|
309
257
|
* In FULL_MOCK_MODE, browser pool tests are not meaningful because:
|
|
@@ -317,22 +265,16 @@ export function resolveHeadlessMode(): boolean | 'virtual' {
|
|
|
317
265
|
* Whether the browser stack's NATIVE dependencies can actually load.
|
|
318
266
|
*
|
|
319
267
|
* `isBrowserAvailable()` answers a different question — is the camoufox binary on
|
|
320
|
-
* disk — and a browser can be fully downloaded and still unable to launch.
|
|
321
|
-
* requires `better-sqlite3`, whose binding is produced by a dependency INSTALL SCRIPT,
|
|
322
|
-
* and npm 12 turns those off by default. The module then imports fine and throws only
|
|
323
|
-
* when first used, so nothing notices until every browser worker dies mid-run with
|
|
324
|
-
* "Could not locate the bindings file" and the run reports a network problem it does
|
|
325
|
-
* not have.
|
|
326
|
-
*
|
|
327
|
-
* Measured, not assumed: of this package's native dependencies, only better-sqlite3
|
|
328
|
-
* fails a scripts-blocked install. onnxruntime-node ships its binding inside its own
|
|
329
|
-
* tarball, and lancedb, impit and html-to-markdown all resolve prebuilt platform
|
|
330
|
-
* packages, so the knowledge store keeps working while search is dead — exactly the
|
|
331
|
-
* split seen in the field.
|
|
268
|
+
* disk — and a browser can be fully downloaded and still unable to launch.
|
|
332
269
|
*
|
|
333
|
-
*
|
|
334
|
-
*
|
|
335
|
-
* first use.
|
|
270
|
+
* History: camoufox-js required `better-sqlite3`, whose binding came from a dependency
|
|
271
|
+
* INSTALL SCRIPT that npm 12 turns off, so the module imported fine and every browser
|
|
272
|
+
* worker died at first use. `@camoufox/camoufox` has no better-sqlite3; the one native
|
|
273
|
+
* module left in the launcher's path is `impit` (its IP/geo lookups), which resolves a
|
|
274
|
+
* prebuilt platform package. That package can still be dropped by npm's optional-
|
|
275
|
+
* dependency bug (npm/cli#4828), and the failure is again at first use, so the probe
|
|
276
|
+
* loads it: resolving the module name or stat-ing a path does not settle it. It costs a
|
|
277
|
+
* few milliseconds and is confined to diagnostics.
|
|
336
278
|
*/
|
|
337
279
|
export function probeBrowserNativeDeps(
|
|
338
280
|
/** Injected for tests. Production callers pass nothing. */
|
|
@@ -350,16 +292,17 @@ export function probeBrowserNativeDeps(
|
|
|
350
292
|
|
|
351
293
|
function defaultNativeDepLoad(): void {
|
|
352
294
|
const require = createRequire(import.meta.url);
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
295
|
+
// Requiring `impit` loads its native binding eagerly and throws if the platform
|
|
296
|
+
// package is missing.
|
|
297
|
+
require('impit');
|
|
356
298
|
}
|
|
357
299
|
|
|
358
300
|
export function isBrowserAvailable(): boolean {
|
|
359
301
|
if (isFullMockMode()) return false;
|
|
360
302
|
try {
|
|
361
|
-
import.meta.resolve('camoufox
|
|
362
|
-
|
|
303
|
+
import.meta.resolve('@camoufox/camoufox');
|
|
304
|
+
const root = findPackageRoot(import.meta.url);
|
|
305
|
+
return isUsable(getCamoufoxBinaryPath(), root ? loadPin(root) : null);
|
|
363
306
|
} catch {
|
|
364
307
|
return false;
|
|
365
308
|
}
|