@lincoln504/pi-research 1.7.7 → 1.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent-skill/pi-research/README.md +3 -3
- package/agent-skill/pi-research/SKILL.md +1 -1
- package/dist/cli-bin.mjs +10 -1
- package/dist/cli.mjs +692 -387
- package/dist/pdf-extract-worker.mjs +12 -3
- package/dist/thread-worker.mjs +41 -12
- package/docs/AGENT-SKILL.md +3 -3
- package/docs/ARCHITECTURE.md +64 -50
- package/docs/CHANGELOG.md +31 -0
- package/docs/CONFIGURATION.md +2 -2
- package/docs/SDK.md +7 -6
- package/docs/THIRD-PARTY-NOTICES.md +11 -7
- package/package.json +10 -8
- package/scripts/camoufox-layout.cjs +455 -0
- package/scripts/cleanup.cjs +72 -29
- package/scripts/setup.cjs +72 -111
- package/src/cli-bin.mjs +10 -1
- package/src/cli.ts +27 -15
- package/src/healthcheck/index.ts +1 -1
- package/src/infrastructure/browser/browser-cleanup.ts +1 -1
- package/src/infrastructure/browser/browser-error-utils.ts +5 -4
- package/src/infrastructure/browser/camoufox-layout.ts +450 -0
- package/src/infrastructure/browser/config.ts +46 -103
- package/src/infrastructure/browser/ensure-browser.ts +63 -49
- package/src/infrastructure/browser/thread-worker-browser.ts +38 -42
- package/src/infrastructure/browser/thread-worker.mjs +41 -12
- package/src/infrastructure/knowledge-store-service.ts +6 -6
- package/src/knowledge/store.ts +1 -1
- package/src/orchestration/researcher-executor.ts +47 -1
- package/src/tui/research-health.ts +1 -1
- package/src/utils/log-utils.ts +19 -4
- package/src/utils/research-export.ts +1 -1
- package/src/web-research/browser-search.ts +8 -10
- package/src/web-research/pdf-extract-worker.mjs +12 -3
- package/src/web-research/scraper-utils.ts +42 -10
- package/src/web-research/web-scraper.ts +1 -1
|
@@ -0,0 +1,455 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Camoufox install-layout helpers shared by scripts/setup.cjs and scripts/cleanup.cjs.
|
|
5
|
+
*
|
|
6
|
+
* TWIN of src/infrastructure/browser/camoufox-layout.ts (the runtime half). The two
|
|
7
|
+
* are kept line-for-line equivalent on purpose and test/unit/infrastructure/
|
|
8
|
+
* camoufox-layout.test.ts runs both against the same fixtures, because a drift
|
|
9
|
+
* between "where setup installed it" and "where the runtime looks" is exactly the
|
|
10
|
+
* install/runtime mismatch this package has shipped before.
|
|
11
|
+
*
|
|
12
|
+
* Why this exists. `@camoufox/camoufox` (0.5.x) differs from the old `camoufox-js`:
|
|
13
|
+
* - It installs under <INSTALL_DIR>/browsers/<repo>/<version>/ (multi-version),
|
|
14
|
+
* with `config.json`, `repo_cache.json` and a `.0.5_FLAG` beside it. camoufox-js
|
|
15
|
+
* put the browser files FLAT in INSTALL_DIR with a root `version.json`.
|
|
16
|
+
* - INSTALL_DIR is derived ONLY from platform cache dirs (XDG_CACHE_HOME on Linux,
|
|
17
|
+
* LOCALAPPDATA on Windows, $HOME/Library/Caches on macOS). It does NOT read
|
|
18
|
+
* CAMOUFOX_INSTALL_DIR or PLAYWRIGHT_BROWSERS_PATH.
|
|
19
|
+
* - `camoufox fetch` does `rm -rf INSTALL_DIR` when it is non-empty and has no
|
|
20
|
+
* `.0.5_FLAG` ("Cleaning old data...") BEFORE it downloads anything, so a failed
|
|
21
|
+
* download over a legacy install leaves the user with no browser at all.
|
|
22
|
+
* - A newer pi-research release that pins a newer browser installs the new build
|
|
23
|
+
* beside the old one and never removes it; the old one is dead weight because a
|
|
24
|
+
* released library only ever launches its own pinned build.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
const fs = require('fs');
|
|
28
|
+
const os = require('os');
|
|
29
|
+
const path = require('path');
|
|
30
|
+
|
|
31
|
+
const COMPAT_FLAG = '.0.5_FLAG';
|
|
32
|
+
|
|
33
|
+
/** Names whose presence, with a root version.json, proves a legacy camoufox-js install. */
|
|
34
|
+
const LEGACY_MARKERS = ['camoufox-bin', 'camoufox', 'camoufox.exe', 'application.ini', 'Camoufox.app', 'properties.json'];
|
|
35
|
+
|
|
36
|
+
function isDir(p) {
|
|
37
|
+
try {
|
|
38
|
+
return fs.statSync(p).isDirectory();
|
|
39
|
+
} catch (_) {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function isFile(p) {
|
|
45
|
+
try {
|
|
46
|
+
return fs.statSync(p).isFile();
|
|
47
|
+
} catch (_) {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The custom location the user asked for, if any (camoufox itself ignores both variables). */
|
|
53
|
+
function customDir(env = process.env) {
|
|
54
|
+
return env.CAMOUFOX_INSTALL_DIR || env.PLAYWRIGHT_BROWSERS_PATH || null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Mirrors @camoufox/camoufox paths.js userCacheDir('camoufox') for a given environment. */
|
|
58
|
+
function defaultInstallDir(env = process.env, platform = process.platform, home = os.homedir()) {
|
|
59
|
+
if (platform === 'win32') {
|
|
60
|
+
const base = env.LOCALAPPDATA && path.win32.isAbsolute(env.LOCALAPPDATA)
|
|
61
|
+
? env.LOCALAPPDATA
|
|
62
|
+
: path.join(home, 'AppData', 'Local');
|
|
63
|
+
return path.join(base, 'camoufox', 'camoufox', 'Cache');
|
|
64
|
+
}
|
|
65
|
+
if (platform === 'darwin') return path.join(home, 'Library', 'Caches', 'camoufox');
|
|
66
|
+
const xdg = (env.XDG_CACHE_HOME || '').trim();
|
|
67
|
+
return path.join(xdg ? xdg : path.join(home, '.cache'), 'camoufox');
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Resolve where the browser lives and which env overrides make camoufox use it.
|
|
72
|
+
*
|
|
73
|
+
* - No custom dir: the platform default, no overrides.
|
|
74
|
+
* - Linux: custom is the cache HOME (XDG_CACHE_HOME=custom, browser in custom/camoufox),
|
|
75
|
+
* except when custom already ends in `camoufox`, where its parent becomes the
|
|
76
|
+
* cache home so the browser lives in custom itself (old camoufox-js semantics).
|
|
77
|
+
* - Windows: same idea through LOCALAPPDATA (browser in <base>\camoufox\camoufox\Cache).
|
|
78
|
+
* - macOS: camoufox derives the cache from $HOME only; relocating it would mean
|
|
79
|
+
* redirecting HOME for the browser process, which also moves Firefox's own
|
|
80
|
+
* state. The custom dir is ignored there (`ignoredCustom` is set so callers can say so).
|
|
81
|
+
*/
|
|
82
|
+
function resolveInstall(env = process.env, platform = process.platform, home = os.homedir()) {
|
|
83
|
+
const custom = customDir(env);
|
|
84
|
+
if (!custom) return { dir: defaultInstallDir(env, platform, home), overrides: {}, custom: null, ignoredCustom: false };
|
|
85
|
+
if (platform === 'darwin') {
|
|
86
|
+
return { dir: defaultInstallDir(env, platform, home), overrides: {}, custom, ignoredCustom: true };
|
|
87
|
+
}
|
|
88
|
+
const resolved = (platform === 'win32' ? path.win32 : path).resolve(custom);
|
|
89
|
+
if (platform === 'win32') {
|
|
90
|
+
// <LOCALAPPDATA>\camoufox\camoufox\Cache : one fixed shape, custom = LOCALAPPDATA.
|
|
91
|
+
const overrides = { LOCALAPPDATA: resolved };
|
|
92
|
+
return { dir: defaultInstallDir({ ...env, ...overrides }, platform, home), overrides, custom, ignoredCustom: false };
|
|
93
|
+
}
|
|
94
|
+
const endsInCamoufox = path.basename(resolved).toLowerCase() === 'camoufox';
|
|
95
|
+
const cacheHome = endsInCamoufox ? path.dirname(resolved) : resolved;
|
|
96
|
+
const overrides = { XDG_CACHE_HOME: cacheHome };
|
|
97
|
+
return { dir: defaultInstallDir({ ...env, ...overrides }, platform, home), overrides, custom, ignoredCustom: false };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Every `browsers/<repo>/<version>/` directory that holds a parseable version.json. */
|
|
101
|
+
function listInstalls(dir) {
|
|
102
|
+
const out = [];
|
|
103
|
+
const browsers = path.join(dir, 'browsers');
|
|
104
|
+
let repos = [];
|
|
105
|
+
try {
|
|
106
|
+
repos = fs.readdirSync(browsers, { withFileTypes: true });
|
|
107
|
+
} catch (_) {
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
110
|
+
for (const repo of repos) {
|
|
111
|
+
if (!repo.isDirectory() || repo.name.startsWith('.')) continue;
|
|
112
|
+
const repoDir = path.join(browsers, repo.name);
|
|
113
|
+
let versions = [];
|
|
114
|
+
try {
|
|
115
|
+
versions = fs.readdirSync(repoDir, { withFileTypes: true });
|
|
116
|
+
} catch (_) {
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
for (const v of versions) {
|
|
120
|
+
if (!v.isDirectory()) continue;
|
|
121
|
+
const vdir = path.join(repoDir, v.name);
|
|
122
|
+
let meta = null;
|
|
123
|
+
try {
|
|
124
|
+
meta = JSON.parse(fs.readFileSync(path.join(vdir, 'version.json'), 'utf8'));
|
|
125
|
+
} catch (_) {
|
|
126
|
+
continue; // missing/corrupt version.json: the library skips it too
|
|
127
|
+
}
|
|
128
|
+
out.push({
|
|
129
|
+
repo: repo.name,
|
|
130
|
+
name: v.name,
|
|
131
|
+
path: vdir,
|
|
132
|
+
rel: `browsers/${repo.name}/${v.name}`,
|
|
133
|
+
version: meta && typeof meta.version === 'string' ? meta.version : '',
|
|
134
|
+
build: meta && typeof meta.build === 'string' ? meta.build : '',
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return out;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** True when the NEW layout holds at least one usable browser version. */
|
|
142
|
+
function hasInstall(dir) {
|
|
143
|
+
return listInstalls(dir).length > 0;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Proof that `dir` is a legacy flat camoufox-js install: a root version.json carrying
|
|
148
|
+
* `version` + `release`, no `.0.5_FLAG`, and at least one file only a camoufox build has.
|
|
149
|
+
* Anything short of that is NOT claimed, so it is never removed or moved.
|
|
150
|
+
*/
|
|
151
|
+
function isLegacyFlat(dir) {
|
|
152
|
+
if (!isDir(dir) || isFile(path.join(dir, COMPAT_FLAG))) return false;
|
|
153
|
+
let meta;
|
|
154
|
+
try {
|
|
155
|
+
meta = JSON.parse(fs.readFileSync(path.join(dir, 'version.json'), 'utf8'));
|
|
156
|
+
} catch (_) {
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
if (!meta || typeof meta.version !== 'string' || typeof meta.release !== 'string') return false;
|
|
160
|
+
return LEGACY_MARKERS.some((m) => fs.existsSync(path.join(dir, m)));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Whether `dir` is provably camoufox's own install dir (new or legacy layout, or the
|
|
165
|
+
* empty shell camoufox leaves behind). Used as the guard before any recursive delete.
|
|
166
|
+
*/
|
|
167
|
+
function isCamoufoxDir(dir) {
|
|
168
|
+
if (!isDir(dir)) return false;
|
|
169
|
+
if (isFile(path.join(dir, COMPAT_FLAG))) return true;
|
|
170
|
+
if (hasInstall(dir)) return true;
|
|
171
|
+
if (isLegacyFlat(dir)) return true;
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** True when the user chose a channel/pin themselves (camoufox then honours it, so keep their versions). */
|
|
176
|
+
function hasExplicitChoice(dir) {
|
|
177
|
+
try {
|
|
178
|
+
const cfg = JSON.parse(fs.readFileSync(path.join(dir, 'config.json'), 'utf8'));
|
|
179
|
+
return Boolean(cfg && (cfg.pinned || cfg.channel));
|
|
180
|
+
} catch (_) {
|
|
181
|
+
return false;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* The browser build @camoufox/camoufox was released with, from its browser-pin.json
|
|
187
|
+
* (`{ repoName, version, build }`, repoName lowercased), or null when unpinned/unreadable.
|
|
188
|
+
* `packageRoot` is the directory of the @camoufox/camoufox package.
|
|
189
|
+
*/
|
|
190
|
+
function loadPin(packageRoot) {
|
|
191
|
+
try {
|
|
192
|
+
const data = JSON.parse(fs.readFileSync(path.join(packageRoot, 'dist', 'data-files', 'browser-pin.json'), 'utf8'));
|
|
193
|
+
if (!data || !data.tag) return null;
|
|
194
|
+
return { repoName: String(data.repo_name).toLowerCase(), version: String(data.version), build: String(data.build) };
|
|
195
|
+
} catch (_) {
|
|
196
|
+
return null;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** The @camoufox/camoufox package directory as seen from `fromDir`, or null. */
|
|
201
|
+
function findPackageRoot(fromDir) {
|
|
202
|
+
try {
|
|
203
|
+
return path.dirname(require.resolve('@camoufox/camoufox/package.json', { paths: [fromDir] }));
|
|
204
|
+
} catch (_) {
|
|
205
|
+
return null;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** Whether `build` is the one `pin` names. */
|
|
210
|
+
function matchesPin(build, pin) {
|
|
211
|
+
return build.repo.toLowerCase() === pin.repoName && build.version === pin.version && build.build === pin.build;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Remove every installed version except the one the library will launch: the paired
|
|
216
|
+
* build when `pin` is given and the user made no explicit choice, otherwise the active one. A released camoufox launches
|
|
217
|
+
* only the build it was released with, so superseded builds are pure disk weight.
|
|
218
|
+
* Does nothing when the user made an explicit `camoufox set` choice, when there is no
|
|
219
|
+
* identifiable active version, or when the active version's folder is missing.
|
|
220
|
+
* Returns the removed relative paths.
|
|
221
|
+
*/
|
|
222
|
+
function pruneSuperseded(dir, log = () => {}, pin = null) {
|
|
223
|
+
if (hasExplicitChoice(dir)) return [];
|
|
224
|
+
const installs = listInstalls(dir);
|
|
225
|
+
let keep = null;
|
|
226
|
+
if (pin) {
|
|
227
|
+
const paired = installs.find((i) => matchesPin(i, pin));
|
|
228
|
+
keep = paired ? paired.rel : null;
|
|
229
|
+
} else {
|
|
230
|
+
try {
|
|
231
|
+
keep = JSON.parse(fs.readFileSync(path.join(dir, 'config.json'), 'utf8')).active_version || null;
|
|
232
|
+
} catch (_) {
|
|
233
|
+
return [];
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
if (!keep || !installs.some((i) => i.rel === keep)) return [];
|
|
237
|
+
const removed = [];
|
|
238
|
+
for (const i of installs) {
|
|
239
|
+
if (i.rel === keep) continue;
|
|
240
|
+
try {
|
|
241
|
+
fs.rmSync(i.path, { recursive: true, force: true });
|
|
242
|
+
removed.push(i.rel);
|
|
243
|
+
log(`pi-research: removed superseded Camoufox build ${i.rel}`);
|
|
244
|
+
} catch (e) {
|
|
245
|
+
log(`pi-research: could not remove superseded Camoufox build ${i.rel}: ${e && e.message ? e.message : e}`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
// Drop now-empty repo folders so listings stay clean.
|
|
249
|
+
try {
|
|
250
|
+
for (const r of fs.readdirSync(path.join(dir, 'browsers'))) {
|
|
251
|
+
const rd = path.join(dir, 'browsers', r);
|
|
252
|
+
if (isDir(rd) && fs.readdirSync(rd).length === 0) fs.rmdirSync(rd);
|
|
253
|
+
}
|
|
254
|
+
} catch (_) { /* best effort */ }
|
|
255
|
+
return removed;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Move a PROVEN legacy flat install out of the way so `camoufox fetch` (which would
|
|
260
|
+
* rm -rf it before downloading) cannot destroy a working browser if the download fails.
|
|
261
|
+
* Returns the stash path, or null when `dir` is not a provable legacy install.
|
|
262
|
+
*/
|
|
263
|
+
function stashLegacy(dir) {
|
|
264
|
+
if (!isLegacyFlat(dir)) return null;
|
|
265
|
+
const stash = `${dir}.legacy-${process.pid}-${Date.now()}`;
|
|
266
|
+
fs.renameSync(dir, stash);
|
|
267
|
+
return stash;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Put a stashed legacy install back (download failed). */
|
|
271
|
+
function restoreLegacy(dir, stash) {
|
|
272
|
+
if (!stash || !fs.existsSync(stash)) return;
|
|
273
|
+
try {
|
|
274
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
275
|
+
} catch (_) { /* partial new dir; best effort */ }
|
|
276
|
+
fs.renameSync(stash, dir);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Delete a stashed legacy install (new layout verified usable). */
|
|
280
|
+
function discardLegacy(stash) {
|
|
281
|
+
if (!stash) return;
|
|
282
|
+
fs.rmSync(stash, { recursive: true, force: true });
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Leftover legacy stash dirs from a crashed run next to `dir`; deleted only when the new layout is usable. */
|
|
286
|
+
function sweepStaleStashes(dir, log = () => {}) {
|
|
287
|
+
if (!hasInstall(dir)) return [];
|
|
288
|
+
const parent = path.dirname(dir);
|
|
289
|
+
const prefix = `${path.basename(dir)}.legacy-`;
|
|
290
|
+
const removed = [];
|
|
291
|
+
let entries = [];
|
|
292
|
+
try {
|
|
293
|
+
entries = fs.readdirSync(parent);
|
|
294
|
+
} catch (_) {
|
|
295
|
+
return removed;
|
|
296
|
+
}
|
|
297
|
+
for (const e of entries) {
|
|
298
|
+
if (!e.startsWith(prefix)) continue;
|
|
299
|
+
const p = path.join(parent, e);
|
|
300
|
+
if (!isLegacyFlat(p)) continue;
|
|
301
|
+
try {
|
|
302
|
+
fs.rmSync(p, { recursive: true, force: true });
|
|
303
|
+
removed.push(p);
|
|
304
|
+
log(`pi-research: removed leftover legacy Camoufox stash ${p}`);
|
|
305
|
+
} catch (_) { /* best effort */ }
|
|
306
|
+
}
|
|
307
|
+
return removed;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Remove the whole install dir, but only when it is provably camoufox's own.
|
|
312
|
+
* Returns 'removed' | 'absent' | 'refused'.
|
|
313
|
+
*/
|
|
314
|
+
function removeInstallDir(dir) {
|
|
315
|
+
if (!fs.existsSync(dir)) return 'absent';
|
|
316
|
+
if (!isCamoufoxDir(dir)) return 'refused';
|
|
317
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
318
|
+
return 'removed';
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** True when the NEW layout can launch: the compat flag plus the paired build (or any build when unpinned). */
|
|
322
|
+
function isUsable(dir, pin = null) {
|
|
323
|
+
if (!isFile(path.join(dir, COMPAT_FLAG))) return false;
|
|
324
|
+
const installs = listInstalls(dir);
|
|
325
|
+
if (installs.length === 0) return false;
|
|
326
|
+
// A released library launches ONLY its paired build (unless the user chose one with
|
|
327
|
+
// `camoufox set`), and downloads it itself if absent. So "usable" means the paired
|
|
328
|
+
// build is there, which makes an upgrade to a new pin re-fetch centrally instead of in
|
|
329
|
+
// every worker at once.
|
|
330
|
+
if (pin && !hasExplicitChoice(dir)) return installs.some((i) => matchesPin(i, pin));
|
|
331
|
+
return true;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** Stash dirs (`<dir>.legacy-<pid>-<ts>`) left beside `dir`, newest first, each proven legacy. */
|
|
335
|
+
function listStashes(dir) {
|
|
336
|
+
const parent = path.dirname(dir);
|
|
337
|
+
const prefix = `${path.basename(dir)}.legacy-`;
|
|
338
|
+
let entries = [];
|
|
339
|
+
try {
|
|
340
|
+
entries = fs.readdirSync(parent);
|
|
341
|
+
} catch (_) {
|
|
342
|
+
return [];
|
|
343
|
+
}
|
|
344
|
+
return entries
|
|
345
|
+
.filter((e) => e.startsWith(prefix))
|
|
346
|
+
.map((e) => path.join(parent, e))
|
|
347
|
+
.filter((p) => isLegacyFlat(p))
|
|
348
|
+
.sort()
|
|
349
|
+
.reverse();
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Crash recovery: a previous run moved a legacy install aside and died before the
|
|
354
|
+
* new build landed. If `dir` is now absent/empty and a proven legacy stash exists,
|
|
355
|
+
* put it back so the user is not left without a browser. Returns the restored stash or null.
|
|
356
|
+
*/
|
|
357
|
+
function recoverStash(dir, log = () => {}) {
|
|
358
|
+
let empty = !fs.existsSync(dir);
|
|
359
|
+
if (!empty) {
|
|
360
|
+
try {
|
|
361
|
+
empty = isDir(dir) && fs.readdirSync(dir).length === 0;
|
|
362
|
+
} catch (_) {
|
|
363
|
+
return null;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
if (!empty) return null;
|
|
367
|
+
const stash = listStashes(dir)[0];
|
|
368
|
+
if (!stash) return null;
|
|
369
|
+
try {
|
|
370
|
+
if (fs.existsSync(dir)) fs.rmdirSync(dir);
|
|
371
|
+
fs.renameSync(stash, dir);
|
|
372
|
+
log(`pi-research: restored the previous Camoufox install from ${stash}`);
|
|
373
|
+
return stash;
|
|
374
|
+
} catch (_) {
|
|
375
|
+
return null;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Install the pinned browser without ever leaving the user with less than they had.
|
|
381
|
+
*
|
|
382
|
+
* `camoufox fetch` (and the library's own first launch) run `rm -rf <dir>` on a
|
|
383
|
+
* non-empty dir that has no compat flag, i.e. on every legacy flat install, BEFORE
|
|
384
|
+
* downloading. So a legacy install is moved aside first, the fetch runs against a clean
|
|
385
|
+
* dir, and the stash is deleted only after the new layout verifies usable; any failure
|
|
386
|
+
* restores it. `fetch` is `() => boolean` (true = exited 0).
|
|
387
|
+
*
|
|
388
|
+
* Returns { status: 'present' | 'installed' | 'failed', pruned: string[], reason? }.
|
|
389
|
+
*/
|
|
390
|
+
function provisionBrowser(dir, fetch, log = () => {}, pin = null) {
|
|
391
|
+
recoverStash(dir, log);
|
|
392
|
+
if (isUsable(dir, pin)) {
|
|
393
|
+
sweepStaleStashes(dir, log);
|
|
394
|
+
return { status: 'present', pruned: pruneSuperseded(dir, log, pin) };
|
|
395
|
+
}
|
|
396
|
+
let stash = null;
|
|
397
|
+
try {
|
|
398
|
+
stash = stashLegacy(dir);
|
|
399
|
+
} catch (e) {
|
|
400
|
+
// Could not move it (in use on Windows, permissions): fetching now would delete it.
|
|
401
|
+
return { status: 'failed', pruned: [], reason: `could not move the existing legacy install aside (${e && e.message ? e.message : e}); left untouched` };
|
|
402
|
+
}
|
|
403
|
+
if (stash) log(`pi-research: legacy Camoufox install moved aside to ${stash} until the new browser is verified`);
|
|
404
|
+
let ok = false;
|
|
405
|
+
let reason;
|
|
406
|
+
try {
|
|
407
|
+
ok = fetch() === true;
|
|
408
|
+
if (!ok) reason = 'camoufox fetch did not exit 0';
|
|
409
|
+
} catch (e) {
|
|
410
|
+
reason = e && e.message ? e.message : String(e);
|
|
411
|
+
}
|
|
412
|
+
// A non-zero exit is NOT proof the browser is unusable. The launcher extracts the
|
|
413
|
+
// browser and writes the compat flag BEFORE it fetches its companion geoip database
|
|
414
|
+
// from a separate GitHub release, so a failure there (observed: `UND_ERR_SOCKET other
|
|
415
|
+
// side closed` on ubuntu-latest, three runs in a row on 2026-10-05) exits non-zero with
|
|
416
|
+
// a fully launchable paired build already in place. Verified: a real headless launch
|
|
417
|
+
// succeeds with the mmdb removed. Keep the install and drop the legacy stash rather
|
|
418
|
+
// than reporting a failure and restoring an older browser on top of a working one.
|
|
419
|
+
if (isUsable(dir, pin)) {
|
|
420
|
+
discardLegacy(stash);
|
|
421
|
+
if (stash) log('pi-research: removed the legacy Camoufox install (replaced by the new build)');
|
|
422
|
+
if (!ok) log(`pi-research: camoufox fetch reported a failure after the browser was installed (${reason}); keeping it. Run "npx camoufox fetch" to retry the companion download (e.g. the geoip database).`);
|
|
423
|
+
sweepStaleStashes(dir, log);
|
|
424
|
+
return { status: 'installed', pruned: pruneSuperseded(dir, log, pin) };
|
|
425
|
+
}
|
|
426
|
+
if (ok) reason = 'camoufox fetch exited 0 but no usable build was found';
|
|
427
|
+
restoreLegacy(dir, stash);
|
|
428
|
+
if (stash) log('pi-research: restored the previous Camoufox install after a failed upgrade');
|
|
429
|
+
return { status: 'failed', pruned: [], reason };
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
module.exports = {
|
|
433
|
+
COMPAT_FLAG,
|
|
434
|
+
customDir,
|
|
435
|
+
defaultInstallDir,
|
|
436
|
+
resolveInstall,
|
|
437
|
+
listInstalls,
|
|
438
|
+
hasInstall,
|
|
439
|
+
isLegacyFlat,
|
|
440
|
+
isCamoufoxDir,
|
|
441
|
+
hasExplicitChoice,
|
|
442
|
+
pruneSuperseded,
|
|
443
|
+
stashLegacy,
|
|
444
|
+
restoreLegacy,
|
|
445
|
+
discardLegacy,
|
|
446
|
+
sweepStaleStashes,
|
|
447
|
+
removeInstallDir,
|
|
448
|
+
isUsable,
|
|
449
|
+
loadPin,
|
|
450
|
+
findPackageRoot,
|
|
451
|
+
matchesPin,
|
|
452
|
+
listStashes,
|
|
453
|
+
recoverStash,
|
|
454
|
+
provisionBrowser,
|
|
455
|
+
};
|
package/scripts/cleanup.cjs
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
const { spawnSync } = require('child_process');
|
|
19
|
+
const fs = require('fs');
|
|
19
20
|
const { rmSync, existsSync, lstatSync, readdirSync, readlinkSync, readFileSync, writeFileSync, unlinkSync } = require('fs');
|
|
20
21
|
const os = require('os');
|
|
21
22
|
const path = require('path');
|
|
@@ -90,46 +91,88 @@ function removeInstalledSkills() {
|
|
|
90
91
|
|
|
91
92
|
removeInstalledSkills();
|
|
92
93
|
|
|
94
|
+
const layout = require('./camoufox-layout.cjs');
|
|
95
|
+
|
|
93
96
|
/**
|
|
94
|
-
* Resolve the camoufox
|
|
95
|
-
*
|
|
96
|
-
* See setup.cjs for rationale — dependencies are hoisted to root node_modules.
|
|
97
|
+
* Resolve the `camoufox` CLI entry of @camoufox/camoufox (see setup.cjs: run through
|
|
98
|
+
* process.execPath with an argv array, never a shell string).
|
|
97
99
|
*/
|
|
98
|
-
function
|
|
100
|
+
function resolveCamoufoxCli() {
|
|
99
101
|
try {
|
|
100
|
-
const
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
102
|
+
const pkgJson = require.resolve('@camoufox/camoufox/package.json', { paths: [projectRoot] });
|
|
103
|
+
const pkg = require(pkgJson);
|
|
104
|
+
const rel = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin && pkg.bin.camoufox;
|
|
105
|
+
if (rel) {
|
|
106
|
+
const entry = path.join(path.dirname(pkgJson), rel);
|
|
107
|
+
if (existsSync(entry)) return entry;
|
|
108
|
+
}
|
|
105
109
|
} catch (_) { /* fall through */ }
|
|
106
110
|
return null;
|
|
107
111
|
}
|
|
108
112
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
113
|
+
/**
|
|
114
|
+
* Remove EVERY Camoufox browser this package could have installed: all
|
|
115
|
+
* `browsers/<repo>/<version>/` builds, the config/geoip/addon data beside them, a legacy
|
|
116
|
+
* flat camoufox-js install, and any legacy stash a crashed upgrade left behind.
|
|
117
|
+
*
|
|
118
|
+
* Only when explicitly requested: the cache (~/.cache/camoufox) is shared with every
|
|
119
|
+
* other Camoufox user on the box, so the default leaves it alone. The directory is
|
|
120
|
+
* removed only when it is provably a Camoufox install (layout.isCamoufoxDir): a
|
|
121
|
+
* CAMOUFOX_INSTALL_DIR pointed at something else is refused, never deleted.
|
|
122
|
+
*/
|
|
123
|
+
function purgeBrowsers(env = process.env, platform = process.platform) {
|
|
124
|
+
const install = layout.resolveInstall(env, platform, os.homedir());
|
|
125
|
+
const dir = install.dir;
|
|
126
|
+
const spawnEnv = { ...env, ...install.overrides };
|
|
127
|
+
delete spawnEnv.CAMOUFOX_INSTALL_DIR;
|
|
128
|
+
delete spawnEnv.PLAYWRIGHT_BROWSERS_PATH;
|
|
129
|
+
let removedAnything = false;
|
|
130
|
+
|
|
131
|
+
if (pathExists(dir)) {
|
|
132
|
+
if (!layout.isCamoufoxDir(dir)) {
|
|
133
|
+
console.warn(`pi-research: ${dir} does not look like a Camoufox install; refusing to delete it.`);
|
|
134
|
+
} else {
|
|
135
|
+
const cli = resolveCamoufoxCli();
|
|
136
|
+
if (cli) {
|
|
137
|
+
// The upstream CLI removes the whole data dir; -y skips its interactive prompt.
|
|
138
|
+
const res = spawnSync(process.execPath, [cli, 'remove', '-y'], { stdio: 'inherit', env: spawnEnv });
|
|
139
|
+
if (res.error || res.status !== 0) {
|
|
140
|
+
console.warn(`pi-research: camoufox remove did not succeed (${res.error ? res.error.message : `exit ${res.status}`}); removing the directory directly.`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
// Either the CLI is unavailable (package already uninstalled) or it did not finish:
|
|
144
|
+
// finish the job ourselves, under the same ownership guard.
|
|
145
|
+
if (pathExists(dir)) layout.removeInstallDir(dir);
|
|
146
|
+
removedAnything = !pathExists(dir);
|
|
147
|
+
if (removedAnything) console.log(`pi-research: removed Camoufox browser data at ${dir}.`);
|
|
148
|
+
else console.warn(`pi-research: could not fully remove ${dir}; remove it manually.`);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
// Stashes from a crashed upgrade are legacy browsers too (proven by listStashes).
|
|
152
|
+
for (const stash of layout.listStashes(dir)) {
|
|
114
153
|
try {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
// directory name containing `$(…)` execute at uninstall time.
|
|
119
|
-
const isWin = process.platform === 'win32';
|
|
120
|
-
const res = bin
|
|
121
|
-
? (isWin
|
|
122
|
-
? spawnSync(`"${bin}"`, ['remove'], { stdio: 'inherit', shell: true })
|
|
123
|
-
: spawnSync(bin, ['remove'], { stdio: 'inherit' }))
|
|
124
|
-
: spawnSync('npx', ['camoufox-js', 'remove'], { stdio: 'inherit', shell: isWin });
|
|
125
|
-
if (res.error) throw res.error;
|
|
126
|
-
if (res.status !== 0) throw new Error(`camoufox remove exited with code ${res.status}`);
|
|
127
|
-
console.log('pi-research: camoufox browser binaries removed.');
|
|
154
|
+
fs.rmSync(stash, { recursive: true, force: true });
|
|
155
|
+
console.log(`pi-research: removed leftover legacy Camoufox stash ${stash}`);
|
|
156
|
+
removedAnything = true;
|
|
128
157
|
} catch (error) {
|
|
129
|
-
console.warn(`pi-research: could not remove
|
|
130
|
-
console.warn('pi-research: to remove manually, run: npx camoufox-js remove');
|
|
158
|
+
console.warn(`pi-research: could not remove ${stash}: ${error instanceof Error ? error.message : String(error)}`);
|
|
131
159
|
}
|
|
132
160
|
}
|
|
161
|
+
if (!removedAnything && !pathExists(dir)) console.log('pi-research: no Camoufox browser data to remove.');
|
|
162
|
+
return removedAnything;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function pathExists(p) {
|
|
166
|
+
try { return existsSync(p); } catch (_) { return false; }
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
if (process.env.PI_RESEARCH_PURGE_BROWSERS === '1') {
|
|
170
|
+
try {
|
|
171
|
+
purgeBrowsers();
|
|
172
|
+
} catch (error) {
|
|
173
|
+
console.warn(`pi-research: could not remove camoufox binaries: ${error instanceof Error ? error.message : String(error)}`);
|
|
174
|
+
console.warn('pi-research: to remove manually, run: npx camoufox remove -y');
|
|
175
|
+
}
|
|
133
176
|
} else {
|
|
134
177
|
console.log('pi-research: leaving shared camoufox binaries in place (set PI_RESEARCH_PURGE_BROWSERS=1 to remove).');
|
|
135
178
|
}
|