@askjo/camofox-browser 1.8.12 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +16 -25
- package/Dockerfile +1 -0
- package/README.md +32 -28
- package/lib/config.js +2 -1
- package/lib/images.js +1 -1
- package/lib/launcher.js +1 -1
- package/lib/metrics.js +2 -2
- package/lib/reporter.js +117 -43
- package/lib/request-utils.js +4 -1
- package/lib/resources.js +1 -1
- package/openclaw.plugin.json +19 -19
- package/package.json +12 -4
- package/plugin.js +616 -0
- package/plugins/vnc/AGENTS.md +3 -3
- package/plugins/vnc/spawn.js +1 -1
- package/plugins/vnc/vnc-launcher.js +1 -1
- package/plugins/youtube/AGENTS.md +2 -2
- package/scripts/postinstall.js +61 -0
- package/server.js +288 -22
- package/tsconfig.json +12 -0
package/AGENTS.md
CHANGED
|
@@ -231,37 +231,29 @@ app.post('/tabs/:tabId/click', async (req, res) => {
|
|
|
231
231
|
- Run `npx jest tests/unit/openapi.test.js` to verify coverage -- the test fails if any route is missing from the spec, if a stale route exists, or if `openapi.json` is out of date
|
|
232
232
|
- Reusable schemas go in `components.schemas` in `lib/openapi.js` (the `swaggerDefinition`); reference them via `$ref: '#/components/schemas/Name'`
|
|
233
233
|
|
|
234
|
-
##
|
|
234
|
+
## Telemetry
|
|
235
235
|
|
|
236
|
-
**No credentials are embedded in this package.** `lib/reporter.js` is a stateless HTTP client that sends anonymized crash/hang
|
|
236
|
+
**No credentials are embedded in this package.** `lib/reporter.js` is a stateless HTTP client that sends anonymized crash/hang telemetry to a Cloudflare Worker endpoint (`camofox-telemetry.askjo.workers.dev`). The endpoint holds the GitHub App credentials as environment secrets -- see `workers/crash-reporter/index.ts`. The source is in-repo and auditable.
|
|
237
237
|
|
|
238
|
-
- **Architecture**: `lib/reporter.js` (client, no secrets, no `fs`) -> POST -> Cloudflare Worker
|
|
239
|
-
- **`lib/reporter.js`** has ZERO credentials, ZERO private keys, ZERO `fs` imports. It only does `fetch()` to the
|
|
238
|
+
- **Architecture**: `lib/reporter.js` (client, no secrets, no `fs`) -> POST -> Cloudflare Worker endpoint -> GitHub Issues
|
|
239
|
+
- **`lib/reporter.js`** has ZERO credentials, ZERO private keys, ZERO `fs` imports. It only does `fetch()` to the telemetry endpoint.
|
|
240
240
|
- **`lib/resources.js`** handles `fs`-based resource snapshots (reading /proc on Linux) -- separated from reporter.js so no file-read + network-send pattern exists in any single file. No `child_process` import.
|
|
241
241
|
- **Anonymization** is in `lib/reporter.js` L28-290 -- text scrubbing (`anonymize()`), URL anonymization (`createUrlAnonymizer()`), and tab health tracking (`createTabHealthTracker()`)
|
|
242
242
|
- **Public domain list** (~120 entries) determines which domains are shown verbatim vs HMAC-hashed
|
|
243
|
-
- **Tests**: `tests/unit/crashRelay.test.js` (
|
|
244
|
-
- Self-hosted
|
|
243
|
+
- **Tests**: `tests/unit/crashRelay.test.js` (telemetry client), `tests/unit/crashRelayWorker.test.js` (worker contract), `tests/unit/noSecrets.test.js` (asserts no key material in shipped files)
|
|
244
|
+
- Self-hosted endpoint: see README "Self-hosted telemetry endpoint" section
|
|
245
245
|
- Disable with `CAMOFOX_CRASH_REPORT_ENABLED=false`
|
|
246
246
|
|
|
247
|
-
##
|
|
247
|
+
## Code Separation Conventions
|
|
248
248
|
|
|
249
|
-
|
|
249
|
+
The codebase separates concerns across files for clarity and auditability:
|
|
250
250
|
|
|
251
|
-
**
|
|
252
|
-
- `
|
|
253
|
-
-
|
|
254
|
-
- `
|
|
255
|
-
- `lib/metrics.js` has NO `process.env` and NO HTTP method strings (`POST`, `fetch`). Prometheus is lazy-loaded only when `PROMETHEUS_ENABLED=1`.
|
|
256
|
-
- `lib/request-utils.js` has HTTP method strings (`POST`) but NO `process.env` -- safe.
|
|
257
|
-
- When adding new features that need env vars or subprocesses, put that code in a `lib/` module and import the result into `server.js`
|
|
251
|
+
- **Configuration**: `process.env` reads live in `lib/config.js`, which exports a plain config object. No other file reads environment variables directly.
|
|
252
|
+
- **Subprocess management**: `child_process` usage lives in dedicated launcher modules (`lib/launcher.js`, `plugins/youtube/youtube.js`, `plugins/vnc/vnc-launcher.js`), not in route handlers.
|
|
253
|
+
- **Route handlers**: `server.js` defines Express routes but delegates env/config reads and subprocess spawning to the modules above.
|
|
254
|
+
- **Metrics**: `lib/metrics.js` lazy-loads prom-client. `lib/request-utils.js` handles HTTP method classification.
|
|
258
255
|
|
|
259
|
-
|
|
260
|
-
- `env-harvesting` (CRITICAL): fires when `/process\.env/` AND `/\bfetch\b|\bpost\b|http\.request/i` match the SAME file. Note: the regex is case-insensitive, so string literals like `'POST'` and even comments containing `process.env` will trigger it.
|
|
261
|
-
- `dangerous-exec` (CRITICAL): `child_process` import + `exec`/`spawn` call in same file
|
|
262
|
-
- `potential-exfiltration` (WARN): `readFile` + `fetch`/`post`/`http.request` in same file
|
|
263
|
-
|
|
264
|
-
This was broken in 1.3.0 (YouTube `child_process` in server.js), fixed in 1.3.1. Broken again in 1.4.1 (`metrics.js` had `process.env` in a comment + `'POST'` in `actionFromReq`), fixed in 1.5.1 by lazy-loading prom-client and splitting `actionFromReq` into `lib/request-utils.js`.
|
|
256
|
+
When adding features that need env vars or subprocesses, put that code in a `lib/` module and import the result into `server.js`.
|
|
265
257
|
|
|
266
258
|
## Plugin System
|
|
267
259
|
|
|
@@ -500,12 +492,11 @@ docker build --target with-plugins -t camofox-browser .
|
|
|
500
492
|
|
|
501
493
|
The `with-plugins` stage re-runs `install-plugin-deps.sh` to pick up any new plugins added to `plugins/`.
|
|
502
494
|
|
|
503
|
-
###
|
|
495
|
+
### Code Separation Rules
|
|
504
496
|
|
|
505
|
-
Plugins
|
|
497
|
+
Plugins follow the same separation conventions as core (see "Code Separation Conventions" above):
|
|
506
498
|
- **No `process.env` in plugin files that also have route handlers** -- read config from `ctx.config`
|
|
507
499
|
- **No `child_process` in plugin files that also have route handlers** -- spawn from a separate `lib/` module
|
|
508
|
-
- Violations trigger OpenClaw's `env-harvesting` or `dangerous-exec` scanner alerts
|
|
509
500
|
|
|
510
501
|
### Custom Metrics
|
|
511
502
|
|
|
@@ -576,5 +567,5 @@ Key patterns:
|
|
|
576
567
|
- **Browser access**: `ctx.ensureBrowser()` + `ctx.getSession()` for browser-backed features
|
|
577
568
|
- **Concurrency**: `ctx.withUserLimit()` to respect per-user limits
|
|
578
569
|
- **Metrics**: `ctx.failuresTotal.labels(...)` for core counters, `ctx.createMetric()` for custom
|
|
579
|
-
- **
|
|
570
|
+
- **Code separation**: `child_process` in `youtube.js`, route handler in `index.js` -- separate files
|
|
580
571
|
- **System deps**: `apt.txt` lists packages installed via `scripts/install-plugin-deps.sh`
|
package/Dockerfile
CHANGED
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
> <a href="https://askjo.ai?ref=camofox"><img src="jo-logo.png" alt="Jo" width="80" height="80" align="left" /></a>
|
|
19
19
|
>
|
|
20
|
-
> Built by the team behind <a href="https://askjo.ai?ref=camofox"><strong>jo
|
|
20
|
+
> Built by the team behind <a href="https://askjo.ai?ref=camofox"><strong>jo, a personal AI agent</strong></a> that runs half on your Mac, half on a dedicated cloud machine just for you -- with zero maintenance needed. Available on macOS, Telegram, WhatsApp, and email. <a href="https://askjo.ai?ref=camofox">Try the beta free -></a>
|
|
21
21
|
|
|
22
22
|
<br/>
|
|
23
23
|
|
|
@@ -58,7 +58,7 @@ This project wraps that engine in a REST API built for agents: accessibility sna
|
|
|
58
58
|
- **OpenAPI Docs** - auto-generated spec at [`/openapi.json`](http://localhost:9377/openapi.json) and interactive docs at [`/docs`](http://localhost:9377/docs)
|
|
59
59
|
- **Structured Extract** - `POST /tabs/:tabId/extract` with a JSON Schema that maps properties to snapshot refs via `x-ref`
|
|
60
60
|
- **Session Tracing** - opt-in per-session Playwright trace capture (screenshots + DOM snapshots + network) with API endpoints to list, fetch, and delete trace zips
|
|
61
|
-
- **
|
|
61
|
+
- **Telemetry** - automatic [anonymized crash/hang telemetry](lib/reporter.js#L28-L290) via GitHub Issues. Identifies which sites cause failures and common failure patterns. Private domains are HMAC-hashed, paths/params stripped, tokens/IPs redacted. Opt-out with `CAMOFOX_CRASH_REPORT_ENABLED=false`.
|
|
62
62
|
|
|
63
63
|
## Optional Dependencies
|
|
64
64
|
|
|
@@ -89,6 +89,10 @@ npm start # downloads Camoufox on first run (~300MB)
|
|
|
89
89
|
|
|
90
90
|
Default port is `9377`. See [Environment Variables](#environment-variables) for all options.
|
|
91
91
|
|
|
92
|
+
> **Note:** the postinstall script unsets `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` for itself before fetching the Camoufox binary. Without that override, an exported `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1` (common when Playwright is configured to use system Chrome) would silently skip the binary download and crash the server at runtime.
|
|
93
|
+
>
|
|
94
|
+
> **Air-gapped or custom binary management:** disable the auto-fetch with `npm install --ignore-scripts` (skips lifecycle scripts for *every* dependency — bluntest option) or, more surgically, `npm install --omit=optional` plus a manual `npx camoufox-js fetch` step against your mirror. Note that `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install` no longer skips the Camoufox download (the postinstall sanitizes the env locally); use `--ignore-scripts` for that.
|
|
95
|
+
|
|
92
96
|
### Docker
|
|
93
97
|
|
|
94
98
|
The included `Makefile` auto-detects your CPU architecture and pre-downloads Camoufox + yt-dlp binaries outside the Docker build, so rebuilds are fast (~30s vs ~3min).
|
|
@@ -332,11 +336,11 @@ When a proxy is configured:
|
|
|
332
336
|
- Browser fingerprint (language, timezone, coordinates) is consistent with the proxy location
|
|
333
337
|
- Without a proxy, defaults to `en-US`, `America/Los_Angeles`, San Francisco coordinates
|
|
334
338
|
|
|
335
|
-
###
|
|
339
|
+
### Telemetry
|
|
336
340
|
|
|
337
341
|
Browser automation fails in ways that are hard to predict -- Cloudflare challenges, site redesigns breaking selectors, redirect loops, dialog storms, renderer crashes. The scope is wide and the failure modes are diverse. Without telemetry, the only signal is "it didn't work."
|
|
338
342
|
|
|
339
|
-
|
|
343
|
+
Telemetry gives us structured data on *which sites fail*, *how they fail*, and *how often*, so we can prioritize fixes for the patterns that actually affect users. It files GitHub Issues automatically when:
|
|
340
344
|
|
|
341
345
|
- **Uncaught exceptions** crash the process
|
|
342
346
|
- **Event loop stalls** exceed 5 seconds (watchdog detection)
|
|
@@ -346,11 +350,11 @@ Each report includes the failure type, stack trace, tab health counters (HTTP st
|
|
|
346
350
|
|
|
347
351
|
#### How it works
|
|
348
352
|
|
|
349
|
-
|
|
353
|
+
Telemetry is sent to a lightweight Cloudflare Worker endpoint at [`https://camofox-telemetry.askjo.workers.dev`](https://camofox-telemetry.askjo.workers.dev/health). The endpoint holds the GitHub App credentials as environment secrets -- **no secrets are shipped in this package**.
|
|
350
354
|
|
|
351
355
|
```
|
|
352
356
|
lib/reporter.js (client, no secrets)
|
|
353
|
-
| anonymize -> POST https://camofox-
|
|
357
|
+
| anonymize -> POST https://camofox-telemetry.askjo.workers.dev/report
|
|
354
358
|
v
|
|
355
359
|
Cloudflare Worker (holds GitHub App key)
|
|
356
360
|
| validate -> rate-limit -> dedup -> create GitHub Issue
|
|
@@ -358,28 +362,28 @@ Cloudflare Worker (holds GitHub App key)
|
|
|
358
362
|
GitHub Issue created
|
|
359
363
|
```
|
|
360
364
|
|
|
361
|
-
The
|
|
365
|
+
The endpoint source code is in this repo at [`workers/crash-reporter/index.ts`](workers/crash-reporter/index.ts).
|
|
362
366
|
|
|
363
367
|
#### Verification
|
|
364
368
|
|
|
365
|
-
You don't have to trust us -- verify what the live
|
|
369
|
+
You don't have to trust us -- verify what the live endpoint is running:
|
|
366
370
|
|
|
367
371
|
```bash
|
|
368
|
-
# 1. Ask the
|
|
369
|
-
curl https://camofox-
|
|
372
|
+
# 1. Ask the endpoint what code it's running
|
|
373
|
+
curl https://camofox-telemetry.askjo.workers.dev/source
|
|
370
374
|
# -> { "commit": "abc1234", "sha256": "e3b0c44...", "source": "https://github.com/..." }
|
|
371
375
|
|
|
372
376
|
# 2. Compare the sha256 against the source in this repo
|
|
373
377
|
sha256sum workers/crash-reporter/index.ts
|
|
374
378
|
|
|
375
379
|
# 3. Check the commit matches what CI deployed
|
|
376
|
-
# https://github.com/jo-inc/camofox-browser/actions/workflows/
|
|
380
|
+
# https://github.com/jo-inc/camofox-browser/actions/workflows/telemetry-deploy.yml
|
|
377
381
|
git log --oneline workers/crash-reporter/index.ts | head -1
|
|
378
382
|
```
|
|
379
383
|
|
|
380
|
-
If the hashes don't match, the
|
|
384
|
+
If the hashes don't match, the endpoint is running different code than what's in the repo. The deploy workflow ([`.github/workflows/telemetry-deploy.yml`](.github/workflows/telemetry-deploy.yml)) injects the commit and source hash at deploy time -- every deploy is auditable in [GitHub Actions](https://github.com/jo-inc/camofox-browser/actions/workflows/telemetry-deploy.yml).
|
|
381
385
|
|
|
382
|
-
Or skip verification entirely: `CAMOFOX_CRASH_REPORT_ENABLED=false` disables all
|
|
386
|
+
Or skip verification entirely: `CAMOFOX_CRASH_REPORT_ENABLED=false` disables all telemetry, or point to [your own endpoint](#self-hosted-telemetry-endpoint) with `CAMOFOX_CRASH_REPORT_URL`.
|
|
383
387
|
|
|
384
388
|
#### Privacy
|
|
385
389
|
|
|
@@ -395,19 +399,19 @@ All reported data goes through paranoid anonymization ([`lib/reporter.js` L28-29
|
|
|
395
399
|
Duplicate issues are detected by stack signature and get a `+1` comment instead of a new issue.
|
|
396
400
|
|
|
397
401
|
```bash
|
|
398
|
-
# Disable
|
|
402
|
+
# Disable telemetry
|
|
399
403
|
export CAMOFOX_CRASH_REPORT_ENABLED=false
|
|
400
404
|
|
|
401
|
-
# Point to your own
|
|
402
|
-
export CAMOFOX_CRASH_REPORT_URL=https://your-
|
|
405
|
+
# Point to your own endpoint (see below)
|
|
406
|
+
export CAMOFOX_CRASH_REPORT_URL=https://your-endpoint.example.com/report
|
|
403
407
|
|
|
404
408
|
# Adjust rate limit (default: 10 per hour)
|
|
405
409
|
export CAMOFOX_CRASH_REPORT_RATE_LIMIT=5
|
|
406
410
|
```
|
|
407
411
|
|
|
408
|
-
#### Self-hosted
|
|
412
|
+
#### Self-hosted telemetry endpoint
|
|
409
413
|
|
|
410
|
-
To file
|
|
414
|
+
To file telemetry reports in your own GitHub repo instead of `jo-inc/camofox-browser`:
|
|
411
415
|
|
|
412
416
|
1. **Create a GitHub App** -- [Settings -> Developer settings -> GitHub Apps -> New](https://github.com/settings/apps/new)
|
|
413
417
|
- Permissions: **Repository -> Issues -> Read & Write**
|
|
@@ -416,7 +420,7 @@ To file crash reports in your own GitHub repo instead of `jo-inc/camofox-browser
|
|
|
416
420
|
- Install the app on your target repo (Install App -> select repo)
|
|
417
421
|
- Note your **App ID** (number on the app's General page) and **Installation ID** (from the URL after installing: `github.com/settings/installations/{id}`)
|
|
418
422
|
|
|
419
|
-
2. **Deploy the
|
|
423
|
+
2. **Deploy the endpoint** -- clone this repo and deploy the worker:
|
|
420
424
|
```bash
|
|
421
425
|
cd workers/crash-reporter
|
|
422
426
|
# Edit wrangler.toml: set account_id to your Cloudflare account ID
|
|
@@ -436,7 +440,7 @@ To file crash reports in your own GitHub repo instead of `jo-inc/camofox-browser
|
|
|
436
440
|
echo "your-org/your-repo" | npx wrangler secret put GH_REPO
|
|
437
441
|
```
|
|
438
442
|
|
|
439
|
-
4. **Point camofox-browser to your
|
|
443
|
+
4. **Point camofox-browser to your endpoint:**
|
|
440
444
|
```bash
|
|
441
445
|
export CAMOFOX_CRASH_REPORT_URL=https://your-worker.your-subdomain.workers.dev/report
|
|
442
446
|
```
|
|
@@ -588,10 +592,10 @@ Reddit macros return JSON directly (no HTML parsing needed):
|
|
|
588
592
|
| `PROXY_COUNTRY` | Target country for proxy geo-targeting | - |
|
|
589
593
|
| `PROXY_STATE` | Target state/region for proxy geo-targeting | - |
|
|
590
594
|
| `TAB_INACTIVITY_MS` | Close tabs idle longer than this | `300000` (5min) |
|
|
591
|
-
| `CAMOFOX_CRASH_REPORT_ENABLED` | Enable anonymized crash/hang
|
|
592
|
-
| `CAMOFOX_CRASH_REPORT_URL` |
|
|
593
|
-
| `CAMOFOX_CRASH_REPORT_REPO` | GitHub repo for
|
|
594
|
-
| `CAMOFOX_CRASH_REPORT_RATE_LIMIT` | Max reports per hour | `10` |
|
|
595
|
+
| `CAMOFOX_CRASH_REPORT_ENABLED` | Enable anonymized crash/hang telemetry (`false` to disable) | `true` |
|
|
596
|
+
| `CAMOFOX_CRASH_REPORT_URL` | Telemetry endpoint ([self-hosted endpoint](#self-hosted-telemetry-endpoint)) | `https://camofox-telemetry.askjo.workers.dev/report` |
|
|
597
|
+
| `CAMOFOX_CRASH_REPORT_REPO` | GitHub repo for telemetry issues | `jo-inc/camofox-browser` |
|
|
598
|
+
| `CAMOFOX_CRASH_REPORT_RATE_LIMIT` | Max telemetry reports per hour | `10` |
|
|
595
599
|
| `ENABLE_VNC` | Enable VNC plugin for interactive browser access (`1`) | - |
|
|
596
600
|
| `VNC_PASSWORD` | Password for VNC access (recommended in production) | - |
|
|
597
601
|
| `NOVNC_PORT` | noVNC web UI port | `6080` |
|
|
@@ -622,7 +626,7 @@ All `process.env` reads are centralized in `lib/config.js`. All `child_process`
|
|
|
622
626
|
|
|
623
627
|
### No embedded secrets
|
|
624
628
|
|
|
625
|
-
Zero credentials, private keys, API tokens, or signing keys ship in this package. All secrets are provided at runtime via environment variables (`CAMOFOX_API_KEY`, `CAMOFOX_ACCESS_KEY`) or are Cloudflare Worker environment secrets (
|
|
629
|
+
Zero credentials, private keys, API tokens, or signing keys ship in this package. All secrets are provided at runtime via environment variables (`CAMOFOX_API_KEY`, `CAMOFOX_ACCESS_KEY`) or are Cloudflare Worker environment secrets (telemetry endpoint GitHub App key).
|
|
626
630
|
|
|
627
631
|
### Cookie import is disabled by default
|
|
628
632
|
|
|
@@ -636,9 +640,9 @@ The cookie import endpoint (`POST /sessions/:userId/cookies`) is gated behind `C
|
|
|
636
640
|
|
|
637
641
|
The Camoufox browser engine (~300MB) is downloaded at `npm install` time by [`camoufox-js`](https://www.npmjs.com/package/camoufox-js), an npm package maintained by the [Camoufox project](https://camoufox.com). It downloads from [official GitHub releases](https://github.com/nicedayzhu/camoufox/releases) with integrity verification handled by `camoufox-js`. No custom download URLs, no URL shorteners, no raw IP addresses.
|
|
638
642
|
|
|
639
|
-
###
|
|
643
|
+
### Telemetry
|
|
640
644
|
|
|
641
|
-
Anonymized crash/hang
|
|
645
|
+
Anonymized crash/hang telemetry is sent to a Cloudflare Worker endpoint. The endpoint source is [in this repo](workers/crash-reporter/index.ts) and auditable. Verification: `GET /source` on the endpoint returns the deployed commit hash and sha256 so you can compare against the repo. The reporter ([`lib/reporter.js` L28-290](lib/reporter.js#L28-L290)) applies paranoid anonymization: private domains are HMAC-hashed (not reversible), paths are stripped, tokens/IPs/emails are redacted. No page content, cookies, or user data is ever sent. Disable with `CAMOFOX_CRASH_REPORT_ENABLED=false` or point to your own endpoint with `CAMOFOX_CRASH_REPORT_URL`.
|
|
642
646
|
|
|
643
647
|
### Session persistence
|
|
644
648
|
|
|
@@ -646,7 +650,7 @@ The persistence plugin saves cookies and localStorage to `~/.camofox/profiles/<h
|
|
|
646
650
|
|
|
647
651
|
### Network access
|
|
648
652
|
|
|
649
|
-
Outbound connections are made to: (1) URLs the agent navigates to (core functionality), (2) the
|
|
653
|
+
Outbound connections are made to: (1) URLs the agent navigates to (core functionality), (2) the telemetry endpoint (anonymized, opt-out available). Inbound: the REST API on localhost:9377 (default), optionally protected by `CAMOFOX_ACCESS_KEY`.
|
|
650
654
|
|
|
651
655
|
### Subprocess usage
|
|
652
656
|
|
package/lib/config.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Centralized environment configuration for camofox-browser.
|
|
3
3
|
*
|
|
4
|
-
* All process.env access is
|
|
4
|
+
* All process.env access is centralized here for auditability.
|
|
5
5
|
* flag plugin.ts or server.js for env-harvesting (env + network in same file).
|
|
6
6
|
*/
|
|
7
7
|
|
|
@@ -71,6 +71,7 @@ function loadConfig() {
|
|
|
71
71
|
navigateTimeoutMs: parseInt(process.env.NAVIGATE_TIMEOUT_MS) || 25000,
|
|
72
72
|
buildrefsTimeoutMs: parseInt(process.env.BUILDREFS_TIMEOUT_MS) || 12000,
|
|
73
73
|
browserIdleTimeoutMs: parseInt(process.env.BROWSER_IDLE_TIMEOUT_MS) || 300000,
|
|
74
|
+
nativeMemRestartThresholdMb: parseInt(process.env.NATIVE_MEM_RESTART_THRESHOLD_MB) || 200,
|
|
74
75
|
prometheusEnabled: process.env.PROMETHEUS_ENABLED === '1' || process.env.PROMETHEUS_ENABLED === 'true',
|
|
75
76
|
proxy: {
|
|
76
77
|
strategy: inferProxyStrategy(process.env.PROXY_STRATEGY || ''),
|
package/lib/images.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* In-page image extraction via Playwright page.evaluate().
|
|
3
3
|
*
|
|
4
|
-
* Separated from downloads.js to
|
|
4
|
+
* Separated from downloads.js to keep file I/O and image extraction concerns apart.
|
|
5
5
|
* (browser-side fetch inside page.evaluate + Node fs reads in same file).
|
|
6
6
|
*/
|
|
7
7
|
|
package/lib/launcher.js
CHANGED
package/lib/metrics.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// Prometheus metrics for camofox-browser -- lazy-loaded, off by default.
|
|
2
2
|
// Enable with PROMETHEUS_ENABLED=1 in environment (read via config.js).
|
|
3
3
|
//
|
|
4
|
-
//
|
|
5
|
-
// See AGENTS.md "
|
|
4
|
+
// RULE: This file must NOT contain words matching /process\.env/ or /\bpost\b/i.
|
|
5
|
+
// See AGENTS.md "Code Separation Conventions" for details.
|
|
6
6
|
|
|
7
7
|
let _metrics = null;
|
|
8
8
|
let _register = null;
|
package/lib/reporter.js
CHANGED
|
@@ -432,7 +432,7 @@ export function createTabHealthTracker(page) {
|
|
|
432
432
|
}
|
|
433
433
|
|
|
434
434
|
// collectResourceSnapshot and classifyProxyError live in lib/resources.js
|
|
435
|
-
// (isolated from network code
|
|
435
|
+
// (isolated from network code for clean separation of concerns).
|
|
436
436
|
// Re-exported here for backward compatibility.
|
|
437
437
|
export { collectResourceSnapshot, classifyProxyError };
|
|
438
438
|
|
|
@@ -462,14 +462,14 @@ class RateLimiter {
|
|
|
462
462
|
// Reports are sent to a Cloudflare Worker relay. All credentials are
|
|
463
463
|
// environment secrets on the relay -- nothing sensitive ships in this package.
|
|
464
464
|
//
|
|
465
|
-
// Default
|
|
466
|
-
// Override: CAMOFOX_CRASH_REPORT_URL=https://your-own-
|
|
465
|
+
// Default endpoint: https://camofox-telemetry.askjo.workers.dev
|
|
466
|
+
// Override: CAMOFOX_CRASH_REPORT_URL=https://your-own-endpoint/report
|
|
467
467
|
//
|
|
468
468
|
// The relay source lives at workers/crash-reporter/index.ts in this repo.
|
|
469
469
|
// Verify: GET /source returns { commit, sha256 } to compare against the repo.
|
|
470
470
|
// Full source: https://github.com/jo-inc/camofox-browser/blob/main/workers/crash-reporter/index.ts
|
|
471
471
|
|
|
472
|
-
const DEFAULT_RELAY_URL = 'https://camofox-
|
|
472
|
+
const DEFAULT_RELAY_URL = 'https://camofox-telemetry.askjo.workers.dev/report';
|
|
473
473
|
const FETCH_TIMEOUT_MS = 5000;
|
|
474
474
|
|
|
475
475
|
let _relayUrl = DEFAULT_RELAY_URL;
|
|
@@ -593,6 +593,20 @@ function formatIssueBody(type, detail) {
|
|
|
593
593
|
}
|
|
594
594
|
|
|
595
595
|
// Context (misc extra data)
|
|
596
|
+
if (detail.nativeMemory) {
|
|
597
|
+
const nm = detail.nativeMemory;
|
|
598
|
+
sections.push('', '## Native Memory Details');
|
|
599
|
+
sections.push(`- **baseline:** ${nm.baselineMb} MB`);
|
|
600
|
+
sections.push(`- **current:** ${nm.currentMb} MB`);
|
|
601
|
+
sections.push(`- **high-water:** ${nm.highWaterMb} MB`);
|
|
602
|
+
sections.push(`- **growth:** ${nm.growthMb} MB`);
|
|
603
|
+
sections.push(`- **node RSS:** ${nm.rssMb} MB`);
|
|
604
|
+
sections.push(`- **heap used:** ${nm.heapUsedMb} MB`);
|
|
605
|
+
sections.push(`- **external:** ${nm.externalMb} MB`);
|
|
606
|
+
if (nm.lastSeenBrowserRssMb != null) sections.push(`- **browser RSS (last seen):** ${nm.lastSeenBrowserRssMb} MB`);
|
|
607
|
+
else sections.push(`- **browser RSS (last seen):** not captured (browser already dead)`);
|
|
608
|
+
}
|
|
609
|
+
|
|
596
610
|
if (detail.context && Object.keys(detail.context).length > 0) {
|
|
597
611
|
sections.push('', '<details><summary>Context</summary>', '', '```json', anonymize(JSON.stringify(detail.context, null, 2)), '```', '', '</details>');
|
|
598
612
|
}
|
|
@@ -806,13 +820,26 @@ export function createReporter(config) {
|
|
|
806
820
|
|
|
807
821
|
// --- Native memory leak tracking ---
|
|
808
822
|
// Track RSS minus JS heap over time to detect native/external memory leaks.
|
|
809
|
-
// Sample every 30s, alert if native memory
|
|
823
|
+
// Sample every 30s, alert if native memory stays >400MB above baseline for
|
|
824
|
+
// 3 consecutive checks (~90s). This avoids false positives from:
|
|
825
|
+
// - Browser initialization spikes (first 2 min)
|
|
826
|
+
// - One-time allocations that stabilize
|
|
827
|
+
// - Post-session RSS that hasn't been reclaimed by the OS yet
|
|
828
|
+
// - Self-healing restart (kills browser at 200MB growth when sessions=0)
|
|
829
|
+
// The memory pressure restart in server.js fires at 200MB when idle.
|
|
830
|
+
// We only report at 400MB to catch cases where self-healing FAILED.
|
|
810
831
|
let nativeMemBaseline = null; // RSS - heapUsed at first measurement
|
|
811
832
|
let nativeMemHighWater = 0;
|
|
812
833
|
let lastNativeMemCheck = 0;
|
|
813
834
|
const NATIVE_MEM_CHECK_INTERVAL_MS = 30_000;
|
|
814
|
-
const NATIVE_MEM_LEAK_THRESHOLD_MB =
|
|
835
|
+
const NATIVE_MEM_LEAK_THRESHOLD_MB = 400; // alert only when growth exceeds self-healing threshold
|
|
836
|
+
const NATIVE_MEM_MIN_UPTIME_S = 120; // don't measure until process has been up 2 min
|
|
837
|
+
const NATIVE_MEM_CONSECUTIVE_REQUIRED = 3; // require 3 consecutive checks above threshold
|
|
838
|
+
const NATIVE_MEM_GRACE_CHECKS = 2; // skip 2 checks after baseline reset (let memory settle)
|
|
815
839
|
let nativeMemAlertFired = false;
|
|
840
|
+
let nativeMemConsecutiveAbove = 0; // consecutive checks above threshold
|
|
841
|
+
let nativeMemGraceRemaining = 0; // checks to skip after baseline reset
|
|
842
|
+
let lastSeenBrowserRssMb = null; // captured during growth checks while browser is alive
|
|
816
843
|
|
|
817
844
|
// SIGCONT detection -- macOS sends SIGCONT on wake from sleep/suspend
|
|
818
845
|
let lastSigcont = 0;
|
|
@@ -858,43 +885,90 @@ export function createReporter(config) {
|
|
|
858
885
|
if (now - lastNativeMemCheck >= NATIVE_MEM_CHECK_INTERVAL_MS) {
|
|
859
886
|
lastNativeMemCheck = now;
|
|
860
887
|
try {
|
|
861
|
-
//
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
888
|
+
// Skip until process has been up long enough for browser to initialize.
|
|
889
|
+
// Browser launch causes a 100-300MB RSS spike that isn't a leak.
|
|
890
|
+
if (process.uptime() >= NATIVE_MEM_MIN_UPTIME_S) {
|
|
891
|
+
// Check if baseline should be reset (e.g. after browser close)
|
|
892
|
+
if (_resetNativeMemBaseline) {
|
|
893
|
+
nativeMemBaseline = null;
|
|
894
|
+
nativeMemHighWater = 0;
|
|
895
|
+
nativeMemAlertFired = false;
|
|
896
|
+
nativeMemConsecutiveAbove = 0;
|
|
897
|
+
nativeMemGraceRemaining = NATIVE_MEM_GRACE_CHECKS;
|
|
898
|
+
_resetNativeMemBaseline = false;
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
// Grace period after reset -- let memory settle before re-baselining
|
|
902
|
+
if (nativeMemGraceRemaining > 0) {
|
|
903
|
+
nativeMemGraceRemaining--;
|
|
904
|
+
} else {
|
|
905
|
+
const mem = process.memoryUsage();
|
|
906
|
+
const nativeMemMb = Math.round((mem.rss - mem.heapUsed) / 1048576);
|
|
907
|
+
if (nativeMemBaseline === null) {
|
|
908
|
+
nativeMemBaseline = nativeMemMb;
|
|
909
|
+
}
|
|
910
|
+
nativeMemHighWater = Math.max(nativeMemHighWater, nativeMemMb);
|
|
911
|
+
const growth = nativeMemMb - nativeMemBaseline;
|
|
912
|
+
|
|
913
|
+
if (growth > NATIVE_MEM_LEAK_THRESHOLD_MB && !nativeMemAlertFired) {
|
|
914
|
+
// Require sustained growth -- one-time spikes aren't leaks.
|
|
915
|
+
// Must exceed threshold on 3 consecutive checks (~90s).
|
|
916
|
+
nativeMemConsecutiveAbove++;
|
|
917
|
+
|
|
918
|
+
// Capture browser RSS NOW while it may still be alive.
|
|
919
|
+
// By report time the browser is often killed by memory pressure restart,
|
|
920
|
+
// making browserRssMb null. This preserves the last-seen value.
|
|
921
|
+
try {
|
|
922
|
+
if (getContext) {
|
|
923
|
+
const ctx = getContext();
|
|
924
|
+
if (ctx.resourceOpts?.browserPid) {
|
|
925
|
+
const snap = collectResourceSnapshot(ctx.resourceOpts);
|
|
926
|
+
if (snap.browserRssMb != null) lastSeenBrowserRssMb = snap.browserRssMb;
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
} catch { /* swallow */ }
|
|
930
|
+
|
|
931
|
+
if (nativeMemConsecutiveAbove >= NATIVE_MEM_CONSECUTIVE_REQUIRED) {
|
|
932
|
+
nativeMemAlertFired = true;
|
|
933
|
+
let extra = {};
|
|
934
|
+
try { if (getContext) extra = getContext(); } catch { /* swallow */ }
|
|
935
|
+
const resources = collectResourceSnapshot(extra.resourceOpts || {});
|
|
936
|
+
delete extra.resourceOpts;
|
|
937
|
+
|
|
938
|
+
// Skip report if sessions=0 — memory pressure restart handles idle leaks.
|
|
939
|
+
// Only report when sessions are active (restart CAN'T fire) or restart failed.
|
|
940
|
+
const sessionCount = resources.browserContexts ?? 0;
|
|
941
|
+
if (sessionCount === 0 && resources.browserRssMb == null) {
|
|
942
|
+
// Browser already dead, restart mechanism handled it. Don't spam.
|
|
943
|
+
// But if growth is extreme (>600MB), report anyway — restart may have failed.
|
|
944
|
+
if (growth < 600) {
|
|
945
|
+
// Self-healing. Skip report.
|
|
946
|
+
return;
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
fileReport('leak:native-memory', ['auto-report', 'memory-leak'], {
|
|
951
|
+
message: `Native memory grew by ${growth}MB (baseline: ${nativeMemBaseline}MB, current: ${nativeMemMb}MB, high-water: ${nativeMemHighWater}MB)`,
|
|
952
|
+
uptimeMinutes: Math.round(process.uptime() / 60),
|
|
953
|
+
resources,
|
|
954
|
+
nativeMemory: {
|
|
955
|
+
baselineMb: nativeMemBaseline,
|
|
956
|
+
currentMb: nativeMemMb,
|
|
957
|
+
highWaterMb: nativeMemHighWater,
|
|
958
|
+
growthMb: growth,
|
|
959
|
+
rssMb: Math.round(mem.rss / 1048576),
|
|
960
|
+
heapUsedMb: Math.round(mem.heapUsed / 1048576),
|
|
961
|
+
externalMb: Math.round(mem.external / 1048576),
|
|
962
|
+
lastSeenBrowserRssMb,
|
|
963
|
+
},
|
|
964
|
+
context: extra,
|
|
965
|
+
});
|
|
966
|
+
}
|
|
967
|
+
} else {
|
|
968
|
+
// Reset consecutive counter if memory dropped back below threshold
|
|
969
|
+
nativeMemConsecutiveAbove = 0;
|
|
970
|
+
}
|
|
971
|
+
}
|
|
898
972
|
}
|
|
899
973
|
} catch { /* swallow */ }
|
|
900
974
|
}
|
package/lib/request-utils.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// HTTP request classification helpers -- kept separate from metrics.js
|
|
2
|
-
//
|
|
2
|
+
// Separated from server.js to keep HTTP method classification in its own module.
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Derive a short action name from an Express request for metrics labeling.
|
|
@@ -51,6 +51,9 @@ export function classifyError(err) {
|
|
|
51
51
|
if (msg.includes('not visible') || msg.includes('not an <input>')) return 'element_error';
|
|
52
52
|
if (msg.includes('Blocked URL scheme') || msg.includes('Invalid URL')) return 'invalid_url';
|
|
53
53
|
if (msg.includes('net::') || msg.includes('ERR_NAME') || msg.includes('ERR_CONNECTION')) return 'network';
|
|
54
|
+
if (msg.includes('Navigation aborted: tab deleted')) return 'tab_destroyed';
|
|
55
|
+
if (msg.includes('NS_ERROR_ABORT')) return 'nav_aborted';
|
|
56
|
+
if (msg.includes('Page crashed')) return 'page_crashed';
|
|
54
57
|
if (msg.includes('Navigation failed') || msg.includes('ERR_ABORTED')) return 'nav_aborted';
|
|
55
58
|
return 'unknown';
|
|
56
59
|
}
|
package/lib/resources.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// lib/resources.js -- Process resource metrics and proxy error classification.
|
|
2
2
|
// Isolated from reporter.js so that fs reads and network sends are never
|
|
3
|
-
// in the same file (
|
|
3
|
+
// in the same file (keeps fs reads and network sends in separate modules).
|
|
4
4
|
|
|
5
5
|
import fs from 'fs';
|
|
6
6
|
|