@phi-code-admin/camofox-browser 1.0.0 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/AGENTS.md +571 -571
  2. package/Dockerfile +86 -86
  3. package/LICENSE +21 -21
  4. package/README.md +691 -691
  5. package/camofox.config.json +10 -10
  6. package/lib/auth.js +134 -134
  7. package/lib/camoufox-executable.js +189 -189
  8. package/lib/config.js +153 -153
  9. package/lib/cookies.js +119 -119
  10. package/lib/downloads.js +168 -168
  11. package/lib/extract.js +74 -74
  12. package/lib/fly.js +54 -54
  13. package/lib/images.js +88 -88
  14. package/lib/inflight.js +16 -16
  15. package/lib/launcher.js +47 -47
  16. package/lib/macros.js +31 -31
  17. package/lib/metrics.js +184 -184
  18. package/lib/openapi.js +105 -105
  19. package/lib/persistence.js +89 -89
  20. package/lib/plugins.js +178 -175
  21. package/lib/proxy.js +277 -277
  22. package/lib/reporter.js +1102 -1102
  23. package/lib/request-utils.js +59 -59
  24. package/lib/resources.js +76 -76
  25. package/lib/snapshot.js +41 -41
  26. package/lib/tmp-cleanup.js +108 -108
  27. package/lib/tracing.js +137 -137
  28. package/openclaw.plugin.json +268 -268
  29. package/package.json +148 -148
  30. package/plugin.ts +758 -758
  31. package/plugins/persistence/AGENTS.md +37 -37
  32. package/plugins/persistence/README.md +48 -48
  33. package/plugins/persistence/index.js +124 -124
  34. package/plugins/vnc/AGENTS.md +42 -42
  35. package/plugins/vnc/README.md +165 -165
  36. package/plugins/vnc/apt.txt +7 -7
  37. package/plugins/vnc/index.js +142 -142
  38. package/plugins/vnc/spawn.js +8 -8
  39. package/plugins/vnc/vnc-launcher.js +64 -64
  40. package/plugins/vnc/vnc-watcher.sh +82 -82
  41. package/plugins/youtube/AGENTS.md +25 -25
  42. package/plugins/youtube/apt.txt +1 -1
  43. package/plugins/youtube/index.js +206 -206
  44. package/plugins/youtube/post-install.sh +5 -5
  45. package/plugins/youtube/youtube.js +301 -301
  46. package/run.sh +37 -37
  47. package/scripts/exec.js +8 -8
  48. package/scripts/generate-openapi.js +24 -24
  49. package/scripts/install-plugin-deps.sh +63 -63
  50. package/scripts/plugin.js +342 -342
  51. package/scripts/sync-version.js +25 -25
  52. package/server.js +6062 -6059
  53. package/tsconfig.json +12 -12
package/lib/reporter.js CHANGED
@@ -1,1102 +1,1102 @@
1
- // lib/reporter.js -- Crash/hang reporter for camofox-browser
2
- // Files GitHub issues with paranoid anonymization. No env reads here.
3
- // Config passed via createReporter(config) from lib/config.js.
4
-
5
- import crypto from 'crypto';
6
- import { monitorEventLoopDelay } from 'perf_hooks';
7
- import { collectResourceSnapshot, classifyProxyError } from './resources.js';
8
-
9
- // ============================================================================
10
- // Anonymization
11
- // ============================================================================
12
-
13
- const SAFE_HOSTS = new Set([
14
- 'github.com', 'api.github.com', 'npmjs.com', 'registry.npmjs.org',
15
- 'nodejs.org',
16
- ]);
17
-
18
- const SECRET_PREFIXES = [
19
- 'ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_',
20
- 'sk-', 'sk_live_', 'sk_test_', 'pk_live_', 'pk_test_',
21
- 'AKIA', 'ASIA',
22
- 'xox', 'Bearer ', 'Basic ',
23
- 'eyJ',
24
- ];
25
-
26
- /**
27
- * Paranoid anonymization of arbitrary text (stack traces, error messages, etc.)
28
- * Better to over-strip than leak. Order matters -- more specific patterns first.
29
- */
30
- export function anonymize(text) {
31
- if (!text || typeof text !== 'string') return text || '';
32
-
33
- let s = text;
34
-
35
- // 1. Strip known secret-prefixed tokens
36
- for (const prefix of SECRET_PREFIXES) {
37
- const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
38
- s = s.replace(new RegExp(escaped + '[A-Za-z0-9_\\-\\.=+/]{8,}', 'g'), '<token>');
39
- }
40
-
41
- // 2. Strip Bearer/Basic auth headers
42
- s = s.replace(/(?:Bearer|Basic)\s+[A-Za-z0-9_\-\.=+/]{8,}/gi, '<token>');
43
-
44
- // 3. Strip proxy URLs with credentials (before email -- email regex eats user:pass@host)
45
- s = s.replace(/(?:https?|socks[45]?):\/\/[^:]+:[^@]+@[^\s]+/gi, '<proxy-url>');
46
-
47
- // 4. Strip email addresses
48
- s = s.replace(/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g, '<email>');
49
-
50
- // 5. Strip full URLs (preserve scheme for context)
51
- s = s.replace(/(https?|wss?|ftp):\/\/[^\s'",)}\]>]+/g, (match, scheme) => {
52
- try {
53
- const u = new URL(match);
54
- if (SAFE_HOSTS.has(u.hostname)) return match;
55
- } catch { /* not a valid URL, strip it */ }
56
- return `<${scheme}-url>`;
57
- });
58
-
59
- // 6. Strip absolute file paths (Unix + Windows), preserve last filename
60
- s = s.replace(
61
- /(?:\/(?:Users|home|root|tmp|var|opt|data|app|srv|etc|mnt|run|snap|proc)\/[^\s:;,'")\]}]+|[A-Z]:\\(?:Users|Documents and Settings)\\[^\s:;,'")\]}]+)/g,
62
- (match) => {
63
- const parts = match.replace(/\\/g, '/').split('/');
64
- const filename = parts[parts.length - 1] || parts[parts.length - 2] || 'unknown';
65
- return `<path>/${filename}`;
66
- }
67
- );
68
-
69
- // 7. Strip IPv4 addresses
70
- s = s.replace(/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, '<ip>');
71
-
72
- // 8. Strip IPv6 addresses
73
- s = s.replace(/\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, '<ipv6>');
74
- s = s.replace(/::(?:ffff:)?(?:\d{1,3}\.){3}\d{1,3}/g, '<ipv6>');
75
-
76
- // 9. Strip hostnames in connection errors
77
- s = s.replace(
78
- /(?:ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|EHOSTUNREACH)\s+([a-zA-Z0-9.\-]+):(\d+)/g,
79
- (match, host, port) => {
80
- if (SAFE_HOSTS.has(host)) return match;
81
- return match.replace(host, '<host>');
82
- }
83
- );
84
-
85
- // 10. Strip Fly machine IDs (14-char hex), Docker container IDs (12+ hex)
86
- s = s.replace(/\b[0-9a-f]{12,64}\b/g, '<id>');
87
-
88
- // 11. Strip jo-* app names
89
- s = s.replace(/\bjo-(?:machine|browser|whatsapp|discord|bot)[a-z0-9\-]*/gi, '<app>');
90
-
91
- // 12. Strip environment variable assignments
92
- s = s.replace(/\b[A-Z][A-Z0-9_]{3,}=[^\s]{4,}/g, '<env-var>');
93
-
94
- // 13. Strip long alphanumeric strings (40+ chars)
95
- s = s.replace(/[A-Za-z0-9_\-]{40,}/g, '<redacted>');
96
-
97
- // 14. Strip base64 blobs (20+ chars with mixed case)
98
- s = s.replace(/[A-Za-z0-9+/]{20,}={0,3}/g, (match) => {
99
- if (/[a-z]/.test(match) && /[A-Z]/.test(match)) return '<redacted>';
100
- return match;
101
- });
102
-
103
- return s;
104
- }
105
-
106
- /**
107
- * Generate a stable signature for dedup. Uses error name + first meaningful
108
- * stack frame (file:line, not column -- columns shift with minor edits).
109
- */
110
- export function stackSignature(type, error) {
111
- const name = error?.name || error?.code || 'unknown';
112
- const message = error?.message || String(error || '');
113
-
114
- const stack = error?.stack || '';
115
- const frames = stack.split('\n').slice(1);
116
- let keyFrame = '';
117
- for (const frame of frames) {
118
- const trimmed = frame.trim();
119
- if (trimmed.startsWith('at ') && !trimmed.includes('node_modules') && !trimmed.includes('node:internal')) {
120
- const fileMatch = trimmed.match(/\(([^)]+)\)/) || trimmed.match(/at\s+(.+)$/);
121
- if (fileMatch) {
122
- const loc = fileMatch[1];
123
- const parts = loc.replace(/\\/g, '/').split('/');
124
- const last = parts[parts.length - 1];
125
- const [file, line] = last.split(':');
126
- keyFrame = `${file}:${line || '?'}`;
127
- break;
128
- }
129
- }
130
- }
131
-
132
- const raw = `${type}|${name}|${keyFrame || anonymize(message).slice(0, 80)}`;
133
- return fnv1a(raw);
134
- }
135
-
136
- /** FNV-1a hash -> 8-char hex. Stable bucketing, not crypto. */
137
- function fnv1a(str) {
138
- let hash = 0x811c9dc5;
139
- for (let i = 0; i < str.length; i++) {
140
- hash ^= str.charCodeAt(i);
141
- hash = (hash * 0x01000193) >>> 0;
142
- }
143
- return hash.toString(16).padStart(8, '0');
144
- }
145
-
146
- // ============================================================================
147
- // URL anonymization (per-report salted HMAC for private domains)
148
- // ============================================================================
149
-
150
- // Public domains safe to show verbatim in reports.
151
- // These are public knowledge -- showing "amazon.com" in a crash report is not PII.
152
- // Matched by suffix. NEVER add multi-tenant hosting (herokuapp.com, vercel.app, etc.)
153
- const PUBLIC_DOMAINS = [
154
- // CDN & edge
155
- 'cloudflare.com', 'cloudflare-dns.com', 'cloudflareinsights.com',
156
- 'fastly.net', 'fastlylb.net',
157
- 'akamaized.net', 'akamai.net', 'cloudfront.net',
158
- 'cdn.jsdelivr.net', 'unpkg.com', 'cdnjs.com',
159
- // Google
160
- 'google.com', 'googleapis.com', 'gstatic.com',
161
- 'googleusercontent.com', 'google-analytics.com', 'googletagmanager.com',
162
- 'googlesyndication.com', 'doubleclick.net', 'youtube.com', 'ytimg.com',
163
- 'recaptcha.net',
164
- // Microsoft
165
- 'microsoft.com', 'msecnd.net', 'azureedge.net', 'bing.com', 'live.com',
166
- 'outlook.com', 'office.com', 'linkedin.com',
167
- // Meta
168
- 'facebook.com', 'facebook.net', 'fbcdn.net', 'instagram.com', 'threads.net',
169
- 'whatsapp.com',
170
- // X/Twitter
171
- 'twitter.com', 'x.com', 'twimg.com',
172
- // GitHub
173
- 'github.com', 'githubusercontent.com', 'githubassets.com',
174
- // Major sites (common anti-bot / frustration sources)
175
- 'amazon.com', 'amazon.co.uk', 'amazon.de', 'amazon.co.jp',
176
- 'reddit.com', 'redd.it',
177
- 'apple.com', 'icloud.com',
178
- 'netflix.com', 'spotify.com', 'discord.com', 'discord.gg',
179
- 'tiktok.com', 'pinterest.com', 'tumblr.com',
180
- 'stackoverflow.com', 'stackexchange.com',
181
- 'medium.com', 'substack.com',
182
- 'nytimes.com', 'washingtonpost.com', 'bbc.co.uk', 'bbc.com', 'cnn.com',
183
- 'ebay.com', 'etsy.com', 'walmart.com', 'target.com', 'shopify.com',
184
- 'stripe.com', 'paypal.com',
185
- 'twitch.tv', 'vimeo.com', 'dailymotion.com',
186
- 'yahoo.com', 'duckduckgo.com', 'baidu.com',
187
- 'zoom.us', 'slack.com', 'notion.so', 'figma.com',
188
- 'dropbox.com', 'box.com',
189
- 'archive.org', 'web.archive.org',
190
- // Prediction markets & crypto (heavy anti-bot, commonly scraped)
191
- 'polymarket.com', 'kalshi.com', 'metaculus.com', 'manifold.markets',
192
- 'predictit.org', 'augur.net',
193
- 'coinbase.com', 'binance.com', 'kraken.com', 'gemini.com',
194
- 'coingecko.com', 'coinmarketcap.com',
195
- 'opensea.io', 'blur.io', 'rarible.com',
196
- 'etherscan.io', 'solscan.io', 'blockchair.com',
197
- 'uniswap.org', 'dexscreener.com', 'dextools.io',
198
- // Data / scraping targets (aggressive anti-bot)
199
- 'zillow.com', 'realtor.com', 'redfin.com', 'trulia.com',
200
- 'indeed.com', 'glassdoor.com', 'lever.co', 'greenhouse.io',
201
- 'airbnb.com', 'booking.com', 'expedia.com', 'tripadvisor.com',
202
- 'yelp.com', 'trustpilot.com',
203
- 'craigslist.org', 'nextdoor.com',
204
- 'ticketmaster.com', 'stubhub.com', 'seatgeek.com',
205
- // Finance / trading
206
- 'tradingview.com', 'investing.com', 'seekingalpha.com',
207
- 'finance.yahoo.com', 'bloomberg.com', 'reuters.com', 'wsj.com',
208
- 'robinhood.com', 'schwab.com', 'fidelity.com', 'etrade.com',
209
- // AI / developer tools
210
- 'openai.com', 'anthropic.com', 'huggingface.co',
211
- 'vercel.com', 'netlify.com', 'render.com', 'fly.io',
212
- 'npmjs.com', 'pypi.org', 'crates.io', 'pkg.go.dev',
213
- // Social / forums
214
- 'quora.com', 'hackernews.com', 'news.ycombinator.com',
215
- 'producthunt.com', 'indiehackers.com',
216
- // Reference
217
- 'wikipedia.org', 'wikimedia.org', 'mozilla.org', 'mozilla.net',
218
- // Anti-bot / CAPTCHA
219
- 'hcaptcha.com',
220
- // Fonts
221
- 'typekit.net', 'fontawesome.com',
222
- ].sort((a, b) => b.length - a.length); // longest-suffix-first
223
-
224
- // Stable key for domain hashing -- NOT a secret, just ensures consistent hashes
225
- // across reports so we can correlate "site-a1b2c3d4 caused 12 hangs this week".
226
- const DOMAIN_HASH_KEY = 'camofox-domain-hash-v1';
227
-
228
- /**
229
- * Create a URL anonymizer.
230
- * Public domains shown verbatim. Private domains get a stable hash
231
- * (same domain -> same hash across all reports, enabling correlation).
232
- */
233
- export function createUrlAnonymizer() {
234
-
235
- function isPublicDomain(hostname) {
236
- for (const d of PUBLIC_DOMAINS) {
237
- if (hostname === d || hostname.endsWith('.' + d)) return true;
238
- }
239
- return false;
240
- }
241
-
242
- function hashHost(hostname) {
243
- return 'site-' + crypto.createHmac('sha256', DOMAIN_HASH_KEY).update(hostname).digest('hex').slice(0, 8);
244
- }
245
-
246
- /**
247
- * Anonymize a URL. Preserves: scheme, public infra hostnames, path depth,
248
- * query param count, fragment presence. Strips everything else.
249
- *
250
- * Examples:
251
- * https://challenges.cloudflare.com/[path]/[path]/[path]
252
- * https://site-a1b2c3d4:8443/[path]/[path] ?[3] #[frag]
253
- */
254
- function anonymizeUrl(rawUrl) {
255
- if (!rawUrl || typeof rawUrl !== 'string') return '[empty]';
256
- if (rawUrl.startsWith('data:')) return '[data-uri]';
257
- if (rawUrl.startsWith('blob:')) return '[blob-uri]';
258
- if (rawUrl.startsWith('about:')) return rawUrl;
259
- if (rawUrl.startsWith('javascript:')) return '[javascript-uri]';
260
-
261
- let url;
262
- try { url = new URL(rawUrl); } catch { return '[invalid-url]'; }
263
-
264
- const parts = [url.protocol + '//'];
265
- const h = url.hostname.toLowerCase();
266
-
267
- if (/^(\d{1,3}\.){3}\d{1,3}$/.test(h) || h.includes(':')) {
268
- parts.push(hashHost(h));
269
- } else if (isPublicDomain(h)) {
270
- parts.push(h);
271
- } else {
272
- parts.push(hashHost(h));
273
- }
274
-
275
- if (url.port) parts.push(':' + url.port);
276
-
277
- const segs = url.pathname.split('/').filter(Boolean);
278
- parts.push(segs.length > 0 ? '/' + segs.map(() => '\u2022').join('/') : '/');
279
-
280
- const paramCount = [...url.searchParams].length;
281
- if (paramCount > 0) parts.push(` ?[${paramCount}]`);
282
- if (url.hash && url.hash.length > 1) parts.push(' #[frag]');
283
-
284
- return parts.join('');
285
- }
286
-
287
- function anonymizeChain(urls) {
288
- if (!Array.isArray(urls) || urls.length === 0) return '[empty-chain]';
289
- return urls.map(u => anonymizeUrl(u)).join(' \u2192 ');
290
- }
291
-
292
- return { anonymizeUrl, anonymizeChain };
293
- }
294
-
295
- // ============================================================================
296
- // Per-tab health tracker (count-only, no content)
297
- // ============================================================================
298
-
299
- // Known bot-detection providers, matched by response header fingerprints.
300
- // Order: most specific first.
301
- const BOT_DETECTION_SIGNATURES = [
302
- { header: 'cf-mitigated', value: 'challenge', provider: 'cloudflare' },
303
- { header: 'x-datadome', provider: 'datadome' },
304
- { header: 'x-px', provider: 'perimeterx' },
305
- { header: 'x-distil-cs', provider: 'distil' },
306
- { header: 'x-sucuri-id', provider: 'sucuri' },
307
- { header: 'server', value: 'akamaighost', provider: 'akamai' },
308
- // cf-ray is on ALL Cloudflare responses (even 200 OK). Must be last so it
309
- // doesn't short-circuit other providers on multi-CDN sites.
310
- { header: 'cf-ray', provider: 'cloudflare' },
311
- ];
312
-
313
- /**
314
- * Detect bot-detection provider from Playwright response headers.
315
- * Returns { detected: bool, provider: string|null, httpStatus: number|null }
316
- */
317
- export function detectBotProtection(response) {
318
- if (!response) return { detected: false, provider: null, httpStatus: null };
319
- const status = response.status();
320
- let headers;
321
- try { headers = response.headers(); } catch { return { detected: false, provider: null, httpStatus: status }; }
322
- for (const sig of BOT_DETECTION_SIGNATURES) {
323
- const val = headers[sig.header];
324
- if (val !== undefined) {
325
- if (sig.value && !val.toLowerCase().includes(sig.value)) continue;
326
- const challenged = status === 403 || status === 429 || status === 503;
327
- return { detected: challenged, provider: sig.provider, httpStatus: status };
328
- }
329
- }
330
- return { detected: false, provider: null, httpStatus: status };
331
- }
332
-
333
- /**
334
- * Create a health tracker for a tab. Attaches to Playwright page events.
335
- * Tracks: crashes, page errors, request failures, redirect status codes,
336
- * HTTP status histogram (4xx+), and anti-bot challenge detection.
337
- * All count-based -- no URLs or content stored.
338
- */
339
- export function createTabHealthTracker(page) {
340
- const health = {
341
- crashes: 0,
342
- pageErrors: 0,
343
- requestFailures: 0,
344
- inflightRequests: 0,
345
- maxRedirectDepth: 0,
346
- redirectStatusCodes: [], // status codes in redirect chain, e.g. [301, 302, 403]
347
- statusCounts: {}, // { 403: 5, 429: 2, ... }
348
- botDetection: null, // { detected, provider, httpStatus } from last nav response
349
- lastNavResponseSize: 0,
350
- _redirectDepth: 0,
351
- };
352
-
353
- // Renderer crash (OOM, segfault)
354
- page.on('crash', () => { health.crashes++; });
355
-
356
- // Uncaught JS exceptions on the page
357
- page.on('pageerror', () => { health.pageErrors++; });
358
-
359
- // Failed requests (blocked, DNS failure, etc.) + decrement in-flight counter
360
- page.on('requestfailed', () => {
361
- health.requestFailures++;
362
- health.inflightRequests = Math.max(0, health.inflightRequests - 1);
363
- });
364
-
365
- // Track in-flight requests for hang diagnostics
366
- page.on('request', () => { health.inflightRequests++; });
367
- page.on('requestfinished', () => { health.inflightRequests = Math.max(0, health.inflightRequests - 1); });
368
-
369
- // HTTP status tracking (non-2xx only)
370
- page.on('response', (resp) => {
371
- const s = resp.status();
372
- if (s >= 400) health.statusCounts[s] = (health.statusCounts[s] || 0) + 1;
373
- });
374
-
375
- // Auto-dismiss dialogs to prevent page hangs (not tracked as a metric -- noise)
376
- page.on('dialog', async (dialog) => {
377
- try { await dialog.dismiss(); } catch { /* page might be closed */ }
378
- });
379
-
380
- // Redirect depth + status code chain per navigation
381
- page.on('request', (req) => {
382
- if (req.isNavigationRequest()) {
383
- if (req.redirectedFrom()) {
384
- health._redirectDepth++;
385
- if (health._redirectDepth > health.maxRedirectDepth) {
386
- health.maxRedirectDepth = health._redirectDepth;
387
- }
388
- } else {
389
- health._redirectDepth = 0;
390
- health.redirectStatusCodes = [];
391
- health.inflightRequests = 0; // reset on new navigation to prevent drift
392
- }
393
- }
394
- });
395
-
396
- // Capture redirect status codes and detect bot protection on nav responses
397
- page.on('response', (resp) => {
398
- try {
399
- const req = resp.request();
400
- if (req.isNavigationRequest()) {
401
- health.redirectStatusCodes.push(resp.status());
402
- health.botDetection = detectBotProtection(resp);
403
- // Approximate response body size from content-length (no body read)
404
- const cl = resp.headers()['content-length'];
405
- if (cl) health.lastNavResponseSize = parseInt(cl, 10) || 0;
406
- }
407
- } catch { /* page closed */ }
408
- });
409
-
410
- /** Snapshot current health counters for inclusion in reports. */
411
- function snapshot() {
412
- const { _redirectDepth, ...clean } = health;
413
- return { ...clean };
414
- }
415
-
416
- /**
417
- * Get document.readyState from the page. Returns null if page is unresponsive.
418
- * Use a tight timeout -- if the renderer is crashed, evaluate will hang.
419
- */
420
- async function getReadyState() {
421
- try {
422
- return await Promise.race([
423
- page.evaluate(() => document.readyState),
424
- new Promise(resolve => setTimeout(() => resolve('unresponsive'), 1000)),
425
- ]);
426
- } catch {
427
- return 'unresponsive';
428
- }
429
- }
430
-
431
- return { health, snapshot, getReadyState };
432
- }
433
-
434
- // collectResourceSnapshot and classifyProxyError live in lib/resources.js
435
- // (isolated from network code for clean separation of concerns).
436
- // Re-exported here for backward compatibility.
437
- export { collectResourceSnapshot, classifyProxyError };
438
-
439
- // ============================================================================
440
- // Rate limiter (sliding window, 1 hour)
441
- // ============================================================================
442
-
443
- class RateLimiter {
444
- constructor(maxPerHour) {
445
- this.maxPerHour = maxPerHour;
446
- this.timestamps = [];
447
- }
448
-
449
- tryAcquire() {
450
- const now = Date.now();
451
- this.timestamps = this.timestamps.filter(t => t > now - 3600_000);
452
- if (this.timestamps.length >= this.maxPerHour) return false;
453
- this.timestamps.push(now);
454
- return true;
455
- }
456
- }
457
-
458
- // ============================================================================
459
- // Crash relay client
460
- // ============================================================================
461
-
462
- // PHI-VENDOR: Crash relay endpoint defaults to empty (telemetry disabled).
463
- // The original upstream endpoint pointed to a Cloudflare Worker maintained
464
- // by Jo Inc (askjo.workers.dev). The phi-code vendored snapshot ships with
465
- // telemetry off by default — users who want their own relay can set
466
- // CAMOFOX_CRASH_REPORT_URL explicitly.
467
- //
468
- // Upstream endpoint (for reference, not used unless re-enabled):
469
- // https://camofox-telemetry.askjo.workers.dev/report
470
-
471
- const DEFAULT_RELAY_URL = process.env.CAMOFOX_CRASH_REPORT_URL || '';
472
- const FETCH_TIMEOUT_MS = 5000;
473
-
474
- let _relayUrl = DEFAULT_RELAY_URL;
475
-
476
- function fetchWithTimeout(url, options) {
477
- const controller = new AbortController();
478
- const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
479
- return fetch(url, { ...options, signal: controller.signal })
480
- .finally(() => clearTimeout(timer));
481
- }
482
-
483
- /**
484
- * Send a crash report to the relay. Returns true if accepted.
485
- * Never throws -- reporter must never crash the server.
486
- */
487
- export async function sendToRelay(payload) {
488
- // PHI-VENDOR: telemetry disabled by default. When CAMOFOX_CRASH_REPORT_URL
489
- // is unset (the new default), this is a no-op — no network call leaves
490
- // the user's machine. Set the env var explicitly to opt back in.
491
- if (!_relayUrl) return false;
492
- try {
493
- const resp = await fetchWithTimeout(_relayUrl, {
494
- method: 'POST',
495
- headers: { 'Content-Type': 'application/json' },
496
- body: JSON.stringify(payload),
497
- });
498
- return resp.ok || resp.status === 429; // rate-limited is fine, not an error
499
- } catch {
500
- return false;
501
- }
502
- }
503
-
504
- // ============================================================================
505
- // Issue formatting
506
- // ============================================================================
507
-
508
- function formatIssueBody(type, detail) {
509
- const sections = [
510
- '> Auto-reported by camofox-crash-reporter. All data is anonymized.',
511
- '',
512
- '## Environment',
513
- `- **version:** ${detail.version || 'unknown'}`,
514
- `- **node:** ${detail.nodeVersion || 'unknown'}`,
515
- `- **platform:** ${detail.platform || 'unknown'}`,
516
- `- **uptime:** ${detail.uptimeMinutes != null ? detail.uptimeMinutes + ' min' : 'unknown'}`,
517
- ];
518
-
519
- // Resource snapshot (memory, handles, browser RSS)
520
- const r = detail.resources;
521
- if (r) {
522
- sections.push('', '## Resources');
523
- sections.push(`- **node RSS:** ${r.nodeRssMb ?? '?'} MB`);
524
- sections.push(`- **node heap:** ${r.nodeHeapUsedMb ?? '?'} / ${r.nodeHeapTotalMb ?? '?'} MB`);
525
- if (r.browserRssMb != null) sections.push(`- **browser RSS:** ${r.browserRssMb} MB`);
526
- if (r.browserContexts != null) sections.push(`- **browser contexts:** ${r.browserContexts}`);
527
- if (r.activeTabs != null) sections.push(`- **active tabs:** ${r.activeTabs}`);
528
- if (r.openFds != null) sections.push(`- **open FDs:** ${r.openFds}`);
529
- if (r.activeHandles != null) sections.push(`- **active handles:** ${r.activeHandles}`);
530
- if (r.eventLoopLagMs != null) sections.push(`- **event loop lag:** ${r.eventLoopLagMs} ms`);
531
- }
532
-
533
- // Error info
534
- if (detail.signal) sections.push('', `**Signal:** ${detail.signal}`);
535
- if (detail.activeRoute) sections.push(`**Active route:** ${detail.activeRoute}`);
536
- if (detail.message) {
537
- sections.push('', '## Error', '```', anonymize(detail.message), '```');
538
- }
539
- if (detail.stack) {
540
- sections.push('', '## Stack Trace', '```', anonymize(detail.stack), '```');
541
- }
542
-
543
- // Hang-specific details
544
- if (detail.hang) {
545
- const h = detail.hang;
546
- sections.push('', '## Hang Details');
547
- sections.push(`- **operation:** ${h.operation}`);
548
- sections.push(`- **duration:** ${Math.round(h.durationMs / 1000)}s`);
549
- if (h.lockQueueMs != null) sections.push(`- **lock queue wait:** ${Math.round(h.lockQueueMs)}ms`);
550
- if (h.documentReadyState) sections.push(`- **document.readyState:** ${h.documentReadyState}`);
551
- if (h.inflightRequests != null) sections.push(`- **in-flight requests:** ${h.inflightRequests}`);
552
- }
553
-
554
- // Anti-bot detection
555
- if (detail.botDetection?.detected) {
556
- const b = detail.botDetection;
557
- sections.push('', '## Anti-Bot Detection');
558
- sections.push(`- **provider:** ${b.provider || 'unknown'}`);
559
- sections.push(`- **HTTP status:** ${b.httpStatus || '?'}`);
560
- if (b.responseBodySizeKb != null) sections.push(`- **response size:** ${b.responseBodySizeKb} KB`);
561
- if (b.redirectChainLength != null) sections.push(`- **redirect chain:** ${b.redirectChainLength} hops`);
562
- if (b.redirectStatusCodes?.length) sections.push(`- **redirect statuses:** ${b.redirectStatusCodes.join(' -> ')}`);
563
- }
564
-
565
- // Proxy info (safe fields only -- no IPs, credentials, or hostnames)
566
- if (detail.proxy) {
567
- const p = detail.proxy;
568
- sections.push('', '## Proxy');
569
- sections.push(`- **configured:** ${p.configured}`);
570
- if (p.configured) {
571
- if (p.type) sections.push(`- **type:** ${p.type}`);
572
- sections.push(`- **auth configured:** ${p.authConfigured ?? 'unknown'}`);
573
- if (p.error) sections.push(`- **error:** ${p.error}`);
574
- if (p.tlsError) sections.push(`- **TLS error:** yes`);
575
- }
576
- }
577
-
578
- // Stall-specific details
579
- if (detail.stall) {
580
- const s = detail.stall;
581
- sections.push('', '## Stall Details');
582
- sections.push(`- **stall duration:** ${Math.round(s.driftMs / 1000)}s`);
583
- if (s.classification) sections.push(`- **classification:** ${s.classification}`);
584
- if (s.cpuElapsedS != null) sections.push(`- **CPU time during stall:** ${s.cpuElapsedS}s`);
585
- if (s.cpuRatio != null) sections.push(`- **CPU/wall ratio:** ${s.cpuRatio}`);
586
- if (s.sigcontInWindow != null) sections.push(`- **SIGCONT in window:** ${s.sigcontInWindow}`);
587
- if (s.hrtimeWallDriftS != null) sections.push(`- **hrtime<->wall drift:** ${s.hrtimeWallDriftS}s`);
588
- if (s.eventLoopDelay) {
589
- const eld = s.eventLoopDelay;
590
- sections.push(`- **event loop delay:** p50=${eld.p50Ms}ms p99=${eld.p99Ms}ms max=${eld.maxMs}ms`);
591
- }
592
- if (s.lastRoute) sections.push(`- **last route:** ${s.lastRoute}`);
593
- if (s.activeHandles != null) sections.push(`- **active handles:** ${s.activeHandles}`);
594
- if (s.activeRequests != null) sections.push(`- **active requests:** ${s.activeRequests}`);
595
- if (s.heapDeltaMb != null) sections.push(`- **heap delta:** ${s.heapDeltaMb > 0 ? '+' : ''}${s.heapDeltaMb} MB`);
596
- }
597
-
598
- // Context (misc extra data)
599
- if (detail.nativeMemory) {
600
- const nm = detail.nativeMemory;
601
- sections.push('', '## Native Memory Details');
602
- sections.push(`- **baseline:** ${nm.baselineMb} MB`);
603
- sections.push(`- **current:** ${nm.currentMb} MB`);
604
- sections.push(`- **high-water:** ${nm.highWaterMb} MB`);
605
- sections.push(`- **growth:** ${nm.growthMb} MB`);
606
- sections.push(`- **node RSS:** ${nm.rssMb} MB`);
607
- sections.push(`- **heap used:** ${nm.heapUsedMb} MB`);
608
- sections.push(`- **external:** ${nm.externalMb} MB`);
609
- if (nm.lastSeenBrowserRssMb != null) sections.push(`- **browser RSS (last seen):** ${nm.lastSeenBrowserRssMb} MB`);
610
- else sections.push(`- **browser RSS (last seen):** not captured (browser already dead)`);
611
- }
612
-
613
- if (detail.context && Object.keys(detail.context).length > 0) {
614
- sections.push('', '<details><summary>Context</summary>', '', '```json', anonymize(JSON.stringify(detail.context, null, 2)), '```', '', '</details>');
615
- }
616
-
617
- return sections.join('\n');
618
- }
619
-
620
-
621
- // ============================================================================
622
- // Core reporter factory
623
- // ============================================================================
624
-
625
- /**
626
- * Create a reporter instance.
627
- *
628
- * @param {object} config
629
- * @param {boolean} config.crashReportEnabled
630
- * @param {string} config.crashReportRepo - "owner/repo" (env override)
631
- * @param {number} config.crashReportRateLimit - max reports per hour
632
- * @param {object} config.crashReporterConfig - from camofox.config.json crashReporter section
633
- * @param {string} [config.version] - package version
634
- */
635
- export function createReporter(config) {
636
- // Set relay URL (env override for self-hosted relays)
637
- _relayUrl = config.crashReportUrl || DEFAULT_RELAY_URL;
638
-
639
- const enabled = config.crashReportEnabled !== false;
640
- const repo = config.crashReportRepo || 'jo-inc/camofox-browser';
641
- const rateLimiters = {
642
- crash: new RateLimiter(5), // 5 crashes/hr
643
- hang: new RateLimiter(5), // 5 hangs/hr
644
- stuck: new RateLimiter(2), // 2 stalls/hr (with active tabs only)
645
- leak: new RateLimiter(2), // 2 leak alerts/hr
646
- _default: new RateLimiter(config.crashReportRateLimit || 10),
647
- };
648
- const version = config.version || 'unknown';
649
-
650
- let watchdogInterval = null;
651
- let _resetNativeMemBaseline = false; // Set by resetNativeMemBaseline(), read by watchdog
652
- let lastTick = Date.now();
653
- const inFlight = new Set();
654
-
655
- // Track last Express route for stall reports
656
- let _lastRoute = null;
657
-
658
- // No-op when disabled
659
- if (!enabled) {
660
- return {
661
- reportCrash: async () => {},
662
- reportHang: async () => {},
663
- reportStuckLoop: async () => {},
664
- startWatchdog: () => {},
665
- trackRoute: () => {},
666
- stop: () => {},
667
- _anonymize: anonymize,
668
- _stackSignature: stackSignature,
669
- };
670
- }
671
-
672
- /** Core: build and send a report to the relay. NEVER throws. */
673
- async function fileReport(type, labels, detail) {
674
- const bucket = type.startsWith('stuck:') ? 'stuck' : type.startsWith('hang:') ? 'hang' : type.startsWith('leak:') ? 'leak' : 'crash';
675
- const limiter = rateLimiters[bucket] || rateLimiters._default;
676
- if (!limiter.tryAcquire()) return;
677
-
678
- const reportPromise = (async () => {
679
- try {
680
- const sig = stackSignature(type, detail.error || { message: detail.message, stack: detail.stack });
681
- const safeMessage = anonymize(detail.message || detail.error?.message || type);
682
- const title = `[${sig}] ${type}: ${safeMessage.slice(0, 120)}`;
683
-
684
- const body = formatIssueBody(type, {
685
- ...detail,
686
- version,
687
- nodeVersion: typeof process !== 'undefined' ? process.version : 'unknown',
688
- platform: typeof process !== 'undefined' ? process.platform : 'unknown',
689
- });
690
-
691
- const issueLabels = Array.isArray(labels) ? labels : [labels, 'auto-report'];
692
-
693
- await sendToRelay({
694
- type,
695
- signature: sig,
696
- title,
697
- body,
698
- labels: issueLabels,
699
- version,
700
- });
701
- } catch {
702
- // Swallow -- reporter must never crash the server
703
- }
704
- })();
705
-
706
- inFlight.add(reportPromise);
707
- reportPromise.finally(() => inFlight.delete(reportPromise));
708
- }
709
-
710
- /**
711
- * Track the last Express route for stall diagnostics.
712
- * Call from middleware: reporter.trackRoute(req.method + ' ' + req.route?.path)
713
- */
714
- function trackRoute(route) {
715
- _lastRoute = route || null;
716
- }
717
-
718
- async function reportCrash(error, opts = {}) {
719
- const err = error instanceof Error ? error : new Error(String(error));
720
- const uptimeMinutes = typeof process !== 'undefined'
721
- ? Math.round(process.uptime() / 60) : undefined;
722
- const resources = collectResourceSnapshot(opts.resourceOpts || {});
723
-
724
- await fileReport(
725
- opts.signal ? `signal:${opts.signal}` : (err.name || 'crash'),
726
- ['crash', 'auto-report'],
727
- {
728
- error: err,
729
- message: err.message,
730
- stack: err.stack,
731
- signal: opts.signal || null,
732
- activeRoute: _lastRoute,
733
- uptimeMinutes,
734
- resources,
735
- proxy: opts.proxy || null,
736
- context: opts.context,
737
- },
738
- );
739
- }
740
-
741
- async function reportHang(operation, durationMs, opts = {}) {
742
- const uptimeMinutes = typeof process !== 'undefined'
743
- ? Math.round(process.uptime() / 60) : undefined;
744
- const resources = collectResourceSnapshot(opts.resourceOpts || {});
745
-
746
- // Build lean context (journal only, no redundant fields)
747
- const context = { ...opts.context };
748
- if (context.journal) {
749
- context.journal = context.journal.map(j => typeof j === 'string' ? j : j);
750
- }
751
- // Remove fields that now have dedicated sections
752
- delete context.operation;
753
- delete context.durationMs;
754
-
755
- // Anti-bot detection from health snapshot
756
- const healthSnap = opts.healthSnapshot;
757
- const botDetection = healthSnap?.botDetection?.detected ? {
758
- ...healthSnap.botDetection,
759
- responseBodySizeKb: healthSnap.lastNavResponseSize
760
- ? Math.round(healthSnap.lastNavResponseSize / 1024) : null,
761
- redirectChainLength: healthSnap.redirectStatusCodes?.length || null,
762
- redirectStatusCodes: healthSnap.redirectStatusCodes?.length
763
- ? healthSnap.redirectStatusCodes : null,
764
- } : null;
765
-
766
- // Get document.readyState if healthTracker provided
767
- let documentReadyState = null;
768
- if (opts.healthTracker?.getReadyState) {
769
- documentReadyState = await opts.healthTracker.getReadyState();
770
- }
771
-
772
- const labels = ['hang', 'auto-report'];
773
- if (botDetection?.detected) labels.push('bot-detection');
774
-
775
- await fileReport(
776
- `hang:${operation}`,
777
- labels,
778
- {
779
- message: `Operation "${operation}" hung for ${Math.round(durationMs / 1000)}s`,
780
- stack: opts.error?.stack,
781
- activeRoute: _lastRoute,
782
- uptimeMinutes,
783
- resources,
784
- hang: {
785
- operation,
786
- durationMs,
787
- lockQueueMs: opts.lockQueueMs ?? null,
788
- documentReadyState,
789
- inflightRequests: healthSnap?.inflightRequests ?? null,
790
- },
791
- botDetection,
792
- proxy: opts.proxy || null,
793
- context,
794
- },
795
- );
796
- }
797
-
798
- async function reportStuckLoop(durationMs, opts = {}) {
799
- const uptimeMinutes = typeof process !== 'undefined'
800
- ? Math.round(process.uptime() / 60) : undefined;
801
- const resources = collectResourceSnapshot(opts.resourceOpts || {});
802
-
803
- await fileReport(
804
- 'stuck:tab-lock',
805
- ['stuck', 'auto-report'],
806
- {
807
- message: `Tab lock held for ${Math.round(durationMs / 1000)}s (tab destroyed)`,
808
- uptimeMinutes,
809
- resources,
810
- context: { durationMs, ...opts.context },
811
- },
812
- );
813
- }
814
-
815
- function startWatchdog(thresholdMs = 5000, getContext) {
816
- if (watchdogInterval) return;
817
-
818
- const checkMs = 1000;
819
- lastTick = Date.now();
820
- let lastCpuUsage = process.cpuUsage();
821
- let lastHrtime = process.hrtime.bigint();
822
- let lastHeapUsed = process.memoryUsage().heapUsed;
823
-
824
- // --- Native memory leak tracking ---
825
- // Track RSS minus JS heap over time to detect native/external memory leaks.
826
- // Sample every 30s, alert if native memory stays >400MB above baseline for
827
- // 3 consecutive checks (~90s). This avoids false positives from:
828
- // - Browser initialization spikes (first 2 min)
829
- // - One-time allocations that stabilize
830
- // - Post-session RSS that hasn't been reclaimed by the OS yet
831
- // - Self-healing restart (kills browser at 200MB growth when sessions=0)
832
- // The memory pressure restart in server.js fires at 200MB when idle.
833
- // We only report at 400MB to catch cases where self-healing FAILED.
834
- let nativeMemBaseline = null; // RSS - heapUsed at first measurement
835
- let nativeMemHighWater = 0;
836
- let lastNativeMemCheck = 0;
837
- const NATIVE_MEM_CHECK_INTERVAL_MS = 30_000;
838
- const NATIVE_MEM_LEAK_THRESHOLD_MB = 400; // alert only when growth exceeds self-healing threshold
839
- const NATIVE_MEM_MIN_UPTIME_S = 120; // don't measure until process has been up 2 min
840
- const NATIVE_MEM_CONSECUTIVE_REQUIRED = 3; // require 3 consecutive checks above threshold
841
- const NATIVE_MEM_GRACE_CHECKS = 2; // skip 2 checks after baseline reset (let memory settle)
842
- let nativeMemAlertFired = false;
843
- let nativeMemConsecutiveAbove = 0; // consecutive checks above threshold
844
- let nativeMemGraceRemaining = 0; // checks to skip after baseline reset
845
- let lastSeenBrowserRssMb = null; // captured during growth checks while browser is alive
846
-
847
- // SIGCONT detection -- macOS sends SIGCONT on wake from sleep/suspend
848
- let lastSigcont = 0;
849
- try { process.on('SIGCONT', () => { lastSigcont = Date.now(); }); } catch { /* unavailable */ }
850
-
851
- // Event loop delay histogram (perf_hooks) -- correlating evidence
852
- let elHistogram = null;
853
- try {
854
- elHistogram = monitorEventLoopDelay({ resolution: 20 });
855
- elHistogram.enable();
856
- } catch { /* unavailable */ }
857
-
858
- // Suppress false positives from OS sleep/suspend (laptop lid close, VM pause).
859
- // Stalls > 120s are almost certainly not event-loop bugs.
860
- const MAX_REPORTABLE_DRIFT_MS = 60_000;
861
- let suppressTicksRemaining = 0;
862
- const SUPPRESS_TICKS_AFTER_WAKE = 5;
863
-
864
- watchdogInterval = setInterval(() => {
865
- const now = Date.now();
866
- const drift = now - lastTick - checkMs;
867
- const cpuDelta = process.cpuUsage(lastCpuUsage);
868
- const hrtimeNow = process.hrtime.bigint();
869
- const hrtimeDeltaMs = Number(hrtimeNow - lastHrtime) / 1e6;
870
-
871
- lastTick = now;
872
- lastCpuUsage = process.cpuUsage();
873
- lastHrtime = hrtimeNow;
874
-
875
- // After a long sleep/suspend, suppress the next few ticks (post-wake jitter)
876
- if (drift > MAX_REPORTABLE_DRIFT_MS) {
877
- suppressTicksRemaining = SUPPRESS_TICKS_AFTER_WAKE;
878
- lastHeapUsed = process.memoryUsage().heapUsed;
879
- return;
880
- }
881
- if (suppressTicksRemaining > 0) {
882
- suppressTicksRemaining--;
883
- lastHeapUsed = process.memoryUsage().heapUsed;
884
- return;
885
- }
886
-
887
- // --- Native memory leak detection (runs every ~30s) ---
888
- if (now - lastNativeMemCheck >= NATIVE_MEM_CHECK_INTERVAL_MS) {
889
- lastNativeMemCheck = now;
890
- try {
891
- // Skip until process has been up long enough for browser to initialize.
892
- // Browser launch causes a 100-300MB RSS spike that isn't a leak.
893
- if (process.uptime() >= NATIVE_MEM_MIN_UPTIME_S) {
894
- // Check if baseline should be reset (e.g. after browser close)
895
- if (_resetNativeMemBaseline) {
896
- nativeMemBaseline = null;
897
- nativeMemHighWater = 0;
898
- nativeMemAlertFired = false;
899
- nativeMemConsecutiveAbove = 0;
900
- nativeMemGraceRemaining = NATIVE_MEM_GRACE_CHECKS;
901
- _resetNativeMemBaseline = false;
902
- }
903
-
904
- // Grace period after reset -- let memory settle before re-baselining
905
- if (nativeMemGraceRemaining > 0) {
906
- nativeMemGraceRemaining--;
907
- } else {
908
- const mem = process.memoryUsage();
909
- const nativeMemMb = Math.round((mem.rss - mem.heapUsed) / 1048576);
910
- if (nativeMemBaseline === null) {
911
- nativeMemBaseline = nativeMemMb;
912
- }
913
- nativeMemHighWater = Math.max(nativeMemHighWater, nativeMemMb);
914
- const growth = nativeMemMb - nativeMemBaseline;
915
-
916
- if (growth > NATIVE_MEM_LEAK_THRESHOLD_MB && !nativeMemAlertFired) {
917
- // Require sustained growth -- one-time spikes aren't leaks.
918
- // Must exceed threshold on 3 consecutive checks (~90s).
919
- nativeMemConsecutiveAbove++;
920
-
921
- // Capture browser RSS NOW while it may still be alive.
922
- // By report time the browser is often killed by memory pressure restart,
923
- // making browserRssMb null. This preserves the last-seen value.
924
- try {
925
- if (getContext) {
926
- const ctx = getContext();
927
- if (ctx.resourceOpts?.browserPid) {
928
- const snap = collectResourceSnapshot(ctx.resourceOpts);
929
- if (snap.browserRssMb != null) lastSeenBrowserRssMb = snap.browserRssMb;
930
- }
931
- }
932
- } catch { /* swallow */ }
933
-
934
- if (nativeMemConsecutiveAbove >= NATIVE_MEM_CONSECUTIVE_REQUIRED) {
935
- nativeMemAlertFired = true;
936
- let extra = {};
937
- try { if (getContext) extra = getContext(); } catch { /* swallow */ }
938
- const resources = collectResourceSnapshot(extra.resourceOpts || {});
939
- delete extra.resourceOpts;
940
-
941
- // Skip report if sessions=0 — memory pressure restart handles idle leaks.
942
- // Only report when sessions are active (restart CAN'T fire) or restart failed.
943
- const sessionCount = resources.browserContexts ?? 0;
944
- if (sessionCount === 0 && resources.browserRssMb == null) {
945
- // Browser already dead, restart mechanism handled it. Don't spam.
946
- // But if growth is extreme (>600MB), report anyway — restart may have failed.
947
- if (growth < 600) {
948
- // Self-healing. Skip report.
949
- return;
950
- }
951
- }
952
-
953
- fileReport('leak:native-memory', ['auto-report', 'memory-leak'], {
954
- message: `Native memory grew by ${growth}MB (baseline: ${nativeMemBaseline}MB, current: ${nativeMemMb}MB, high-water: ${nativeMemHighWater}MB)`,
955
- uptimeMinutes: Math.round(process.uptime() / 60),
956
- resources,
957
- nativeMemory: {
958
- baselineMb: nativeMemBaseline,
959
- currentMb: nativeMemMb,
960
- highWaterMb: nativeMemHighWater,
961
- growthMb: growth,
962
- rssMb: Math.round(mem.rss / 1048576),
963
- heapUsedMb: Math.round(mem.heapUsed / 1048576),
964
- externalMb: Math.round(mem.external / 1048576),
965
- lastSeenBrowserRssMb,
966
- },
967
- context: extra,
968
- });
969
- }
970
- } else {
971
- // Reset consecutive counter if memory dropped back below threshold
972
- nativeMemConsecutiveAbove = 0;
973
- }
974
- }
975
- }
976
- } catch { /* swallow */ }
977
- }
978
-
979
- if (drift > thresholdMs) {
980
- // CPU time consumed during the stall interval (user + system, in seconds)
981
- const cpuElapsedS = (cpuDelta.user + cpuDelta.system) / 1e6;
982
- const wallElapsedS = drift / 1000;
983
- const cpuRatio = wallElapsedS > 0 ? cpuElapsedS / wallElapsedS : 0;
984
-
985
- // SIGCONT within the stall window = OS sleep/resume
986
- const sigcontInWindow = lastSigcont > 0 && (now - lastSigcont) < drift + 2000;
987
-
988
- // hrtime vs wall clock drift (macOS: hrtime doesn't advance during sleep)
989
- const hrtimeWallDriftS = Math.abs((drift - (hrtimeDeltaMs - checkMs))) / 1000;
990
-
991
- // Classify: sleep vs real stall
992
- let classification;
993
- if (cpuRatio < 0.01 && sigcontInWindow) classification = 'sleep';
994
- else if (cpuRatio < 0.001) classification = 'likely_sleep';
995
- else if (cpuRatio < 0.01) classification = 'likely_sleep';
996
- else if (cpuRatio > 0.1) classification = 'real_stall';
997
- else classification = 'ambiguous';
998
-
999
- // Don't file reports for sleep/suspend-resume stalls
1000
- if (classification === 'sleep' || classification === 'likely_sleep') {
1001
- lastHeapUsed = process.memoryUsage().heapUsed;
1002
- return;
1003
- }
1004
-
1005
- // Capture heap delta during stall (GC indicator)
1006
- const currentHeap = process.memoryUsage().heapUsed;
1007
- const heapDeltaMb = Math.round((currentHeap - lastHeapUsed) / 1048576);
1008
- lastHeapUsed = currentHeap;
1009
-
1010
- let extra = {};
1011
- try { if (getContext) extra = getContext(); } catch { /* swallow */ }
1012
-
1013
- const resources = collectResourceSnapshot(extra.resourceOpts || {});
1014
- // Remove resourceOpts from extra so it doesn't end up in context
1015
- delete extra.resourceOpts;
1016
-
1017
- // Don't report idle-server stalls -- no user impact
1018
- if ((resources.activeTabs || 0) === 0 && (resources.browserContexts || 0) === 0) {
1019
- return;
1020
- }
1021
-
1022
- // Event loop delay histogram snapshot
1023
- let elDelay = null;
1024
- if (elHistogram) {
1025
- try {
1026
- elDelay = {
1027
- p50Ms: Math.round(elHistogram.percentile(50) / 1e6),
1028
- p99Ms: Math.round(elHistogram.percentile(99) / 1e6),
1029
- maxMs: Math.round(elHistogram.max / 1e6),
1030
- };
1031
- elHistogram.reset();
1032
- } catch { /* unavailable */ }
1033
- }
1034
-
1035
- const labels = ['stuck', 'auto-report'];
1036
-
1037
- fileReport('stuck:event-loop', labels, {
1038
- message: `Event loop stalled for ${Math.round(drift / 1000)}s (threshold: ${Math.round(thresholdMs / 1000)}s)`,
1039
- // Stable signature: duration is NOT included -- all stalls on the same route dedup
1040
- error: { name: 'EventLoopStall', message: _lastRoute || 'idle', stack: '' },
1041
- uptimeMinutes: typeof process !== 'undefined'
1042
- ? Math.round(process.uptime() / 60) : undefined,
1043
- resources,
1044
- stall: {
1045
- driftMs: drift,
1046
- thresholdMs,
1047
- classification,
1048
- cpuElapsedS: Math.round(cpuElapsedS * 1000) / 1000,
1049
- cpuRatio: Math.round(cpuRatio * 10000) / 10000,
1050
- sigcontInWindow,
1051
- hrtimeWallDriftS: Math.round(hrtimeWallDriftS * 100) / 100,
1052
- eventLoopDelay: elDelay,
1053
- lastRoute: _lastRoute,
1054
- activeHandles: resources.activeHandles,
1055
- activeRequests: resources.activeRequests,
1056
- heapDeltaMb,
1057
- nativeMemGrowthMb: nativeMemBaseline !== null
1058
- ? Math.round((resources.nodeRssMb - resources.nodeHeapUsedMb) - nativeMemBaseline)
1059
- : null,
1060
- nativeMemBaselineMb: nativeMemBaseline,
1061
- },
1062
- context: extra,
1063
- });
1064
- } else {
1065
- lastHeapUsed = process.memoryUsage().heapUsed;
1066
- }
1067
- }, checkMs);
1068
-
1069
- if (watchdogInterval.unref) watchdogInterval.unref();
1070
- }
1071
-
1072
- function stop() {
1073
- if (watchdogInterval) {
1074
- clearInterval(watchdogInterval);
1075
- watchdogInterval = null;
1076
- }
1077
- return Promise.allSettled([...inFlight]);
1078
- }
1079
-
1080
- /**
1081
- * Reset native memory baseline. Call after browser close so the next
1082
- * browser session measures from a fresh baseline, not the old one.
1083
- */
1084
- function resetNativeMemBaseline() {
1085
- // These are closure vars in startWatchdog -- we need to reach them.
1086
- // Since this runs in the same module, we set a flag the watchdog reads.
1087
- _resetNativeMemBaseline = true;
1088
- }
1089
-
1090
- return {
1091
- reportCrash,
1092
- reportHang,
1093
- reportStuckLoop,
1094
- startWatchdog,
1095
- trackRoute,
1096
- stop,
1097
- resetNativeMemBaseline,
1098
- _anonymize: anonymize,
1099
- _stackSignature: stackSignature,
1100
- _rateLimiter: rateLimiters,
1101
- };
1102
- }
1
+ // lib/reporter.js -- Crash/hang reporter for camofox-browser
2
+ // Files GitHub issues with paranoid anonymization. No env reads here.
3
+ // Config passed via createReporter(config) from lib/config.js.
4
+
5
+ import crypto from 'crypto';
6
+ import { monitorEventLoopDelay } from 'perf_hooks';
7
+ import { collectResourceSnapshot, classifyProxyError } from './resources.js';
8
+
9
+ // ============================================================================
10
+ // Anonymization
11
+ // ============================================================================
12
+
13
+ const SAFE_HOSTS = new Set([
14
+ 'github.com', 'api.github.com', 'npmjs.com', 'registry.npmjs.org',
15
+ 'nodejs.org',
16
+ ]);
17
+
18
+ const SECRET_PREFIXES = [
19
+ 'ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_',
20
+ 'sk-', 'sk_live_', 'sk_test_', 'pk_live_', 'pk_test_',
21
+ 'AKIA', 'ASIA',
22
+ 'xox', 'Bearer ', 'Basic ',
23
+ 'eyJ',
24
+ ];
25
+
26
+ /**
27
+ * Paranoid anonymization of arbitrary text (stack traces, error messages, etc.)
28
+ * Better to over-strip than leak. Order matters -- more specific patterns first.
29
+ */
30
+ export function anonymize(text) {
31
+ if (!text || typeof text !== 'string') return text || '';
32
+
33
+ let s = text;
34
+
35
+ // 1. Strip known secret-prefixed tokens
36
+ for (const prefix of SECRET_PREFIXES) {
37
+ const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
38
+ s = s.replace(new RegExp(escaped + '[A-Za-z0-9_\\-\\.=+/]{8,}', 'g'), '<token>');
39
+ }
40
+
41
+ // 2. Strip Bearer/Basic auth headers
42
+ s = s.replace(/(?:Bearer|Basic)\s+[A-Za-z0-9_\-\.=+/]{8,}/gi, '<token>');
43
+
44
+ // 3. Strip proxy URLs with credentials (before email -- email regex eats user:pass@host)
45
+ s = s.replace(/(?:https?|socks[45]?):\/\/[^:]+:[^@]+@[^\s]+/gi, '<proxy-url>');
46
+
47
+ // 4. Strip email addresses
48
+ s = s.replace(/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g, '<email>');
49
+
50
+ // 5. Strip full URLs (preserve scheme for context)
51
+ s = s.replace(/(https?|wss?|ftp):\/\/[^\s'",)}\]>]+/g, (match, scheme) => {
52
+ try {
53
+ const u = new URL(match);
54
+ if (SAFE_HOSTS.has(u.hostname)) return match;
55
+ } catch { /* not a valid URL, strip it */ }
56
+ return `<${scheme}-url>`;
57
+ });
58
+
59
+ // 6. Strip absolute file paths (Unix + Windows), preserve last filename
60
+ s = s.replace(
61
+ /(?:\/(?:Users|home|root|tmp|var|opt|data|app|srv|etc|mnt|run|snap|proc)\/[^\s:;,'")\]}]+|[A-Z]:\\(?:Users|Documents and Settings)\\[^\s:;,'")\]}]+)/g,
62
+ (match) => {
63
+ const parts = match.replace(/\\/g, '/').split('/');
64
+ const filename = parts[parts.length - 1] || parts[parts.length - 2] || 'unknown';
65
+ return `<path>/${filename}`;
66
+ }
67
+ );
68
+
69
+ // 7. Strip IPv4 addresses
70
+ s = s.replace(/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, '<ip>');
71
+
72
+ // 8. Strip IPv6 addresses
73
+ s = s.replace(/\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, '<ipv6>');
74
+ s = s.replace(/::(?:ffff:)?(?:\d{1,3}\.){3}\d{1,3}/g, '<ipv6>');
75
+
76
+ // 9. Strip hostnames in connection errors
77
+ s = s.replace(
78
+ /(?:ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|EHOSTUNREACH)\s+([a-zA-Z0-9.\-]+):(\d+)/g,
79
+ (match, host, port) => {
80
+ if (SAFE_HOSTS.has(host)) return match;
81
+ return match.replace(host, '<host>');
82
+ }
83
+ );
84
+
85
+ // 10. Strip Fly machine IDs (14-char hex), Docker container IDs (12+ hex)
86
+ s = s.replace(/\b[0-9a-f]{12,64}\b/g, '<id>');
87
+
88
+ // 11. Strip jo-* app names
89
+ s = s.replace(/\bjo-(?:machine|browser|whatsapp|discord|bot)[a-z0-9\-]*/gi, '<app>');
90
+
91
+ // 12. Strip environment variable assignments
92
+ s = s.replace(/\b[A-Z][A-Z0-9_]{3,}=[^\s]{4,}/g, '<env-var>');
93
+
94
+ // 13. Strip long alphanumeric strings (40+ chars)
95
+ s = s.replace(/[A-Za-z0-9_\-]{40,}/g, '<redacted>');
96
+
97
+ // 14. Strip base64 blobs (20+ chars with mixed case)
98
+ s = s.replace(/[A-Za-z0-9+/]{20,}={0,3}/g, (match) => {
99
+ if (/[a-z]/.test(match) && /[A-Z]/.test(match)) return '<redacted>';
100
+ return match;
101
+ });
102
+
103
+ return s;
104
+ }
105
+
106
+ /**
107
+ * Generate a stable signature for dedup. Uses error name + first meaningful
108
+ * stack frame (file:line, not column -- columns shift with minor edits).
109
+ */
110
+ export function stackSignature(type, error) {
111
+ const name = error?.name || error?.code || 'unknown';
112
+ const message = error?.message || String(error || '');
113
+
114
+ const stack = error?.stack || '';
115
+ const frames = stack.split('\n').slice(1);
116
+ let keyFrame = '';
117
+ for (const frame of frames) {
118
+ const trimmed = frame.trim();
119
+ if (trimmed.startsWith('at ') && !trimmed.includes('node_modules') && !trimmed.includes('node:internal')) {
120
+ const fileMatch = trimmed.match(/\(([^)]+)\)/) || trimmed.match(/at\s+(.+)$/);
121
+ if (fileMatch) {
122
+ const loc = fileMatch[1];
123
+ const parts = loc.replace(/\\/g, '/').split('/');
124
+ const last = parts[parts.length - 1];
125
+ const [file, line] = last.split(':');
126
+ keyFrame = `${file}:${line || '?'}`;
127
+ break;
128
+ }
129
+ }
130
+ }
131
+
132
+ const raw = `${type}|${name}|${keyFrame || anonymize(message).slice(0, 80)}`;
133
+ return fnv1a(raw);
134
+ }
135
+
136
+ /** FNV-1a hash -> 8-char hex. Stable bucketing, not crypto. */
137
+ function fnv1a(str) {
138
+ let hash = 0x811c9dc5;
139
+ for (let i = 0; i < str.length; i++) {
140
+ hash ^= str.charCodeAt(i);
141
+ hash = (hash * 0x01000193) >>> 0;
142
+ }
143
+ return hash.toString(16).padStart(8, '0');
144
+ }
145
+
146
+ // ============================================================================
147
+ // URL anonymization (per-report salted HMAC for private domains)
148
+ // ============================================================================
149
+
150
+ // Public domains safe to show verbatim in reports.
151
+ // These are public knowledge -- showing "amazon.com" in a crash report is not PII.
152
+ // Matched by suffix. NEVER add multi-tenant hosting (herokuapp.com, vercel.app, etc.)
153
+ const PUBLIC_DOMAINS = [
154
+ // CDN & edge
155
+ 'cloudflare.com', 'cloudflare-dns.com', 'cloudflareinsights.com',
156
+ 'fastly.net', 'fastlylb.net',
157
+ 'akamaized.net', 'akamai.net', 'cloudfront.net',
158
+ 'cdn.jsdelivr.net', 'unpkg.com', 'cdnjs.com',
159
+ // Google
160
+ 'google.com', 'googleapis.com', 'gstatic.com',
161
+ 'googleusercontent.com', 'google-analytics.com', 'googletagmanager.com',
162
+ 'googlesyndication.com', 'doubleclick.net', 'youtube.com', 'ytimg.com',
163
+ 'recaptcha.net',
164
+ // Microsoft
165
+ 'microsoft.com', 'msecnd.net', 'azureedge.net', 'bing.com', 'live.com',
166
+ 'outlook.com', 'office.com', 'linkedin.com',
167
+ // Meta
168
+ 'facebook.com', 'facebook.net', 'fbcdn.net', 'instagram.com', 'threads.net',
169
+ 'whatsapp.com',
170
+ // X/Twitter
171
+ 'twitter.com', 'x.com', 'twimg.com',
172
+ // GitHub
173
+ 'github.com', 'githubusercontent.com', 'githubassets.com',
174
+ // Major sites (common anti-bot / frustration sources)
175
+ 'amazon.com', 'amazon.co.uk', 'amazon.de', 'amazon.co.jp',
176
+ 'reddit.com', 'redd.it',
177
+ 'apple.com', 'icloud.com',
178
+ 'netflix.com', 'spotify.com', 'discord.com', 'discord.gg',
179
+ 'tiktok.com', 'pinterest.com', 'tumblr.com',
180
+ 'stackoverflow.com', 'stackexchange.com',
181
+ 'medium.com', 'substack.com',
182
+ 'nytimes.com', 'washingtonpost.com', 'bbc.co.uk', 'bbc.com', 'cnn.com',
183
+ 'ebay.com', 'etsy.com', 'walmart.com', 'target.com', 'shopify.com',
184
+ 'stripe.com', 'paypal.com',
185
+ 'twitch.tv', 'vimeo.com', 'dailymotion.com',
186
+ 'yahoo.com', 'duckduckgo.com', 'baidu.com',
187
+ 'zoom.us', 'slack.com', 'notion.so', 'figma.com',
188
+ 'dropbox.com', 'box.com',
189
+ 'archive.org', 'web.archive.org',
190
+ // Prediction markets & crypto (heavy anti-bot, commonly scraped)
191
+ 'polymarket.com', 'kalshi.com', 'metaculus.com', 'manifold.markets',
192
+ 'predictit.org', 'augur.net',
193
+ 'coinbase.com', 'binance.com', 'kraken.com', 'gemini.com',
194
+ 'coingecko.com', 'coinmarketcap.com',
195
+ 'opensea.io', 'blur.io', 'rarible.com',
196
+ 'etherscan.io', 'solscan.io', 'blockchair.com',
197
+ 'uniswap.org', 'dexscreener.com', 'dextools.io',
198
+ // Data / scraping targets (aggressive anti-bot)
199
+ 'zillow.com', 'realtor.com', 'redfin.com', 'trulia.com',
200
+ 'indeed.com', 'glassdoor.com', 'lever.co', 'greenhouse.io',
201
+ 'airbnb.com', 'booking.com', 'expedia.com', 'tripadvisor.com',
202
+ 'yelp.com', 'trustpilot.com',
203
+ 'craigslist.org', 'nextdoor.com',
204
+ 'ticketmaster.com', 'stubhub.com', 'seatgeek.com',
205
+ // Finance / trading
206
+ 'tradingview.com', 'investing.com', 'seekingalpha.com',
207
+ 'finance.yahoo.com', 'bloomberg.com', 'reuters.com', 'wsj.com',
208
+ 'robinhood.com', 'schwab.com', 'fidelity.com', 'etrade.com',
209
+ // AI / developer tools
210
+ 'openai.com', 'anthropic.com', 'huggingface.co',
211
+ 'vercel.com', 'netlify.com', 'render.com', 'fly.io',
212
+ 'npmjs.com', 'pypi.org', 'crates.io', 'pkg.go.dev',
213
+ // Social / forums
214
+ 'quora.com', 'hackernews.com', 'news.ycombinator.com',
215
+ 'producthunt.com', 'indiehackers.com',
216
+ // Reference
217
+ 'wikipedia.org', 'wikimedia.org', 'mozilla.org', 'mozilla.net',
218
+ // Anti-bot / CAPTCHA
219
+ 'hcaptcha.com',
220
+ // Fonts
221
+ 'typekit.net', 'fontawesome.com',
222
+ ].sort((a, b) => b.length - a.length); // longest-suffix-first
223
+
224
+ // Stable key for domain hashing -- NOT a secret, just ensures consistent hashes
225
+ // across reports so we can correlate "site-a1b2c3d4 caused 12 hangs this week".
226
+ const DOMAIN_HASH_KEY = 'camofox-domain-hash-v1';
227
+
228
+ /**
229
+ * Create a URL anonymizer.
230
+ * Public domains shown verbatim. Private domains get a stable hash
231
+ * (same domain -> same hash across all reports, enabling correlation).
232
+ */
233
+ export function createUrlAnonymizer() {
234
+
235
+ function isPublicDomain(hostname) {
236
+ for (const d of PUBLIC_DOMAINS) {
237
+ if (hostname === d || hostname.endsWith('.' + d)) return true;
238
+ }
239
+ return false;
240
+ }
241
+
242
+ function hashHost(hostname) {
243
+ return 'site-' + crypto.createHmac('sha256', DOMAIN_HASH_KEY).update(hostname).digest('hex').slice(0, 8);
244
+ }
245
+
246
+ /**
247
+ * Anonymize a URL. Preserves: scheme, public infra hostnames, path depth,
248
+ * query param count, fragment presence. Strips everything else.
249
+ *
250
+ * Examples:
251
+ * https://challenges.cloudflare.com/[path]/[path]/[path]
252
+ * https://site-a1b2c3d4:8443/[path]/[path] ?[3] #[frag]
253
+ */
254
+ function anonymizeUrl(rawUrl) {
255
+ if (!rawUrl || typeof rawUrl !== 'string') return '[empty]';
256
+ if (rawUrl.startsWith('data:')) return '[data-uri]';
257
+ if (rawUrl.startsWith('blob:')) return '[blob-uri]';
258
+ if (rawUrl.startsWith('about:')) return rawUrl;
259
+ if (rawUrl.startsWith('javascript:')) return '[javascript-uri]';
260
+
261
+ let url;
262
+ try { url = new URL(rawUrl); } catch { return '[invalid-url]'; }
263
+
264
+ const parts = [url.protocol + '//'];
265
+ const h = url.hostname.toLowerCase();
266
+
267
+ if (/^(\d{1,3}\.){3}\d{1,3}$/.test(h) || h.includes(':')) {
268
+ parts.push(hashHost(h));
269
+ } else if (isPublicDomain(h)) {
270
+ parts.push(h);
271
+ } else {
272
+ parts.push(hashHost(h));
273
+ }
274
+
275
+ if (url.port) parts.push(':' + url.port);
276
+
277
+ const segs = url.pathname.split('/').filter(Boolean);
278
+ parts.push(segs.length > 0 ? '/' + segs.map(() => '\u2022').join('/') : '/');
279
+
280
+ const paramCount = [...url.searchParams].length;
281
+ if (paramCount > 0) parts.push(` ?[${paramCount}]`);
282
+ if (url.hash && url.hash.length > 1) parts.push(' #[frag]');
283
+
284
+ return parts.join('');
285
+ }
286
+
287
+ function anonymizeChain(urls) {
288
+ if (!Array.isArray(urls) || urls.length === 0) return '[empty-chain]';
289
+ return urls.map(u => anonymizeUrl(u)).join(' \u2192 ');
290
+ }
291
+
292
+ return { anonymizeUrl, anonymizeChain };
293
+ }
294
+
295
+ // ============================================================================
296
+ // Per-tab health tracker (count-only, no content)
297
+ // ============================================================================
298
+
299
+ // Known bot-detection providers, matched by response header fingerprints.
300
+ // Order: most specific first.
301
+ const BOT_DETECTION_SIGNATURES = [
302
+ { header: 'cf-mitigated', value: 'challenge', provider: 'cloudflare' },
303
+ { header: 'x-datadome', provider: 'datadome' },
304
+ { header: 'x-px', provider: 'perimeterx' },
305
+ { header: 'x-distil-cs', provider: 'distil' },
306
+ { header: 'x-sucuri-id', provider: 'sucuri' },
307
+ { header: 'server', value: 'akamaighost', provider: 'akamai' },
308
+ // cf-ray is on ALL Cloudflare responses (even 200 OK). Must be last so it
309
+ // doesn't short-circuit other providers on multi-CDN sites.
310
+ { header: 'cf-ray', provider: 'cloudflare' },
311
+ ];
312
+
313
+ /**
314
+ * Detect bot-detection provider from Playwright response headers.
315
+ * Returns { detected: bool, provider: string|null, httpStatus: number|null }
316
+ */
317
+ export function detectBotProtection(response) {
318
+ if (!response) return { detected: false, provider: null, httpStatus: null };
319
+ const status = response.status();
320
+ let headers;
321
+ try { headers = response.headers(); } catch { return { detected: false, provider: null, httpStatus: status }; }
322
+ for (const sig of BOT_DETECTION_SIGNATURES) {
323
+ const val = headers[sig.header];
324
+ if (val !== undefined) {
325
+ if (sig.value && !val.toLowerCase().includes(sig.value)) continue;
326
+ const challenged = status === 403 || status === 429 || status === 503;
327
+ return { detected: challenged, provider: sig.provider, httpStatus: status };
328
+ }
329
+ }
330
+ return { detected: false, provider: null, httpStatus: status };
331
+ }
332
+
333
+ /**
334
+ * Create a health tracker for a tab. Attaches to Playwright page events.
335
+ * Tracks: crashes, page errors, request failures, redirect status codes,
336
+ * HTTP status histogram (4xx+), and anti-bot challenge detection.
337
+ * All count-based -- no URLs or content stored.
338
+ */
339
+ export function createTabHealthTracker(page) {
340
+ const health = {
341
+ crashes: 0,
342
+ pageErrors: 0,
343
+ requestFailures: 0,
344
+ inflightRequests: 0,
345
+ maxRedirectDepth: 0,
346
+ redirectStatusCodes: [], // status codes in redirect chain, e.g. [301, 302, 403]
347
+ statusCounts: {}, // { 403: 5, 429: 2, ... }
348
+ botDetection: null, // { detected, provider, httpStatus } from last nav response
349
+ lastNavResponseSize: 0,
350
+ _redirectDepth: 0,
351
+ };
352
+
353
+ // Renderer crash (OOM, segfault)
354
+ page.on('crash', () => { health.crashes++; });
355
+
356
+ // Uncaught JS exceptions on the page
357
+ page.on('pageerror', () => { health.pageErrors++; });
358
+
359
+ // Failed requests (blocked, DNS failure, etc.) + decrement in-flight counter
360
+ page.on('requestfailed', () => {
361
+ health.requestFailures++;
362
+ health.inflightRequests = Math.max(0, health.inflightRequests - 1);
363
+ });
364
+
365
+ // Track in-flight requests for hang diagnostics
366
+ page.on('request', () => { health.inflightRequests++; });
367
+ page.on('requestfinished', () => { health.inflightRequests = Math.max(0, health.inflightRequests - 1); });
368
+
369
+ // HTTP status tracking (non-2xx only)
370
+ page.on('response', (resp) => {
371
+ const s = resp.status();
372
+ if (s >= 400) health.statusCounts[s] = (health.statusCounts[s] || 0) + 1;
373
+ });
374
+
375
+ // Auto-dismiss dialogs to prevent page hangs (not tracked as a metric -- noise)
376
+ page.on('dialog', async (dialog) => {
377
+ try { await dialog.dismiss(); } catch { /* page might be closed */ }
378
+ });
379
+
380
+ // Redirect depth + status code chain per navigation
381
+ page.on('request', (req) => {
382
+ if (req.isNavigationRequest()) {
383
+ if (req.redirectedFrom()) {
384
+ health._redirectDepth++;
385
+ if (health._redirectDepth > health.maxRedirectDepth) {
386
+ health.maxRedirectDepth = health._redirectDepth;
387
+ }
388
+ } else {
389
+ health._redirectDepth = 0;
390
+ health.redirectStatusCodes = [];
391
+ health.inflightRequests = 0; // reset on new navigation to prevent drift
392
+ }
393
+ }
394
+ });
395
+
396
+ // Capture redirect status codes and detect bot protection on nav responses
397
+ page.on('response', (resp) => {
398
+ try {
399
+ const req = resp.request();
400
+ if (req.isNavigationRequest()) {
401
+ health.redirectStatusCodes.push(resp.status());
402
+ health.botDetection = detectBotProtection(resp);
403
+ // Approximate response body size from content-length (no body read)
404
+ const cl = resp.headers()['content-length'];
405
+ if (cl) health.lastNavResponseSize = parseInt(cl, 10) || 0;
406
+ }
407
+ } catch { /* page closed */ }
408
+ });
409
+
410
+ /** Snapshot current health counters for inclusion in reports. */
411
+ function snapshot() {
412
+ const { _redirectDepth, ...clean } = health;
413
+ return { ...clean };
414
+ }
415
+
416
+ /**
417
+ * Get document.readyState from the page. Returns null if page is unresponsive.
418
+ * Use a tight timeout -- if the renderer is crashed, evaluate will hang.
419
+ */
420
+ async function getReadyState() {
421
+ try {
422
+ return await Promise.race([
423
+ page.evaluate(() => document.readyState),
424
+ new Promise(resolve => setTimeout(() => resolve('unresponsive'), 1000)),
425
+ ]);
426
+ } catch {
427
+ return 'unresponsive';
428
+ }
429
+ }
430
+
431
+ return { health, snapshot, getReadyState };
432
+ }
433
+
434
+ // collectResourceSnapshot and classifyProxyError live in lib/resources.js
435
+ // (isolated from network code for clean separation of concerns).
436
+ // Re-exported here for backward compatibility.
437
+ export { collectResourceSnapshot, classifyProxyError };
438
+
439
+ // ============================================================================
440
+ // Rate limiter (sliding window, 1 hour)
441
+ // ============================================================================
442
+
443
+ class RateLimiter {
444
+ constructor(maxPerHour) {
445
+ this.maxPerHour = maxPerHour;
446
+ this.timestamps = [];
447
+ }
448
+
449
+ tryAcquire() {
450
+ const now = Date.now();
451
+ this.timestamps = this.timestamps.filter(t => t > now - 3600_000);
452
+ if (this.timestamps.length >= this.maxPerHour) return false;
453
+ this.timestamps.push(now);
454
+ return true;
455
+ }
456
+ }
457
+
458
+ // ============================================================================
459
+ // Crash relay client
460
+ // ============================================================================
461
+
462
+ // PHI-VENDOR: Crash relay endpoint defaults to empty (telemetry disabled).
463
+ // The original upstream endpoint pointed to a Cloudflare Worker maintained
464
+ // by Jo Inc (askjo.workers.dev). The phi-code vendored snapshot ships with
465
+ // telemetry off by default — users who want their own relay can set
466
+ // CAMOFOX_CRASH_REPORT_URL explicitly.
467
+ //
468
+ // Upstream endpoint (for reference, not used unless re-enabled):
469
+ // https://camofox-telemetry.askjo.workers.dev/report
470
+
471
+ const DEFAULT_RELAY_URL = process.env.CAMOFOX_CRASH_REPORT_URL || '';
472
+ const FETCH_TIMEOUT_MS = 5000;
473
+
474
+ let _relayUrl = DEFAULT_RELAY_URL;
475
+
476
+ function fetchWithTimeout(url, options) {
477
+ const controller = new AbortController();
478
+ const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
479
+ return fetch(url, { ...options, signal: controller.signal })
480
+ .finally(() => clearTimeout(timer));
481
+ }
482
+
483
+ /**
484
+ * Send a crash report to the relay. Returns true if accepted.
485
+ * Never throws -- reporter must never crash the server.
486
+ */
487
+ export async function sendToRelay(payload) {
488
+ // PHI-VENDOR: telemetry disabled by default. When CAMOFOX_CRASH_REPORT_URL
489
+ // is unset (the new default), this is a no-op — no network call leaves
490
+ // the user's machine. Set the env var explicitly to opt back in.
491
+ if (!_relayUrl) return false;
492
+ try {
493
+ const resp = await fetchWithTimeout(_relayUrl, {
494
+ method: 'POST',
495
+ headers: { 'Content-Type': 'application/json' },
496
+ body: JSON.stringify(payload),
497
+ });
498
+ return resp.ok || resp.status === 429; // rate-limited is fine, not an error
499
+ } catch {
500
+ return false;
501
+ }
502
+ }
503
+
504
+ // ============================================================================
505
+ // Issue formatting
506
+ // ============================================================================
507
+
508
+ function formatIssueBody(type, detail) {
509
+ const sections = [
510
+ '> Auto-reported by camofox-crash-reporter. All data is anonymized.',
511
+ '',
512
+ '## Environment',
513
+ `- **version:** ${detail.version || 'unknown'}`,
514
+ `- **node:** ${detail.nodeVersion || 'unknown'}`,
515
+ `- **platform:** ${detail.platform || 'unknown'}`,
516
+ `- **uptime:** ${detail.uptimeMinutes != null ? detail.uptimeMinutes + ' min' : 'unknown'}`,
517
+ ];
518
+
519
+ // Resource snapshot (memory, handles, browser RSS)
520
+ const r = detail.resources;
521
+ if (r) {
522
+ sections.push('', '## Resources');
523
+ sections.push(`- **node RSS:** ${r.nodeRssMb ?? '?'} MB`);
524
+ sections.push(`- **node heap:** ${r.nodeHeapUsedMb ?? '?'} / ${r.nodeHeapTotalMb ?? '?'} MB`);
525
+ if (r.browserRssMb != null) sections.push(`- **browser RSS:** ${r.browserRssMb} MB`);
526
+ if (r.browserContexts != null) sections.push(`- **browser contexts:** ${r.browserContexts}`);
527
+ if (r.activeTabs != null) sections.push(`- **active tabs:** ${r.activeTabs}`);
528
+ if (r.openFds != null) sections.push(`- **open FDs:** ${r.openFds}`);
529
+ if (r.activeHandles != null) sections.push(`- **active handles:** ${r.activeHandles}`);
530
+ if (r.eventLoopLagMs != null) sections.push(`- **event loop lag:** ${r.eventLoopLagMs} ms`);
531
+ }
532
+
533
+ // Error info
534
+ if (detail.signal) sections.push('', `**Signal:** ${detail.signal}`);
535
+ if (detail.activeRoute) sections.push(`**Active route:** ${detail.activeRoute}`);
536
+ if (detail.message) {
537
+ sections.push('', '## Error', '```', anonymize(detail.message), '```');
538
+ }
539
+ if (detail.stack) {
540
+ sections.push('', '## Stack Trace', '```', anonymize(detail.stack), '```');
541
+ }
542
+
543
+ // Hang-specific details
544
+ if (detail.hang) {
545
+ const h = detail.hang;
546
+ sections.push('', '## Hang Details');
547
+ sections.push(`- **operation:** ${h.operation}`);
548
+ sections.push(`- **duration:** ${Math.round(h.durationMs / 1000)}s`);
549
+ if (h.lockQueueMs != null) sections.push(`- **lock queue wait:** ${Math.round(h.lockQueueMs)}ms`);
550
+ if (h.documentReadyState) sections.push(`- **document.readyState:** ${h.documentReadyState}`);
551
+ if (h.inflightRequests != null) sections.push(`- **in-flight requests:** ${h.inflightRequests}`);
552
+ }
553
+
554
+ // Anti-bot detection
555
+ if (detail.botDetection?.detected) {
556
+ const b = detail.botDetection;
557
+ sections.push('', '## Anti-Bot Detection');
558
+ sections.push(`- **provider:** ${b.provider || 'unknown'}`);
559
+ sections.push(`- **HTTP status:** ${b.httpStatus || '?'}`);
560
+ if (b.responseBodySizeKb != null) sections.push(`- **response size:** ${b.responseBodySizeKb} KB`);
561
+ if (b.redirectChainLength != null) sections.push(`- **redirect chain:** ${b.redirectChainLength} hops`);
562
+ if (b.redirectStatusCodes?.length) sections.push(`- **redirect statuses:** ${b.redirectStatusCodes.join(' -> ')}`);
563
+ }
564
+
565
+ // Proxy info (safe fields only -- no IPs, credentials, or hostnames)
566
+ if (detail.proxy) {
567
+ const p = detail.proxy;
568
+ sections.push('', '## Proxy');
569
+ sections.push(`- **configured:** ${p.configured}`);
570
+ if (p.configured) {
571
+ if (p.type) sections.push(`- **type:** ${p.type}`);
572
+ sections.push(`- **auth configured:** ${p.authConfigured ?? 'unknown'}`);
573
+ if (p.error) sections.push(`- **error:** ${p.error}`);
574
+ if (p.tlsError) sections.push(`- **TLS error:** yes`);
575
+ }
576
+ }
577
+
578
+ // Stall-specific details
579
+ if (detail.stall) {
580
+ const s = detail.stall;
581
+ sections.push('', '## Stall Details');
582
+ sections.push(`- **stall duration:** ${Math.round(s.driftMs / 1000)}s`);
583
+ if (s.classification) sections.push(`- **classification:** ${s.classification}`);
584
+ if (s.cpuElapsedS != null) sections.push(`- **CPU time during stall:** ${s.cpuElapsedS}s`);
585
+ if (s.cpuRatio != null) sections.push(`- **CPU/wall ratio:** ${s.cpuRatio}`);
586
+ if (s.sigcontInWindow != null) sections.push(`- **SIGCONT in window:** ${s.sigcontInWindow}`);
587
+ if (s.hrtimeWallDriftS != null) sections.push(`- **hrtime<->wall drift:** ${s.hrtimeWallDriftS}s`);
588
+ if (s.eventLoopDelay) {
589
+ const eld = s.eventLoopDelay;
590
+ sections.push(`- **event loop delay:** p50=${eld.p50Ms}ms p99=${eld.p99Ms}ms max=${eld.maxMs}ms`);
591
+ }
592
+ if (s.lastRoute) sections.push(`- **last route:** ${s.lastRoute}`);
593
+ if (s.activeHandles != null) sections.push(`- **active handles:** ${s.activeHandles}`);
594
+ if (s.activeRequests != null) sections.push(`- **active requests:** ${s.activeRequests}`);
595
+ if (s.heapDeltaMb != null) sections.push(`- **heap delta:** ${s.heapDeltaMb > 0 ? '+' : ''}${s.heapDeltaMb} MB`);
596
+ }
597
+
598
+ // Context (misc extra data)
599
+ if (detail.nativeMemory) {
600
+ const nm = detail.nativeMemory;
601
+ sections.push('', '## Native Memory Details');
602
+ sections.push(`- **baseline:** ${nm.baselineMb} MB`);
603
+ sections.push(`- **current:** ${nm.currentMb} MB`);
604
+ sections.push(`- **high-water:** ${nm.highWaterMb} MB`);
605
+ sections.push(`- **growth:** ${nm.growthMb} MB`);
606
+ sections.push(`- **node RSS:** ${nm.rssMb} MB`);
607
+ sections.push(`- **heap used:** ${nm.heapUsedMb} MB`);
608
+ sections.push(`- **external:** ${nm.externalMb} MB`);
609
+ if (nm.lastSeenBrowserRssMb != null) sections.push(`- **browser RSS (last seen):** ${nm.lastSeenBrowserRssMb} MB`);
610
+ else sections.push(`- **browser RSS (last seen):** not captured (browser already dead)`);
611
+ }
612
+
613
+ if (detail.context && Object.keys(detail.context).length > 0) {
614
+ sections.push('', '<details><summary>Context</summary>', '', '```json', anonymize(JSON.stringify(detail.context, null, 2)), '```', '', '</details>');
615
+ }
616
+
617
+ return sections.join('\n');
618
+ }
619
+
620
+
621
+ // ============================================================================
622
+ // Core reporter factory
623
+ // ============================================================================
624
+
625
+ /**
626
+ * Create a reporter instance.
627
+ *
628
+ * @param {object} config
629
+ * @param {boolean} config.crashReportEnabled
630
+ * @param {string} config.crashReportRepo - "owner/repo" (env override)
631
+ * @param {number} config.crashReportRateLimit - max reports per hour
632
+ * @param {object} config.crashReporterConfig - from camofox.config.json crashReporter section
633
+ * @param {string} [config.version] - package version
634
+ */
635
+ export function createReporter(config) {
636
+ // Set relay URL (env override for self-hosted relays)
637
+ _relayUrl = config.crashReportUrl || DEFAULT_RELAY_URL;
638
+
639
+ const enabled = config.crashReportEnabled !== false;
640
+ const repo = config.crashReportRepo || 'jo-inc/camofox-browser';
641
+ const rateLimiters = {
642
+ crash: new RateLimiter(5), // 5 crashes/hr
643
+ hang: new RateLimiter(5), // 5 hangs/hr
644
+ stuck: new RateLimiter(2), // 2 stalls/hr (with active tabs only)
645
+ leak: new RateLimiter(2), // 2 leak alerts/hr
646
+ _default: new RateLimiter(config.crashReportRateLimit || 10),
647
+ };
648
+ const version = config.version || 'unknown';
649
+
650
+ let watchdogInterval = null;
651
+ let _resetNativeMemBaseline = false; // Set by resetNativeMemBaseline(), read by watchdog
652
+ let lastTick = Date.now();
653
+ const inFlight = new Set();
654
+
655
+ // Track last Express route for stall reports
656
+ let _lastRoute = null;
657
+
658
+ // No-op when disabled
659
+ if (!enabled) {
660
+ return {
661
+ reportCrash: async () => {},
662
+ reportHang: async () => {},
663
+ reportStuckLoop: async () => {},
664
+ startWatchdog: () => {},
665
+ trackRoute: () => {},
666
+ stop: () => {},
667
+ _anonymize: anonymize,
668
+ _stackSignature: stackSignature,
669
+ };
670
+ }
671
+
672
+ /** Core: build and send a report to the relay. NEVER throws. */
673
+ async function fileReport(type, labels, detail) {
674
+ const bucket = type.startsWith('stuck:') ? 'stuck' : type.startsWith('hang:') ? 'hang' : type.startsWith('leak:') ? 'leak' : 'crash';
675
+ const limiter = rateLimiters[bucket] || rateLimiters._default;
676
+ if (!limiter.tryAcquire()) return;
677
+
678
+ const reportPromise = (async () => {
679
+ try {
680
+ const sig = stackSignature(type, detail.error || { message: detail.message, stack: detail.stack });
681
+ const safeMessage = anonymize(detail.message || detail.error?.message || type);
682
+ const title = `[${sig}] ${type}: ${safeMessage.slice(0, 120)}`;
683
+
684
+ const body = formatIssueBody(type, {
685
+ ...detail,
686
+ version,
687
+ nodeVersion: typeof process !== 'undefined' ? process.version : 'unknown',
688
+ platform: typeof process !== 'undefined' ? process.platform : 'unknown',
689
+ });
690
+
691
+ const issueLabels = Array.isArray(labels) ? labels : [labels, 'auto-report'];
692
+
693
+ await sendToRelay({
694
+ type,
695
+ signature: sig,
696
+ title,
697
+ body,
698
+ labels: issueLabels,
699
+ version,
700
+ });
701
+ } catch {
702
+ // Swallow -- reporter must never crash the server
703
+ }
704
+ })();
705
+
706
+ inFlight.add(reportPromise);
707
+ reportPromise.finally(() => inFlight.delete(reportPromise));
708
+ }
709
+
710
+ /**
711
+ * Track the last Express route for stall diagnostics.
712
+ * Call from middleware: reporter.trackRoute(req.method + ' ' + req.route?.path)
713
+ */
714
+ function trackRoute(route) {
715
+ _lastRoute = route || null;
716
+ }
717
+
718
+ async function reportCrash(error, opts = {}) {
719
+ const err = error instanceof Error ? error : new Error(String(error));
720
+ const uptimeMinutes = typeof process !== 'undefined'
721
+ ? Math.round(process.uptime() / 60) : undefined;
722
+ const resources = collectResourceSnapshot(opts.resourceOpts || {});
723
+
724
+ await fileReport(
725
+ opts.signal ? `signal:${opts.signal}` : (err.name || 'crash'),
726
+ ['crash', 'auto-report'],
727
+ {
728
+ error: err,
729
+ message: err.message,
730
+ stack: err.stack,
731
+ signal: opts.signal || null,
732
+ activeRoute: _lastRoute,
733
+ uptimeMinutes,
734
+ resources,
735
+ proxy: opts.proxy || null,
736
+ context: opts.context,
737
+ },
738
+ );
739
+ }
740
+
741
+ async function reportHang(operation, durationMs, opts = {}) {
742
+ const uptimeMinutes = typeof process !== 'undefined'
743
+ ? Math.round(process.uptime() / 60) : undefined;
744
+ const resources = collectResourceSnapshot(opts.resourceOpts || {});
745
+
746
+ // Build lean context (journal only, no redundant fields)
747
+ const context = { ...opts.context };
748
+ if (context.journal) {
749
+ context.journal = context.journal.map(j => typeof j === 'string' ? j : j);
750
+ }
751
+ // Remove fields that now have dedicated sections
752
+ delete context.operation;
753
+ delete context.durationMs;
754
+
755
+ // Anti-bot detection from health snapshot
756
+ const healthSnap = opts.healthSnapshot;
757
+ const botDetection = healthSnap?.botDetection?.detected ? {
758
+ ...healthSnap.botDetection,
759
+ responseBodySizeKb: healthSnap.lastNavResponseSize
760
+ ? Math.round(healthSnap.lastNavResponseSize / 1024) : null,
761
+ redirectChainLength: healthSnap.redirectStatusCodes?.length || null,
762
+ redirectStatusCodes: healthSnap.redirectStatusCodes?.length
763
+ ? healthSnap.redirectStatusCodes : null,
764
+ } : null;
765
+
766
+ // Get document.readyState if healthTracker provided
767
+ let documentReadyState = null;
768
+ if (opts.healthTracker?.getReadyState) {
769
+ documentReadyState = await opts.healthTracker.getReadyState();
770
+ }
771
+
772
+ const labels = ['hang', 'auto-report'];
773
+ if (botDetection?.detected) labels.push('bot-detection');
774
+
775
+ await fileReport(
776
+ `hang:${operation}`,
777
+ labels,
778
+ {
779
+ message: `Operation "${operation}" hung for ${Math.round(durationMs / 1000)}s`,
780
+ stack: opts.error?.stack,
781
+ activeRoute: _lastRoute,
782
+ uptimeMinutes,
783
+ resources,
784
+ hang: {
785
+ operation,
786
+ durationMs,
787
+ lockQueueMs: opts.lockQueueMs ?? null,
788
+ documentReadyState,
789
+ inflightRequests: healthSnap?.inflightRequests ?? null,
790
+ },
791
+ botDetection,
792
+ proxy: opts.proxy || null,
793
+ context,
794
+ },
795
+ );
796
+ }
797
+
798
+ async function reportStuckLoop(durationMs, opts = {}) {
799
+ const uptimeMinutes = typeof process !== 'undefined'
800
+ ? Math.round(process.uptime() / 60) : undefined;
801
+ const resources = collectResourceSnapshot(opts.resourceOpts || {});
802
+
803
+ await fileReport(
804
+ 'stuck:tab-lock',
805
+ ['stuck', 'auto-report'],
806
+ {
807
+ message: `Tab lock held for ${Math.round(durationMs / 1000)}s (tab destroyed)`,
808
+ uptimeMinutes,
809
+ resources,
810
+ context: { durationMs, ...opts.context },
811
+ },
812
+ );
813
+ }
814
+
815
+ function startWatchdog(thresholdMs = 5000, getContext) {
816
+ if (watchdogInterval) return;
817
+
818
+ const checkMs = 1000;
819
+ lastTick = Date.now();
820
+ let lastCpuUsage = process.cpuUsage();
821
+ let lastHrtime = process.hrtime.bigint();
822
+ let lastHeapUsed = process.memoryUsage().heapUsed;
823
+
824
+ // --- Native memory leak tracking ---
825
+ // Track RSS minus JS heap over time to detect native/external memory leaks.
826
+ // Sample every 30s, alert if native memory stays >400MB above baseline for
827
+ // 3 consecutive checks (~90s). This avoids false positives from:
828
+ // - Browser initialization spikes (first 2 min)
829
+ // - One-time allocations that stabilize
830
+ // - Post-session RSS that hasn't been reclaimed by the OS yet
831
+ // - Self-healing restart (kills browser at 200MB growth when sessions=0)
832
+ // The memory pressure restart in server.js fires at 200MB when idle.
833
+ // We only report at 400MB to catch cases where self-healing FAILED.
834
+ let nativeMemBaseline = null; // RSS - heapUsed at first measurement
835
+ let nativeMemHighWater = 0;
836
+ let lastNativeMemCheck = 0;
837
+ const NATIVE_MEM_CHECK_INTERVAL_MS = 30_000;
838
+ const NATIVE_MEM_LEAK_THRESHOLD_MB = 400; // alert only when growth exceeds self-healing threshold
839
+ const NATIVE_MEM_MIN_UPTIME_S = 120; // don't measure until process has been up 2 min
840
+ const NATIVE_MEM_CONSECUTIVE_REQUIRED = 3; // require 3 consecutive checks above threshold
841
+ const NATIVE_MEM_GRACE_CHECKS = 2; // skip 2 checks after baseline reset (let memory settle)
842
+ let nativeMemAlertFired = false;
843
+ let nativeMemConsecutiveAbove = 0; // consecutive checks above threshold
844
+ let nativeMemGraceRemaining = 0; // checks to skip after baseline reset
845
+ let lastSeenBrowserRssMb = null; // captured during growth checks while browser is alive
846
+
847
+ // SIGCONT detection -- macOS sends SIGCONT on wake from sleep/suspend
848
+ let lastSigcont = 0;
849
+ try { process.on('SIGCONT', () => { lastSigcont = Date.now(); }); } catch { /* unavailable */ }
850
+
851
+ // Event loop delay histogram (perf_hooks) -- correlating evidence
852
+ let elHistogram = null;
853
+ try {
854
+ elHistogram = monitorEventLoopDelay({ resolution: 20 });
855
+ elHistogram.enable();
856
+ } catch { /* unavailable */ }
857
+
858
+ // Suppress false positives from OS sleep/suspend (laptop lid close, VM pause).
859
+ // Stalls > 120s are almost certainly not event-loop bugs.
860
+ const MAX_REPORTABLE_DRIFT_MS = 60_000;
861
+ let suppressTicksRemaining = 0;
862
+ const SUPPRESS_TICKS_AFTER_WAKE = 5;
863
+
864
+ watchdogInterval = setInterval(() => {
865
+ const now = Date.now();
866
+ const drift = now - lastTick - checkMs;
867
+ const cpuDelta = process.cpuUsage(lastCpuUsage);
868
+ const hrtimeNow = process.hrtime.bigint();
869
+ const hrtimeDeltaMs = Number(hrtimeNow - lastHrtime) / 1e6;
870
+
871
+ lastTick = now;
872
+ lastCpuUsage = process.cpuUsage();
873
+ lastHrtime = hrtimeNow;
874
+
875
+ // After a long sleep/suspend, suppress the next few ticks (post-wake jitter)
876
+ if (drift > MAX_REPORTABLE_DRIFT_MS) {
877
+ suppressTicksRemaining = SUPPRESS_TICKS_AFTER_WAKE;
878
+ lastHeapUsed = process.memoryUsage().heapUsed;
879
+ return;
880
+ }
881
+ if (suppressTicksRemaining > 0) {
882
+ suppressTicksRemaining--;
883
+ lastHeapUsed = process.memoryUsage().heapUsed;
884
+ return;
885
+ }
886
+
887
+ // --- Native memory leak detection (runs every ~30s) ---
888
+ if (now - lastNativeMemCheck >= NATIVE_MEM_CHECK_INTERVAL_MS) {
889
+ lastNativeMemCheck = now;
890
+ try {
891
+ // Skip until process has been up long enough for browser to initialize.
892
+ // Browser launch causes a 100-300MB RSS spike that isn't a leak.
893
+ if (process.uptime() >= NATIVE_MEM_MIN_UPTIME_S) {
894
+ // Check if baseline should be reset (e.g. after browser close)
895
+ if (_resetNativeMemBaseline) {
896
+ nativeMemBaseline = null;
897
+ nativeMemHighWater = 0;
898
+ nativeMemAlertFired = false;
899
+ nativeMemConsecutiveAbove = 0;
900
+ nativeMemGraceRemaining = NATIVE_MEM_GRACE_CHECKS;
901
+ _resetNativeMemBaseline = false;
902
+ }
903
+
904
+ // Grace period after reset -- let memory settle before re-baselining
905
+ if (nativeMemGraceRemaining > 0) {
906
+ nativeMemGraceRemaining--;
907
+ } else {
908
+ const mem = process.memoryUsage();
909
+ const nativeMemMb = Math.round((mem.rss - mem.heapUsed) / 1048576);
910
+ if (nativeMemBaseline === null) {
911
+ nativeMemBaseline = nativeMemMb;
912
+ }
913
+ nativeMemHighWater = Math.max(nativeMemHighWater, nativeMemMb);
914
+ const growth = nativeMemMb - nativeMemBaseline;
915
+
916
+ if (growth > NATIVE_MEM_LEAK_THRESHOLD_MB && !nativeMemAlertFired) {
917
+ // Require sustained growth -- one-time spikes aren't leaks.
918
+ // Must exceed threshold on 3 consecutive checks (~90s).
919
+ nativeMemConsecutiveAbove++;
920
+
921
+ // Capture browser RSS NOW while it may still be alive.
922
+ // By report time the browser is often killed by memory pressure restart,
923
+ // making browserRssMb null. This preserves the last-seen value.
924
+ try {
925
+ if (getContext) {
926
+ const ctx = getContext();
927
+ if (ctx.resourceOpts?.browserPid) {
928
+ const snap = collectResourceSnapshot(ctx.resourceOpts);
929
+ if (snap.browserRssMb != null) lastSeenBrowserRssMb = snap.browserRssMb;
930
+ }
931
+ }
932
+ } catch { /* swallow */ }
933
+
934
+ if (nativeMemConsecutiveAbove >= NATIVE_MEM_CONSECUTIVE_REQUIRED) {
935
+ nativeMemAlertFired = true;
936
+ let extra = {};
937
+ try { if (getContext) extra = getContext(); } catch { /* swallow */ }
938
+ const resources = collectResourceSnapshot(extra.resourceOpts || {});
939
+ delete extra.resourceOpts;
940
+
941
+ // Skip report if sessions=0 — memory pressure restart handles idle leaks.
942
+ // Only report when sessions are active (restart CAN'T fire) or restart failed.
943
+ const sessionCount = resources.browserContexts ?? 0;
944
+ if (sessionCount === 0 && resources.browserRssMb == null) {
945
+ // Browser already dead, restart mechanism handled it. Don't spam.
946
+ // But if growth is extreme (>600MB), report anyway — restart may have failed.
947
+ if (growth < 600) {
948
+ // Self-healing. Skip report.
949
+ return;
950
+ }
951
+ }
952
+
953
+ fileReport('leak:native-memory', ['auto-report', 'memory-leak'], {
954
+ message: `Native memory grew by ${growth}MB (baseline: ${nativeMemBaseline}MB, current: ${nativeMemMb}MB, high-water: ${nativeMemHighWater}MB)`,
955
+ uptimeMinutes: Math.round(process.uptime() / 60),
956
+ resources,
957
+ nativeMemory: {
958
+ baselineMb: nativeMemBaseline,
959
+ currentMb: nativeMemMb,
960
+ highWaterMb: nativeMemHighWater,
961
+ growthMb: growth,
962
+ rssMb: Math.round(mem.rss / 1048576),
963
+ heapUsedMb: Math.round(mem.heapUsed / 1048576),
964
+ externalMb: Math.round(mem.external / 1048576),
965
+ lastSeenBrowserRssMb,
966
+ },
967
+ context: extra,
968
+ });
969
+ }
970
+ } else {
971
+ // Reset consecutive counter if memory dropped back below threshold
972
+ nativeMemConsecutiveAbove = 0;
973
+ }
974
+ }
975
+ }
976
+ } catch { /* swallow */ }
977
+ }
978
+
979
+ if (drift > thresholdMs) {
980
+ // CPU time consumed during the stall interval (user + system, in seconds)
981
+ const cpuElapsedS = (cpuDelta.user + cpuDelta.system) / 1e6;
982
+ const wallElapsedS = drift / 1000;
983
+ const cpuRatio = wallElapsedS > 0 ? cpuElapsedS / wallElapsedS : 0;
984
+
985
+ // SIGCONT within the stall window = OS sleep/resume
986
+ const sigcontInWindow = lastSigcont > 0 && (now - lastSigcont) < drift + 2000;
987
+
988
+ // hrtime vs wall clock drift (macOS: hrtime doesn't advance during sleep)
989
+ const hrtimeWallDriftS = Math.abs((drift - (hrtimeDeltaMs - checkMs))) / 1000;
990
+
991
+ // Classify: sleep vs real stall
992
+ let classification;
993
+ if (cpuRatio < 0.01 && sigcontInWindow) classification = 'sleep';
994
+ else if (cpuRatio < 0.001) classification = 'likely_sleep';
995
+ else if (cpuRatio < 0.01) classification = 'likely_sleep';
996
+ else if (cpuRatio > 0.1) classification = 'real_stall';
997
+ else classification = 'ambiguous';
998
+
999
+ // Don't file reports for sleep/suspend-resume stalls
1000
+ if (classification === 'sleep' || classification === 'likely_sleep') {
1001
+ lastHeapUsed = process.memoryUsage().heapUsed;
1002
+ return;
1003
+ }
1004
+
1005
+ // Capture heap delta during stall (GC indicator)
1006
+ const currentHeap = process.memoryUsage().heapUsed;
1007
+ const heapDeltaMb = Math.round((currentHeap - lastHeapUsed) / 1048576);
1008
+ lastHeapUsed = currentHeap;
1009
+
1010
+ let extra = {};
1011
+ try { if (getContext) extra = getContext(); } catch { /* swallow */ }
1012
+
1013
+ const resources = collectResourceSnapshot(extra.resourceOpts || {});
1014
+ // Remove resourceOpts from extra so it doesn't end up in context
1015
+ delete extra.resourceOpts;
1016
+
1017
+ // Don't report idle-server stalls -- no user impact
1018
+ if ((resources.activeTabs || 0) === 0 && (resources.browserContexts || 0) === 0) {
1019
+ return;
1020
+ }
1021
+
1022
+ // Event loop delay histogram snapshot
1023
+ let elDelay = null;
1024
+ if (elHistogram) {
1025
+ try {
1026
+ elDelay = {
1027
+ p50Ms: Math.round(elHistogram.percentile(50) / 1e6),
1028
+ p99Ms: Math.round(elHistogram.percentile(99) / 1e6),
1029
+ maxMs: Math.round(elHistogram.max / 1e6),
1030
+ };
1031
+ elHistogram.reset();
1032
+ } catch { /* unavailable */ }
1033
+ }
1034
+
1035
+ const labels = ['stuck', 'auto-report'];
1036
+
1037
+ fileReport('stuck:event-loop', labels, {
1038
+ message: `Event loop stalled for ${Math.round(drift / 1000)}s (threshold: ${Math.round(thresholdMs / 1000)}s)`,
1039
+ // Stable signature: duration is NOT included -- all stalls on the same route dedup
1040
+ error: { name: 'EventLoopStall', message: _lastRoute || 'idle', stack: '' },
1041
+ uptimeMinutes: typeof process !== 'undefined'
1042
+ ? Math.round(process.uptime() / 60) : undefined,
1043
+ resources,
1044
+ stall: {
1045
+ driftMs: drift,
1046
+ thresholdMs,
1047
+ classification,
1048
+ cpuElapsedS: Math.round(cpuElapsedS * 1000) / 1000,
1049
+ cpuRatio: Math.round(cpuRatio * 10000) / 10000,
1050
+ sigcontInWindow,
1051
+ hrtimeWallDriftS: Math.round(hrtimeWallDriftS * 100) / 100,
1052
+ eventLoopDelay: elDelay,
1053
+ lastRoute: _lastRoute,
1054
+ activeHandles: resources.activeHandles,
1055
+ activeRequests: resources.activeRequests,
1056
+ heapDeltaMb,
1057
+ nativeMemGrowthMb: nativeMemBaseline !== null
1058
+ ? Math.round((resources.nodeRssMb - resources.nodeHeapUsedMb) - nativeMemBaseline)
1059
+ : null,
1060
+ nativeMemBaselineMb: nativeMemBaseline,
1061
+ },
1062
+ context: extra,
1063
+ });
1064
+ } else {
1065
+ lastHeapUsed = process.memoryUsage().heapUsed;
1066
+ }
1067
+ }, checkMs);
1068
+
1069
+ if (watchdogInterval.unref) watchdogInterval.unref();
1070
+ }
1071
+
1072
+ function stop() {
1073
+ if (watchdogInterval) {
1074
+ clearInterval(watchdogInterval);
1075
+ watchdogInterval = null;
1076
+ }
1077
+ return Promise.allSettled([...inFlight]);
1078
+ }
1079
+
1080
+ /**
1081
+ * Reset native memory baseline. Call after browser close so the next
1082
+ * browser session measures from a fresh baseline, not the old one.
1083
+ */
1084
+ function resetNativeMemBaseline() {
1085
+ // These are closure vars in startWatchdog -- we need to reach them.
1086
+ // Since this runs in the same module, we set a flag the watchdog reads.
1087
+ _resetNativeMemBaseline = true;
1088
+ }
1089
+
1090
+ return {
1091
+ reportCrash,
1092
+ reportHang,
1093
+ reportStuckLoop,
1094
+ startWatchdog,
1095
+ trackRoute,
1096
+ stop,
1097
+ resetNativeMemBaseline,
1098
+ _anonymize: anonymize,
1099
+ _stackSignature: stackSignature,
1100
+ _rateLimiter: rateLimiters,
1101
+ };
1102
+ }