@phi-code-admin/camofox-browser 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +571 -571
- package/Dockerfile +86 -86
- package/LICENSE +21 -21
- package/README.md +691 -691
- package/camofox.config.json +10 -10
- package/lib/auth.js +134 -134
- package/lib/camoufox-executable.js +189 -189
- package/lib/config.js +153 -153
- package/lib/cookies.js +119 -119
- package/lib/downloads.js +168 -168
- package/lib/extract.js +74 -74
- package/lib/fly.js +54 -54
- package/lib/images.js +88 -88
- package/lib/inflight.js +16 -16
- package/lib/launcher.js +47 -47
- package/lib/macros.js +31 -31
- package/lib/metrics.js +184 -184
- package/lib/openapi.js +105 -105
- package/lib/persistence.js +89 -89
- package/lib/plugins.js +178 -175
- package/lib/proxy.js +277 -277
- package/lib/reporter.js +1102 -1102
- package/lib/request-utils.js +59 -59
- package/lib/resources.js +76 -76
- package/lib/snapshot.js +41 -41
- package/lib/tmp-cleanup.js +108 -108
- package/lib/tracing.js +137 -137
- package/openclaw.plugin.json +268 -268
- package/package.json +148 -148
- package/plugin.ts +758 -758
- package/plugins/persistence/AGENTS.md +37 -37
- package/plugins/persistence/README.md +48 -48
- package/plugins/persistence/index.js +124 -124
- package/plugins/vnc/AGENTS.md +42 -42
- package/plugins/vnc/README.md +165 -165
- package/plugins/vnc/apt.txt +7 -7
- package/plugins/vnc/index.js +142 -142
- package/plugins/vnc/spawn.js +8 -8
- package/plugins/vnc/vnc-launcher.js +64 -64
- package/plugins/vnc/vnc-watcher.sh +82 -82
- package/plugins/youtube/AGENTS.md +25 -25
- package/plugins/youtube/apt.txt +1 -1
- package/plugins/youtube/index.js +206 -206
- package/plugins/youtube/post-install.sh +5 -5
- package/plugins/youtube/youtube.js +301 -301
- package/run.sh +37 -37
- package/scripts/exec.js +8 -8
- package/scripts/generate-openapi.js +24 -24
- package/scripts/install-plugin-deps.sh +63 -63
- package/scripts/plugin.js +342 -342
- package/scripts/sync-version.js +25 -25
- package/server.js +6062 -6059
- package/tsconfig.json +12 -12
package/lib/reporter.js
CHANGED
|
@@ -1,1102 +1,1102 @@
|
|
|
1
|
-
// lib/reporter.js -- Crash/hang reporter for camofox-browser
|
|
2
|
-
// Files GitHub issues with paranoid anonymization. No env reads here.
|
|
3
|
-
// Config passed via createReporter(config) from lib/config.js.
|
|
4
|
-
|
|
5
|
-
import crypto from 'crypto';
|
|
6
|
-
import { monitorEventLoopDelay } from 'perf_hooks';
|
|
7
|
-
import { collectResourceSnapshot, classifyProxyError } from './resources.js';
|
|
8
|
-
|
|
9
|
-
// ============================================================================
|
|
10
|
-
// Anonymization
|
|
11
|
-
// ============================================================================
|
|
12
|
-
|
|
13
|
-
const SAFE_HOSTS = new Set([
|
|
14
|
-
'github.com', 'api.github.com', 'npmjs.com', 'registry.npmjs.org',
|
|
15
|
-
'nodejs.org',
|
|
16
|
-
]);
|
|
17
|
-
|
|
18
|
-
const SECRET_PREFIXES = [
|
|
19
|
-
'ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_',
|
|
20
|
-
'sk-', 'sk_live_', 'sk_test_', 'pk_live_', 'pk_test_',
|
|
21
|
-
'AKIA', 'ASIA',
|
|
22
|
-
'xox', 'Bearer ', 'Basic ',
|
|
23
|
-
'eyJ',
|
|
24
|
-
];
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Paranoid anonymization of arbitrary text (stack traces, error messages, etc.)
|
|
28
|
-
* Better to over-strip than leak. Order matters -- more specific patterns first.
|
|
29
|
-
*/
|
|
30
|
-
export function anonymize(text) {
|
|
31
|
-
if (!text || typeof text !== 'string') return text || '';
|
|
32
|
-
|
|
33
|
-
let s = text;
|
|
34
|
-
|
|
35
|
-
// 1. Strip known secret-prefixed tokens
|
|
36
|
-
for (const prefix of SECRET_PREFIXES) {
|
|
37
|
-
const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
38
|
-
s = s.replace(new RegExp(escaped + '[A-Za-z0-9_\\-\\.=+/]{8,}', 'g'), '<token>');
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
// 2. Strip Bearer/Basic auth headers
|
|
42
|
-
s = s.replace(/(?:Bearer|Basic)\s+[A-Za-z0-9_\-\.=+/]{8,}/gi, '<token>');
|
|
43
|
-
|
|
44
|
-
// 3. Strip proxy URLs with credentials (before email -- email regex eats user:pass@host)
|
|
45
|
-
s = s.replace(/(?:https?|socks[45]?):\/\/[^:]+:[^@]+@[^\s]+/gi, '<proxy-url>');
|
|
46
|
-
|
|
47
|
-
// 4. Strip email addresses
|
|
48
|
-
s = s.replace(/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g, '<email>');
|
|
49
|
-
|
|
50
|
-
// 5. Strip full URLs (preserve scheme for context)
|
|
51
|
-
s = s.replace(/(https?|wss?|ftp):\/\/[^\s'",)}\]>]+/g, (match, scheme) => {
|
|
52
|
-
try {
|
|
53
|
-
const u = new URL(match);
|
|
54
|
-
if (SAFE_HOSTS.has(u.hostname)) return match;
|
|
55
|
-
} catch { /* not a valid URL, strip it */ }
|
|
56
|
-
return `<${scheme}-url>`;
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
// 6. Strip absolute file paths (Unix + Windows), preserve last filename
|
|
60
|
-
s = s.replace(
|
|
61
|
-
/(?:\/(?:Users|home|root|tmp|var|opt|data|app|srv|etc|mnt|run|snap|proc)\/[^\s:;,'")\]}]+|[A-Z]:\\(?:Users|Documents and Settings)\\[^\s:;,'")\]}]+)/g,
|
|
62
|
-
(match) => {
|
|
63
|
-
const parts = match.replace(/\\/g, '/').split('/');
|
|
64
|
-
const filename = parts[parts.length - 1] || parts[parts.length - 2] || 'unknown';
|
|
65
|
-
return `<path>/${filename}`;
|
|
66
|
-
}
|
|
67
|
-
);
|
|
68
|
-
|
|
69
|
-
// 7. Strip IPv4 addresses
|
|
70
|
-
s = s.replace(/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, '<ip>');
|
|
71
|
-
|
|
72
|
-
// 8. Strip IPv6 addresses
|
|
73
|
-
s = s.replace(/\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, '<ipv6>');
|
|
74
|
-
s = s.replace(/::(?:ffff:)?(?:\d{1,3}\.){3}\d{1,3}/g, '<ipv6>');
|
|
75
|
-
|
|
76
|
-
// 9. Strip hostnames in connection errors
|
|
77
|
-
s = s.replace(
|
|
78
|
-
/(?:ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|EHOSTUNREACH)\s+([a-zA-Z0-9.\-]+):(\d+)/g,
|
|
79
|
-
(match, host, port) => {
|
|
80
|
-
if (SAFE_HOSTS.has(host)) return match;
|
|
81
|
-
return match.replace(host, '<host>');
|
|
82
|
-
}
|
|
83
|
-
);
|
|
84
|
-
|
|
85
|
-
// 10. Strip Fly machine IDs (14-char hex), Docker container IDs (12+ hex)
|
|
86
|
-
s = s.replace(/\b[0-9a-f]{12,64}\b/g, '<id>');
|
|
87
|
-
|
|
88
|
-
// 11. Strip jo-* app names
|
|
89
|
-
s = s.replace(/\bjo-(?:machine|browser|whatsapp|discord|bot)[a-z0-9\-]*/gi, '<app>');
|
|
90
|
-
|
|
91
|
-
// 12. Strip environment variable assignments
|
|
92
|
-
s = s.replace(/\b[A-Z][A-Z0-9_]{3,}=[^\s]{4,}/g, '<env-var>');
|
|
93
|
-
|
|
94
|
-
// 13. Strip long alphanumeric strings (40+ chars)
|
|
95
|
-
s = s.replace(/[A-Za-z0-9_\-]{40,}/g, '<redacted>');
|
|
96
|
-
|
|
97
|
-
// 14. Strip base64 blobs (20+ chars with mixed case)
|
|
98
|
-
s = s.replace(/[A-Za-z0-9+/]{20,}={0,3}/g, (match) => {
|
|
99
|
-
if (/[a-z]/.test(match) && /[A-Z]/.test(match)) return '<redacted>';
|
|
100
|
-
return match;
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
return s;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
/**
|
|
107
|
-
* Generate a stable signature for dedup. Uses error name + first meaningful
|
|
108
|
-
* stack frame (file:line, not column -- columns shift with minor edits).
|
|
109
|
-
*/
|
|
110
|
-
export function stackSignature(type, error) {
|
|
111
|
-
const name = error?.name || error?.code || 'unknown';
|
|
112
|
-
const message = error?.message || String(error || '');
|
|
113
|
-
|
|
114
|
-
const stack = error?.stack || '';
|
|
115
|
-
const frames = stack.split('\n').slice(1);
|
|
116
|
-
let keyFrame = '';
|
|
117
|
-
for (const frame of frames) {
|
|
118
|
-
const trimmed = frame.trim();
|
|
119
|
-
if (trimmed.startsWith('at ') && !trimmed.includes('node_modules') && !trimmed.includes('node:internal')) {
|
|
120
|
-
const fileMatch = trimmed.match(/\(([^)]+)\)/) || trimmed.match(/at\s+(.+)$/);
|
|
121
|
-
if (fileMatch) {
|
|
122
|
-
const loc = fileMatch[1];
|
|
123
|
-
const parts = loc.replace(/\\/g, '/').split('/');
|
|
124
|
-
const last = parts[parts.length - 1];
|
|
125
|
-
const [file, line] = last.split(':');
|
|
126
|
-
keyFrame = `${file}:${line || '?'}`;
|
|
127
|
-
break;
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
const raw = `${type}|${name}|${keyFrame || anonymize(message).slice(0, 80)}`;
|
|
133
|
-
return fnv1a(raw);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
/** FNV-1a hash -> 8-char hex. Stable bucketing, not crypto. */
|
|
137
|
-
function fnv1a(str) {
|
|
138
|
-
let hash = 0x811c9dc5;
|
|
139
|
-
for (let i = 0; i < str.length; i++) {
|
|
140
|
-
hash ^= str.charCodeAt(i);
|
|
141
|
-
hash = (hash * 0x01000193) >>> 0;
|
|
142
|
-
}
|
|
143
|
-
return hash.toString(16).padStart(8, '0');
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
// ============================================================================
|
|
147
|
-
// URL anonymization (per-report salted HMAC for private domains)
|
|
148
|
-
// ============================================================================
|
|
149
|
-
|
|
150
|
-
// Public domains safe to show verbatim in reports.
|
|
151
|
-
// These are public knowledge -- showing "amazon.com" in a crash report is not PII.
|
|
152
|
-
// Matched by suffix. NEVER add multi-tenant hosting (herokuapp.com, vercel.app, etc.)
|
|
153
|
-
const PUBLIC_DOMAINS = [
|
|
154
|
-
// CDN & edge
|
|
155
|
-
'cloudflare.com', 'cloudflare-dns.com', 'cloudflareinsights.com',
|
|
156
|
-
'fastly.net', 'fastlylb.net',
|
|
157
|
-
'akamaized.net', 'akamai.net', 'cloudfront.net',
|
|
158
|
-
'cdn.jsdelivr.net', 'unpkg.com', 'cdnjs.com',
|
|
159
|
-
// Google
|
|
160
|
-
'google.com', 'googleapis.com', 'gstatic.com',
|
|
161
|
-
'googleusercontent.com', 'google-analytics.com', 'googletagmanager.com',
|
|
162
|
-
'googlesyndication.com', 'doubleclick.net', 'youtube.com', 'ytimg.com',
|
|
163
|
-
'recaptcha.net',
|
|
164
|
-
// Microsoft
|
|
165
|
-
'microsoft.com', 'msecnd.net', 'azureedge.net', 'bing.com', 'live.com',
|
|
166
|
-
'outlook.com', 'office.com', 'linkedin.com',
|
|
167
|
-
// Meta
|
|
168
|
-
'facebook.com', 'facebook.net', 'fbcdn.net', 'instagram.com', 'threads.net',
|
|
169
|
-
'whatsapp.com',
|
|
170
|
-
// X/Twitter
|
|
171
|
-
'twitter.com', 'x.com', 'twimg.com',
|
|
172
|
-
// GitHub
|
|
173
|
-
'github.com', 'githubusercontent.com', 'githubassets.com',
|
|
174
|
-
// Major sites (common anti-bot / frustration sources)
|
|
175
|
-
'amazon.com', 'amazon.co.uk', 'amazon.de', 'amazon.co.jp',
|
|
176
|
-
'reddit.com', 'redd.it',
|
|
177
|
-
'apple.com', 'icloud.com',
|
|
178
|
-
'netflix.com', 'spotify.com', 'discord.com', 'discord.gg',
|
|
179
|
-
'tiktok.com', 'pinterest.com', 'tumblr.com',
|
|
180
|
-
'stackoverflow.com', 'stackexchange.com',
|
|
181
|
-
'medium.com', 'substack.com',
|
|
182
|
-
'nytimes.com', 'washingtonpost.com', 'bbc.co.uk', 'bbc.com', 'cnn.com',
|
|
183
|
-
'ebay.com', 'etsy.com', 'walmart.com', 'target.com', 'shopify.com',
|
|
184
|
-
'stripe.com', 'paypal.com',
|
|
185
|
-
'twitch.tv', 'vimeo.com', 'dailymotion.com',
|
|
186
|
-
'yahoo.com', 'duckduckgo.com', 'baidu.com',
|
|
187
|
-
'zoom.us', 'slack.com', 'notion.so', 'figma.com',
|
|
188
|
-
'dropbox.com', 'box.com',
|
|
189
|
-
'archive.org', 'web.archive.org',
|
|
190
|
-
// Prediction markets & crypto (heavy anti-bot, commonly scraped)
|
|
191
|
-
'polymarket.com', 'kalshi.com', 'metaculus.com', 'manifold.markets',
|
|
192
|
-
'predictit.org', 'augur.net',
|
|
193
|
-
'coinbase.com', 'binance.com', 'kraken.com', 'gemini.com',
|
|
194
|
-
'coingecko.com', 'coinmarketcap.com',
|
|
195
|
-
'opensea.io', 'blur.io', 'rarible.com',
|
|
196
|
-
'etherscan.io', 'solscan.io', 'blockchair.com',
|
|
197
|
-
'uniswap.org', 'dexscreener.com', 'dextools.io',
|
|
198
|
-
// Data / scraping targets (aggressive anti-bot)
|
|
199
|
-
'zillow.com', 'realtor.com', 'redfin.com', 'trulia.com',
|
|
200
|
-
'indeed.com', 'glassdoor.com', 'lever.co', 'greenhouse.io',
|
|
201
|
-
'airbnb.com', 'booking.com', 'expedia.com', 'tripadvisor.com',
|
|
202
|
-
'yelp.com', 'trustpilot.com',
|
|
203
|
-
'craigslist.org', 'nextdoor.com',
|
|
204
|
-
'ticketmaster.com', 'stubhub.com', 'seatgeek.com',
|
|
205
|
-
// Finance / trading
|
|
206
|
-
'tradingview.com', 'investing.com', 'seekingalpha.com',
|
|
207
|
-
'finance.yahoo.com', 'bloomberg.com', 'reuters.com', 'wsj.com',
|
|
208
|
-
'robinhood.com', 'schwab.com', 'fidelity.com', 'etrade.com',
|
|
209
|
-
// AI / developer tools
|
|
210
|
-
'openai.com', 'anthropic.com', 'huggingface.co',
|
|
211
|
-
'vercel.com', 'netlify.com', 'render.com', 'fly.io',
|
|
212
|
-
'npmjs.com', 'pypi.org', 'crates.io', 'pkg.go.dev',
|
|
213
|
-
// Social / forums
|
|
214
|
-
'quora.com', 'hackernews.com', 'news.ycombinator.com',
|
|
215
|
-
'producthunt.com', 'indiehackers.com',
|
|
216
|
-
// Reference
|
|
217
|
-
'wikipedia.org', 'wikimedia.org', 'mozilla.org', 'mozilla.net',
|
|
218
|
-
// Anti-bot / CAPTCHA
|
|
219
|
-
'hcaptcha.com',
|
|
220
|
-
// Fonts
|
|
221
|
-
'typekit.net', 'fontawesome.com',
|
|
222
|
-
].sort((a, b) => b.length - a.length); // longest-suffix-first
|
|
223
|
-
|
|
224
|
-
// Stable key for domain hashing -- NOT a secret, just ensures consistent hashes
|
|
225
|
-
// across reports so we can correlate "site-a1b2c3d4 caused 12 hangs this week".
|
|
226
|
-
const DOMAIN_HASH_KEY = 'camofox-domain-hash-v1';
|
|
227
|
-
|
|
228
|
-
/**
|
|
229
|
-
* Create a URL anonymizer.
|
|
230
|
-
* Public domains shown verbatim. Private domains get a stable hash
|
|
231
|
-
* (same domain -> same hash across all reports, enabling correlation).
|
|
232
|
-
*/
|
|
233
|
-
export function createUrlAnonymizer() {
|
|
234
|
-
|
|
235
|
-
function isPublicDomain(hostname) {
|
|
236
|
-
for (const d of PUBLIC_DOMAINS) {
|
|
237
|
-
if (hostname === d || hostname.endsWith('.' + d)) return true;
|
|
238
|
-
}
|
|
239
|
-
return false;
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
function hashHost(hostname) {
|
|
243
|
-
return 'site-' + crypto.createHmac('sha256', DOMAIN_HASH_KEY).update(hostname).digest('hex').slice(0, 8);
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
/**
|
|
247
|
-
* Anonymize a URL. Preserves: scheme, public infra hostnames, path depth,
|
|
248
|
-
* query param count, fragment presence. Strips everything else.
|
|
249
|
-
*
|
|
250
|
-
* Examples:
|
|
251
|
-
* https://challenges.cloudflare.com/[path]/[path]/[path]
|
|
252
|
-
* https://site-a1b2c3d4:8443/[path]/[path] ?[3] #[frag]
|
|
253
|
-
*/
|
|
254
|
-
function anonymizeUrl(rawUrl) {
|
|
255
|
-
if (!rawUrl || typeof rawUrl !== 'string') return '[empty]';
|
|
256
|
-
if (rawUrl.startsWith('data:')) return '[data-uri]';
|
|
257
|
-
if (rawUrl.startsWith('blob:')) return '[blob-uri]';
|
|
258
|
-
if (rawUrl.startsWith('about:')) return rawUrl;
|
|
259
|
-
if (rawUrl.startsWith('javascript:')) return '[javascript-uri]';
|
|
260
|
-
|
|
261
|
-
let url;
|
|
262
|
-
try { url = new URL(rawUrl); } catch { return '[invalid-url]'; }
|
|
263
|
-
|
|
264
|
-
const parts = [url.protocol + '//'];
|
|
265
|
-
const h = url.hostname.toLowerCase();
|
|
266
|
-
|
|
267
|
-
if (/^(\d{1,3}\.){3}\d{1,3}$/.test(h) || h.includes(':')) {
|
|
268
|
-
parts.push(hashHost(h));
|
|
269
|
-
} else if (isPublicDomain(h)) {
|
|
270
|
-
parts.push(h);
|
|
271
|
-
} else {
|
|
272
|
-
parts.push(hashHost(h));
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
if (url.port) parts.push(':' + url.port);
|
|
276
|
-
|
|
277
|
-
const segs = url.pathname.split('/').filter(Boolean);
|
|
278
|
-
parts.push(segs.length > 0 ? '/' + segs.map(() => '\u2022').join('/') : '/');
|
|
279
|
-
|
|
280
|
-
const paramCount = [...url.searchParams].length;
|
|
281
|
-
if (paramCount > 0) parts.push(` ?[${paramCount}]`);
|
|
282
|
-
if (url.hash && url.hash.length > 1) parts.push(' #[frag]');
|
|
283
|
-
|
|
284
|
-
return parts.join('');
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
function anonymizeChain(urls) {
|
|
288
|
-
if (!Array.isArray(urls) || urls.length === 0) return '[empty-chain]';
|
|
289
|
-
return urls.map(u => anonymizeUrl(u)).join(' \u2192 ');
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
return { anonymizeUrl, anonymizeChain };
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
// ============================================================================
|
|
296
|
-
// Per-tab health tracker (count-only, no content)
|
|
297
|
-
// ============================================================================
|
|
298
|
-
|
|
299
|
-
// Known bot-detection providers, matched by response header fingerprints.
|
|
300
|
-
// Order: most specific first.
|
|
301
|
-
const BOT_DETECTION_SIGNATURES = [
|
|
302
|
-
{ header: 'cf-mitigated', value: 'challenge', provider: 'cloudflare' },
|
|
303
|
-
{ header: 'x-datadome', provider: 'datadome' },
|
|
304
|
-
{ header: 'x-px', provider: 'perimeterx' },
|
|
305
|
-
{ header: 'x-distil-cs', provider: 'distil' },
|
|
306
|
-
{ header: 'x-sucuri-id', provider: 'sucuri' },
|
|
307
|
-
{ header: 'server', value: 'akamaighost', provider: 'akamai' },
|
|
308
|
-
// cf-ray is on ALL Cloudflare responses (even 200 OK). Must be last so it
|
|
309
|
-
// doesn't short-circuit other providers on multi-CDN sites.
|
|
310
|
-
{ header: 'cf-ray', provider: 'cloudflare' },
|
|
311
|
-
];
|
|
312
|
-
|
|
313
|
-
/**
|
|
314
|
-
* Detect bot-detection provider from Playwright response headers.
|
|
315
|
-
* Returns { detected: bool, provider: string|null, httpStatus: number|null }
|
|
316
|
-
*/
|
|
317
|
-
export function detectBotProtection(response) {
|
|
318
|
-
if (!response) return { detected: false, provider: null, httpStatus: null };
|
|
319
|
-
const status = response.status();
|
|
320
|
-
let headers;
|
|
321
|
-
try { headers = response.headers(); } catch { return { detected: false, provider: null, httpStatus: status }; }
|
|
322
|
-
for (const sig of BOT_DETECTION_SIGNATURES) {
|
|
323
|
-
const val = headers[sig.header];
|
|
324
|
-
if (val !== undefined) {
|
|
325
|
-
if (sig.value && !val.toLowerCase().includes(sig.value)) continue;
|
|
326
|
-
const challenged = status === 403 || status === 429 || status === 503;
|
|
327
|
-
return { detected: challenged, provider: sig.provider, httpStatus: status };
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
return { detected: false, provider: null, httpStatus: status };
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
/**
|
|
334
|
-
* Create a health tracker for a tab. Attaches to Playwright page events.
|
|
335
|
-
* Tracks: crashes, page errors, request failures, redirect status codes,
|
|
336
|
-
* HTTP status histogram (4xx+), and anti-bot challenge detection.
|
|
337
|
-
* All count-based -- no URLs or content stored.
|
|
338
|
-
*/
|
|
339
|
-
export function createTabHealthTracker(page) {
|
|
340
|
-
const health = {
|
|
341
|
-
crashes: 0,
|
|
342
|
-
pageErrors: 0,
|
|
343
|
-
requestFailures: 0,
|
|
344
|
-
inflightRequests: 0,
|
|
345
|
-
maxRedirectDepth: 0,
|
|
346
|
-
redirectStatusCodes: [], // status codes in redirect chain, e.g. [301, 302, 403]
|
|
347
|
-
statusCounts: {}, // { 403: 5, 429: 2, ... }
|
|
348
|
-
botDetection: null, // { detected, provider, httpStatus } from last nav response
|
|
349
|
-
lastNavResponseSize: 0,
|
|
350
|
-
_redirectDepth: 0,
|
|
351
|
-
};
|
|
352
|
-
|
|
353
|
-
// Renderer crash (OOM, segfault)
|
|
354
|
-
page.on('crash', () => { health.crashes++; });
|
|
355
|
-
|
|
356
|
-
// Uncaught JS exceptions on the page
|
|
357
|
-
page.on('pageerror', () => { health.pageErrors++; });
|
|
358
|
-
|
|
359
|
-
// Failed requests (blocked, DNS failure, etc.) + decrement in-flight counter
|
|
360
|
-
page.on('requestfailed', () => {
|
|
361
|
-
health.requestFailures++;
|
|
362
|
-
health.inflightRequests = Math.max(0, health.inflightRequests - 1);
|
|
363
|
-
});
|
|
364
|
-
|
|
365
|
-
// Track in-flight requests for hang diagnostics
|
|
366
|
-
page.on('request', () => { health.inflightRequests++; });
|
|
367
|
-
page.on('requestfinished', () => { health.inflightRequests = Math.max(0, health.inflightRequests - 1); });
|
|
368
|
-
|
|
369
|
-
// HTTP status tracking (non-2xx only)
|
|
370
|
-
page.on('response', (resp) => {
|
|
371
|
-
const s = resp.status();
|
|
372
|
-
if (s >= 400) health.statusCounts[s] = (health.statusCounts[s] || 0) + 1;
|
|
373
|
-
});
|
|
374
|
-
|
|
375
|
-
// Auto-dismiss dialogs to prevent page hangs (not tracked as a metric -- noise)
|
|
376
|
-
page.on('dialog', async (dialog) => {
|
|
377
|
-
try { await dialog.dismiss(); } catch { /* page might be closed */ }
|
|
378
|
-
});
|
|
379
|
-
|
|
380
|
-
// Redirect depth + status code chain per navigation
|
|
381
|
-
page.on('request', (req) => {
|
|
382
|
-
if (req.isNavigationRequest()) {
|
|
383
|
-
if (req.redirectedFrom()) {
|
|
384
|
-
health._redirectDepth++;
|
|
385
|
-
if (health._redirectDepth > health.maxRedirectDepth) {
|
|
386
|
-
health.maxRedirectDepth = health._redirectDepth;
|
|
387
|
-
}
|
|
388
|
-
} else {
|
|
389
|
-
health._redirectDepth = 0;
|
|
390
|
-
health.redirectStatusCodes = [];
|
|
391
|
-
health.inflightRequests = 0; // reset on new navigation to prevent drift
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
});
|
|
395
|
-
|
|
396
|
-
// Capture redirect status codes and detect bot protection on nav responses
|
|
397
|
-
page.on('response', (resp) => {
|
|
398
|
-
try {
|
|
399
|
-
const req = resp.request();
|
|
400
|
-
if (req.isNavigationRequest()) {
|
|
401
|
-
health.redirectStatusCodes.push(resp.status());
|
|
402
|
-
health.botDetection = detectBotProtection(resp);
|
|
403
|
-
// Approximate response body size from content-length (no body read)
|
|
404
|
-
const cl = resp.headers()['content-length'];
|
|
405
|
-
if (cl) health.lastNavResponseSize = parseInt(cl, 10) || 0;
|
|
406
|
-
}
|
|
407
|
-
} catch { /* page closed */ }
|
|
408
|
-
});
|
|
409
|
-
|
|
410
|
-
/** Snapshot current health counters for inclusion in reports. */
|
|
411
|
-
function snapshot() {
|
|
412
|
-
const { _redirectDepth, ...clean } = health;
|
|
413
|
-
return { ...clean };
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
/**
|
|
417
|
-
* Get document.readyState from the page. Returns null if page is unresponsive.
|
|
418
|
-
* Use a tight timeout -- if the renderer is crashed, evaluate will hang.
|
|
419
|
-
*/
|
|
420
|
-
async function getReadyState() {
|
|
421
|
-
try {
|
|
422
|
-
return await Promise.race([
|
|
423
|
-
page.evaluate(() => document.readyState),
|
|
424
|
-
new Promise(resolve => setTimeout(() => resolve('unresponsive'), 1000)),
|
|
425
|
-
]);
|
|
426
|
-
} catch {
|
|
427
|
-
return 'unresponsive';
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
return { health, snapshot, getReadyState };
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
// collectResourceSnapshot and classifyProxyError live in lib/resources.js
|
|
435
|
-
// (isolated from network code for clean separation of concerns).
|
|
436
|
-
// Re-exported here for backward compatibility.
|
|
437
|
-
export { collectResourceSnapshot, classifyProxyError };
|
|
438
|
-
|
|
439
|
-
// ============================================================================
|
|
440
|
-
// Rate limiter (sliding window, 1 hour)
|
|
441
|
-
// ============================================================================
|
|
442
|
-
|
|
443
|
-
class RateLimiter {
|
|
444
|
-
constructor(maxPerHour) {
|
|
445
|
-
this.maxPerHour = maxPerHour;
|
|
446
|
-
this.timestamps = [];
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
tryAcquire() {
|
|
450
|
-
const now = Date.now();
|
|
451
|
-
this.timestamps = this.timestamps.filter(t => t > now - 3600_000);
|
|
452
|
-
if (this.timestamps.length >= this.maxPerHour) return false;
|
|
453
|
-
this.timestamps.push(now);
|
|
454
|
-
return true;
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
// ============================================================================
|
|
459
|
-
// Crash relay client
|
|
460
|
-
// ============================================================================
|
|
461
|
-
|
|
462
|
-
// PHI-VENDOR: Crash relay endpoint defaults to empty (telemetry disabled).
|
|
463
|
-
// The original upstream endpoint pointed to a Cloudflare Worker maintained
|
|
464
|
-
// by Jo Inc (askjo.workers.dev). The phi-code vendored snapshot ships with
|
|
465
|
-
// telemetry off by default — users who want their own relay can set
|
|
466
|
-
// CAMOFOX_CRASH_REPORT_URL explicitly.
|
|
467
|
-
//
|
|
468
|
-
// Upstream endpoint (for reference, not used unless re-enabled):
|
|
469
|
-
// https://camofox-telemetry.askjo.workers.dev/report
|
|
470
|
-
|
|
471
|
-
const DEFAULT_RELAY_URL = process.env.CAMOFOX_CRASH_REPORT_URL || '';
|
|
472
|
-
const FETCH_TIMEOUT_MS = 5000;
|
|
473
|
-
|
|
474
|
-
let _relayUrl = DEFAULT_RELAY_URL;
|
|
475
|
-
|
|
476
|
-
function fetchWithTimeout(url, options) {
|
|
477
|
-
const controller = new AbortController();
|
|
478
|
-
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
|
479
|
-
return fetch(url, { ...options, signal: controller.signal })
|
|
480
|
-
.finally(() => clearTimeout(timer));
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
/**
|
|
484
|
-
* Send a crash report to the relay. Returns true if accepted.
|
|
485
|
-
* Never throws -- reporter must never crash the server.
|
|
486
|
-
*/
|
|
487
|
-
export async function sendToRelay(payload) {
|
|
488
|
-
// PHI-VENDOR: telemetry disabled by default. When CAMOFOX_CRASH_REPORT_URL
|
|
489
|
-
// is unset (the new default), this is a no-op — no network call leaves
|
|
490
|
-
// the user's machine. Set the env var explicitly to opt back in.
|
|
491
|
-
if (!_relayUrl) return false;
|
|
492
|
-
try {
|
|
493
|
-
const resp = await fetchWithTimeout(_relayUrl, {
|
|
494
|
-
method: 'POST',
|
|
495
|
-
headers: { 'Content-Type': 'application/json' },
|
|
496
|
-
body: JSON.stringify(payload),
|
|
497
|
-
});
|
|
498
|
-
return resp.ok || resp.status === 429; // rate-limited is fine, not an error
|
|
499
|
-
} catch {
|
|
500
|
-
return false;
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
// ============================================================================
|
|
505
|
-
// Issue formatting
|
|
506
|
-
// ============================================================================
|
|
507
|
-
|
|
508
|
-
function formatIssueBody(type, detail) {
|
|
509
|
-
const sections = [
|
|
510
|
-
'> Auto-reported by camofox-crash-reporter. All data is anonymized.',
|
|
511
|
-
'',
|
|
512
|
-
'## Environment',
|
|
513
|
-
`- **version:** ${detail.version || 'unknown'}`,
|
|
514
|
-
`- **node:** ${detail.nodeVersion || 'unknown'}`,
|
|
515
|
-
`- **platform:** ${detail.platform || 'unknown'}`,
|
|
516
|
-
`- **uptime:** ${detail.uptimeMinutes != null ? detail.uptimeMinutes + ' min' : 'unknown'}`,
|
|
517
|
-
];
|
|
518
|
-
|
|
519
|
-
// Resource snapshot (memory, handles, browser RSS)
|
|
520
|
-
const r = detail.resources;
|
|
521
|
-
if (r) {
|
|
522
|
-
sections.push('', '## Resources');
|
|
523
|
-
sections.push(`- **node RSS:** ${r.nodeRssMb ?? '?'} MB`);
|
|
524
|
-
sections.push(`- **node heap:** ${r.nodeHeapUsedMb ?? '?'} / ${r.nodeHeapTotalMb ?? '?'} MB`);
|
|
525
|
-
if (r.browserRssMb != null) sections.push(`- **browser RSS:** ${r.browserRssMb} MB`);
|
|
526
|
-
if (r.browserContexts != null) sections.push(`- **browser contexts:** ${r.browserContexts}`);
|
|
527
|
-
if (r.activeTabs != null) sections.push(`- **active tabs:** ${r.activeTabs}`);
|
|
528
|
-
if (r.openFds != null) sections.push(`- **open FDs:** ${r.openFds}`);
|
|
529
|
-
if (r.activeHandles != null) sections.push(`- **active handles:** ${r.activeHandles}`);
|
|
530
|
-
if (r.eventLoopLagMs != null) sections.push(`- **event loop lag:** ${r.eventLoopLagMs} ms`);
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
// Error info
|
|
534
|
-
if (detail.signal) sections.push('', `**Signal:** ${detail.signal}`);
|
|
535
|
-
if (detail.activeRoute) sections.push(`**Active route:** ${detail.activeRoute}`);
|
|
536
|
-
if (detail.message) {
|
|
537
|
-
sections.push('', '## Error', '```', anonymize(detail.message), '```');
|
|
538
|
-
}
|
|
539
|
-
if (detail.stack) {
|
|
540
|
-
sections.push('', '## Stack Trace', '```', anonymize(detail.stack), '```');
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
// Hang-specific details
|
|
544
|
-
if (detail.hang) {
|
|
545
|
-
const h = detail.hang;
|
|
546
|
-
sections.push('', '## Hang Details');
|
|
547
|
-
sections.push(`- **operation:** ${h.operation}`);
|
|
548
|
-
sections.push(`- **duration:** ${Math.round(h.durationMs / 1000)}s`);
|
|
549
|
-
if (h.lockQueueMs != null) sections.push(`- **lock queue wait:** ${Math.round(h.lockQueueMs)}ms`);
|
|
550
|
-
if (h.documentReadyState) sections.push(`- **document.readyState:** ${h.documentReadyState}`);
|
|
551
|
-
if (h.inflightRequests != null) sections.push(`- **in-flight requests:** ${h.inflightRequests}`);
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
// Anti-bot detection
|
|
555
|
-
if (detail.botDetection?.detected) {
|
|
556
|
-
const b = detail.botDetection;
|
|
557
|
-
sections.push('', '## Anti-Bot Detection');
|
|
558
|
-
sections.push(`- **provider:** ${b.provider || 'unknown'}`);
|
|
559
|
-
sections.push(`- **HTTP status:** ${b.httpStatus || '?'}`);
|
|
560
|
-
if (b.responseBodySizeKb != null) sections.push(`- **response size:** ${b.responseBodySizeKb} KB`);
|
|
561
|
-
if (b.redirectChainLength != null) sections.push(`- **redirect chain:** ${b.redirectChainLength} hops`);
|
|
562
|
-
if (b.redirectStatusCodes?.length) sections.push(`- **redirect statuses:** ${b.redirectStatusCodes.join(' -> ')}`);
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
// Proxy info (safe fields only -- no IPs, credentials, or hostnames)
|
|
566
|
-
if (detail.proxy) {
|
|
567
|
-
const p = detail.proxy;
|
|
568
|
-
sections.push('', '## Proxy');
|
|
569
|
-
sections.push(`- **configured:** ${p.configured}`);
|
|
570
|
-
if (p.configured) {
|
|
571
|
-
if (p.type) sections.push(`- **type:** ${p.type}`);
|
|
572
|
-
sections.push(`- **auth configured:** ${p.authConfigured ?? 'unknown'}`);
|
|
573
|
-
if (p.error) sections.push(`- **error:** ${p.error}`);
|
|
574
|
-
if (p.tlsError) sections.push(`- **TLS error:** yes`);
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
|
-
|
|
578
|
-
// Stall-specific details
|
|
579
|
-
if (detail.stall) {
|
|
580
|
-
const s = detail.stall;
|
|
581
|
-
sections.push('', '## Stall Details');
|
|
582
|
-
sections.push(`- **stall duration:** ${Math.round(s.driftMs / 1000)}s`);
|
|
583
|
-
if (s.classification) sections.push(`- **classification:** ${s.classification}`);
|
|
584
|
-
if (s.cpuElapsedS != null) sections.push(`- **CPU time during stall:** ${s.cpuElapsedS}s`);
|
|
585
|
-
if (s.cpuRatio != null) sections.push(`- **CPU/wall ratio:** ${s.cpuRatio}`);
|
|
586
|
-
if (s.sigcontInWindow != null) sections.push(`- **SIGCONT in window:** ${s.sigcontInWindow}`);
|
|
587
|
-
if (s.hrtimeWallDriftS != null) sections.push(`- **hrtime<->wall drift:** ${s.hrtimeWallDriftS}s`);
|
|
588
|
-
if (s.eventLoopDelay) {
|
|
589
|
-
const eld = s.eventLoopDelay;
|
|
590
|
-
sections.push(`- **event loop delay:** p50=${eld.p50Ms}ms p99=${eld.p99Ms}ms max=${eld.maxMs}ms`);
|
|
591
|
-
}
|
|
592
|
-
if (s.lastRoute) sections.push(`- **last route:** ${s.lastRoute}`);
|
|
593
|
-
if (s.activeHandles != null) sections.push(`- **active handles:** ${s.activeHandles}`);
|
|
594
|
-
if (s.activeRequests != null) sections.push(`- **active requests:** ${s.activeRequests}`);
|
|
595
|
-
if (s.heapDeltaMb != null) sections.push(`- **heap delta:** ${s.heapDeltaMb > 0 ? '+' : ''}${s.heapDeltaMb} MB`);
|
|
596
|
-
}
|
|
597
|
-
|
|
598
|
-
// Context (misc extra data)
|
|
599
|
-
if (detail.nativeMemory) {
|
|
600
|
-
const nm = detail.nativeMemory;
|
|
601
|
-
sections.push('', '## Native Memory Details');
|
|
602
|
-
sections.push(`- **baseline:** ${nm.baselineMb} MB`);
|
|
603
|
-
sections.push(`- **current:** ${nm.currentMb} MB`);
|
|
604
|
-
sections.push(`- **high-water:** ${nm.highWaterMb} MB`);
|
|
605
|
-
sections.push(`- **growth:** ${nm.growthMb} MB`);
|
|
606
|
-
sections.push(`- **node RSS:** ${nm.rssMb} MB`);
|
|
607
|
-
sections.push(`- **heap used:** ${nm.heapUsedMb} MB`);
|
|
608
|
-
sections.push(`- **external:** ${nm.externalMb} MB`);
|
|
609
|
-
if (nm.lastSeenBrowserRssMb != null) sections.push(`- **browser RSS (last seen):** ${nm.lastSeenBrowserRssMb} MB`);
|
|
610
|
-
else sections.push(`- **browser RSS (last seen):** not captured (browser already dead)`);
|
|
611
|
-
}
|
|
612
|
-
|
|
613
|
-
if (detail.context && Object.keys(detail.context).length > 0) {
|
|
614
|
-
sections.push('', '<details><summary>Context</summary>', '', '```json', anonymize(JSON.stringify(detail.context, null, 2)), '```', '', '</details>');
|
|
615
|
-
}
|
|
616
|
-
|
|
617
|
-
return sections.join('\n');
|
|
618
|
-
}
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
// ============================================================================
|
|
622
|
-
// Core reporter factory
|
|
623
|
-
// ============================================================================
|
|
624
|
-
|
|
625
|
-
/**
|
|
626
|
-
* Create a reporter instance.
|
|
627
|
-
*
|
|
628
|
-
* @param {object} config
|
|
629
|
-
* @param {boolean} config.crashReportEnabled
|
|
630
|
-
* @param {string} config.crashReportRepo - "owner/repo" (env override)
|
|
631
|
-
* @param {number} config.crashReportRateLimit - max reports per hour
|
|
632
|
-
* @param {object} config.crashReporterConfig - from camofox.config.json crashReporter section
|
|
633
|
-
* @param {string} [config.version] - package version
|
|
634
|
-
*/
|
|
635
|
-
export function createReporter(config) {
|
|
636
|
-
// Set relay URL (env override for self-hosted relays)
|
|
637
|
-
_relayUrl = config.crashReportUrl || DEFAULT_RELAY_URL;
|
|
638
|
-
|
|
639
|
-
const enabled = config.crashReportEnabled !== false;
|
|
640
|
-
const repo = config.crashReportRepo || 'jo-inc/camofox-browser';
|
|
641
|
-
const rateLimiters = {
|
|
642
|
-
crash: new RateLimiter(5), // 5 crashes/hr
|
|
643
|
-
hang: new RateLimiter(5), // 5 hangs/hr
|
|
644
|
-
stuck: new RateLimiter(2), // 2 stalls/hr (with active tabs only)
|
|
645
|
-
leak: new RateLimiter(2), // 2 leak alerts/hr
|
|
646
|
-
_default: new RateLimiter(config.crashReportRateLimit || 10),
|
|
647
|
-
};
|
|
648
|
-
const version = config.version || 'unknown';
|
|
649
|
-
|
|
650
|
-
let watchdogInterval = null;
|
|
651
|
-
let _resetNativeMemBaseline = false; // Set by resetNativeMemBaseline(), read by watchdog
|
|
652
|
-
let lastTick = Date.now();
|
|
653
|
-
const inFlight = new Set();
|
|
654
|
-
|
|
655
|
-
// Track last Express route for stall reports
|
|
656
|
-
let _lastRoute = null;
|
|
657
|
-
|
|
658
|
-
// No-op when disabled
|
|
659
|
-
if (!enabled) {
|
|
660
|
-
return {
|
|
661
|
-
reportCrash: async () => {},
|
|
662
|
-
reportHang: async () => {},
|
|
663
|
-
reportStuckLoop: async () => {},
|
|
664
|
-
startWatchdog: () => {},
|
|
665
|
-
trackRoute: () => {},
|
|
666
|
-
stop: () => {},
|
|
667
|
-
_anonymize: anonymize,
|
|
668
|
-
_stackSignature: stackSignature,
|
|
669
|
-
};
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
/** Core: build and send a report to the relay. NEVER throws. */
|
|
673
|
-
async function fileReport(type, labels, detail) {
|
|
674
|
-
const bucket = type.startsWith('stuck:') ? 'stuck' : type.startsWith('hang:') ? 'hang' : type.startsWith('leak:') ? 'leak' : 'crash';
|
|
675
|
-
const limiter = rateLimiters[bucket] || rateLimiters._default;
|
|
676
|
-
if (!limiter.tryAcquire()) return;
|
|
677
|
-
|
|
678
|
-
const reportPromise = (async () => {
|
|
679
|
-
try {
|
|
680
|
-
const sig = stackSignature(type, detail.error || { message: detail.message, stack: detail.stack });
|
|
681
|
-
const safeMessage = anonymize(detail.message || detail.error?.message || type);
|
|
682
|
-
const title = `[${sig}] ${type}: ${safeMessage.slice(0, 120)}`;
|
|
683
|
-
|
|
684
|
-
const body = formatIssueBody(type, {
|
|
685
|
-
...detail,
|
|
686
|
-
version,
|
|
687
|
-
nodeVersion: typeof process !== 'undefined' ? process.version : 'unknown',
|
|
688
|
-
platform: typeof process !== 'undefined' ? process.platform : 'unknown',
|
|
689
|
-
});
|
|
690
|
-
|
|
691
|
-
const issueLabels = Array.isArray(labels) ? labels : [labels, 'auto-report'];
|
|
692
|
-
|
|
693
|
-
await sendToRelay({
|
|
694
|
-
type,
|
|
695
|
-
signature: sig,
|
|
696
|
-
title,
|
|
697
|
-
body,
|
|
698
|
-
labels: issueLabels,
|
|
699
|
-
version,
|
|
700
|
-
});
|
|
701
|
-
} catch {
|
|
702
|
-
// Swallow -- reporter must never crash the server
|
|
703
|
-
}
|
|
704
|
-
})();
|
|
705
|
-
|
|
706
|
-
inFlight.add(reportPromise);
|
|
707
|
-
reportPromise.finally(() => inFlight.delete(reportPromise));
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
/**
|
|
711
|
-
* Track the last Express route for stall diagnostics.
|
|
712
|
-
* Call from middleware: reporter.trackRoute(req.method + ' ' + req.route?.path)
|
|
713
|
-
*/
|
|
714
|
-
function trackRoute(route) {
|
|
715
|
-
_lastRoute = route || null;
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
async function reportCrash(error, opts = {}) {
|
|
719
|
-
const err = error instanceof Error ? error : new Error(String(error));
|
|
720
|
-
const uptimeMinutes = typeof process !== 'undefined'
|
|
721
|
-
? Math.round(process.uptime() / 60) : undefined;
|
|
722
|
-
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
723
|
-
|
|
724
|
-
await fileReport(
|
|
725
|
-
opts.signal ? `signal:${opts.signal}` : (err.name || 'crash'),
|
|
726
|
-
['crash', 'auto-report'],
|
|
727
|
-
{
|
|
728
|
-
error: err,
|
|
729
|
-
message: err.message,
|
|
730
|
-
stack: err.stack,
|
|
731
|
-
signal: opts.signal || null,
|
|
732
|
-
activeRoute: _lastRoute,
|
|
733
|
-
uptimeMinutes,
|
|
734
|
-
resources,
|
|
735
|
-
proxy: opts.proxy || null,
|
|
736
|
-
context: opts.context,
|
|
737
|
-
},
|
|
738
|
-
);
|
|
739
|
-
}
|
|
740
|
-
|
|
741
|
-
async function reportHang(operation, durationMs, opts = {}) {
|
|
742
|
-
const uptimeMinutes = typeof process !== 'undefined'
|
|
743
|
-
? Math.round(process.uptime() / 60) : undefined;
|
|
744
|
-
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
745
|
-
|
|
746
|
-
// Build lean context (journal only, no redundant fields)
|
|
747
|
-
const context = { ...opts.context };
|
|
748
|
-
if (context.journal) {
|
|
749
|
-
context.journal = context.journal.map(j => typeof j === 'string' ? j : j);
|
|
750
|
-
}
|
|
751
|
-
// Remove fields that now have dedicated sections
|
|
752
|
-
delete context.operation;
|
|
753
|
-
delete context.durationMs;
|
|
754
|
-
|
|
755
|
-
// Anti-bot detection from health snapshot
|
|
756
|
-
const healthSnap = opts.healthSnapshot;
|
|
757
|
-
const botDetection = healthSnap?.botDetection?.detected ? {
|
|
758
|
-
...healthSnap.botDetection,
|
|
759
|
-
responseBodySizeKb: healthSnap.lastNavResponseSize
|
|
760
|
-
? Math.round(healthSnap.lastNavResponseSize / 1024) : null,
|
|
761
|
-
redirectChainLength: healthSnap.redirectStatusCodes?.length || null,
|
|
762
|
-
redirectStatusCodes: healthSnap.redirectStatusCodes?.length
|
|
763
|
-
? healthSnap.redirectStatusCodes : null,
|
|
764
|
-
} : null;
|
|
765
|
-
|
|
766
|
-
// Get document.readyState if healthTracker provided
|
|
767
|
-
let documentReadyState = null;
|
|
768
|
-
if (opts.healthTracker?.getReadyState) {
|
|
769
|
-
documentReadyState = await opts.healthTracker.getReadyState();
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
const labels = ['hang', 'auto-report'];
|
|
773
|
-
if (botDetection?.detected) labels.push('bot-detection');
|
|
774
|
-
|
|
775
|
-
await fileReport(
|
|
776
|
-
`hang:${operation}`,
|
|
777
|
-
labels,
|
|
778
|
-
{
|
|
779
|
-
message: `Operation "${operation}" hung for ${Math.round(durationMs / 1000)}s`,
|
|
780
|
-
stack: opts.error?.stack,
|
|
781
|
-
activeRoute: _lastRoute,
|
|
782
|
-
uptimeMinutes,
|
|
783
|
-
resources,
|
|
784
|
-
hang: {
|
|
785
|
-
operation,
|
|
786
|
-
durationMs,
|
|
787
|
-
lockQueueMs: opts.lockQueueMs ?? null,
|
|
788
|
-
documentReadyState,
|
|
789
|
-
inflightRequests: healthSnap?.inflightRequests ?? null,
|
|
790
|
-
},
|
|
791
|
-
botDetection,
|
|
792
|
-
proxy: opts.proxy || null,
|
|
793
|
-
context,
|
|
794
|
-
},
|
|
795
|
-
);
|
|
796
|
-
}
|
|
797
|
-
|
|
798
|
-
async function reportStuckLoop(durationMs, opts = {}) {
|
|
799
|
-
const uptimeMinutes = typeof process !== 'undefined'
|
|
800
|
-
? Math.round(process.uptime() / 60) : undefined;
|
|
801
|
-
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
802
|
-
|
|
803
|
-
await fileReport(
|
|
804
|
-
'stuck:tab-lock',
|
|
805
|
-
['stuck', 'auto-report'],
|
|
806
|
-
{
|
|
807
|
-
message: `Tab lock held for ${Math.round(durationMs / 1000)}s (tab destroyed)`,
|
|
808
|
-
uptimeMinutes,
|
|
809
|
-
resources,
|
|
810
|
-
context: { durationMs, ...opts.context },
|
|
811
|
-
},
|
|
812
|
-
);
|
|
813
|
-
}
|
|
814
|
-
|
|
815
|
-
function startWatchdog(thresholdMs = 5000, getContext) {
|
|
816
|
-
if (watchdogInterval) return;
|
|
817
|
-
|
|
818
|
-
const checkMs = 1000;
|
|
819
|
-
lastTick = Date.now();
|
|
820
|
-
let lastCpuUsage = process.cpuUsage();
|
|
821
|
-
let lastHrtime = process.hrtime.bigint();
|
|
822
|
-
let lastHeapUsed = process.memoryUsage().heapUsed;
|
|
823
|
-
|
|
824
|
-
// --- Native memory leak tracking ---
|
|
825
|
-
// Track RSS minus JS heap over time to detect native/external memory leaks.
|
|
826
|
-
// Sample every 30s, alert if native memory stays >400MB above baseline for
|
|
827
|
-
// 3 consecutive checks (~90s). This avoids false positives from:
|
|
828
|
-
// - Browser initialization spikes (first 2 min)
|
|
829
|
-
// - One-time allocations that stabilize
|
|
830
|
-
// - Post-session RSS that hasn't been reclaimed by the OS yet
|
|
831
|
-
// - Self-healing restart (kills browser at 200MB growth when sessions=0)
|
|
832
|
-
// The memory pressure restart in server.js fires at 200MB when idle.
|
|
833
|
-
// We only report at 400MB to catch cases where self-healing FAILED.
|
|
834
|
-
let nativeMemBaseline = null; // RSS - heapUsed at first measurement
|
|
835
|
-
let nativeMemHighWater = 0;
|
|
836
|
-
let lastNativeMemCheck = 0;
|
|
837
|
-
const NATIVE_MEM_CHECK_INTERVAL_MS = 30_000;
|
|
838
|
-
const NATIVE_MEM_LEAK_THRESHOLD_MB = 400; // alert only when growth exceeds self-healing threshold
|
|
839
|
-
const NATIVE_MEM_MIN_UPTIME_S = 120; // don't measure until process has been up 2 min
|
|
840
|
-
const NATIVE_MEM_CONSECUTIVE_REQUIRED = 3; // require 3 consecutive checks above threshold
|
|
841
|
-
const NATIVE_MEM_GRACE_CHECKS = 2; // skip 2 checks after baseline reset (let memory settle)
|
|
842
|
-
let nativeMemAlertFired = false;
|
|
843
|
-
let nativeMemConsecutiveAbove = 0; // consecutive checks above threshold
|
|
844
|
-
let nativeMemGraceRemaining = 0; // checks to skip after baseline reset
|
|
845
|
-
let lastSeenBrowserRssMb = null; // captured during growth checks while browser is alive
|
|
846
|
-
|
|
847
|
-
// SIGCONT detection -- macOS sends SIGCONT on wake from sleep/suspend
|
|
848
|
-
let lastSigcont = 0;
|
|
849
|
-
try { process.on('SIGCONT', () => { lastSigcont = Date.now(); }); } catch { /* unavailable */ }
|
|
850
|
-
|
|
851
|
-
// Event loop delay histogram (perf_hooks) -- correlating evidence
|
|
852
|
-
let elHistogram = null;
|
|
853
|
-
try {
|
|
854
|
-
elHistogram = monitorEventLoopDelay({ resolution: 20 });
|
|
855
|
-
elHistogram.enable();
|
|
856
|
-
} catch { /* unavailable */ }
|
|
857
|
-
|
|
858
|
-
// Suppress false positives from OS sleep/suspend (laptop lid close, VM pause).
|
|
859
|
-
// Stalls > 120s are almost certainly not event-loop bugs.
|
|
860
|
-
const MAX_REPORTABLE_DRIFT_MS = 60_000;
|
|
861
|
-
let suppressTicksRemaining = 0;
|
|
862
|
-
const SUPPRESS_TICKS_AFTER_WAKE = 5;
|
|
863
|
-
|
|
864
|
-
watchdogInterval = setInterval(() => {
|
|
865
|
-
const now = Date.now();
|
|
866
|
-
const drift = now - lastTick - checkMs;
|
|
867
|
-
const cpuDelta = process.cpuUsage(lastCpuUsage);
|
|
868
|
-
const hrtimeNow = process.hrtime.bigint();
|
|
869
|
-
const hrtimeDeltaMs = Number(hrtimeNow - lastHrtime) / 1e6;
|
|
870
|
-
|
|
871
|
-
lastTick = now;
|
|
872
|
-
lastCpuUsage = process.cpuUsage();
|
|
873
|
-
lastHrtime = hrtimeNow;
|
|
874
|
-
|
|
875
|
-
// After a long sleep/suspend, suppress the next few ticks (post-wake jitter)
|
|
876
|
-
if (drift > MAX_REPORTABLE_DRIFT_MS) {
|
|
877
|
-
suppressTicksRemaining = SUPPRESS_TICKS_AFTER_WAKE;
|
|
878
|
-
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
879
|
-
return;
|
|
880
|
-
}
|
|
881
|
-
if (suppressTicksRemaining > 0) {
|
|
882
|
-
suppressTicksRemaining--;
|
|
883
|
-
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
884
|
-
return;
|
|
885
|
-
}
|
|
886
|
-
|
|
887
|
-
// --- Native memory leak detection (runs every ~30s) ---
|
|
888
|
-
if (now - lastNativeMemCheck >= NATIVE_MEM_CHECK_INTERVAL_MS) {
|
|
889
|
-
lastNativeMemCheck = now;
|
|
890
|
-
try {
|
|
891
|
-
// Skip until process has been up long enough for browser to initialize.
|
|
892
|
-
// Browser launch causes a 100-300MB RSS spike that isn't a leak.
|
|
893
|
-
if (process.uptime() >= NATIVE_MEM_MIN_UPTIME_S) {
|
|
894
|
-
// Check if baseline should be reset (e.g. after browser close)
|
|
895
|
-
if (_resetNativeMemBaseline) {
|
|
896
|
-
nativeMemBaseline = null;
|
|
897
|
-
nativeMemHighWater = 0;
|
|
898
|
-
nativeMemAlertFired = false;
|
|
899
|
-
nativeMemConsecutiveAbove = 0;
|
|
900
|
-
nativeMemGraceRemaining = NATIVE_MEM_GRACE_CHECKS;
|
|
901
|
-
_resetNativeMemBaseline = false;
|
|
902
|
-
}
|
|
903
|
-
|
|
904
|
-
// Grace period after reset -- let memory settle before re-baselining
|
|
905
|
-
if (nativeMemGraceRemaining > 0) {
|
|
906
|
-
nativeMemGraceRemaining--;
|
|
907
|
-
} else {
|
|
908
|
-
const mem = process.memoryUsage();
|
|
909
|
-
const nativeMemMb = Math.round((mem.rss - mem.heapUsed) / 1048576);
|
|
910
|
-
if (nativeMemBaseline === null) {
|
|
911
|
-
nativeMemBaseline = nativeMemMb;
|
|
912
|
-
}
|
|
913
|
-
nativeMemHighWater = Math.max(nativeMemHighWater, nativeMemMb);
|
|
914
|
-
const growth = nativeMemMb - nativeMemBaseline;
|
|
915
|
-
|
|
916
|
-
if (growth > NATIVE_MEM_LEAK_THRESHOLD_MB && !nativeMemAlertFired) {
|
|
917
|
-
// Require sustained growth -- one-time spikes aren't leaks.
|
|
918
|
-
// Must exceed threshold on 3 consecutive checks (~90s).
|
|
919
|
-
nativeMemConsecutiveAbove++;
|
|
920
|
-
|
|
921
|
-
// Capture browser RSS NOW while it may still be alive.
|
|
922
|
-
// By report time the browser is often killed by memory pressure restart,
|
|
923
|
-
// making browserRssMb null. This preserves the last-seen value.
|
|
924
|
-
try {
|
|
925
|
-
if (getContext) {
|
|
926
|
-
const ctx = getContext();
|
|
927
|
-
if (ctx.resourceOpts?.browserPid) {
|
|
928
|
-
const snap = collectResourceSnapshot(ctx.resourceOpts);
|
|
929
|
-
if (snap.browserRssMb != null) lastSeenBrowserRssMb = snap.browserRssMb;
|
|
930
|
-
}
|
|
931
|
-
}
|
|
932
|
-
} catch { /* swallow */ }
|
|
933
|
-
|
|
934
|
-
if (nativeMemConsecutiveAbove >= NATIVE_MEM_CONSECUTIVE_REQUIRED) {
|
|
935
|
-
nativeMemAlertFired = true;
|
|
936
|
-
let extra = {};
|
|
937
|
-
try { if (getContext) extra = getContext(); } catch { /* swallow */ }
|
|
938
|
-
const resources = collectResourceSnapshot(extra.resourceOpts || {});
|
|
939
|
-
delete extra.resourceOpts;
|
|
940
|
-
|
|
941
|
-
// Skip report if sessions=0 — memory pressure restart handles idle leaks.
|
|
942
|
-
// Only report when sessions are active (restart CAN'T fire) or restart failed.
|
|
943
|
-
const sessionCount = resources.browserContexts ?? 0;
|
|
944
|
-
if (sessionCount === 0 && resources.browserRssMb == null) {
|
|
945
|
-
// Browser already dead, restart mechanism handled it. Don't spam.
|
|
946
|
-
// But if growth is extreme (>600MB), report anyway — restart may have failed.
|
|
947
|
-
if (growth < 600) {
|
|
948
|
-
// Self-healing. Skip report.
|
|
949
|
-
return;
|
|
950
|
-
}
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
fileReport('leak:native-memory', ['auto-report', 'memory-leak'], {
|
|
954
|
-
message: `Native memory grew by ${growth}MB (baseline: ${nativeMemBaseline}MB, current: ${nativeMemMb}MB, high-water: ${nativeMemHighWater}MB)`,
|
|
955
|
-
uptimeMinutes: Math.round(process.uptime() / 60),
|
|
956
|
-
resources,
|
|
957
|
-
nativeMemory: {
|
|
958
|
-
baselineMb: nativeMemBaseline,
|
|
959
|
-
currentMb: nativeMemMb,
|
|
960
|
-
highWaterMb: nativeMemHighWater,
|
|
961
|
-
growthMb: growth,
|
|
962
|
-
rssMb: Math.round(mem.rss / 1048576),
|
|
963
|
-
heapUsedMb: Math.round(mem.heapUsed / 1048576),
|
|
964
|
-
externalMb: Math.round(mem.external / 1048576),
|
|
965
|
-
lastSeenBrowserRssMb,
|
|
966
|
-
},
|
|
967
|
-
context: extra,
|
|
968
|
-
});
|
|
969
|
-
}
|
|
970
|
-
} else {
|
|
971
|
-
// Reset consecutive counter if memory dropped back below threshold
|
|
972
|
-
nativeMemConsecutiveAbove = 0;
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
} catch { /* swallow */ }
|
|
977
|
-
}
|
|
978
|
-
|
|
979
|
-
if (drift > thresholdMs) {
|
|
980
|
-
// CPU time consumed during the stall interval (user + system, in seconds)
|
|
981
|
-
const cpuElapsedS = (cpuDelta.user + cpuDelta.system) / 1e6;
|
|
982
|
-
const wallElapsedS = drift / 1000;
|
|
983
|
-
const cpuRatio = wallElapsedS > 0 ? cpuElapsedS / wallElapsedS : 0;
|
|
984
|
-
|
|
985
|
-
// SIGCONT within the stall window = OS sleep/resume
|
|
986
|
-
const sigcontInWindow = lastSigcont > 0 && (now - lastSigcont) < drift + 2000;
|
|
987
|
-
|
|
988
|
-
// hrtime vs wall clock drift (macOS: hrtime doesn't advance during sleep)
|
|
989
|
-
const hrtimeWallDriftS = Math.abs((drift - (hrtimeDeltaMs - checkMs))) / 1000;
|
|
990
|
-
|
|
991
|
-
// Classify: sleep vs real stall
|
|
992
|
-
let classification;
|
|
993
|
-
if (cpuRatio < 0.01 && sigcontInWindow) classification = 'sleep';
|
|
994
|
-
else if (cpuRatio < 0.001) classification = 'likely_sleep';
|
|
995
|
-
else if (cpuRatio < 0.01) classification = 'likely_sleep';
|
|
996
|
-
else if (cpuRatio > 0.1) classification = 'real_stall';
|
|
997
|
-
else classification = 'ambiguous';
|
|
998
|
-
|
|
999
|
-
// Don't file reports for sleep/suspend-resume stalls
|
|
1000
|
-
if (classification === 'sleep' || classification === 'likely_sleep') {
|
|
1001
|
-
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
1002
|
-
return;
|
|
1003
|
-
}
|
|
1004
|
-
|
|
1005
|
-
// Capture heap delta during stall (GC indicator)
|
|
1006
|
-
const currentHeap = process.memoryUsage().heapUsed;
|
|
1007
|
-
const heapDeltaMb = Math.round((currentHeap - lastHeapUsed) / 1048576);
|
|
1008
|
-
lastHeapUsed = currentHeap;
|
|
1009
|
-
|
|
1010
|
-
let extra = {};
|
|
1011
|
-
try { if (getContext) extra = getContext(); } catch { /* swallow */ }
|
|
1012
|
-
|
|
1013
|
-
const resources = collectResourceSnapshot(extra.resourceOpts || {});
|
|
1014
|
-
// Remove resourceOpts from extra so it doesn't end up in context
|
|
1015
|
-
delete extra.resourceOpts;
|
|
1016
|
-
|
|
1017
|
-
// Don't report idle-server stalls -- no user impact
|
|
1018
|
-
if ((resources.activeTabs || 0) === 0 && (resources.browserContexts || 0) === 0) {
|
|
1019
|
-
return;
|
|
1020
|
-
}
|
|
1021
|
-
|
|
1022
|
-
// Event loop delay histogram snapshot
|
|
1023
|
-
let elDelay = null;
|
|
1024
|
-
if (elHistogram) {
|
|
1025
|
-
try {
|
|
1026
|
-
elDelay = {
|
|
1027
|
-
p50Ms: Math.round(elHistogram.percentile(50) / 1e6),
|
|
1028
|
-
p99Ms: Math.round(elHistogram.percentile(99) / 1e6),
|
|
1029
|
-
maxMs: Math.round(elHistogram.max / 1e6),
|
|
1030
|
-
};
|
|
1031
|
-
elHistogram.reset();
|
|
1032
|
-
} catch { /* unavailable */ }
|
|
1033
|
-
}
|
|
1034
|
-
|
|
1035
|
-
const labels = ['stuck', 'auto-report'];
|
|
1036
|
-
|
|
1037
|
-
fileReport('stuck:event-loop', labels, {
|
|
1038
|
-
message: `Event loop stalled for ${Math.round(drift / 1000)}s (threshold: ${Math.round(thresholdMs / 1000)}s)`,
|
|
1039
|
-
// Stable signature: duration is NOT included -- all stalls on the same route dedup
|
|
1040
|
-
error: { name: 'EventLoopStall', message: _lastRoute || 'idle', stack: '' },
|
|
1041
|
-
uptimeMinutes: typeof process !== 'undefined'
|
|
1042
|
-
? Math.round(process.uptime() / 60) : undefined,
|
|
1043
|
-
resources,
|
|
1044
|
-
stall: {
|
|
1045
|
-
driftMs: drift,
|
|
1046
|
-
thresholdMs,
|
|
1047
|
-
classification,
|
|
1048
|
-
cpuElapsedS: Math.round(cpuElapsedS * 1000) / 1000,
|
|
1049
|
-
cpuRatio: Math.round(cpuRatio * 10000) / 10000,
|
|
1050
|
-
sigcontInWindow,
|
|
1051
|
-
hrtimeWallDriftS: Math.round(hrtimeWallDriftS * 100) / 100,
|
|
1052
|
-
eventLoopDelay: elDelay,
|
|
1053
|
-
lastRoute: _lastRoute,
|
|
1054
|
-
activeHandles: resources.activeHandles,
|
|
1055
|
-
activeRequests: resources.activeRequests,
|
|
1056
|
-
heapDeltaMb,
|
|
1057
|
-
nativeMemGrowthMb: nativeMemBaseline !== null
|
|
1058
|
-
? Math.round((resources.nodeRssMb - resources.nodeHeapUsedMb) - nativeMemBaseline)
|
|
1059
|
-
: null,
|
|
1060
|
-
nativeMemBaselineMb: nativeMemBaseline,
|
|
1061
|
-
},
|
|
1062
|
-
context: extra,
|
|
1063
|
-
});
|
|
1064
|
-
} else {
|
|
1065
|
-
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
1066
|
-
}
|
|
1067
|
-
}, checkMs);
|
|
1068
|
-
|
|
1069
|
-
if (watchdogInterval.unref) watchdogInterval.unref();
|
|
1070
|
-
}
|
|
1071
|
-
|
|
1072
|
-
function stop() {
|
|
1073
|
-
if (watchdogInterval) {
|
|
1074
|
-
clearInterval(watchdogInterval);
|
|
1075
|
-
watchdogInterval = null;
|
|
1076
|
-
}
|
|
1077
|
-
return Promise.allSettled([...inFlight]);
|
|
1078
|
-
}
|
|
1079
|
-
|
|
1080
|
-
/**
|
|
1081
|
-
* Reset native memory baseline. Call after browser close so the next
|
|
1082
|
-
* browser session measures from a fresh baseline, not the old one.
|
|
1083
|
-
*/
|
|
1084
|
-
function resetNativeMemBaseline() {
|
|
1085
|
-
// These are closure vars in startWatchdog -- we need to reach them.
|
|
1086
|
-
// Since this runs in the same module, we set a flag the watchdog reads.
|
|
1087
|
-
_resetNativeMemBaseline = true;
|
|
1088
|
-
}
|
|
1089
|
-
|
|
1090
|
-
return {
|
|
1091
|
-
reportCrash,
|
|
1092
|
-
reportHang,
|
|
1093
|
-
reportStuckLoop,
|
|
1094
|
-
startWatchdog,
|
|
1095
|
-
trackRoute,
|
|
1096
|
-
stop,
|
|
1097
|
-
resetNativeMemBaseline,
|
|
1098
|
-
_anonymize: anonymize,
|
|
1099
|
-
_stackSignature: stackSignature,
|
|
1100
|
-
_rateLimiter: rateLimiters,
|
|
1101
|
-
};
|
|
1102
|
-
}
|
|
1
|
+
// lib/reporter.js -- Crash/hang reporter for camofox-browser
|
|
2
|
+
// Files GitHub issues with paranoid anonymization. No env reads here.
|
|
3
|
+
// Config passed via createReporter(config) from lib/config.js.
|
|
4
|
+
|
|
5
|
+
import crypto from 'crypto';
|
|
6
|
+
import { monitorEventLoopDelay } from 'perf_hooks';
|
|
7
|
+
import { collectResourceSnapshot, classifyProxyError } from './resources.js';
|
|
8
|
+
|
|
9
|
+
// ============================================================================
|
|
10
|
+
// Anonymization
|
|
11
|
+
// ============================================================================
|
|
12
|
+
|
|
13
|
+
const SAFE_HOSTS = new Set([
|
|
14
|
+
'github.com', 'api.github.com', 'npmjs.com', 'registry.npmjs.org',
|
|
15
|
+
'nodejs.org',
|
|
16
|
+
]);
|
|
17
|
+
|
|
18
|
+
const SECRET_PREFIXES = [
|
|
19
|
+
'ghp_', 'gho_', 'ghu_', 'ghs_', 'ghr_',
|
|
20
|
+
'sk-', 'sk_live_', 'sk_test_', 'pk_live_', 'pk_test_',
|
|
21
|
+
'AKIA', 'ASIA',
|
|
22
|
+
'xox', 'Bearer ', 'Basic ',
|
|
23
|
+
'eyJ',
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Paranoid anonymization of arbitrary text (stack traces, error messages, etc.)
|
|
28
|
+
* Better to over-strip than leak. Order matters -- more specific patterns first.
|
|
29
|
+
*/
|
|
30
|
+
export function anonymize(text) {
|
|
31
|
+
if (!text || typeof text !== 'string') return text || '';
|
|
32
|
+
|
|
33
|
+
let s = text;
|
|
34
|
+
|
|
35
|
+
// 1. Strip known secret-prefixed tokens
|
|
36
|
+
for (const prefix of SECRET_PREFIXES) {
|
|
37
|
+
const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
38
|
+
s = s.replace(new RegExp(escaped + '[A-Za-z0-9_\\-\\.=+/]{8,}', 'g'), '<token>');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// 2. Strip Bearer/Basic auth headers
|
|
42
|
+
s = s.replace(/(?:Bearer|Basic)\s+[A-Za-z0-9_\-\.=+/]{8,}/gi, '<token>');
|
|
43
|
+
|
|
44
|
+
// 3. Strip proxy URLs with credentials (before email -- email regex eats user:pass@host)
|
|
45
|
+
s = s.replace(/(?:https?|socks[45]?):\/\/[^:]+:[^@]+@[^\s]+/gi, '<proxy-url>');
|
|
46
|
+
|
|
47
|
+
// 4. Strip email addresses
|
|
48
|
+
s = s.replace(/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g, '<email>');
|
|
49
|
+
|
|
50
|
+
// 5. Strip full URLs (preserve scheme for context)
|
|
51
|
+
s = s.replace(/(https?|wss?|ftp):\/\/[^\s'",)}\]>]+/g, (match, scheme) => {
|
|
52
|
+
try {
|
|
53
|
+
const u = new URL(match);
|
|
54
|
+
if (SAFE_HOSTS.has(u.hostname)) return match;
|
|
55
|
+
} catch { /* not a valid URL, strip it */ }
|
|
56
|
+
return `<${scheme}-url>`;
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
// 6. Strip absolute file paths (Unix + Windows), preserve last filename
|
|
60
|
+
s = s.replace(
|
|
61
|
+
/(?:\/(?:Users|home|root|tmp|var|opt|data|app|srv|etc|mnt|run|snap|proc)\/[^\s:;,'")\]}]+|[A-Z]:\\(?:Users|Documents and Settings)\\[^\s:;,'")\]}]+)/g,
|
|
62
|
+
(match) => {
|
|
63
|
+
const parts = match.replace(/\\/g, '/').split('/');
|
|
64
|
+
const filename = parts[parts.length - 1] || parts[parts.length - 2] || 'unknown';
|
|
65
|
+
return `<path>/${filename}`;
|
|
66
|
+
}
|
|
67
|
+
);
|
|
68
|
+
|
|
69
|
+
// 7. Strip IPv4 addresses
|
|
70
|
+
s = s.replace(/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, '<ip>');
|
|
71
|
+
|
|
72
|
+
// 8. Strip IPv6 addresses
|
|
73
|
+
s = s.replace(/\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, '<ipv6>');
|
|
74
|
+
s = s.replace(/::(?:ffff:)?(?:\d{1,3}\.){3}\d{1,3}/g, '<ipv6>');
|
|
75
|
+
|
|
76
|
+
// 9. Strip hostnames in connection errors
|
|
77
|
+
s = s.replace(
|
|
78
|
+
/(?:ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|EHOSTUNREACH)\s+([a-zA-Z0-9.\-]+):(\d+)/g,
|
|
79
|
+
(match, host, port) => {
|
|
80
|
+
if (SAFE_HOSTS.has(host)) return match;
|
|
81
|
+
return match.replace(host, '<host>');
|
|
82
|
+
}
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
// 10. Strip Fly machine IDs (14-char hex), Docker container IDs (12+ hex)
|
|
86
|
+
s = s.replace(/\b[0-9a-f]{12,64}\b/g, '<id>');
|
|
87
|
+
|
|
88
|
+
// 11. Strip jo-* app names
|
|
89
|
+
s = s.replace(/\bjo-(?:machine|browser|whatsapp|discord|bot)[a-z0-9\-]*/gi, '<app>');
|
|
90
|
+
|
|
91
|
+
// 12. Strip environment variable assignments
|
|
92
|
+
s = s.replace(/\b[A-Z][A-Z0-9_]{3,}=[^\s]{4,}/g, '<env-var>');
|
|
93
|
+
|
|
94
|
+
// 13. Strip long alphanumeric strings (40+ chars)
|
|
95
|
+
s = s.replace(/[A-Za-z0-9_\-]{40,}/g, '<redacted>');
|
|
96
|
+
|
|
97
|
+
// 14. Strip base64 blobs (20+ chars with mixed case)
|
|
98
|
+
s = s.replace(/[A-Za-z0-9+/]{20,}={0,3}/g, (match) => {
|
|
99
|
+
if (/[a-z]/.test(match) && /[A-Z]/.test(match)) return '<redacted>';
|
|
100
|
+
return match;
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
return s;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Generate a stable signature for dedup. Uses error name + first meaningful
|
|
108
|
+
* stack frame (file:line, not column -- columns shift with minor edits).
|
|
109
|
+
*/
|
|
110
|
+
export function stackSignature(type, error) {
|
|
111
|
+
const name = error?.name || error?.code || 'unknown';
|
|
112
|
+
const message = error?.message || String(error || '');
|
|
113
|
+
|
|
114
|
+
const stack = error?.stack || '';
|
|
115
|
+
const frames = stack.split('\n').slice(1);
|
|
116
|
+
let keyFrame = '';
|
|
117
|
+
for (const frame of frames) {
|
|
118
|
+
const trimmed = frame.trim();
|
|
119
|
+
if (trimmed.startsWith('at ') && !trimmed.includes('node_modules') && !trimmed.includes('node:internal')) {
|
|
120
|
+
const fileMatch = trimmed.match(/\(([^)]+)\)/) || trimmed.match(/at\s+(.+)$/);
|
|
121
|
+
if (fileMatch) {
|
|
122
|
+
const loc = fileMatch[1];
|
|
123
|
+
const parts = loc.replace(/\\/g, '/').split('/');
|
|
124
|
+
const last = parts[parts.length - 1];
|
|
125
|
+
const [file, line] = last.split(':');
|
|
126
|
+
keyFrame = `${file}:${line || '?'}`;
|
|
127
|
+
break;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const raw = `${type}|${name}|${keyFrame || anonymize(message).slice(0, 80)}`;
|
|
133
|
+
return fnv1a(raw);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** FNV-1a hash -> 8-char hex. Stable bucketing, not crypto. */
|
|
137
|
+
function fnv1a(str) {
|
|
138
|
+
let hash = 0x811c9dc5;
|
|
139
|
+
for (let i = 0; i < str.length; i++) {
|
|
140
|
+
hash ^= str.charCodeAt(i);
|
|
141
|
+
hash = (hash * 0x01000193) >>> 0;
|
|
142
|
+
}
|
|
143
|
+
return hash.toString(16).padStart(8, '0');
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ============================================================================
|
|
147
|
+
// URL anonymization (per-report salted HMAC for private domains)
|
|
148
|
+
// ============================================================================
|
|
149
|
+
|
|
150
|
+
// Public domains safe to show verbatim in reports.
|
|
151
|
+
// These are public knowledge -- showing "amazon.com" in a crash report is not PII.
|
|
152
|
+
// Matched by suffix. NEVER add multi-tenant hosting (herokuapp.com, vercel.app, etc.)
|
|
153
|
+
const PUBLIC_DOMAINS = [
|
|
154
|
+
// CDN & edge
|
|
155
|
+
'cloudflare.com', 'cloudflare-dns.com', 'cloudflareinsights.com',
|
|
156
|
+
'fastly.net', 'fastlylb.net',
|
|
157
|
+
'akamaized.net', 'akamai.net', 'cloudfront.net',
|
|
158
|
+
'cdn.jsdelivr.net', 'unpkg.com', 'cdnjs.com',
|
|
159
|
+
// Google
|
|
160
|
+
'google.com', 'googleapis.com', 'gstatic.com',
|
|
161
|
+
'googleusercontent.com', 'google-analytics.com', 'googletagmanager.com',
|
|
162
|
+
'googlesyndication.com', 'doubleclick.net', 'youtube.com', 'ytimg.com',
|
|
163
|
+
'recaptcha.net',
|
|
164
|
+
// Microsoft
|
|
165
|
+
'microsoft.com', 'msecnd.net', 'azureedge.net', 'bing.com', 'live.com',
|
|
166
|
+
'outlook.com', 'office.com', 'linkedin.com',
|
|
167
|
+
// Meta
|
|
168
|
+
'facebook.com', 'facebook.net', 'fbcdn.net', 'instagram.com', 'threads.net',
|
|
169
|
+
'whatsapp.com',
|
|
170
|
+
// X/Twitter
|
|
171
|
+
'twitter.com', 'x.com', 'twimg.com',
|
|
172
|
+
// GitHub
|
|
173
|
+
'github.com', 'githubusercontent.com', 'githubassets.com',
|
|
174
|
+
// Major sites (common anti-bot / frustration sources)
|
|
175
|
+
'amazon.com', 'amazon.co.uk', 'amazon.de', 'amazon.co.jp',
|
|
176
|
+
'reddit.com', 'redd.it',
|
|
177
|
+
'apple.com', 'icloud.com',
|
|
178
|
+
'netflix.com', 'spotify.com', 'discord.com', 'discord.gg',
|
|
179
|
+
'tiktok.com', 'pinterest.com', 'tumblr.com',
|
|
180
|
+
'stackoverflow.com', 'stackexchange.com',
|
|
181
|
+
'medium.com', 'substack.com',
|
|
182
|
+
'nytimes.com', 'washingtonpost.com', 'bbc.co.uk', 'bbc.com', 'cnn.com',
|
|
183
|
+
'ebay.com', 'etsy.com', 'walmart.com', 'target.com', 'shopify.com',
|
|
184
|
+
'stripe.com', 'paypal.com',
|
|
185
|
+
'twitch.tv', 'vimeo.com', 'dailymotion.com',
|
|
186
|
+
'yahoo.com', 'duckduckgo.com', 'baidu.com',
|
|
187
|
+
'zoom.us', 'slack.com', 'notion.so', 'figma.com',
|
|
188
|
+
'dropbox.com', 'box.com',
|
|
189
|
+
'archive.org', 'web.archive.org',
|
|
190
|
+
// Prediction markets & crypto (heavy anti-bot, commonly scraped)
|
|
191
|
+
'polymarket.com', 'kalshi.com', 'metaculus.com', 'manifold.markets',
|
|
192
|
+
'predictit.org', 'augur.net',
|
|
193
|
+
'coinbase.com', 'binance.com', 'kraken.com', 'gemini.com',
|
|
194
|
+
'coingecko.com', 'coinmarketcap.com',
|
|
195
|
+
'opensea.io', 'blur.io', 'rarible.com',
|
|
196
|
+
'etherscan.io', 'solscan.io', 'blockchair.com',
|
|
197
|
+
'uniswap.org', 'dexscreener.com', 'dextools.io',
|
|
198
|
+
// Data / scraping targets (aggressive anti-bot)
|
|
199
|
+
'zillow.com', 'realtor.com', 'redfin.com', 'trulia.com',
|
|
200
|
+
'indeed.com', 'glassdoor.com', 'lever.co', 'greenhouse.io',
|
|
201
|
+
'airbnb.com', 'booking.com', 'expedia.com', 'tripadvisor.com',
|
|
202
|
+
'yelp.com', 'trustpilot.com',
|
|
203
|
+
'craigslist.org', 'nextdoor.com',
|
|
204
|
+
'ticketmaster.com', 'stubhub.com', 'seatgeek.com',
|
|
205
|
+
// Finance / trading
|
|
206
|
+
'tradingview.com', 'investing.com', 'seekingalpha.com',
|
|
207
|
+
'finance.yahoo.com', 'bloomberg.com', 'reuters.com', 'wsj.com',
|
|
208
|
+
'robinhood.com', 'schwab.com', 'fidelity.com', 'etrade.com',
|
|
209
|
+
// AI / developer tools
|
|
210
|
+
'openai.com', 'anthropic.com', 'huggingface.co',
|
|
211
|
+
'vercel.com', 'netlify.com', 'render.com', 'fly.io',
|
|
212
|
+
'npmjs.com', 'pypi.org', 'crates.io', 'pkg.go.dev',
|
|
213
|
+
// Social / forums
|
|
214
|
+
'quora.com', 'hackernews.com', 'news.ycombinator.com',
|
|
215
|
+
'producthunt.com', 'indiehackers.com',
|
|
216
|
+
// Reference
|
|
217
|
+
'wikipedia.org', 'wikimedia.org', 'mozilla.org', 'mozilla.net',
|
|
218
|
+
// Anti-bot / CAPTCHA
|
|
219
|
+
'hcaptcha.com',
|
|
220
|
+
// Fonts
|
|
221
|
+
'typekit.net', 'fontawesome.com',
|
|
222
|
+
].sort((a, b) => b.length - a.length); // longest-suffix-first
|
|
223
|
+
|
|
224
|
+
// Stable key for domain hashing -- NOT a secret, just ensures consistent hashes
|
|
225
|
+
// across reports so we can correlate "site-a1b2c3d4 caused 12 hangs this week".
|
|
226
|
+
const DOMAIN_HASH_KEY = 'camofox-domain-hash-v1';
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Create a URL anonymizer.
|
|
230
|
+
* Public domains shown verbatim. Private domains get a stable hash
|
|
231
|
+
* (same domain -> same hash across all reports, enabling correlation).
|
|
232
|
+
*/
|
|
233
|
+
export function createUrlAnonymizer() {
|
|
234
|
+
|
|
235
|
+
function isPublicDomain(hostname) {
|
|
236
|
+
for (const d of PUBLIC_DOMAINS) {
|
|
237
|
+
if (hostname === d || hostname.endsWith('.' + d)) return true;
|
|
238
|
+
}
|
|
239
|
+
return false;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function hashHost(hostname) {
|
|
243
|
+
return 'site-' + crypto.createHmac('sha256', DOMAIN_HASH_KEY).update(hostname).digest('hex').slice(0, 8);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Anonymize a URL. Preserves: scheme, public infra hostnames, path depth,
|
|
248
|
+
* query param count, fragment presence. Strips everything else.
|
|
249
|
+
*
|
|
250
|
+
* Examples:
|
|
251
|
+
* https://challenges.cloudflare.com/[path]/[path]/[path]
|
|
252
|
+
* https://site-a1b2c3d4:8443/[path]/[path] ?[3] #[frag]
|
|
253
|
+
*/
|
|
254
|
+
function anonymizeUrl(rawUrl) {
|
|
255
|
+
if (!rawUrl || typeof rawUrl !== 'string') return '[empty]';
|
|
256
|
+
if (rawUrl.startsWith('data:')) return '[data-uri]';
|
|
257
|
+
if (rawUrl.startsWith('blob:')) return '[blob-uri]';
|
|
258
|
+
if (rawUrl.startsWith('about:')) return rawUrl;
|
|
259
|
+
if (rawUrl.startsWith('javascript:')) return '[javascript-uri]';
|
|
260
|
+
|
|
261
|
+
let url;
|
|
262
|
+
try { url = new URL(rawUrl); } catch { return '[invalid-url]'; }
|
|
263
|
+
|
|
264
|
+
const parts = [url.protocol + '//'];
|
|
265
|
+
const h = url.hostname.toLowerCase();
|
|
266
|
+
|
|
267
|
+
if (/^(\d{1,3}\.){3}\d{1,3}$/.test(h) || h.includes(':')) {
|
|
268
|
+
parts.push(hashHost(h));
|
|
269
|
+
} else if (isPublicDomain(h)) {
|
|
270
|
+
parts.push(h);
|
|
271
|
+
} else {
|
|
272
|
+
parts.push(hashHost(h));
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
if (url.port) parts.push(':' + url.port);
|
|
276
|
+
|
|
277
|
+
const segs = url.pathname.split('/').filter(Boolean);
|
|
278
|
+
parts.push(segs.length > 0 ? '/' + segs.map(() => '\u2022').join('/') : '/');
|
|
279
|
+
|
|
280
|
+
const paramCount = [...url.searchParams].length;
|
|
281
|
+
if (paramCount > 0) parts.push(` ?[${paramCount}]`);
|
|
282
|
+
if (url.hash && url.hash.length > 1) parts.push(' #[frag]');
|
|
283
|
+
|
|
284
|
+
return parts.join('');
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function anonymizeChain(urls) {
|
|
288
|
+
if (!Array.isArray(urls) || urls.length === 0) return '[empty-chain]';
|
|
289
|
+
return urls.map(u => anonymizeUrl(u)).join(' \u2192 ');
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
return { anonymizeUrl, anonymizeChain };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// ============================================================================
|
|
296
|
+
// Per-tab health tracker (count-only, no content)
|
|
297
|
+
// ============================================================================
|
|
298
|
+
|
|
299
|
+
// Known bot-detection providers, matched by response header fingerprints.
|
|
300
|
+
// Order: most specific first.
|
|
301
|
+
const BOT_DETECTION_SIGNATURES = [
|
|
302
|
+
{ header: 'cf-mitigated', value: 'challenge', provider: 'cloudflare' },
|
|
303
|
+
{ header: 'x-datadome', provider: 'datadome' },
|
|
304
|
+
{ header: 'x-px', provider: 'perimeterx' },
|
|
305
|
+
{ header: 'x-distil-cs', provider: 'distil' },
|
|
306
|
+
{ header: 'x-sucuri-id', provider: 'sucuri' },
|
|
307
|
+
{ header: 'server', value: 'akamaighost', provider: 'akamai' },
|
|
308
|
+
// cf-ray is on ALL Cloudflare responses (even 200 OK). Must be last so it
|
|
309
|
+
// doesn't short-circuit other providers on multi-CDN sites.
|
|
310
|
+
{ header: 'cf-ray', provider: 'cloudflare' },
|
|
311
|
+
];
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Detect bot-detection provider from Playwright response headers.
|
|
315
|
+
* Returns { detected: bool, provider: string|null, httpStatus: number|null }
|
|
316
|
+
*/
|
|
317
|
+
export function detectBotProtection(response) {
|
|
318
|
+
if (!response) return { detected: false, provider: null, httpStatus: null };
|
|
319
|
+
const status = response.status();
|
|
320
|
+
let headers;
|
|
321
|
+
try { headers = response.headers(); } catch { return { detected: false, provider: null, httpStatus: status }; }
|
|
322
|
+
for (const sig of BOT_DETECTION_SIGNATURES) {
|
|
323
|
+
const val = headers[sig.header];
|
|
324
|
+
if (val !== undefined) {
|
|
325
|
+
if (sig.value && !val.toLowerCase().includes(sig.value)) continue;
|
|
326
|
+
const challenged = status === 403 || status === 429 || status === 503;
|
|
327
|
+
return { detected: challenged, provider: sig.provider, httpStatus: status };
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return { detected: false, provider: null, httpStatus: status };
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Create a health tracker for a tab. Attaches to Playwright page events.
|
|
335
|
+
* Tracks: crashes, page errors, request failures, redirect status codes,
|
|
336
|
+
* HTTP status histogram (4xx+), and anti-bot challenge detection.
|
|
337
|
+
* All count-based -- no URLs or content stored.
|
|
338
|
+
*/
|
|
339
|
+
export function createTabHealthTracker(page) {
|
|
340
|
+
const health = {
|
|
341
|
+
crashes: 0,
|
|
342
|
+
pageErrors: 0,
|
|
343
|
+
requestFailures: 0,
|
|
344
|
+
inflightRequests: 0,
|
|
345
|
+
maxRedirectDepth: 0,
|
|
346
|
+
redirectStatusCodes: [], // status codes in redirect chain, e.g. [301, 302, 403]
|
|
347
|
+
statusCounts: {}, // { 403: 5, 429: 2, ... }
|
|
348
|
+
botDetection: null, // { detected, provider, httpStatus } from last nav response
|
|
349
|
+
lastNavResponseSize: 0,
|
|
350
|
+
_redirectDepth: 0,
|
|
351
|
+
};
|
|
352
|
+
|
|
353
|
+
// Renderer crash (OOM, segfault)
|
|
354
|
+
page.on('crash', () => { health.crashes++; });
|
|
355
|
+
|
|
356
|
+
// Uncaught JS exceptions on the page
|
|
357
|
+
page.on('pageerror', () => { health.pageErrors++; });
|
|
358
|
+
|
|
359
|
+
// Failed requests (blocked, DNS failure, etc.) + decrement in-flight counter
|
|
360
|
+
page.on('requestfailed', () => {
|
|
361
|
+
health.requestFailures++;
|
|
362
|
+
health.inflightRequests = Math.max(0, health.inflightRequests - 1);
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
// Track in-flight requests for hang diagnostics
|
|
366
|
+
page.on('request', () => { health.inflightRequests++; });
|
|
367
|
+
page.on('requestfinished', () => { health.inflightRequests = Math.max(0, health.inflightRequests - 1); });
|
|
368
|
+
|
|
369
|
+
// HTTP status tracking (non-2xx only)
|
|
370
|
+
page.on('response', (resp) => {
|
|
371
|
+
const s = resp.status();
|
|
372
|
+
if (s >= 400) health.statusCounts[s] = (health.statusCounts[s] || 0) + 1;
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
// Auto-dismiss dialogs to prevent page hangs (not tracked as a metric -- noise)
|
|
376
|
+
page.on('dialog', async (dialog) => {
|
|
377
|
+
try { await dialog.dismiss(); } catch { /* page might be closed */ }
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
// Redirect depth + status code chain per navigation
|
|
381
|
+
page.on('request', (req) => {
|
|
382
|
+
if (req.isNavigationRequest()) {
|
|
383
|
+
if (req.redirectedFrom()) {
|
|
384
|
+
health._redirectDepth++;
|
|
385
|
+
if (health._redirectDepth > health.maxRedirectDepth) {
|
|
386
|
+
health.maxRedirectDepth = health._redirectDepth;
|
|
387
|
+
}
|
|
388
|
+
} else {
|
|
389
|
+
health._redirectDepth = 0;
|
|
390
|
+
health.redirectStatusCodes = [];
|
|
391
|
+
health.inflightRequests = 0; // reset on new navigation to prevent drift
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
// Capture redirect status codes and detect bot protection on nav responses
|
|
397
|
+
page.on('response', (resp) => {
|
|
398
|
+
try {
|
|
399
|
+
const req = resp.request();
|
|
400
|
+
if (req.isNavigationRequest()) {
|
|
401
|
+
health.redirectStatusCodes.push(resp.status());
|
|
402
|
+
health.botDetection = detectBotProtection(resp);
|
|
403
|
+
// Approximate response body size from content-length (no body read)
|
|
404
|
+
const cl = resp.headers()['content-length'];
|
|
405
|
+
if (cl) health.lastNavResponseSize = parseInt(cl, 10) || 0;
|
|
406
|
+
}
|
|
407
|
+
} catch { /* page closed */ }
|
|
408
|
+
});
|
|
409
|
+
|
|
410
|
+
/** Snapshot current health counters for inclusion in reports. */
|
|
411
|
+
function snapshot() {
|
|
412
|
+
const { _redirectDepth, ...clean } = health;
|
|
413
|
+
return { ...clean };
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Get document.readyState from the page. Returns null if page is unresponsive.
|
|
418
|
+
* Use a tight timeout -- if the renderer is crashed, evaluate will hang.
|
|
419
|
+
*/
|
|
420
|
+
async function getReadyState() {
|
|
421
|
+
try {
|
|
422
|
+
return await Promise.race([
|
|
423
|
+
page.evaluate(() => document.readyState),
|
|
424
|
+
new Promise(resolve => setTimeout(() => resolve('unresponsive'), 1000)),
|
|
425
|
+
]);
|
|
426
|
+
} catch {
|
|
427
|
+
return 'unresponsive';
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
return { health, snapshot, getReadyState };
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// collectResourceSnapshot and classifyProxyError live in lib/resources.js
|
|
435
|
+
// (isolated from network code for clean separation of concerns).
|
|
436
|
+
// Re-exported here for backward compatibility.
|
|
437
|
+
export { collectResourceSnapshot, classifyProxyError };
|
|
438
|
+
|
|
439
|
+
// ============================================================================
|
|
440
|
+
// Rate limiter (sliding window, 1 hour)
|
|
441
|
+
// ============================================================================
|
|
442
|
+
|
|
443
|
+
class RateLimiter {
|
|
444
|
+
constructor(maxPerHour) {
|
|
445
|
+
this.maxPerHour = maxPerHour;
|
|
446
|
+
this.timestamps = [];
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
tryAcquire() {
|
|
450
|
+
const now = Date.now();
|
|
451
|
+
this.timestamps = this.timestamps.filter(t => t > now - 3600_000);
|
|
452
|
+
if (this.timestamps.length >= this.maxPerHour) return false;
|
|
453
|
+
this.timestamps.push(now);
|
|
454
|
+
return true;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
// ============================================================================
|
|
459
|
+
// Crash relay client
|
|
460
|
+
// ============================================================================
|
|
461
|
+
|
|
462
|
+
// PHI-VENDOR: Crash relay endpoint defaults to empty (telemetry disabled).
|
|
463
|
+
// The original upstream endpoint pointed to a Cloudflare Worker maintained
|
|
464
|
+
// by Jo Inc (askjo.workers.dev). The phi-code vendored snapshot ships with
|
|
465
|
+
// telemetry off by default — users who want their own relay can set
|
|
466
|
+
// CAMOFOX_CRASH_REPORT_URL explicitly.
|
|
467
|
+
//
|
|
468
|
+
// Upstream endpoint (for reference, not used unless re-enabled):
|
|
469
|
+
// https://camofox-telemetry.askjo.workers.dev/report
|
|
470
|
+
|
|
471
|
+
const DEFAULT_RELAY_URL = process.env.CAMOFOX_CRASH_REPORT_URL || '';
|
|
472
|
+
const FETCH_TIMEOUT_MS = 5000;
|
|
473
|
+
|
|
474
|
+
let _relayUrl = DEFAULT_RELAY_URL;
|
|
475
|
+
|
|
476
|
+
function fetchWithTimeout(url, options) {
|
|
477
|
+
const controller = new AbortController();
|
|
478
|
+
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
|
479
|
+
return fetch(url, { ...options, signal: controller.signal })
|
|
480
|
+
.finally(() => clearTimeout(timer));
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/**
|
|
484
|
+
* Send a crash report to the relay. Returns true if accepted.
|
|
485
|
+
* Never throws -- reporter must never crash the server.
|
|
486
|
+
*/
|
|
487
|
+
export async function sendToRelay(payload) {
|
|
488
|
+
// PHI-VENDOR: telemetry disabled by default. When CAMOFOX_CRASH_REPORT_URL
|
|
489
|
+
// is unset (the new default), this is a no-op — no network call leaves
|
|
490
|
+
// the user's machine. Set the env var explicitly to opt back in.
|
|
491
|
+
if (!_relayUrl) return false;
|
|
492
|
+
try {
|
|
493
|
+
const resp = await fetchWithTimeout(_relayUrl, {
|
|
494
|
+
method: 'POST',
|
|
495
|
+
headers: { 'Content-Type': 'application/json' },
|
|
496
|
+
body: JSON.stringify(payload),
|
|
497
|
+
});
|
|
498
|
+
return resp.ok || resp.status === 429; // rate-limited is fine, not an error
|
|
499
|
+
} catch {
|
|
500
|
+
return false;
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// ============================================================================
|
|
505
|
+
// Issue formatting
|
|
506
|
+
// ============================================================================
|
|
507
|
+
|
|
508
|
+
function formatIssueBody(type, detail) {
|
|
509
|
+
const sections = [
|
|
510
|
+
'> Auto-reported by camofox-crash-reporter. All data is anonymized.',
|
|
511
|
+
'',
|
|
512
|
+
'## Environment',
|
|
513
|
+
`- **version:** ${detail.version || 'unknown'}`,
|
|
514
|
+
`- **node:** ${detail.nodeVersion || 'unknown'}`,
|
|
515
|
+
`- **platform:** ${detail.platform || 'unknown'}`,
|
|
516
|
+
`- **uptime:** ${detail.uptimeMinutes != null ? detail.uptimeMinutes + ' min' : 'unknown'}`,
|
|
517
|
+
];
|
|
518
|
+
|
|
519
|
+
// Resource snapshot (memory, handles, browser RSS)
|
|
520
|
+
const r = detail.resources;
|
|
521
|
+
if (r) {
|
|
522
|
+
sections.push('', '## Resources');
|
|
523
|
+
sections.push(`- **node RSS:** ${r.nodeRssMb ?? '?'} MB`);
|
|
524
|
+
sections.push(`- **node heap:** ${r.nodeHeapUsedMb ?? '?'} / ${r.nodeHeapTotalMb ?? '?'} MB`);
|
|
525
|
+
if (r.browserRssMb != null) sections.push(`- **browser RSS:** ${r.browserRssMb} MB`);
|
|
526
|
+
if (r.browserContexts != null) sections.push(`- **browser contexts:** ${r.browserContexts}`);
|
|
527
|
+
if (r.activeTabs != null) sections.push(`- **active tabs:** ${r.activeTabs}`);
|
|
528
|
+
if (r.openFds != null) sections.push(`- **open FDs:** ${r.openFds}`);
|
|
529
|
+
if (r.activeHandles != null) sections.push(`- **active handles:** ${r.activeHandles}`);
|
|
530
|
+
if (r.eventLoopLagMs != null) sections.push(`- **event loop lag:** ${r.eventLoopLagMs} ms`);
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
// Error info
|
|
534
|
+
if (detail.signal) sections.push('', `**Signal:** ${detail.signal}`);
|
|
535
|
+
if (detail.activeRoute) sections.push(`**Active route:** ${detail.activeRoute}`);
|
|
536
|
+
if (detail.message) {
|
|
537
|
+
sections.push('', '## Error', '```', anonymize(detail.message), '```');
|
|
538
|
+
}
|
|
539
|
+
if (detail.stack) {
|
|
540
|
+
sections.push('', '## Stack Trace', '```', anonymize(detail.stack), '```');
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// Hang-specific details
|
|
544
|
+
if (detail.hang) {
|
|
545
|
+
const h = detail.hang;
|
|
546
|
+
sections.push('', '## Hang Details');
|
|
547
|
+
sections.push(`- **operation:** ${h.operation}`);
|
|
548
|
+
sections.push(`- **duration:** ${Math.round(h.durationMs / 1000)}s`);
|
|
549
|
+
if (h.lockQueueMs != null) sections.push(`- **lock queue wait:** ${Math.round(h.lockQueueMs)}ms`);
|
|
550
|
+
if (h.documentReadyState) sections.push(`- **document.readyState:** ${h.documentReadyState}`);
|
|
551
|
+
if (h.inflightRequests != null) sections.push(`- **in-flight requests:** ${h.inflightRequests}`);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
// Anti-bot detection
|
|
555
|
+
if (detail.botDetection?.detected) {
|
|
556
|
+
const b = detail.botDetection;
|
|
557
|
+
sections.push('', '## Anti-Bot Detection');
|
|
558
|
+
sections.push(`- **provider:** ${b.provider || 'unknown'}`);
|
|
559
|
+
sections.push(`- **HTTP status:** ${b.httpStatus || '?'}`);
|
|
560
|
+
if (b.responseBodySizeKb != null) sections.push(`- **response size:** ${b.responseBodySizeKb} KB`);
|
|
561
|
+
if (b.redirectChainLength != null) sections.push(`- **redirect chain:** ${b.redirectChainLength} hops`);
|
|
562
|
+
if (b.redirectStatusCodes?.length) sections.push(`- **redirect statuses:** ${b.redirectStatusCodes.join(' -> ')}`);
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Proxy info (safe fields only -- no IPs, credentials, or hostnames)
|
|
566
|
+
if (detail.proxy) {
|
|
567
|
+
const p = detail.proxy;
|
|
568
|
+
sections.push('', '## Proxy');
|
|
569
|
+
sections.push(`- **configured:** ${p.configured}`);
|
|
570
|
+
if (p.configured) {
|
|
571
|
+
if (p.type) sections.push(`- **type:** ${p.type}`);
|
|
572
|
+
sections.push(`- **auth configured:** ${p.authConfigured ?? 'unknown'}`);
|
|
573
|
+
if (p.error) sections.push(`- **error:** ${p.error}`);
|
|
574
|
+
if (p.tlsError) sections.push(`- **TLS error:** yes`);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
// Stall-specific details
|
|
579
|
+
if (detail.stall) {
|
|
580
|
+
const s = detail.stall;
|
|
581
|
+
sections.push('', '## Stall Details');
|
|
582
|
+
sections.push(`- **stall duration:** ${Math.round(s.driftMs / 1000)}s`);
|
|
583
|
+
if (s.classification) sections.push(`- **classification:** ${s.classification}`);
|
|
584
|
+
if (s.cpuElapsedS != null) sections.push(`- **CPU time during stall:** ${s.cpuElapsedS}s`);
|
|
585
|
+
if (s.cpuRatio != null) sections.push(`- **CPU/wall ratio:** ${s.cpuRatio}`);
|
|
586
|
+
if (s.sigcontInWindow != null) sections.push(`- **SIGCONT in window:** ${s.sigcontInWindow}`);
|
|
587
|
+
if (s.hrtimeWallDriftS != null) sections.push(`- **hrtime<->wall drift:** ${s.hrtimeWallDriftS}s`);
|
|
588
|
+
if (s.eventLoopDelay) {
|
|
589
|
+
const eld = s.eventLoopDelay;
|
|
590
|
+
sections.push(`- **event loop delay:** p50=${eld.p50Ms}ms p99=${eld.p99Ms}ms max=${eld.maxMs}ms`);
|
|
591
|
+
}
|
|
592
|
+
if (s.lastRoute) sections.push(`- **last route:** ${s.lastRoute}`);
|
|
593
|
+
if (s.activeHandles != null) sections.push(`- **active handles:** ${s.activeHandles}`);
|
|
594
|
+
if (s.activeRequests != null) sections.push(`- **active requests:** ${s.activeRequests}`);
|
|
595
|
+
if (s.heapDeltaMb != null) sections.push(`- **heap delta:** ${s.heapDeltaMb > 0 ? '+' : ''}${s.heapDeltaMb} MB`);
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// Context (misc extra data)
|
|
599
|
+
if (detail.nativeMemory) {
|
|
600
|
+
const nm = detail.nativeMemory;
|
|
601
|
+
sections.push('', '## Native Memory Details');
|
|
602
|
+
sections.push(`- **baseline:** ${nm.baselineMb} MB`);
|
|
603
|
+
sections.push(`- **current:** ${nm.currentMb} MB`);
|
|
604
|
+
sections.push(`- **high-water:** ${nm.highWaterMb} MB`);
|
|
605
|
+
sections.push(`- **growth:** ${nm.growthMb} MB`);
|
|
606
|
+
sections.push(`- **node RSS:** ${nm.rssMb} MB`);
|
|
607
|
+
sections.push(`- **heap used:** ${nm.heapUsedMb} MB`);
|
|
608
|
+
sections.push(`- **external:** ${nm.externalMb} MB`);
|
|
609
|
+
if (nm.lastSeenBrowserRssMb != null) sections.push(`- **browser RSS (last seen):** ${nm.lastSeenBrowserRssMb} MB`);
|
|
610
|
+
else sections.push(`- **browser RSS (last seen):** not captured (browser already dead)`);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
if (detail.context && Object.keys(detail.context).length > 0) {
|
|
614
|
+
sections.push('', '<details><summary>Context</summary>', '', '```json', anonymize(JSON.stringify(detail.context, null, 2)), '```', '', '</details>');
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
return sections.join('\n');
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
// ============================================================================
|
|
622
|
+
// Core reporter factory
|
|
623
|
+
// ============================================================================
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* Create a reporter instance.
|
|
627
|
+
*
|
|
628
|
+
* @param {object} config
|
|
629
|
+
* @param {boolean} config.crashReportEnabled
|
|
630
|
+
* @param {string} config.crashReportRepo - "owner/repo" (env override)
|
|
631
|
+
* @param {number} config.crashReportRateLimit - max reports per hour
|
|
632
|
+
* @param {object} config.crashReporterConfig - from camofox.config.json crashReporter section
|
|
633
|
+
* @param {string} [config.version] - package version
|
|
634
|
+
*/
|
|
635
|
+
export function createReporter(config) {
|
|
636
|
+
// Set relay URL (env override for self-hosted relays)
|
|
637
|
+
_relayUrl = config.crashReportUrl || DEFAULT_RELAY_URL;
|
|
638
|
+
|
|
639
|
+
const enabled = config.crashReportEnabled !== false;
|
|
640
|
+
const repo = config.crashReportRepo || 'jo-inc/camofox-browser';
|
|
641
|
+
const rateLimiters = {
|
|
642
|
+
crash: new RateLimiter(5), // 5 crashes/hr
|
|
643
|
+
hang: new RateLimiter(5), // 5 hangs/hr
|
|
644
|
+
stuck: new RateLimiter(2), // 2 stalls/hr (with active tabs only)
|
|
645
|
+
leak: new RateLimiter(2), // 2 leak alerts/hr
|
|
646
|
+
_default: new RateLimiter(config.crashReportRateLimit || 10),
|
|
647
|
+
};
|
|
648
|
+
const version = config.version || 'unknown';
|
|
649
|
+
|
|
650
|
+
let watchdogInterval = null;
|
|
651
|
+
let _resetNativeMemBaseline = false; // Set by resetNativeMemBaseline(), read by watchdog
|
|
652
|
+
let lastTick = Date.now();
|
|
653
|
+
const inFlight = new Set();
|
|
654
|
+
|
|
655
|
+
// Track last Express route for stall reports
|
|
656
|
+
let _lastRoute = null;
|
|
657
|
+
|
|
658
|
+
// No-op when disabled
|
|
659
|
+
if (!enabled) {
|
|
660
|
+
return {
|
|
661
|
+
reportCrash: async () => {},
|
|
662
|
+
reportHang: async () => {},
|
|
663
|
+
reportStuckLoop: async () => {},
|
|
664
|
+
startWatchdog: () => {},
|
|
665
|
+
trackRoute: () => {},
|
|
666
|
+
stop: () => {},
|
|
667
|
+
_anonymize: anonymize,
|
|
668
|
+
_stackSignature: stackSignature,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
/** Core: build and send a report to the relay. NEVER throws. */
|
|
673
|
+
async function fileReport(type, labels, detail) {
|
|
674
|
+
const bucket = type.startsWith('stuck:') ? 'stuck' : type.startsWith('hang:') ? 'hang' : type.startsWith('leak:') ? 'leak' : 'crash';
|
|
675
|
+
const limiter = rateLimiters[bucket] || rateLimiters._default;
|
|
676
|
+
if (!limiter.tryAcquire()) return;
|
|
677
|
+
|
|
678
|
+
const reportPromise = (async () => {
|
|
679
|
+
try {
|
|
680
|
+
const sig = stackSignature(type, detail.error || { message: detail.message, stack: detail.stack });
|
|
681
|
+
const safeMessage = anonymize(detail.message || detail.error?.message || type);
|
|
682
|
+
const title = `[${sig}] ${type}: ${safeMessage.slice(0, 120)}`;
|
|
683
|
+
|
|
684
|
+
const body = formatIssueBody(type, {
|
|
685
|
+
...detail,
|
|
686
|
+
version,
|
|
687
|
+
nodeVersion: typeof process !== 'undefined' ? process.version : 'unknown',
|
|
688
|
+
platform: typeof process !== 'undefined' ? process.platform : 'unknown',
|
|
689
|
+
});
|
|
690
|
+
|
|
691
|
+
const issueLabels = Array.isArray(labels) ? labels : [labels, 'auto-report'];
|
|
692
|
+
|
|
693
|
+
await sendToRelay({
|
|
694
|
+
type,
|
|
695
|
+
signature: sig,
|
|
696
|
+
title,
|
|
697
|
+
body,
|
|
698
|
+
labels: issueLabels,
|
|
699
|
+
version,
|
|
700
|
+
});
|
|
701
|
+
} catch {
|
|
702
|
+
// Swallow -- reporter must never crash the server
|
|
703
|
+
}
|
|
704
|
+
})();
|
|
705
|
+
|
|
706
|
+
inFlight.add(reportPromise);
|
|
707
|
+
reportPromise.finally(() => inFlight.delete(reportPromise));
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
/**
|
|
711
|
+
* Track the last Express route for stall diagnostics.
|
|
712
|
+
* Call from middleware: reporter.trackRoute(req.method + ' ' + req.route?.path)
|
|
713
|
+
*/
|
|
714
|
+
function trackRoute(route) {
|
|
715
|
+
_lastRoute = route || null;
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
async function reportCrash(error, opts = {}) {
|
|
719
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
720
|
+
const uptimeMinutes = typeof process !== 'undefined'
|
|
721
|
+
? Math.round(process.uptime() / 60) : undefined;
|
|
722
|
+
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
723
|
+
|
|
724
|
+
await fileReport(
|
|
725
|
+
opts.signal ? `signal:${opts.signal}` : (err.name || 'crash'),
|
|
726
|
+
['crash', 'auto-report'],
|
|
727
|
+
{
|
|
728
|
+
error: err,
|
|
729
|
+
message: err.message,
|
|
730
|
+
stack: err.stack,
|
|
731
|
+
signal: opts.signal || null,
|
|
732
|
+
activeRoute: _lastRoute,
|
|
733
|
+
uptimeMinutes,
|
|
734
|
+
resources,
|
|
735
|
+
proxy: opts.proxy || null,
|
|
736
|
+
context: opts.context,
|
|
737
|
+
},
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
async function reportHang(operation, durationMs, opts = {}) {
|
|
742
|
+
const uptimeMinutes = typeof process !== 'undefined'
|
|
743
|
+
? Math.round(process.uptime() / 60) : undefined;
|
|
744
|
+
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
745
|
+
|
|
746
|
+
// Build lean context (journal only, no redundant fields)
|
|
747
|
+
const context = { ...opts.context };
|
|
748
|
+
if (context.journal) {
|
|
749
|
+
context.journal = context.journal.map(j => typeof j === 'string' ? j : j);
|
|
750
|
+
}
|
|
751
|
+
// Remove fields that now have dedicated sections
|
|
752
|
+
delete context.operation;
|
|
753
|
+
delete context.durationMs;
|
|
754
|
+
|
|
755
|
+
// Anti-bot detection from health snapshot
|
|
756
|
+
const healthSnap = opts.healthSnapshot;
|
|
757
|
+
const botDetection = healthSnap?.botDetection?.detected ? {
|
|
758
|
+
...healthSnap.botDetection,
|
|
759
|
+
responseBodySizeKb: healthSnap.lastNavResponseSize
|
|
760
|
+
? Math.round(healthSnap.lastNavResponseSize / 1024) : null,
|
|
761
|
+
redirectChainLength: healthSnap.redirectStatusCodes?.length || null,
|
|
762
|
+
redirectStatusCodes: healthSnap.redirectStatusCodes?.length
|
|
763
|
+
? healthSnap.redirectStatusCodes : null,
|
|
764
|
+
} : null;
|
|
765
|
+
|
|
766
|
+
// Get document.readyState if healthTracker provided
|
|
767
|
+
let documentReadyState = null;
|
|
768
|
+
if (opts.healthTracker?.getReadyState) {
|
|
769
|
+
documentReadyState = await opts.healthTracker.getReadyState();
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
const labels = ['hang', 'auto-report'];
|
|
773
|
+
if (botDetection?.detected) labels.push('bot-detection');
|
|
774
|
+
|
|
775
|
+
await fileReport(
|
|
776
|
+
`hang:${operation}`,
|
|
777
|
+
labels,
|
|
778
|
+
{
|
|
779
|
+
message: `Operation "${operation}" hung for ${Math.round(durationMs / 1000)}s`,
|
|
780
|
+
stack: opts.error?.stack,
|
|
781
|
+
activeRoute: _lastRoute,
|
|
782
|
+
uptimeMinutes,
|
|
783
|
+
resources,
|
|
784
|
+
hang: {
|
|
785
|
+
operation,
|
|
786
|
+
durationMs,
|
|
787
|
+
lockQueueMs: opts.lockQueueMs ?? null,
|
|
788
|
+
documentReadyState,
|
|
789
|
+
inflightRequests: healthSnap?.inflightRequests ?? null,
|
|
790
|
+
},
|
|
791
|
+
botDetection,
|
|
792
|
+
proxy: opts.proxy || null,
|
|
793
|
+
context,
|
|
794
|
+
},
|
|
795
|
+
);
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
async function reportStuckLoop(durationMs, opts = {}) {
|
|
799
|
+
const uptimeMinutes = typeof process !== 'undefined'
|
|
800
|
+
? Math.round(process.uptime() / 60) : undefined;
|
|
801
|
+
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
802
|
+
|
|
803
|
+
await fileReport(
|
|
804
|
+
'stuck:tab-lock',
|
|
805
|
+
['stuck', 'auto-report'],
|
|
806
|
+
{
|
|
807
|
+
message: `Tab lock held for ${Math.round(durationMs / 1000)}s (tab destroyed)`,
|
|
808
|
+
uptimeMinutes,
|
|
809
|
+
resources,
|
|
810
|
+
context: { durationMs, ...opts.context },
|
|
811
|
+
},
|
|
812
|
+
);
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
function startWatchdog(thresholdMs = 5000, getContext) {
|
|
816
|
+
if (watchdogInterval) return;
|
|
817
|
+
|
|
818
|
+
const checkMs = 1000;
|
|
819
|
+
lastTick = Date.now();
|
|
820
|
+
let lastCpuUsage = process.cpuUsage();
|
|
821
|
+
let lastHrtime = process.hrtime.bigint();
|
|
822
|
+
let lastHeapUsed = process.memoryUsage().heapUsed;
|
|
823
|
+
|
|
824
|
+
// --- Native memory leak tracking ---
|
|
825
|
+
// Track RSS minus JS heap over time to detect native/external memory leaks.
|
|
826
|
+
// Sample every 30s, alert if native memory stays >400MB above baseline for
|
|
827
|
+
// 3 consecutive checks (~90s). This avoids false positives from:
|
|
828
|
+
// - Browser initialization spikes (first 2 min)
|
|
829
|
+
// - One-time allocations that stabilize
|
|
830
|
+
// - Post-session RSS that hasn't been reclaimed by the OS yet
|
|
831
|
+
// - Self-healing restart (kills browser at 200MB growth when sessions=0)
|
|
832
|
+
// The memory pressure restart in server.js fires at 200MB when idle.
|
|
833
|
+
// We only report at 400MB to catch cases where self-healing FAILED.
|
|
834
|
+
let nativeMemBaseline = null; // RSS - heapUsed at first measurement
|
|
835
|
+
let nativeMemHighWater = 0;
|
|
836
|
+
let lastNativeMemCheck = 0;
|
|
837
|
+
const NATIVE_MEM_CHECK_INTERVAL_MS = 30_000;
|
|
838
|
+
const NATIVE_MEM_LEAK_THRESHOLD_MB = 400; // alert only when growth exceeds self-healing threshold
|
|
839
|
+
const NATIVE_MEM_MIN_UPTIME_S = 120; // don't measure until process has been up 2 min
|
|
840
|
+
const NATIVE_MEM_CONSECUTIVE_REQUIRED = 3; // require 3 consecutive checks above threshold
|
|
841
|
+
const NATIVE_MEM_GRACE_CHECKS = 2; // skip 2 checks after baseline reset (let memory settle)
|
|
842
|
+
let nativeMemAlertFired = false;
|
|
843
|
+
let nativeMemConsecutiveAbove = 0; // consecutive checks above threshold
|
|
844
|
+
let nativeMemGraceRemaining = 0; // checks to skip after baseline reset
|
|
845
|
+
let lastSeenBrowserRssMb = null; // captured during growth checks while browser is alive
|
|
846
|
+
|
|
847
|
+
// SIGCONT detection -- macOS sends SIGCONT on wake from sleep/suspend
|
|
848
|
+
let lastSigcont = 0;
|
|
849
|
+
try { process.on('SIGCONT', () => { lastSigcont = Date.now(); }); } catch { /* unavailable */ }
|
|
850
|
+
|
|
851
|
+
// Event loop delay histogram (perf_hooks) -- correlating evidence
|
|
852
|
+
let elHistogram = null;
|
|
853
|
+
try {
|
|
854
|
+
elHistogram = monitorEventLoopDelay({ resolution: 20 });
|
|
855
|
+
elHistogram.enable();
|
|
856
|
+
} catch { /* unavailable */ }
|
|
857
|
+
|
|
858
|
+
// Suppress false positives from OS sleep/suspend (laptop lid close, VM pause).
|
|
859
|
+
// Stalls > 120s are almost certainly not event-loop bugs.
|
|
860
|
+
const MAX_REPORTABLE_DRIFT_MS = 60_000;
|
|
861
|
+
let suppressTicksRemaining = 0;
|
|
862
|
+
const SUPPRESS_TICKS_AFTER_WAKE = 5;
|
|
863
|
+
|
|
864
|
+
watchdogInterval = setInterval(() => {
|
|
865
|
+
const now = Date.now();
|
|
866
|
+
const drift = now - lastTick - checkMs;
|
|
867
|
+
const cpuDelta = process.cpuUsage(lastCpuUsage);
|
|
868
|
+
const hrtimeNow = process.hrtime.bigint();
|
|
869
|
+
const hrtimeDeltaMs = Number(hrtimeNow - lastHrtime) / 1e6;
|
|
870
|
+
|
|
871
|
+
lastTick = now;
|
|
872
|
+
lastCpuUsage = process.cpuUsage();
|
|
873
|
+
lastHrtime = hrtimeNow;
|
|
874
|
+
|
|
875
|
+
// After a long sleep/suspend, suppress the next few ticks (post-wake jitter)
|
|
876
|
+
if (drift > MAX_REPORTABLE_DRIFT_MS) {
|
|
877
|
+
suppressTicksRemaining = SUPPRESS_TICKS_AFTER_WAKE;
|
|
878
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
879
|
+
return;
|
|
880
|
+
}
|
|
881
|
+
if (suppressTicksRemaining > 0) {
|
|
882
|
+
suppressTicksRemaining--;
|
|
883
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
884
|
+
return;
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
// --- Native memory leak detection (runs every ~30s) ---
|
|
888
|
+
if (now - lastNativeMemCheck >= NATIVE_MEM_CHECK_INTERVAL_MS) {
|
|
889
|
+
lastNativeMemCheck = now;
|
|
890
|
+
try {
|
|
891
|
+
// Skip until process has been up long enough for browser to initialize.
|
|
892
|
+
// Browser launch causes a 100-300MB RSS spike that isn't a leak.
|
|
893
|
+
if (process.uptime() >= NATIVE_MEM_MIN_UPTIME_S) {
|
|
894
|
+
// Check if baseline should be reset (e.g. after browser close)
|
|
895
|
+
if (_resetNativeMemBaseline) {
|
|
896
|
+
nativeMemBaseline = null;
|
|
897
|
+
nativeMemHighWater = 0;
|
|
898
|
+
nativeMemAlertFired = false;
|
|
899
|
+
nativeMemConsecutiveAbove = 0;
|
|
900
|
+
nativeMemGraceRemaining = NATIVE_MEM_GRACE_CHECKS;
|
|
901
|
+
_resetNativeMemBaseline = false;
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
// Grace period after reset -- let memory settle before re-baselining
|
|
905
|
+
if (nativeMemGraceRemaining > 0) {
|
|
906
|
+
nativeMemGraceRemaining--;
|
|
907
|
+
} else {
|
|
908
|
+
const mem = process.memoryUsage();
|
|
909
|
+
const nativeMemMb = Math.round((mem.rss - mem.heapUsed) / 1048576);
|
|
910
|
+
if (nativeMemBaseline === null) {
|
|
911
|
+
nativeMemBaseline = nativeMemMb;
|
|
912
|
+
}
|
|
913
|
+
nativeMemHighWater = Math.max(nativeMemHighWater, nativeMemMb);
|
|
914
|
+
const growth = nativeMemMb - nativeMemBaseline;
|
|
915
|
+
|
|
916
|
+
if (growth > NATIVE_MEM_LEAK_THRESHOLD_MB && !nativeMemAlertFired) {
|
|
917
|
+
// Require sustained growth -- one-time spikes aren't leaks.
|
|
918
|
+
// Must exceed threshold on 3 consecutive checks (~90s).
|
|
919
|
+
nativeMemConsecutiveAbove++;
|
|
920
|
+
|
|
921
|
+
// Capture browser RSS NOW while it may still be alive.
|
|
922
|
+
// By report time the browser is often killed by memory pressure restart,
|
|
923
|
+
// making browserRssMb null. This preserves the last-seen value.
|
|
924
|
+
try {
|
|
925
|
+
if (getContext) {
|
|
926
|
+
const ctx = getContext();
|
|
927
|
+
if (ctx.resourceOpts?.browserPid) {
|
|
928
|
+
const snap = collectResourceSnapshot(ctx.resourceOpts);
|
|
929
|
+
if (snap.browserRssMb != null) lastSeenBrowserRssMb = snap.browserRssMb;
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
} catch { /* swallow */ }
|
|
933
|
+
|
|
934
|
+
if (nativeMemConsecutiveAbove >= NATIVE_MEM_CONSECUTIVE_REQUIRED) {
|
|
935
|
+
nativeMemAlertFired = true;
|
|
936
|
+
let extra = {};
|
|
937
|
+
try { if (getContext) extra = getContext(); } catch { /* swallow */ }
|
|
938
|
+
const resources = collectResourceSnapshot(extra.resourceOpts || {});
|
|
939
|
+
delete extra.resourceOpts;
|
|
940
|
+
|
|
941
|
+
// Skip report if sessions=0 — memory pressure restart handles idle leaks.
|
|
942
|
+
// Only report when sessions are active (restart CAN'T fire) or restart failed.
|
|
943
|
+
const sessionCount = resources.browserContexts ?? 0;
|
|
944
|
+
if (sessionCount === 0 && resources.browserRssMb == null) {
|
|
945
|
+
// Browser already dead, restart mechanism handled it. Don't spam.
|
|
946
|
+
// But if growth is extreme (>600MB), report anyway — restart may have failed.
|
|
947
|
+
if (growth < 600) {
|
|
948
|
+
// Self-healing. Skip report.
|
|
949
|
+
return;
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
fileReport('leak:native-memory', ['auto-report', 'memory-leak'], {
|
|
954
|
+
message: `Native memory grew by ${growth}MB (baseline: ${nativeMemBaseline}MB, current: ${nativeMemMb}MB, high-water: ${nativeMemHighWater}MB)`,
|
|
955
|
+
uptimeMinutes: Math.round(process.uptime() / 60),
|
|
956
|
+
resources,
|
|
957
|
+
nativeMemory: {
|
|
958
|
+
baselineMb: nativeMemBaseline,
|
|
959
|
+
currentMb: nativeMemMb,
|
|
960
|
+
highWaterMb: nativeMemHighWater,
|
|
961
|
+
growthMb: growth,
|
|
962
|
+
rssMb: Math.round(mem.rss / 1048576),
|
|
963
|
+
heapUsedMb: Math.round(mem.heapUsed / 1048576),
|
|
964
|
+
externalMb: Math.round(mem.external / 1048576),
|
|
965
|
+
lastSeenBrowserRssMb,
|
|
966
|
+
},
|
|
967
|
+
context: extra,
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
} else {
|
|
971
|
+
// Reset consecutive counter if memory dropped back below threshold
|
|
972
|
+
nativeMemConsecutiveAbove = 0;
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
} catch { /* swallow */ }
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
if (drift > thresholdMs) {
|
|
980
|
+
// CPU time consumed during the stall interval (user + system, in seconds)
|
|
981
|
+
const cpuElapsedS = (cpuDelta.user + cpuDelta.system) / 1e6;
|
|
982
|
+
const wallElapsedS = drift / 1000;
|
|
983
|
+
const cpuRatio = wallElapsedS > 0 ? cpuElapsedS / wallElapsedS : 0;
|
|
984
|
+
|
|
985
|
+
// SIGCONT within the stall window = OS sleep/resume
|
|
986
|
+
const sigcontInWindow = lastSigcont > 0 && (now - lastSigcont) < drift + 2000;
|
|
987
|
+
|
|
988
|
+
// hrtime vs wall clock drift (macOS: hrtime doesn't advance during sleep)
|
|
989
|
+
const hrtimeWallDriftS = Math.abs((drift - (hrtimeDeltaMs - checkMs))) / 1000;
|
|
990
|
+
|
|
991
|
+
// Classify: sleep vs real stall
|
|
992
|
+
let classification;
|
|
993
|
+
if (cpuRatio < 0.01 && sigcontInWindow) classification = 'sleep';
|
|
994
|
+
else if (cpuRatio < 0.001) classification = 'likely_sleep';
|
|
995
|
+
else if (cpuRatio < 0.01) classification = 'likely_sleep';
|
|
996
|
+
else if (cpuRatio > 0.1) classification = 'real_stall';
|
|
997
|
+
else classification = 'ambiguous';
|
|
998
|
+
|
|
999
|
+
// Don't file reports for sleep/suspend-resume stalls
|
|
1000
|
+
if (classification === 'sleep' || classification === 'likely_sleep') {
|
|
1001
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
1002
|
+
return;
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
// Capture heap delta during stall (GC indicator)
|
|
1006
|
+
const currentHeap = process.memoryUsage().heapUsed;
|
|
1007
|
+
const heapDeltaMb = Math.round((currentHeap - lastHeapUsed) / 1048576);
|
|
1008
|
+
lastHeapUsed = currentHeap;
|
|
1009
|
+
|
|
1010
|
+
let extra = {};
|
|
1011
|
+
try { if (getContext) extra = getContext(); } catch { /* swallow */ }
|
|
1012
|
+
|
|
1013
|
+
const resources = collectResourceSnapshot(extra.resourceOpts || {});
|
|
1014
|
+
// Remove resourceOpts from extra so it doesn't end up in context
|
|
1015
|
+
delete extra.resourceOpts;
|
|
1016
|
+
|
|
1017
|
+
// Don't report idle-server stalls -- no user impact
|
|
1018
|
+
if ((resources.activeTabs || 0) === 0 && (resources.browserContexts || 0) === 0) {
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
// Event loop delay histogram snapshot
|
|
1023
|
+
let elDelay = null;
|
|
1024
|
+
if (elHistogram) {
|
|
1025
|
+
try {
|
|
1026
|
+
elDelay = {
|
|
1027
|
+
p50Ms: Math.round(elHistogram.percentile(50) / 1e6),
|
|
1028
|
+
p99Ms: Math.round(elHistogram.percentile(99) / 1e6),
|
|
1029
|
+
maxMs: Math.round(elHistogram.max / 1e6),
|
|
1030
|
+
};
|
|
1031
|
+
elHistogram.reset();
|
|
1032
|
+
} catch { /* unavailable */ }
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
const labels = ['stuck', 'auto-report'];
|
|
1036
|
+
|
|
1037
|
+
fileReport('stuck:event-loop', labels, {
|
|
1038
|
+
message: `Event loop stalled for ${Math.round(drift / 1000)}s (threshold: ${Math.round(thresholdMs / 1000)}s)`,
|
|
1039
|
+
// Stable signature: duration is NOT included -- all stalls on the same route dedup
|
|
1040
|
+
error: { name: 'EventLoopStall', message: _lastRoute || 'idle', stack: '' },
|
|
1041
|
+
uptimeMinutes: typeof process !== 'undefined'
|
|
1042
|
+
? Math.round(process.uptime() / 60) : undefined,
|
|
1043
|
+
resources,
|
|
1044
|
+
stall: {
|
|
1045
|
+
driftMs: drift,
|
|
1046
|
+
thresholdMs,
|
|
1047
|
+
classification,
|
|
1048
|
+
cpuElapsedS: Math.round(cpuElapsedS * 1000) / 1000,
|
|
1049
|
+
cpuRatio: Math.round(cpuRatio * 10000) / 10000,
|
|
1050
|
+
sigcontInWindow,
|
|
1051
|
+
hrtimeWallDriftS: Math.round(hrtimeWallDriftS * 100) / 100,
|
|
1052
|
+
eventLoopDelay: elDelay,
|
|
1053
|
+
lastRoute: _lastRoute,
|
|
1054
|
+
activeHandles: resources.activeHandles,
|
|
1055
|
+
activeRequests: resources.activeRequests,
|
|
1056
|
+
heapDeltaMb,
|
|
1057
|
+
nativeMemGrowthMb: nativeMemBaseline !== null
|
|
1058
|
+
? Math.round((resources.nodeRssMb - resources.nodeHeapUsedMb) - nativeMemBaseline)
|
|
1059
|
+
: null,
|
|
1060
|
+
nativeMemBaselineMb: nativeMemBaseline,
|
|
1061
|
+
},
|
|
1062
|
+
context: extra,
|
|
1063
|
+
});
|
|
1064
|
+
} else {
|
|
1065
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
1066
|
+
}
|
|
1067
|
+
}, checkMs);
|
|
1068
|
+
|
|
1069
|
+
if (watchdogInterval.unref) watchdogInterval.unref();
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
function stop() {
|
|
1073
|
+
if (watchdogInterval) {
|
|
1074
|
+
clearInterval(watchdogInterval);
|
|
1075
|
+
watchdogInterval = null;
|
|
1076
|
+
}
|
|
1077
|
+
return Promise.allSettled([...inFlight]);
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
/**
|
|
1081
|
+
* Reset native memory baseline. Call after browser close so the next
|
|
1082
|
+
* browser session measures from a fresh baseline, not the old one.
|
|
1083
|
+
*/
|
|
1084
|
+
function resetNativeMemBaseline() {
|
|
1085
|
+
// These are closure vars in startWatchdog -- we need to reach them.
|
|
1086
|
+
// Since this runs in the same module, we set a flag the watchdog reads.
|
|
1087
|
+
_resetNativeMemBaseline = true;
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
return {
|
|
1091
|
+
reportCrash,
|
|
1092
|
+
reportHang,
|
|
1093
|
+
reportStuckLoop,
|
|
1094
|
+
startWatchdog,
|
|
1095
|
+
trackRoute,
|
|
1096
|
+
stop,
|
|
1097
|
+
resetNativeMemBaseline,
|
|
1098
|
+
_anonymize: anonymize,
|
|
1099
|
+
_stackSignature: stackSignature,
|
|
1100
|
+
_rateLimiter: rateLimiters,
|
|
1101
|
+
};
|
|
1102
|
+
}
|