@askjo/camofox-browser 1.7.0 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/reporter.js +347 -63
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/server.js +27 -14
package/lib/reporter.js
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
// Config passed via createReporter(config) from lib/config.js.
|
|
4
4
|
|
|
5
5
|
import crypto from 'crypto';
|
|
6
|
+
import fs from 'fs';
|
|
7
|
+
import { execSync } from 'child_process';
|
|
6
8
|
|
|
7
9
|
// ============================================================================
|
|
8
10
|
// Anonymization
|
|
@@ -10,7 +12,7 @@ import crypto from 'crypto';
|
|
|
10
12
|
|
|
11
13
|
const SAFE_HOSTS = new Set([
|
|
12
14
|
'github.com', 'api.github.com', 'npmjs.com', 'registry.npmjs.org',
|
|
13
|
-
'nodejs.org',
|
|
15
|
+
'nodejs.org',
|
|
14
16
|
]);
|
|
15
17
|
|
|
16
18
|
const SECRET_PREFIXES = [
|
|
@@ -64,17 +66,11 @@ export function anonymize(text) {
|
|
|
64
66
|
}
|
|
65
67
|
);
|
|
66
68
|
|
|
67
|
-
// 7. Strip IPv4 addresses
|
|
68
|
-
s = s.replace(/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g,
|
|
69
|
-
if (match === '127.0.0.1') return match;
|
|
70
|
-
return '<ip>';
|
|
71
|
-
});
|
|
69
|
+
// 7. Strip IPv4 addresses
|
|
70
|
+
s = s.replace(/\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g, '<ip>');
|
|
72
71
|
|
|
73
|
-
// 8. Strip IPv6 addresses
|
|
74
|
-
s = s.replace(/\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g,
|
|
75
|
-
if (match === '::1') return match;
|
|
76
|
-
return '<ipv6>';
|
|
77
|
-
});
|
|
72
|
+
// 8. Strip IPv6 addresses
|
|
73
|
+
s = s.replace(/\b(?:[0-9a-fA-F]{1,4}:){2,7}[0-9a-fA-F]{1,4}\b/g, '<ipv6>');
|
|
78
74
|
s = s.replace(/::(?:ffff:)?(?:\d{1,3}\.){3}\d{1,3}/g, '<ipv6>');
|
|
79
75
|
|
|
80
76
|
// 9. Strip hostnames in connection errors
|
|
@@ -268,9 +264,7 @@ export function createUrlAnonymizer() {
|
|
|
268
264
|
const parts = [url.protocol + '//'];
|
|
269
265
|
const h = url.hostname.toLowerCase();
|
|
270
266
|
|
|
271
|
-
if (h
|
|
272
|
-
parts.push('localhost');
|
|
273
|
-
} else if (/^(\d{1,3}\.){3}\d{1,3}$/.test(h) || h.includes(':')) {
|
|
267
|
+
if (/^(\d{1,3}\.){3}\d{1,3}$/.test(h) || h.includes(':')) {
|
|
274
268
|
parts.push(hashHost(h));
|
|
275
269
|
} else if (isPublicDomain(h)) {
|
|
276
270
|
parts.push(h);
|
|
@@ -302,22 +296,57 @@ export function createUrlAnonymizer() {
|
|
|
302
296
|
// Per-tab health tracker (count-only, no content)
|
|
303
297
|
// ============================================================================
|
|
304
298
|
|
|
299
|
+
// Known bot-detection providers, matched by response header fingerprints.
|
|
300
|
+
// Order: most specific first.
|
|
301
|
+
const BOT_DETECTION_SIGNATURES = [
|
|
302
|
+
{ header: 'cf-mitigated', value: 'challenge', provider: 'cloudflare' },
|
|
303
|
+
{ header: 'x-datadome', provider: 'datadome' },
|
|
304
|
+
{ header: 'x-px', provider: 'perimeterx' },
|
|
305
|
+
{ header: 'x-distil-cs', provider: 'distil' },
|
|
306
|
+
{ header: 'x-sucuri-id', provider: 'sucuri' },
|
|
307
|
+
{ header: 'server', value: 'akamaighost', provider: 'akamai' },
|
|
308
|
+
// cf-ray is on ALL Cloudflare responses (even 200 OK). Must be last so it
|
|
309
|
+
// doesn't short-circuit other providers on multi-CDN sites.
|
|
310
|
+
{ header: 'cf-ray', provider: 'cloudflare' },
|
|
311
|
+
];
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Detect bot-detection provider from Playwright response headers.
|
|
315
|
+
* Returns { detected: bool, provider: string|null, httpStatus: number|null }
|
|
316
|
+
*/
|
|
317
|
+
export function detectBotProtection(response) {
|
|
318
|
+
if (!response) return { detected: false, provider: null, httpStatus: null };
|
|
319
|
+
const status = response.status();
|
|
320
|
+
let headers;
|
|
321
|
+
try { headers = response.headers(); } catch { return { detected: false, provider: null, httpStatus: status }; }
|
|
322
|
+
for (const sig of BOT_DETECTION_SIGNATURES) {
|
|
323
|
+
const val = headers[sig.header];
|
|
324
|
+
if (val !== undefined) {
|
|
325
|
+
if (sig.value && !val.toLowerCase().includes(sig.value)) continue;
|
|
326
|
+
const challenged = status === 403 || status === 429 || status === 503;
|
|
327
|
+
return { detected: challenged, provider: sig.provider, httpStatus: status };
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return { detected: false, provider: null, httpStatus: status };
|
|
331
|
+
}
|
|
332
|
+
|
|
305
333
|
/**
|
|
306
334
|
* Create a health tracker for a tab. Attaches to Playwright page events.
|
|
307
|
-
* Tracks: crashes, page errors,
|
|
308
|
-
*
|
|
335
|
+
* Tracks: crashes, page errors, request failures, redirect status codes,
|
|
336
|
+
* HTTP status histogram (4xx+), and anti-bot challenge detection.
|
|
309
337
|
* All count-based — no URLs or content stored.
|
|
310
338
|
*/
|
|
311
339
|
export function createTabHealthTracker(page) {
|
|
312
340
|
const health = {
|
|
313
341
|
crashes: 0,
|
|
314
342
|
pageErrors: 0,
|
|
315
|
-
consoleErrors: 0,
|
|
316
343
|
requestFailures: 0,
|
|
317
|
-
|
|
344
|
+
inflightRequests: 0,
|
|
318
345
|
maxRedirectDepth: 0,
|
|
319
|
-
|
|
320
|
-
|
|
346
|
+
redirectStatusCodes: [], // status codes in redirect chain, e.g. [301, 302, 403]
|
|
347
|
+
statusCounts: {}, // { 403: 5, 429: 2, ... }
|
|
348
|
+
botDetection: null, // { detected, provider, httpStatus } from last nav response
|
|
349
|
+
lastNavResponseSize: 0,
|
|
321
350
|
_redirectDepth: 0,
|
|
322
351
|
};
|
|
323
352
|
|
|
@@ -327,13 +356,15 @@ export function createTabHealthTracker(page) {
|
|
|
327
356
|
// Uncaught JS exceptions on the page
|
|
328
357
|
page.on('pageerror', () => { health.pageErrors++; });
|
|
329
358
|
|
|
330
|
-
//
|
|
331
|
-
page.on('
|
|
332
|
-
|
|
359
|
+
// Failed requests (blocked, DNS failure, etc.) + decrement in-flight counter
|
|
360
|
+
page.on('requestfailed', () => {
|
|
361
|
+
health.requestFailures++;
|
|
362
|
+
health.inflightRequests = Math.max(0, health.inflightRequests - 1);
|
|
333
363
|
});
|
|
334
364
|
|
|
335
|
-
//
|
|
336
|
-
page.on('
|
|
365
|
+
// Track in-flight requests for hang diagnostics
|
|
366
|
+
page.on('request', () => { health.inflightRequests++; });
|
|
367
|
+
page.on('requestfinished', () => { health.inflightRequests = Math.max(0, health.inflightRequests - 1); });
|
|
337
368
|
|
|
338
369
|
// HTTP status tracking (non-2xx only)
|
|
339
370
|
page.on('response', (resp) => {
|
|
@@ -341,13 +372,12 @@ export function createTabHealthTracker(page) {
|
|
|
341
372
|
if (s >= 400) health.statusCounts[s] = (health.statusCounts[s] || 0) + 1;
|
|
342
373
|
});
|
|
343
374
|
|
|
344
|
-
//
|
|
375
|
+
// Auto-dismiss dialogs to prevent page hangs (not tracked as a metric — noise)
|
|
345
376
|
page.on('dialog', async (dialog) => {
|
|
346
|
-
health.dialogCount++;
|
|
347
377
|
try { await dialog.dismiss(); } catch { /* page might be closed */ }
|
|
348
378
|
});
|
|
349
379
|
|
|
350
|
-
// Redirect depth per navigation
|
|
380
|
+
// Redirect depth + status code chain per navigation
|
|
351
381
|
page.on('request', (req) => {
|
|
352
382
|
if (req.isNavigationRequest()) {
|
|
353
383
|
if (req.redirectedFrom()) {
|
|
@@ -356,19 +386,120 @@ export function createTabHealthTracker(page) {
|
|
|
356
386
|
health.maxRedirectDepth = health._redirectDepth;
|
|
357
387
|
}
|
|
358
388
|
} else {
|
|
359
|
-
health._redirectDepth = 0;
|
|
389
|
+
health._redirectDepth = 0;
|
|
390
|
+
health.redirectStatusCodes = [];
|
|
391
|
+
health.inflightRequests = 0; // reset on new navigation to prevent drift
|
|
360
392
|
}
|
|
361
393
|
}
|
|
362
394
|
});
|
|
363
395
|
|
|
396
|
+
// Capture redirect status codes and detect bot protection on nav responses
|
|
397
|
+
page.on('response', (resp) => {
|
|
398
|
+
try {
|
|
399
|
+
const req = resp.request();
|
|
400
|
+
if (req.isNavigationRequest()) {
|
|
401
|
+
health.redirectStatusCodes.push(resp.status());
|
|
402
|
+
health.botDetection = detectBotProtection(resp);
|
|
403
|
+
// Approximate response body size from content-length (no body read)
|
|
404
|
+
const cl = resp.headers()['content-length'];
|
|
405
|
+
if (cl) health.lastNavResponseSize = parseInt(cl, 10) || 0;
|
|
406
|
+
}
|
|
407
|
+
} catch { /* page closed */ }
|
|
408
|
+
});
|
|
409
|
+
|
|
364
410
|
/** Snapshot current health counters for inclusion in reports. */
|
|
365
411
|
function snapshot() {
|
|
366
|
-
try { health.frameCount = page.frames().length; } catch { /* closed */ }
|
|
367
412
|
const { _redirectDepth, ...clean } = health;
|
|
368
413
|
return { ...clean };
|
|
369
414
|
}
|
|
370
415
|
|
|
371
|
-
|
|
416
|
+
/**
|
|
417
|
+
* Get document.readyState from the page. Returns null if page is unresponsive.
|
|
418
|
+
* Use a tight timeout — if the renderer is crashed, evaluate will hang.
|
|
419
|
+
*/
|
|
420
|
+
async function getReadyState() {
|
|
421
|
+
try {
|
|
422
|
+
return await Promise.race([
|
|
423
|
+
page.evaluate(() => document.readyState),
|
|
424
|
+
new Promise(resolve => setTimeout(() => resolve('unresponsive'), 1000)),
|
|
425
|
+
]);
|
|
426
|
+
} catch {
|
|
427
|
+
return 'unresponsive';
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
return { health, snapshot, getReadyState };
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// ============================================================================
|
|
435
|
+
// Process resource snapshot (memory, handles, FDs, browser RSS)
|
|
436
|
+
// ============================================================================
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* Collect process-level resource metrics. Safe to call at any time.
|
|
440
|
+
* Returns anonymized metrics — no PIDs, paths, or user data.
|
|
441
|
+
*/
|
|
442
|
+
export function collectResourceSnapshot(opts = {}) {
|
|
443
|
+
const mem = process.memoryUsage();
|
|
444
|
+
const snap = {
|
|
445
|
+
nodeRssMb: Math.round(mem.rss / 1048576),
|
|
446
|
+
nodeHeapUsedMb: Math.round(mem.heapUsed / 1048576),
|
|
447
|
+
nodeHeapTotalMb: Math.round(mem.heapTotal / 1048576),
|
|
448
|
+
nodeExternalMb: Math.round(mem.external / 1048576),
|
|
449
|
+
eventLoopLagMs: null,
|
|
450
|
+
activeHandles: null,
|
|
451
|
+
activeRequests: null,
|
|
452
|
+
openFds: null,
|
|
453
|
+
browserRssMb: null,
|
|
454
|
+
};
|
|
455
|
+
|
|
456
|
+
// Active libuv handles/requests (private API, guarded)
|
|
457
|
+
try { snap.activeHandles = process._getActiveHandles().length; } catch { /* unavailable */ }
|
|
458
|
+
try { snap.activeRequests = process._getActiveRequests().length; } catch { /* unavailable */ }
|
|
459
|
+
|
|
460
|
+
// Open file descriptors (Linux only)
|
|
461
|
+
try {
|
|
462
|
+
if (process.platform === 'linux') {
|
|
463
|
+
snap.openFds = fs.readdirSync('/proc/self/fd').length;
|
|
464
|
+
}
|
|
465
|
+
} catch { /* not available or permission denied */ }
|
|
466
|
+
|
|
467
|
+
// Browser process RSS (the one people miss — browser OOMs, not Node)
|
|
468
|
+
if (opts.browserPid && Number.isInteger(opts.browserPid) && opts.browserPid > 0) {
|
|
469
|
+
try {
|
|
470
|
+
if (process.platform === 'linux') {
|
|
471
|
+
const status = fs.readFileSync(`/proc/${opts.browserPid}/status`, 'utf8');
|
|
472
|
+
const match = status.match(/VmRSS:\s+(\d+)\s+kB/);
|
|
473
|
+
if (match) snap.browserRssMb = Math.round(parseInt(match[1], 10) / 1024);
|
|
474
|
+
} else if (process.platform === 'darwin') {
|
|
475
|
+
const out = execSync(`ps -o rss= -p ${opts.browserPid}`, { timeout: 1000 }).toString().trim();
|
|
476
|
+
if (out) snap.browserRssMb = Math.round(parseInt(out, 10) / 1024);
|
|
477
|
+
}
|
|
478
|
+
} catch { /* process gone or permission denied */ }
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// Session/tab counts from caller
|
|
482
|
+
if (opts.sessionCount != null) snap.browserContexts = opts.sessionCount;
|
|
483
|
+
if (opts.tabCount != null) snap.activeTabs = opts.tabCount;
|
|
484
|
+
|
|
485
|
+
return snap;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* Classify proxy errors from Playwright navigation error messages.
|
|
490
|
+
* Returns { proxyError: string|null, proxyTlsError: bool } — no IPs or credentials.
|
|
491
|
+
*/
|
|
492
|
+
export function classifyProxyError(errorMessage) {
|
|
493
|
+
if (!errorMessage || typeof errorMessage !== 'string') return { proxyError: null, proxyTlsError: false };
|
|
494
|
+
const msg = errorMessage.toUpperCase();
|
|
495
|
+
// Explicit proxy errors from Chromium/Firefox net stack
|
|
496
|
+
if (msg.includes('ERR_PROXY_CONNECTION_FAILED')) return { proxyError: 'ERR_PROXY_CONNECTION_FAILED', proxyTlsError: false };
|
|
497
|
+
if (msg.includes('ERR_TUNNEL_CONNECTION_FAILED')) return { proxyError: 'ERR_TUNNEL_CONNECTION_FAILED', proxyTlsError: false };
|
|
498
|
+
if (msg.includes('ERR_PROXY_AUTH_REQUESTED') || msg.includes('407')) return { proxyError: 'ERR_PROXY_AUTH_REQUESTED', proxyTlsError: false };
|
|
499
|
+
if (msg.includes('ERR_PROXY_CERTIFICATE_INVALID') || (msg.includes('PROXY') && msg.includes('SSL'))) return { proxyError: 'ERR_PROXY_TLS', proxyTlsError: true };
|
|
500
|
+
if (msg.includes('ECONNREFUSED') && msg.includes('PROXY')) return { proxyError: 'ECONNREFUSED', proxyTlsError: false };
|
|
501
|
+
if (msg.includes('ETIMEDOUT') && msg.includes('PROXY')) return { proxyError: 'ETIMEDOUT', proxyTlsError: false };
|
|
502
|
+
return { proxyError: null, proxyTlsError: false };
|
|
372
503
|
}
|
|
373
504
|
|
|
374
505
|
// ============================================================================
|
|
@@ -518,24 +649,86 @@ function formatIssueBody(type, detail) {
|
|
|
518
649
|
const sections = [
|
|
519
650
|
'> Auto-reported by ' + _GH_USER_AGENT + '. All data is anonymized.',
|
|
520
651
|
'',
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
652
|
+
'## Environment',
|
|
653
|
+
`- **version:** ${detail.version || 'unknown'}`,
|
|
654
|
+
`- **node:** ${detail.nodeVersion || 'unknown'}`,
|
|
655
|
+
`- **platform:** ${detail.platform || 'unknown'}`,
|
|
656
|
+
`- **uptime:** ${detail.uptimeMinutes != null ? detail.uptimeMinutes + ' min' : 'unknown'}`,
|
|
526
657
|
];
|
|
527
658
|
|
|
659
|
+
// Resource snapshot (memory, handles, browser RSS)
|
|
660
|
+
const r = detail.resources;
|
|
661
|
+
if (r) {
|
|
662
|
+
sections.push('', '## Resources');
|
|
663
|
+
sections.push(`- **node RSS:** ${r.nodeRssMb ?? '?'} MB`);
|
|
664
|
+
sections.push(`- **node heap:** ${r.nodeHeapUsedMb ?? '?'} / ${r.nodeHeapTotalMb ?? '?'} MB`);
|
|
665
|
+
if (r.browserRssMb != null) sections.push(`- **browser RSS:** ${r.browserRssMb} MB`);
|
|
666
|
+
if (r.browserContexts != null) sections.push(`- **browser contexts:** ${r.browserContexts}`);
|
|
667
|
+
if (r.activeTabs != null) sections.push(`- **active tabs:** ${r.activeTabs}`);
|
|
668
|
+
if (r.openFds != null) sections.push(`- **open FDs:** ${r.openFds}`);
|
|
669
|
+
if (r.activeHandles != null) sections.push(`- **active handles:** ${r.activeHandles}`);
|
|
670
|
+
if (r.eventLoopLagMs != null) sections.push(`- **event loop lag:** ${r.eventLoopLagMs} ms`);
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// Error info
|
|
674
|
+
if (detail.signal) sections.push('', `**Signal:** ${detail.signal}`);
|
|
675
|
+
if (detail.activeRoute) sections.push(`**Active route:** ${detail.activeRoute}`);
|
|
528
676
|
if (detail.message) {
|
|
529
|
-
sections.push('', '
|
|
677
|
+
sections.push('', '## Error', '```', anonymize(detail.message), '```');
|
|
530
678
|
}
|
|
531
679
|
if (detail.stack) {
|
|
532
|
-
sections.push('', '
|
|
680
|
+
sections.push('', '## Stack Trace', '```', anonymize(detail.stack), '```');
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
// Hang-specific details
|
|
684
|
+
if (detail.hang) {
|
|
685
|
+
const h = detail.hang;
|
|
686
|
+
sections.push('', '## Hang Details');
|
|
687
|
+
sections.push(`- **operation:** ${h.operation}`);
|
|
688
|
+
sections.push(`- **duration:** ${Math.round(h.durationMs / 1000)}s`);
|
|
689
|
+
if (h.lockQueueMs != null) sections.push(`- **lock queue wait:** ${Math.round(h.lockQueueMs)}ms`);
|
|
690
|
+
if (h.documentReadyState) sections.push(`- **document.readyState:** ${h.documentReadyState}`);
|
|
691
|
+
if (h.inflightRequests != null) sections.push(`- **in-flight requests:** ${h.inflightRequests}`);
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
// Anti-bot detection
|
|
695
|
+
if (detail.botDetection?.detected) {
|
|
696
|
+
const b = detail.botDetection;
|
|
697
|
+
sections.push('', '## Anti-Bot Detection');
|
|
698
|
+
sections.push(`- **provider:** ${b.provider || 'unknown'}`);
|
|
699
|
+
sections.push(`- **HTTP status:** ${b.httpStatus || '?'}`);
|
|
700
|
+
if (b.responseBodySizeKb != null) sections.push(`- **response size:** ${b.responseBodySizeKb} KB`);
|
|
701
|
+
if (b.redirectChainLength != null) sections.push(`- **redirect chain:** ${b.redirectChainLength} hops`);
|
|
702
|
+
if (b.redirectStatusCodes?.length) sections.push(`- **redirect statuses:** ${b.redirectStatusCodes.join(' → ')}`);
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// Proxy info (safe fields only — no IPs, credentials, or hostnames)
|
|
706
|
+
if (detail.proxy) {
|
|
707
|
+
const p = detail.proxy;
|
|
708
|
+
sections.push('', '## Proxy');
|
|
709
|
+
sections.push(`- **configured:** ${p.configured}`);
|
|
710
|
+
if (p.configured) {
|
|
711
|
+
if (p.type) sections.push(`- **type:** ${p.type}`);
|
|
712
|
+
sections.push(`- **auth configured:** ${p.authConfigured ?? 'unknown'}`);
|
|
713
|
+
if (p.error) sections.push(`- **error:** ${p.error}`);
|
|
714
|
+
if (p.tlsError) sections.push(`- **TLS error:** yes`);
|
|
715
|
+
}
|
|
533
716
|
}
|
|
534
|
-
|
|
535
|
-
|
|
717
|
+
|
|
718
|
+
// Stall-specific details
|
|
719
|
+
if (detail.stall) {
|
|
720
|
+
const s = detail.stall;
|
|
721
|
+
sections.push('', '## Stall Details');
|
|
722
|
+
sections.push(`- **stall duration:** ${Math.round(s.driftMs / 1000)}s`);
|
|
723
|
+
if (s.lastRoute) sections.push(`- **last route:** ${s.lastRoute}`);
|
|
724
|
+
if (s.activeHandles != null) sections.push(`- **active handles:** ${s.activeHandles}`);
|
|
725
|
+
if (s.activeRequests != null) sections.push(`- **active requests:** ${s.activeRequests}`);
|
|
726
|
+
if (s.heapDeltaMb != null) sections.push(`- **heap delta:** ${s.heapDeltaMb > 0 ? '+' : ''}${s.heapDeltaMb} MB`);
|
|
536
727
|
}
|
|
537
|
-
|
|
538
|
-
|
|
728
|
+
|
|
729
|
+
// Context (misc extra data)
|
|
730
|
+
if (detail.context && Object.keys(detail.context).length > 0) {
|
|
731
|
+
sections.push('', '<details><summary>Context</summary>', '', '```json', anonymize(JSON.stringify(detail.context, null, 2)), '```', '', '</details>');
|
|
539
732
|
}
|
|
540
733
|
|
|
541
734
|
return sections.join('\n');
|
|
@@ -547,6 +740,14 @@ function formatCommentBody(type, detail) {
|
|
|
547
740
|
`**+1** — ${ts}`,
|
|
548
741
|
`Version: ${detail.version || 'unknown'}, Uptime: ${detail.uptimeMinutes != null ? detail.uptimeMinutes + ' min' : '?'}`,
|
|
549
742
|
];
|
|
743
|
+
// Include resource snapshot in +1 comments too
|
|
744
|
+
const r = detail.resources;
|
|
745
|
+
if (r) {
|
|
746
|
+
const parts = [`RSS: ${r.nodeRssMb ?? '?'}MB`];
|
|
747
|
+
if (r.browserRssMb != null) parts.push(`Browser: ${r.browserRssMb}MB`);
|
|
748
|
+
if (r.activeTabs != null) parts.push(`Tabs: ${r.activeTabs}`);
|
|
749
|
+
lines.push(parts.join(', '));
|
|
750
|
+
}
|
|
550
751
|
if (detail.message) {
|
|
551
752
|
lines.push('```', anonymize(detail.message).slice(0, 500), '```');
|
|
552
753
|
}
|
|
@@ -586,6 +787,9 @@ export function createReporter(config) {
|
|
|
586
787
|
let lastTick = Date.now();
|
|
587
788
|
const inFlight = new Set();
|
|
588
789
|
|
|
790
|
+
// Track last Express route for stall reports
|
|
791
|
+
let _lastRoute = null;
|
|
792
|
+
|
|
589
793
|
// No-op when disabled
|
|
590
794
|
if (!enabled) {
|
|
591
795
|
return {
|
|
@@ -593,6 +797,7 @@ export function createReporter(config) {
|
|
|
593
797
|
reportHang: async () => {},
|
|
594
798
|
reportStuckLoop: async () => {},
|
|
595
799
|
startWatchdog: () => {},
|
|
800
|
+
trackRoute: () => {},
|
|
596
801
|
stop: () => {},
|
|
597
802
|
_anonymize: anonymize,
|
|
598
803
|
_stackSignature: stackSignature,
|
|
@@ -600,7 +805,7 @@ export function createReporter(config) {
|
|
|
600
805
|
}
|
|
601
806
|
|
|
602
807
|
/** Core: file or deduplicate a report. NEVER throws. */
|
|
603
|
-
async function fileReport(type,
|
|
808
|
+
async function fileReport(type, labels, detail) {
|
|
604
809
|
if (!rateLimiter.tryAcquire()) return;
|
|
605
810
|
|
|
606
811
|
const reportPromise = (async () => {
|
|
@@ -627,7 +832,8 @@ export function createReporter(config) {
|
|
|
627
832
|
platform: typeof process !== 'undefined' ? process.platform : 'unknown',
|
|
628
833
|
});
|
|
629
834
|
|
|
630
|
-
|
|
835
|
+
const issueLabels = Array.isArray(labels) ? labels : [labels, 'auto-report'];
|
|
836
|
+
await createIssue(repo, title, body, issueLabels);
|
|
631
837
|
} catch {
|
|
632
838
|
// Swallow — reporter must never crash the server
|
|
633
839
|
}
|
|
@@ -637,19 +843,32 @@ export function createReporter(config) {
|
|
|
637
843
|
reportPromise.finally(() => inFlight.delete(reportPromise));
|
|
638
844
|
}
|
|
639
845
|
|
|
846
|
+
/**
|
|
847
|
+
* Track the last Express route for stall diagnostics.
|
|
848
|
+
* Call from middleware: reporter.trackRoute(req.method + ' ' + req.route?.path)
|
|
849
|
+
*/
|
|
850
|
+
function trackRoute(route) {
|
|
851
|
+
_lastRoute = route || null;
|
|
852
|
+
}
|
|
853
|
+
|
|
640
854
|
async function reportCrash(error, opts = {}) {
|
|
641
855
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
642
856
|
const uptimeMinutes = typeof process !== 'undefined'
|
|
643
857
|
? Math.round(process.uptime() / 60) : undefined;
|
|
858
|
+
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
644
859
|
|
|
645
860
|
await fileReport(
|
|
646
861
|
opts.signal ? `signal:${opts.signal}` : (err.name || 'crash'),
|
|
647
|
-
'crash',
|
|
862
|
+
['crash', 'auto-report'],
|
|
648
863
|
{
|
|
649
864
|
error: err,
|
|
650
865
|
message: err.message,
|
|
651
866
|
stack: err.stack,
|
|
867
|
+
signal: opts.signal || null,
|
|
868
|
+
activeRoute: _lastRoute,
|
|
652
869
|
uptimeMinutes,
|
|
870
|
+
resources,
|
|
871
|
+
proxy: opts.proxy || null,
|
|
653
872
|
context: opts.context,
|
|
654
873
|
},
|
|
655
874
|
);
|
|
@@ -658,32 +877,55 @@ export function createReporter(config) {
|
|
|
658
877
|
async function reportHang(operation, durationMs, opts = {}) {
|
|
659
878
|
const uptimeMinutes = typeof process !== 'undefined'
|
|
660
879
|
? Math.round(process.uptime() / 60) : undefined;
|
|
880
|
+
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
661
881
|
|
|
662
|
-
//
|
|
663
|
-
const
|
|
664
|
-
const context = { operation, durationMs, ...opts.context };
|
|
665
|
-
|
|
666
|
-
// Anonymize any URLs in the journal
|
|
882
|
+
// Build lean context (journal only, no redundant fields)
|
|
883
|
+
const context = { ...opts.context };
|
|
667
884
|
if (context.journal) {
|
|
668
|
-
context.journal = context.journal.map(j =>
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
885
|
+
context.journal = context.journal.map(j => typeof j === 'string' ? j : j);
|
|
886
|
+
}
|
|
887
|
+
// Remove fields that now have dedicated sections
|
|
888
|
+
delete context.operation;
|
|
889
|
+
delete context.durationMs;
|
|
890
|
+
|
|
891
|
+
// Anti-bot detection from health snapshot
|
|
892
|
+
const healthSnap = opts.healthSnapshot;
|
|
893
|
+
const botDetection = healthSnap?.botDetection?.detected ? {
|
|
894
|
+
...healthSnap.botDetection,
|
|
895
|
+
responseBodySizeKb: healthSnap.lastNavResponseSize
|
|
896
|
+
? Math.round(healthSnap.lastNavResponseSize / 1024) : null,
|
|
897
|
+
redirectChainLength: healthSnap.redirectStatusCodes?.length || null,
|
|
898
|
+
redirectStatusCodes: healthSnap.redirectStatusCodes?.length
|
|
899
|
+
? healthSnap.redirectStatusCodes : null,
|
|
900
|
+
} : null;
|
|
901
|
+
|
|
902
|
+
// Get document.readyState if healthTracker provided
|
|
903
|
+
let documentReadyState = null;
|
|
904
|
+
if (opts.healthTracker?.getReadyState) {
|
|
905
|
+
documentReadyState = await opts.healthTracker.getReadyState();
|
|
672
906
|
}
|
|
673
|
-
// Include anonymized URL if provided
|
|
674
|
-
if (opts.url) context.url = urlAnon.anonymizeUrl(opts.url);
|
|
675
|
-
if (opts.redirectChain) context.redirectChain = urlAnon.anonymizeChain(opts.redirectChain);
|
|
676
907
|
|
|
677
|
-
|
|
678
|
-
if (
|
|
908
|
+
const labels = ['hang', 'auto-report'];
|
|
909
|
+
if (botDetection?.detected) labels.push('bot-detection');
|
|
679
910
|
|
|
680
911
|
await fileReport(
|
|
681
912
|
`hang:${operation}`,
|
|
682
|
-
|
|
913
|
+
labels,
|
|
683
914
|
{
|
|
684
915
|
message: `Operation "${operation}" hung for ${Math.round(durationMs / 1000)}s`,
|
|
685
916
|
stack: opts.error?.stack,
|
|
917
|
+
activeRoute: _lastRoute,
|
|
686
918
|
uptimeMinutes,
|
|
919
|
+
resources,
|
|
920
|
+
hang: {
|
|
921
|
+
operation,
|
|
922
|
+
durationMs,
|
|
923
|
+
lockQueueMs: opts.lockQueueMs ?? null,
|
|
924
|
+
documentReadyState,
|
|
925
|
+
inflightRequests: healthSnap?.inflightRequests ?? null,
|
|
926
|
+
},
|
|
927
|
+
botDetection,
|
|
928
|
+
proxy: opts.proxy || null,
|
|
687
929
|
context,
|
|
688
930
|
},
|
|
689
931
|
);
|
|
@@ -692,13 +934,15 @@ export function createReporter(config) {
|
|
|
692
934
|
async function reportStuckLoop(durationMs, opts = {}) {
|
|
693
935
|
const uptimeMinutes = typeof process !== 'undefined'
|
|
694
936
|
? Math.round(process.uptime() / 60) : undefined;
|
|
937
|
+
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
695
938
|
|
|
696
939
|
await fileReport(
|
|
697
940
|
'stuck:tab-lock',
|
|
698
|
-
'stuck',
|
|
941
|
+
['stuck', 'auto-report'],
|
|
699
942
|
{
|
|
700
943
|
message: `Tab lock held for ${Math.round(durationMs / 1000)}s (tab destroyed)`,
|
|
701
944
|
uptimeMinutes,
|
|
945
|
+
resources,
|
|
702
946
|
context: { durationMs, ...opts.context },
|
|
703
947
|
},
|
|
704
948
|
);
|
|
@@ -709,21 +953,60 @@ export function createReporter(config) {
|
|
|
709
953
|
|
|
710
954
|
const checkMs = 1000;
|
|
711
955
|
lastTick = Date.now();
|
|
956
|
+
let lastHeapUsed = process.memoryUsage().heapUsed;
|
|
957
|
+
// Suppress false positives from OS sleep/suspend (laptop lid close, VM pause).
|
|
958
|
+
// Stalls > 120s are almost certainly not event-loop bugs.
|
|
959
|
+
const MAX_REPORTABLE_DRIFT_MS = 120_000;
|
|
960
|
+
let suppressTicksRemaining = 0;
|
|
961
|
+
const SUPPRESS_TICKS_AFTER_WAKE = 5;
|
|
712
962
|
|
|
713
963
|
watchdogInterval = setInterval(() => {
|
|
714
964
|
const now = Date.now();
|
|
715
965
|
const drift = now - lastTick - checkMs;
|
|
716
966
|
lastTick = now;
|
|
717
967
|
|
|
968
|
+
// After a long sleep/suspend, suppress the next few ticks (post-wake jitter)
|
|
969
|
+
if (drift > MAX_REPORTABLE_DRIFT_MS) {
|
|
970
|
+
suppressTicksRemaining = SUPPRESS_TICKS_AFTER_WAKE;
|
|
971
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
972
|
+
return;
|
|
973
|
+
}
|
|
974
|
+
if (suppressTicksRemaining > 0) {
|
|
975
|
+
suppressTicksRemaining--;
|
|
976
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
977
|
+
return;
|
|
978
|
+
}
|
|
979
|
+
|
|
718
980
|
if (drift > thresholdMs) {
|
|
981
|
+
// Capture heap delta during stall (GC indicator)
|
|
982
|
+
const currentHeap = process.memoryUsage().heapUsed;
|
|
983
|
+
const heapDeltaMb = Math.round((currentHeap - lastHeapUsed) / 1048576);
|
|
984
|
+
lastHeapUsed = currentHeap;
|
|
985
|
+
|
|
719
986
|
let extra = {};
|
|
720
987
|
try { if (getContext) extra = getContext(); } catch { /* swallow */ }
|
|
721
|
-
|
|
988
|
+
|
|
989
|
+
const resources = collectResourceSnapshot(extra.resourceOpts || {});
|
|
990
|
+
// Remove resourceOpts from extra so it doesn't end up in context
|
|
991
|
+
delete extra.resourceOpts;
|
|
992
|
+
|
|
993
|
+
fileReport('stuck:event-loop', ['stuck', 'auto-report'], {
|
|
722
994
|
message: `Event loop stalled for ${Math.round(drift / 1000)}s (threshold: ${Math.round(thresholdMs / 1000)}s)`,
|
|
723
995
|
uptimeMinutes: typeof process !== 'undefined'
|
|
724
996
|
? Math.round(process.uptime() / 60) : undefined,
|
|
725
|
-
|
|
997
|
+
resources,
|
|
998
|
+
stall: {
|
|
999
|
+
driftMs: drift,
|
|
1000
|
+
thresholdMs,
|
|
1001
|
+
lastRoute: _lastRoute,
|
|
1002
|
+
activeHandles: resources.activeHandles,
|
|
1003
|
+
activeRequests: resources.activeRequests,
|
|
1004
|
+
heapDeltaMb,
|
|
1005
|
+
},
|
|
1006
|
+
context: extra,
|
|
726
1007
|
});
|
|
1008
|
+
} else {
|
|
1009
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
727
1010
|
}
|
|
728
1011
|
}, checkMs);
|
|
729
1012
|
|
|
@@ -743,6 +1026,7 @@ export function createReporter(config) {
|
|
|
743
1026
|
reportHang,
|
|
744
1027
|
reportStuckLoop,
|
|
745
1028
|
startWatchdog,
|
|
1029
|
+
trackRoute,
|
|
746
1030
|
stop,
|
|
747
1031
|
_anonymize: anonymize,
|
|
748
1032
|
_stackSignature: stackSignature,
|
package/openclaw.plugin.json
CHANGED
package/package.json
CHANGED
package/server.js
CHANGED
|
@@ -33,7 +33,7 @@ import {
|
|
|
33
33
|
import { actionFromReq, classifyError } from './lib/request-utils.js';
|
|
34
34
|
import { cleanupOrphanedTempFiles } from './lib/tmp-cleanup.js';
|
|
35
35
|
import { coalesceInflight } from './lib/inflight.js';
|
|
36
|
-
import { createReporter, createTabHealthTracker } from './lib/reporter.js';
|
|
36
|
+
import { createReporter, createTabHealthTracker, collectResourceSnapshot, classifyProxyError } from './lib/reporter.js';
|
|
37
37
|
import { mountDocs } from './lib/openapi.js';
|
|
38
38
|
|
|
39
39
|
const CONFIG = loadConfig();
|
|
@@ -42,18 +42,21 @@ const CONFIG = loadConfig();
|
|
|
42
42
|
import { readFileSync } from 'fs';
|
|
43
43
|
const _pkgVersion = (() => { try { return JSON.parse(readFileSync(new URL('./package.json', import.meta.url), 'utf8')).version; } catch { return 'unknown'; } })();
|
|
44
44
|
const reporter = createReporter({ ...CONFIG, version: _pkgVersion });
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
for (const
|
|
48
|
-
const
|
|
49
|
-
for (const group of session.tabGroups.values()) {
|
|
50
|
-
for (const tab of group.values()) {
|
|
51
|
-
try { if (tab.page) urls.push(tab.page.url()); } catch {}
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
summary.push({ userId, urls });
|
|
45
|
+
function _countTabs() {
|
|
46
|
+
let total = 0;
|
|
47
|
+
for (const session of sessions.values()) {
|
|
48
|
+
for (const group of session.tabGroups.values()) total += group.size;
|
|
55
49
|
}
|
|
56
|
-
return
|
|
50
|
+
return total;
|
|
51
|
+
}
|
|
52
|
+
function _browserPid() {
|
|
53
|
+
try { return browser?.process?.()?.pid ?? null; } catch { return null; }
|
|
54
|
+
}
|
|
55
|
+
function _resourceOpts() {
|
|
56
|
+
return { sessionCount: sessions.size, tabCount: _countTabs(), browserPid: _browserPid() };
|
|
57
|
+
}
|
|
58
|
+
reporter.startWatchdog(5000, () => {
|
|
59
|
+
return { resourceOpts: _resourceOpts() };
|
|
57
60
|
});
|
|
58
61
|
|
|
59
62
|
// --- Plugin event bus ---
|
|
@@ -101,6 +104,7 @@ app.use((req, res, next) => {
|
|
|
101
104
|
}
|
|
102
105
|
|
|
103
106
|
const action = actionFromReq(req);
|
|
107
|
+
reporter.trackRoute(`${req.method} ${req.route?.path || '[unmatched]'}`);
|
|
104
108
|
const done = requestDuration.startTimer({ action });
|
|
105
109
|
|
|
106
110
|
const origEnd = res.end.bind(res);
|
|
@@ -1033,10 +1037,19 @@ function handleRouteError(err, req, res, extraFields = {}) {
|
|
|
1033
1037
|
if (ts.failureJournal.length > 20) ts.failureJournal = ts.failureJournal.slice(-20);
|
|
1034
1038
|
|
|
1035
1039
|
if (ts.consecutiveFailures === 3) {
|
|
1040
|
+
const _proxyErr = classifyProxyError(err?.message);
|
|
1036
1041
|
reporter.reportHang(action, req.startTime ? Date.now() - req.startTime : 0, {
|
|
1037
1042
|
error: err,
|
|
1038
|
-
url: ts.lastRequestedUrl || undefined,
|
|
1039
1043
|
healthSnapshot: ts.healthTracker ? ts.healthTracker.snapshot() : undefined,
|
|
1044
|
+
healthTracker: ts.healthTracker || null,
|
|
1045
|
+
resourceOpts: _resourceOpts(),
|
|
1046
|
+
proxy: proxyPool ? {
|
|
1047
|
+
configured: true,
|
|
1048
|
+
type: proxyPool.mode || null,
|
|
1049
|
+
authConfigured: !!CONFIG.proxy?.username,
|
|
1050
|
+
error: _proxyErr.proxyError,
|
|
1051
|
+
tlsError: _proxyErr.proxyTlsError,
|
|
1052
|
+
} : { configured: false },
|
|
1040
1053
|
context: {
|
|
1041
1054
|
failureType,
|
|
1042
1055
|
consecutiveFailures: ts.consecutiveFailures,
|
|
@@ -4928,7 +4941,7 @@ setInterval(async () => {
|
|
|
4928
4941
|
process.on('uncaughtException', (err) => {
|
|
4929
4942
|
pluginEvents.emit('browser:error', { error: err });
|
|
4930
4943
|
log('error', 'uncaughtException', { error: err.message, stack: err.stack });
|
|
4931
|
-
reporter.reportCrash(err);
|
|
4944
|
+
reporter.reportCrash(err, { resourceOpts: _resourceOpts() });
|
|
4932
4945
|
process.exit(1);
|
|
4933
4946
|
});
|
|
4934
4947
|
process.on('unhandledRejection', (reason) => {
|