@askjo/camofox-browser 1.7.1 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/reporter.js +341 -49
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/server.js +27 -14
package/lib/reporter.js
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
// Config passed via createReporter(config) from lib/config.js.
|
|
4
4
|
|
|
5
5
|
import crypto from 'crypto';
|
|
6
|
+
import fs from 'fs';
|
|
7
|
+
import { execSync } from 'child_process';
|
|
6
8
|
|
|
7
9
|
// ============================================================================
|
|
8
10
|
// Anonymization
|
|
@@ -294,22 +296,57 @@ export function createUrlAnonymizer() {
|
|
|
294
296
|
// Per-tab health tracker (count-only, no content)
|
|
295
297
|
// ============================================================================
|
|
296
298
|
|
|
299
|
+
// Known bot-detection providers, matched by response header fingerprints.
|
|
300
|
+
// Order: most specific first.
|
|
301
|
+
const BOT_DETECTION_SIGNATURES = [
|
|
302
|
+
{ header: 'cf-mitigated', value: 'challenge', provider: 'cloudflare' },
|
|
303
|
+
{ header: 'x-datadome', provider: 'datadome' },
|
|
304
|
+
{ header: 'x-px', provider: 'perimeterx' },
|
|
305
|
+
{ header: 'x-distil-cs', provider: 'distil' },
|
|
306
|
+
{ header: 'x-sucuri-id', provider: 'sucuri' },
|
|
307
|
+
{ header: 'server', value: 'akamaighost', provider: 'akamai' },
|
|
308
|
+
// cf-ray is on ALL Cloudflare responses (even 200 OK). Must be last so it
|
|
309
|
+
// doesn't short-circuit other providers on multi-CDN sites.
|
|
310
|
+
{ header: 'cf-ray', provider: 'cloudflare' },
|
|
311
|
+
];
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Detect bot-detection provider from Playwright response headers.
|
|
315
|
+
* Returns { detected: bool, provider: string|null, httpStatus: number|null }
|
|
316
|
+
*/
|
|
317
|
+
export function detectBotProtection(response) {
|
|
318
|
+
if (!response) return { detected: false, provider: null, httpStatus: null };
|
|
319
|
+
const status = response.status();
|
|
320
|
+
let headers;
|
|
321
|
+
try { headers = response.headers(); } catch { return { detected: false, provider: null, httpStatus: status }; }
|
|
322
|
+
for (const sig of BOT_DETECTION_SIGNATURES) {
|
|
323
|
+
const val = headers[sig.header];
|
|
324
|
+
if (val !== undefined) {
|
|
325
|
+
if (sig.value && !val.toLowerCase().includes(sig.value)) continue;
|
|
326
|
+
const challenged = status === 403 || status === 429 || status === 503;
|
|
327
|
+
return { detected: challenged, provider: sig.provider, httpStatus: status };
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return { detected: false, provider: null, httpStatus: status };
|
|
331
|
+
}
|
|
332
|
+
|
|
297
333
|
/**
|
|
298
334
|
* Create a health tracker for a tab. Attaches to Playwright page events.
|
|
299
|
-
* Tracks: crashes, page errors,
|
|
300
|
-
*
|
|
335
|
+
* Tracks: crashes, page errors, request failures, redirect status codes,
|
|
336
|
+
* HTTP status histogram (4xx+), and anti-bot challenge detection.
|
|
301
337
|
* All count-based — no URLs or content stored.
|
|
302
338
|
*/
|
|
303
339
|
export function createTabHealthTracker(page) {
|
|
304
340
|
const health = {
|
|
305
341
|
crashes: 0,
|
|
306
342
|
pageErrors: 0,
|
|
307
|
-
consoleErrors: 0,
|
|
308
343
|
requestFailures: 0,
|
|
309
|
-
|
|
344
|
+
inflightRequests: 0,
|
|
310
345
|
maxRedirectDepth: 0,
|
|
311
|
-
|
|
312
|
-
|
|
346
|
+
redirectStatusCodes: [], // status codes in redirect chain, e.g. [301, 302, 403]
|
|
347
|
+
statusCounts: {}, // { 403: 5, 429: 2, ... }
|
|
348
|
+
botDetection: null, // { detected, provider, httpStatus } from last nav response
|
|
349
|
+
lastNavResponseSize: 0,
|
|
313
350
|
_redirectDepth: 0,
|
|
314
351
|
};
|
|
315
352
|
|
|
@@ -319,13 +356,15 @@ export function createTabHealthTracker(page) {
|
|
|
319
356
|
// Uncaught JS exceptions on the page
|
|
320
357
|
page.on('pageerror', () => { health.pageErrors++; });
|
|
321
358
|
|
|
322
|
-
//
|
|
323
|
-
page.on('
|
|
324
|
-
|
|
359
|
+
// Failed requests (blocked, DNS failure, etc.) + decrement in-flight counter
|
|
360
|
+
page.on('requestfailed', () => {
|
|
361
|
+
health.requestFailures++;
|
|
362
|
+
health.inflightRequests = Math.max(0, health.inflightRequests - 1);
|
|
325
363
|
});
|
|
326
364
|
|
|
327
|
-
//
|
|
328
|
-
page.on('
|
|
365
|
+
// Track in-flight requests for hang diagnostics
|
|
366
|
+
page.on('request', () => { health.inflightRequests++; });
|
|
367
|
+
page.on('requestfinished', () => { health.inflightRequests = Math.max(0, health.inflightRequests - 1); });
|
|
329
368
|
|
|
330
369
|
// HTTP status tracking (non-2xx only)
|
|
331
370
|
page.on('response', (resp) => {
|
|
@@ -333,13 +372,12 @@ export function createTabHealthTracker(page) {
|
|
|
333
372
|
if (s >= 400) health.statusCounts[s] = (health.statusCounts[s] || 0) + 1;
|
|
334
373
|
});
|
|
335
374
|
|
|
336
|
-
//
|
|
375
|
+
// Auto-dismiss dialogs to prevent page hangs (not tracked as a metric — noise)
|
|
337
376
|
page.on('dialog', async (dialog) => {
|
|
338
|
-
health.dialogCount++;
|
|
339
377
|
try { await dialog.dismiss(); } catch { /* page might be closed */ }
|
|
340
378
|
});
|
|
341
379
|
|
|
342
|
-
// Redirect depth per navigation
|
|
380
|
+
// Redirect depth + status code chain per navigation
|
|
343
381
|
page.on('request', (req) => {
|
|
344
382
|
if (req.isNavigationRequest()) {
|
|
345
383
|
if (req.redirectedFrom()) {
|
|
@@ -348,19 +386,120 @@ export function createTabHealthTracker(page) {
|
|
|
348
386
|
health.maxRedirectDepth = health._redirectDepth;
|
|
349
387
|
}
|
|
350
388
|
} else {
|
|
351
|
-
health._redirectDepth = 0;
|
|
389
|
+
health._redirectDepth = 0;
|
|
390
|
+
health.redirectStatusCodes = [];
|
|
391
|
+
health.inflightRequests = 0; // reset on new navigation to prevent drift
|
|
352
392
|
}
|
|
353
393
|
}
|
|
354
394
|
});
|
|
355
395
|
|
|
396
|
+
// Capture redirect status codes and detect bot protection on nav responses
|
|
397
|
+
page.on('response', (resp) => {
|
|
398
|
+
try {
|
|
399
|
+
const req = resp.request();
|
|
400
|
+
if (req.isNavigationRequest()) {
|
|
401
|
+
health.redirectStatusCodes.push(resp.status());
|
|
402
|
+
health.botDetection = detectBotProtection(resp);
|
|
403
|
+
// Approximate response body size from content-length (no body read)
|
|
404
|
+
const cl = resp.headers()['content-length'];
|
|
405
|
+
if (cl) health.lastNavResponseSize = parseInt(cl, 10) || 0;
|
|
406
|
+
}
|
|
407
|
+
} catch { /* page closed */ }
|
|
408
|
+
});
|
|
409
|
+
|
|
356
410
|
/** Snapshot current health counters for inclusion in reports. */
|
|
357
411
|
function snapshot() {
|
|
358
|
-
try { health.frameCount = page.frames().length; } catch { /* closed */ }
|
|
359
412
|
const { _redirectDepth, ...clean } = health;
|
|
360
413
|
return { ...clean };
|
|
361
414
|
}
|
|
362
415
|
|
|
363
|
-
|
|
416
|
+
/**
|
|
417
|
+
* Get document.readyState from the page. Returns null if page is unresponsive.
|
|
418
|
+
* Use a tight timeout — if the renderer is crashed, evaluate will hang.
|
|
419
|
+
*/
|
|
420
|
+
async function getReadyState() {
|
|
421
|
+
try {
|
|
422
|
+
return await Promise.race([
|
|
423
|
+
page.evaluate(() => document.readyState),
|
|
424
|
+
new Promise(resolve => setTimeout(() => resolve('unresponsive'), 1000)),
|
|
425
|
+
]);
|
|
426
|
+
} catch {
|
|
427
|
+
return 'unresponsive';
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
return { health, snapshot, getReadyState };
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// ============================================================================
|
|
435
|
+
// Process resource snapshot (memory, handles, FDs, browser RSS)
|
|
436
|
+
// ============================================================================
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* Collect process-level resource metrics. Safe to call at any time.
|
|
440
|
+
* Returns anonymized metrics — no PIDs, paths, or user data.
|
|
441
|
+
*/
|
|
442
|
+
export function collectResourceSnapshot(opts = {}) {
|
|
443
|
+
const mem = process.memoryUsage();
|
|
444
|
+
const snap = {
|
|
445
|
+
nodeRssMb: Math.round(mem.rss / 1048576),
|
|
446
|
+
nodeHeapUsedMb: Math.round(mem.heapUsed / 1048576),
|
|
447
|
+
nodeHeapTotalMb: Math.round(mem.heapTotal / 1048576),
|
|
448
|
+
nodeExternalMb: Math.round(mem.external / 1048576),
|
|
449
|
+
eventLoopLagMs: null,
|
|
450
|
+
activeHandles: null,
|
|
451
|
+
activeRequests: null,
|
|
452
|
+
openFds: null,
|
|
453
|
+
browserRssMb: null,
|
|
454
|
+
};
|
|
455
|
+
|
|
456
|
+
// Active libuv handles/requests (private API, guarded)
|
|
457
|
+
try { snap.activeHandles = process._getActiveHandles().length; } catch { /* unavailable */ }
|
|
458
|
+
try { snap.activeRequests = process._getActiveRequests().length; } catch { /* unavailable */ }
|
|
459
|
+
|
|
460
|
+
// Open file descriptors (Linux only)
|
|
461
|
+
try {
|
|
462
|
+
if (process.platform === 'linux') {
|
|
463
|
+
snap.openFds = fs.readdirSync('/proc/self/fd').length;
|
|
464
|
+
}
|
|
465
|
+
} catch { /* not available or permission denied */ }
|
|
466
|
+
|
|
467
|
+
// Browser process RSS (the one people miss — browser OOMs, not Node)
|
|
468
|
+
if (opts.browserPid && Number.isInteger(opts.browserPid) && opts.browserPid > 0) {
|
|
469
|
+
try {
|
|
470
|
+
if (process.platform === 'linux') {
|
|
471
|
+
const status = fs.readFileSync(`/proc/${opts.browserPid}/status`, 'utf8');
|
|
472
|
+
const match = status.match(/VmRSS:\s+(\d+)\s+kB/);
|
|
473
|
+
if (match) snap.browserRssMb = Math.round(parseInt(match[1], 10) / 1024);
|
|
474
|
+
} else if (process.platform === 'darwin') {
|
|
475
|
+
const out = execSync(`ps -o rss= -p ${opts.browserPid}`, { timeout: 1000 }).toString().trim();
|
|
476
|
+
if (out) snap.browserRssMb = Math.round(parseInt(out, 10) / 1024);
|
|
477
|
+
}
|
|
478
|
+
} catch { /* process gone or permission denied */ }
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// Session/tab counts from caller
|
|
482
|
+
if (opts.sessionCount != null) snap.browserContexts = opts.sessionCount;
|
|
483
|
+
if (opts.tabCount != null) snap.activeTabs = opts.tabCount;
|
|
484
|
+
|
|
485
|
+
return snap;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* Classify proxy errors from Playwright navigation error messages.
|
|
490
|
+
* Returns { proxyError: string|null, proxyTlsError: bool } — no IPs or credentials.
|
|
491
|
+
*/
|
|
492
|
+
export function classifyProxyError(errorMessage) {
|
|
493
|
+
if (!errorMessage || typeof errorMessage !== 'string') return { proxyError: null, proxyTlsError: false };
|
|
494
|
+
const msg = errorMessage.toUpperCase();
|
|
495
|
+
// Explicit proxy errors from Chromium/Firefox net stack
|
|
496
|
+
if (msg.includes('ERR_PROXY_CONNECTION_FAILED')) return { proxyError: 'ERR_PROXY_CONNECTION_FAILED', proxyTlsError: false };
|
|
497
|
+
if (msg.includes('ERR_TUNNEL_CONNECTION_FAILED')) return { proxyError: 'ERR_TUNNEL_CONNECTION_FAILED', proxyTlsError: false };
|
|
498
|
+
if (msg.includes('ERR_PROXY_AUTH_REQUESTED') || msg.includes('407')) return { proxyError: 'ERR_PROXY_AUTH_REQUESTED', proxyTlsError: false };
|
|
499
|
+
if (msg.includes('ERR_PROXY_CERTIFICATE_INVALID') || (msg.includes('PROXY') && msg.includes('SSL'))) return { proxyError: 'ERR_PROXY_TLS', proxyTlsError: true };
|
|
500
|
+
if (msg.includes('ECONNREFUSED') && msg.includes('PROXY')) return { proxyError: 'ECONNREFUSED', proxyTlsError: false };
|
|
501
|
+
if (msg.includes('ETIMEDOUT') && msg.includes('PROXY')) return { proxyError: 'ETIMEDOUT', proxyTlsError: false };
|
|
502
|
+
return { proxyError: null, proxyTlsError: false };
|
|
364
503
|
}
|
|
365
504
|
|
|
366
505
|
// ============================================================================
|
|
@@ -510,24 +649,86 @@ function formatIssueBody(type, detail) {
|
|
|
510
649
|
const sections = [
|
|
511
650
|
'> Auto-reported by ' + _GH_USER_AGENT + '. All data is anonymized.',
|
|
512
651
|
'',
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
652
|
+
'## Environment',
|
|
653
|
+
`- **version:** ${detail.version || 'unknown'}`,
|
|
654
|
+
`- **node:** ${detail.nodeVersion || 'unknown'}`,
|
|
655
|
+
`- **platform:** ${detail.platform || 'unknown'}`,
|
|
656
|
+
`- **uptime:** ${detail.uptimeMinutes != null ? detail.uptimeMinutes + ' min' : 'unknown'}`,
|
|
518
657
|
];
|
|
519
658
|
|
|
659
|
+
// Resource snapshot (memory, handles, browser RSS)
|
|
660
|
+
const r = detail.resources;
|
|
661
|
+
if (r) {
|
|
662
|
+
sections.push('', '## Resources');
|
|
663
|
+
sections.push(`- **node RSS:** ${r.nodeRssMb ?? '?'} MB`);
|
|
664
|
+
sections.push(`- **node heap:** ${r.nodeHeapUsedMb ?? '?'} / ${r.nodeHeapTotalMb ?? '?'} MB`);
|
|
665
|
+
if (r.browserRssMb != null) sections.push(`- **browser RSS:** ${r.browserRssMb} MB`);
|
|
666
|
+
if (r.browserContexts != null) sections.push(`- **browser contexts:** ${r.browserContexts}`);
|
|
667
|
+
if (r.activeTabs != null) sections.push(`- **active tabs:** ${r.activeTabs}`);
|
|
668
|
+
if (r.openFds != null) sections.push(`- **open FDs:** ${r.openFds}`);
|
|
669
|
+
if (r.activeHandles != null) sections.push(`- **active handles:** ${r.activeHandles}`);
|
|
670
|
+
if (r.eventLoopLagMs != null) sections.push(`- **event loop lag:** ${r.eventLoopLagMs} ms`);
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// Error info
|
|
674
|
+
if (detail.signal) sections.push('', `**Signal:** ${detail.signal}`);
|
|
675
|
+
if (detail.activeRoute) sections.push(`**Active route:** ${detail.activeRoute}`);
|
|
520
676
|
if (detail.message) {
|
|
521
|
-
sections.push('', '
|
|
677
|
+
sections.push('', '## Error', '```', anonymize(detail.message), '```');
|
|
522
678
|
}
|
|
523
679
|
if (detail.stack) {
|
|
524
|
-
sections.push('', '
|
|
680
|
+
sections.push('', '## Stack Trace', '```', anonymize(detail.stack), '```');
|
|
525
681
|
}
|
|
526
|
-
|
|
527
|
-
|
|
682
|
+
|
|
683
|
+
// Hang-specific details
|
|
684
|
+
if (detail.hang) {
|
|
685
|
+
const h = detail.hang;
|
|
686
|
+
sections.push('', '## Hang Details');
|
|
687
|
+
sections.push(`- **operation:** ${h.operation}`);
|
|
688
|
+
sections.push(`- **duration:** ${Math.round(h.durationMs / 1000)}s`);
|
|
689
|
+
if (h.lockQueueMs != null) sections.push(`- **lock queue wait:** ${Math.round(h.lockQueueMs)}ms`);
|
|
690
|
+
if (h.documentReadyState) sections.push(`- **document.readyState:** ${h.documentReadyState}`);
|
|
691
|
+
if (h.inflightRequests != null) sections.push(`- **in-flight requests:** ${h.inflightRequests}`);
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
// Anti-bot detection
|
|
695
|
+
if (detail.botDetection?.detected) {
|
|
696
|
+
const b = detail.botDetection;
|
|
697
|
+
sections.push('', '## Anti-Bot Detection');
|
|
698
|
+
sections.push(`- **provider:** ${b.provider || 'unknown'}`);
|
|
699
|
+
sections.push(`- **HTTP status:** ${b.httpStatus || '?'}`);
|
|
700
|
+
if (b.responseBodySizeKb != null) sections.push(`- **response size:** ${b.responseBodySizeKb} KB`);
|
|
701
|
+
if (b.redirectChainLength != null) sections.push(`- **redirect chain:** ${b.redirectChainLength} hops`);
|
|
702
|
+
if (b.redirectStatusCodes?.length) sections.push(`- **redirect statuses:** ${b.redirectStatusCodes.join(' → ')}`);
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// Proxy info (safe fields only — no IPs, credentials, or hostnames)
|
|
706
|
+
if (detail.proxy) {
|
|
707
|
+
const p = detail.proxy;
|
|
708
|
+
sections.push('', '## Proxy');
|
|
709
|
+
sections.push(`- **configured:** ${p.configured}`);
|
|
710
|
+
if (p.configured) {
|
|
711
|
+
if (p.type) sections.push(`- **type:** ${p.type}`);
|
|
712
|
+
sections.push(`- **auth configured:** ${p.authConfigured ?? 'unknown'}`);
|
|
713
|
+
if (p.error) sections.push(`- **error:** ${p.error}`);
|
|
714
|
+
if (p.tlsError) sections.push(`- **TLS error:** yes`);
|
|
715
|
+
}
|
|
528
716
|
}
|
|
529
|
-
|
|
530
|
-
|
|
717
|
+
|
|
718
|
+
// Stall-specific details
|
|
719
|
+
if (detail.stall) {
|
|
720
|
+
const s = detail.stall;
|
|
721
|
+
sections.push('', '## Stall Details');
|
|
722
|
+
sections.push(`- **stall duration:** ${Math.round(s.driftMs / 1000)}s`);
|
|
723
|
+
if (s.lastRoute) sections.push(`- **last route:** ${s.lastRoute}`);
|
|
724
|
+
if (s.activeHandles != null) sections.push(`- **active handles:** ${s.activeHandles}`);
|
|
725
|
+
if (s.activeRequests != null) sections.push(`- **active requests:** ${s.activeRequests}`);
|
|
726
|
+
if (s.heapDeltaMb != null) sections.push(`- **heap delta:** ${s.heapDeltaMb > 0 ? '+' : ''}${s.heapDeltaMb} MB`);
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
// Context (misc extra data)
|
|
730
|
+
if (detail.context && Object.keys(detail.context).length > 0) {
|
|
731
|
+
sections.push('', '<details><summary>Context</summary>', '', '```json', anonymize(JSON.stringify(detail.context, null, 2)), '```', '', '</details>');
|
|
531
732
|
}
|
|
532
733
|
|
|
533
734
|
return sections.join('\n');
|
|
@@ -539,6 +740,14 @@ function formatCommentBody(type, detail) {
|
|
|
539
740
|
`**+1** — ${ts}`,
|
|
540
741
|
`Version: ${detail.version || 'unknown'}, Uptime: ${detail.uptimeMinutes != null ? detail.uptimeMinutes + ' min' : '?'}`,
|
|
541
742
|
];
|
|
743
|
+
// Include resource snapshot in +1 comments too
|
|
744
|
+
const r = detail.resources;
|
|
745
|
+
if (r) {
|
|
746
|
+
const parts = [`RSS: ${r.nodeRssMb ?? '?'}MB`];
|
|
747
|
+
if (r.browserRssMb != null) parts.push(`Browser: ${r.browserRssMb}MB`);
|
|
748
|
+
if (r.activeTabs != null) parts.push(`Tabs: ${r.activeTabs}`);
|
|
749
|
+
lines.push(parts.join(', '));
|
|
750
|
+
}
|
|
542
751
|
if (detail.message) {
|
|
543
752
|
lines.push('```', anonymize(detail.message).slice(0, 500), '```');
|
|
544
753
|
}
|
|
@@ -578,6 +787,9 @@ export function createReporter(config) {
|
|
|
578
787
|
let lastTick = Date.now();
|
|
579
788
|
const inFlight = new Set();
|
|
580
789
|
|
|
790
|
+
// Track last Express route for stall reports
|
|
791
|
+
let _lastRoute = null;
|
|
792
|
+
|
|
581
793
|
// No-op when disabled
|
|
582
794
|
if (!enabled) {
|
|
583
795
|
return {
|
|
@@ -585,6 +797,7 @@ export function createReporter(config) {
|
|
|
585
797
|
reportHang: async () => {},
|
|
586
798
|
reportStuckLoop: async () => {},
|
|
587
799
|
startWatchdog: () => {},
|
|
800
|
+
trackRoute: () => {},
|
|
588
801
|
stop: () => {},
|
|
589
802
|
_anonymize: anonymize,
|
|
590
803
|
_stackSignature: stackSignature,
|
|
@@ -592,7 +805,7 @@ export function createReporter(config) {
|
|
|
592
805
|
}
|
|
593
806
|
|
|
594
807
|
/** Core: file or deduplicate a report. NEVER throws. */
|
|
595
|
-
async function fileReport(type,
|
|
808
|
+
async function fileReport(type, labels, detail) {
|
|
596
809
|
if (!rateLimiter.tryAcquire()) return;
|
|
597
810
|
|
|
598
811
|
const reportPromise = (async () => {
|
|
@@ -619,7 +832,8 @@ export function createReporter(config) {
|
|
|
619
832
|
platform: typeof process !== 'undefined' ? process.platform : 'unknown',
|
|
620
833
|
});
|
|
621
834
|
|
|
622
|
-
|
|
835
|
+
const issueLabels = Array.isArray(labels) ? labels : [labels, 'auto-report'];
|
|
836
|
+
await createIssue(repo, title, body, issueLabels);
|
|
623
837
|
} catch {
|
|
624
838
|
// Swallow — reporter must never crash the server
|
|
625
839
|
}
|
|
@@ -629,19 +843,32 @@ export function createReporter(config) {
|
|
|
629
843
|
reportPromise.finally(() => inFlight.delete(reportPromise));
|
|
630
844
|
}
|
|
631
845
|
|
|
846
|
+
/**
|
|
847
|
+
* Track the last Express route for stall diagnostics.
|
|
848
|
+
* Call from middleware: reporter.trackRoute(req.method + ' ' + req.route?.path)
|
|
849
|
+
*/
|
|
850
|
+
function trackRoute(route) {
|
|
851
|
+
_lastRoute = route || null;
|
|
852
|
+
}
|
|
853
|
+
|
|
632
854
|
async function reportCrash(error, opts = {}) {
|
|
633
855
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
634
856
|
const uptimeMinutes = typeof process !== 'undefined'
|
|
635
857
|
? Math.round(process.uptime() / 60) : undefined;
|
|
858
|
+
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
636
859
|
|
|
637
860
|
await fileReport(
|
|
638
861
|
opts.signal ? `signal:${opts.signal}` : (err.name || 'crash'),
|
|
639
|
-
'crash',
|
|
862
|
+
['crash', 'auto-report'],
|
|
640
863
|
{
|
|
641
864
|
error: err,
|
|
642
865
|
message: err.message,
|
|
643
866
|
stack: err.stack,
|
|
867
|
+
signal: opts.signal || null,
|
|
868
|
+
activeRoute: _lastRoute,
|
|
644
869
|
uptimeMinutes,
|
|
870
|
+
resources,
|
|
871
|
+
proxy: opts.proxy || null,
|
|
645
872
|
context: opts.context,
|
|
646
873
|
},
|
|
647
874
|
);
|
|
@@ -650,32 +877,55 @@ export function createReporter(config) {
|
|
|
650
877
|
async function reportHang(operation, durationMs, opts = {}) {
|
|
651
878
|
const uptimeMinutes = typeof process !== 'undefined'
|
|
652
879
|
? Math.round(process.uptime() / 60) : undefined;
|
|
880
|
+
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
653
881
|
|
|
654
|
-
//
|
|
655
|
-
const
|
|
656
|
-
const context = { operation, durationMs, ...opts.context };
|
|
657
|
-
|
|
658
|
-
// Anonymize any URLs in the journal
|
|
882
|
+
// Build lean context (journal only, no redundant fields)
|
|
883
|
+
const context = { ...opts.context };
|
|
659
884
|
if (context.journal) {
|
|
660
|
-
context.journal = context.journal.map(j =>
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
885
|
+
context.journal = context.journal.map(j => typeof j === 'string' ? j : j);
|
|
886
|
+
}
|
|
887
|
+
// Remove fields that now have dedicated sections
|
|
888
|
+
delete context.operation;
|
|
889
|
+
delete context.durationMs;
|
|
890
|
+
|
|
891
|
+
// Anti-bot detection from health snapshot
|
|
892
|
+
const healthSnap = opts.healthSnapshot;
|
|
893
|
+
const botDetection = healthSnap?.botDetection?.detected ? {
|
|
894
|
+
...healthSnap.botDetection,
|
|
895
|
+
responseBodySizeKb: healthSnap.lastNavResponseSize
|
|
896
|
+
? Math.round(healthSnap.lastNavResponseSize / 1024) : null,
|
|
897
|
+
redirectChainLength: healthSnap.redirectStatusCodes?.length || null,
|
|
898
|
+
redirectStatusCodes: healthSnap.redirectStatusCodes?.length
|
|
899
|
+
? healthSnap.redirectStatusCodes : null,
|
|
900
|
+
} : null;
|
|
901
|
+
|
|
902
|
+
// Get document.readyState if healthTracker provided
|
|
903
|
+
let documentReadyState = null;
|
|
904
|
+
if (opts.healthTracker?.getReadyState) {
|
|
905
|
+
documentReadyState = await opts.healthTracker.getReadyState();
|
|
664
906
|
}
|
|
665
|
-
// Include anonymized URL if provided
|
|
666
|
-
if (opts.url) context.url = urlAnon.anonymizeUrl(opts.url);
|
|
667
|
-
if (opts.redirectChain) context.redirectChain = urlAnon.anonymizeChain(opts.redirectChain);
|
|
668
907
|
|
|
669
|
-
|
|
670
|
-
if (
|
|
908
|
+
const labels = ['hang', 'auto-report'];
|
|
909
|
+
if (botDetection?.detected) labels.push('bot-detection');
|
|
671
910
|
|
|
672
911
|
await fileReport(
|
|
673
912
|
`hang:${operation}`,
|
|
674
|
-
|
|
913
|
+
labels,
|
|
675
914
|
{
|
|
676
915
|
message: `Operation "${operation}" hung for ${Math.round(durationMs / 1000)}s`,
|
|
677
916
|
stack: opts.error?.stack,
|
|
917
|
+
activeRoute: _lastRoute,
|
|
678
918
|
uptimeMinutes,
|
|
919
|
+
resources,
|
|
920
|
+
hang: {
|
|
921
|
+
operation,
|
|
922
|
+
durationMs,
|
|
923
|
+
lockQueueMs: opts.lockQueueMs ?? null,
|
|
924
|
+
documentReadyState,
|
|
925
|
+
inflightRequests: healthSnap?.inflightRequests ?? null,
|
|
926
|
+
},
|
|
927
|
+
botDetection,
|
|
928
|
+
proxy: opts.proxy || null,
|
|
679
929
|
context,
|
|
680
930
|
},
|
|
681
931
|
);
|
|
@@ -684,13 +934,15 @@ export function createReporter(config) {
|
|
|
684
934
|
async function reportStuckLoop(durationMs, opts = {}) {
|
|
685
935
|
const uptimeMinutes = typeof process !== 'undefined'
|
|
686
936
|
? Math.round(process.uptime() / 60) : undefined;
|
|
937
|
+
const resources = collectResourceSnapshot(opts.resourceOpts || {});
|
|
687
938
|
|
|
688
939
|
await fileReport(
|
|
689
940
|
'stuck:tab-lock',
|
|
690
|
-
'stuck',
|
|
941
|
+
['stuck', 'auto-report'],
|
|
691
942
|
{
|
|
692
943
|
message: `Tab lock held for ${Math.round(durationMs / 1000)}s (tab destroyed)`,
|
|
693
944
|
uptimeMinutes,
|
|
945
|
+
resources,
|
|
694
946
|
context: { durationMs, ...opts.context },
|
|
695
947
|
},
|
|
696
948
|
);
|
|
@@ -701,21 +953,60 @@ export function createReporter(config) {
|
|
|
701
953
|
|
|
702
954
|
const checkMs = 1000;
|
|
703
955
|
lastTick = Date.now();
|
|
956
|
+
let lastHeapUsed = process.memoryUsage().heapUsed;
|
|
957
|
+
// Suppress false positives from OS sleep/suspend (laptop lid close, VM pause).
|
|
958
|
+
// Stalls > 120s are almost certainly not event-loop bugs.
|
|
959
|
+
const MAX_REPORTABLE_DRIFT_MS = 120_000;
|
|
960
|
+
let suppressTicksRemaining = 0;
|
|
961
|
+
const SUPPRESS_TICKS_AFTER_WAKE = 5;
|
|
704
962
|
|
|
705
963
|
watchdogInterval = setInterval(() => {
|
|
706
964
|
const now = Date.now();
|
|
707
965
|
const drift = now - lastTick - checkMs;
|
|
708
966
|
lastTick = now;
|
|
709
967
|
|
|
968
|
+
// After a long sleep/suspend, suppress the next few ticks (post-wake jitter)
|
|
969
|
+
if (drift > MAX_REPORTABLE_DRIFT_MS) {
|
|
970
|
+
suppressTicksRemaining = SUPPRESS_TICKS_AFTER_WAKE;
|
|
971
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
972
|
+
return;
|
|
973
|
+
}
|
|
974
|
+
if (suppressTicksRemaining > 0) {
|
|
975
|
+
suppressTicksRemaining--;
|
|
976
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
977
|
+
return;
|
|
978
|
+
}
|
|
979
|
+
|
|
710
980
|
if (drift > thresholdMs) {
|
|
981
|
+
// Capture heap delta during stall (GC indicator)
|
|
982
|
+
const currentHeap = process.memoryUsage().heapUsed;
|
|
983
|
+
const heapDeltaMb = Math.round((currentHeap - lastHeapUsed) / 1048576);
|
|
984
|
+
lastHeapUsed = currentHeap;
|
|
985
|
+
|
|
711
986
|
let extra = {};
|
|
712
987
|
try { if (getContext) extra = getContext(); } catch { /* swallow */ }
|
|
713
|
-
|
|
988
|
+
|
|
989
|
+
const resources = collectResourceSnapshot(extra.resourceOpts || {});
|
|
990
|
+
// Remove resourceOpts from extra so it doesn't end up in context
|
|
991
|
+
delete extra.resourceOpts;
|
|
992
|
+
|
|
993
|
+
fileReport('stuck:event-loop', ['stuck', 'auto-report'], {
|
|
714
994
|
message: `Event loop stalled for ${Math.round(drift / 1000)}s (threshold: ${Math.round(thresholdMs / 1000)}s)`,
|
|
715
995
|
uptimeMinutes: typeof process !== 'undefined'
|
|
716
996
|
? Math.round(process.uptime() / 60) : undefined,
|
|
717
|
-
|
|
997
|
+
resources,
|
|
998
|
+
stall: {
|
|
999
|
+
driftMs: drift,
|
|
1000
|
+
thresholdMs,
|
|
1001
|
+
lastRoute: _lastRoute,
|
|
1002
|
+
activeHandles: resources.activeHandles,
|
|
1003
|
+
activeRequests: resources.activeRequests,
|
|
1004
|
+
heapDeltaMb,
|
|
1005
|
+
},
|
|
1006
|
+
context: extra,
|
|
718
1007
|
});
|
|
1008
|
+
} else {
|
|
1009
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
719
1010
|
}
|
|
720
1011
|
}, checkMs);
|
|
721
1012
|
|
|
@@ -735,6 +1026,7 @@ export function createReporter(config) {
|
|
|
735
1026
|
reportHang,
|
|
736
1027
|
reportStuckLoop,
|
|
737
1028
|
startWatchdog,
|
|
1029
|
+
trackRoute,
|
|
738
1030
|
stop,
|
|
739
1031
|
_anonymize: anonymize,
|
|
740
1032
|
_stackSignature: stackSignature,
|
package/openclaw.plugin.json
CHANGED
package/package.json
CHANGED
package/server.js
CHANGED
|
@@ -33,7 +33,7 @@ import {
|
|
|
33
33
|
import { actionFromReq, classifyError } from './lib/request-utils.js';
|
|
34
34
|
import { cleanupOrphanedTempFiles } from './lib/tmp-cleanup.js';
|
|
35
35
|
import { coalesceInflight } from './lib/inflight.js';
|
|
36
|
-
import { createReporter, createTabHealthTracker } from './lib/reporter.js';
|
|
36
|
+
import { createReporter, createTabHealthTracker, collectResourceSnapshot, classifyProxyError } from './lib/reporter.js';
|
|
37
37
|
import { mountDocs } from './lib/openapi.js';
|
|
38
38
|
|
|
39
39
|
const CONFIG = loadConfig();
|
|
@@ -42,18 +42,21 @@ const CONFIG = loadConfig();
|
|
|
42
42
|
import { readFileSync } from 'fs';
|
|
43
43
|
const _pkgVersion = (() => { try { return JSON.parse(readFileSync(new URL('./package.json', import.meta.url), 'utf8')).version; } catch { return 'unknown'; } })();
|
|
44
44
|
const reporter = createReporter({ ...CONFIG, version: _pkgVersion });
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
for (const
|
|
48
|
-
const
|
|
49
|
-
for (const group of session.tabGroups.values()) {
|
|
50
|
-
for (const tab of group.values()) {
|
|
51
|
-
try { if (tab.page) urls.push(tab.page.url()); } catch {}
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
summary.push({ userId, urls });
|
|
45
|
+
function _countTabs() {
|
|
46
|
+
let total = 0;
|
|
47
|
+
for (const session of sessions.values()) {
|
|
48
|
+
for (const group of session.tabGroups.values()) total += group.size;
|
|
55
49
|
}
|
|
56
|
-
return
|
|
50
|
+
return total;
|
|
51
|
+
}
|
|
52
|
+
function _browserPid() {
|
|
53
|
+
try { return browser?.process?.()?.pid ?? null; } catch { return null; }
|
|
54
|
+
}
|
|
55
|
+
function _resourceOpts() {
|
|
56
|
+
return { sessionCount: sessions.size, tabCount: _countTabs(), browserPid: _browserPid() };
|
|
57
|
+
}
|
|
58
|
+
reporter.startWatchdog(5000, () => {
|
|
59
|
+
return { resourceOpts: _resourceOpts() };
|
|
57
60
|
});
|
|
58
61
|
|
|
59
62
|
// --- Plugin event bus ---
|
|
@@ -101,6 +104,7 @@ app.use((req, res, next) => {
|
|
|
101
104
|
}
|
|
102
105
|
|
|
103
106
|
const action = actionFromReq(req);
|
|
107
|
+
reporter.trackRoute(`${req.method} ${req.route?.path || '[unmatched]'}`);
|
|
104
108
|
const done = requestDuration.startTimer({ action });
|
|
105
109
|
|
|
106
110
|
const origEnd = res.end.bind(res);
|
|
@@ -1033,10 +1037,19 @@ function handleRouteError(err, req, res, extraFields = {}) {
|
|
|
1033
1037
|
if (ts.failureJournal.length > 20) ts.failureJournal = ts.failureJournal.slice(-20);
|
|
1034
1038
|
|
|
1035
1039
|
if (ts.consecutiveFailures === 3) {
|
|
1040
|
+
const _proxyErr = classifyProxyError(err?.message);
|
|
1036
1041
|
reporter.reportHang(action, req.startTime ? Date.now() - req.startTime : 0, {
|
|
1037
1042
|
error: err,
|
|
1038
|
-
url: ts.lastRequestedUrl || undefined,
|
|
1039
1043
|
healthSnapshot: ts.healthTracker ? ts.healthTracker.snapshot() : undefined,
|
|
1044
|
+
healthTracker: ts.healthTracker || null,
|
|
1045
|
+
resourceOpts: _resourceOpts(),
|
|
1046
|
+
proxy: proxyPool ? {
|
|
1047
|
+
configured: true,
|
|
1048
|
+
type: proxyPool.mode || null,
|
|
1049
|
+
authConfigured: !!CONFIG.proxy?.username,
|
|
1050
|
+
error: _proxyErr.proxyError,
|
|
1051
|
+
tlsError: _proxyErr.proxyTlsError,
|
|
1052
|
+
} : { configured: false },
|
|
1040
1053
|
context: {
|
|
1041
1054
|
failureType,
|
|
1042
1055
|
consecutiveFailures: ts.consecutiveFailures,
|
|
@@ -4928,7 +4941,7 @@ setInterval(async () => {
|
|
|
4928
4941
|
process.on('uncaughtException', (err) => {
|
|
4929
4942
|
pluginEvents.emit('browser:error', { error: err });
|
|
4930
4943
|
log('error', 'uncaughtException', { error: err.message, stack: err.stack });
|
|
4931
|
-
reporter.reportCrash(err);
|
|
4944
|
+
reporter.reportCrash(err, { resourceOpts: _resourceOpts() });
|
|
4932
4945
|
process.exit(1);
|
|
4933
4946
|
});
|
|
4934
4947
|
process.on('unhandledRejection', (reason) => {
|