@askjo/camofox-browser 1.7.2 → 1.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/reporter.js CHANGED
@@ -4,7 +4,7 @@
4
4
 
5
5
  import crypto from 'crypto';
6
6
  import fs from 'fs';
7
- import { execSync } from 'child_process';
7
+ import { monitorEventLoopDelay } from 'perf_hooks';
8
8
 
9
9
  // ============================================================================
10
10
  // Anonymization
@@ -720,6 +720,15 @@ function formatIssueBody(type, detail) {
720
720
  const s = detail.stall;
721
721
  sections.push('', '## Stall Details');
722
722
  sections.push(`- **stall duration:** ${Math.round(s.driftMs / 1000)}s`);
723
+ if (s.classification) sections.push(`- **classification:** ${s.classification}`);
724
+ if (s.cpuElapsedS != null) sections.push(`- **CPU time during stall:** ${s.cpuElapsedS}s`);
725
+ if (s.cpuRatio != null) sections.push(`- **CPU/wall ratio:** ${s.cpuRatio}`);
726
+ if (s.sigcontInWindow != null) sections.push(`- **SIGCONT in window:** ${s.sigcontInWindow}`);
727
+ if (s.hrtimeWallDriftS != null) sections.push(`- **hrtime↔wall drift:** ${s.hrtimeWallDriftS}s`);
728
+ if (s.eventLoopDelay) {
729
+ const eld = s.eventLoopDelay;
730
+ sections.push(`- **event loop delay:** p50=${eld.p50Ms}ms p99=${eld.p99Ms}ms max=${eld.maxMs}ms`);
731
+ }
723
732
  if (s.lastRoute) sections.push(`- **last route:** ${s.lastRoute}`);
724
733
  if (s.activeHandles != null) sections.push(`- **active handles:** ${s.activeHandles}`);
725
734
  if (s.activeRequests != null) sections.push(`- **active requests:** ${s.activeRequests}`);
@@ -784,6 +793,7 @@ export function createReporter(config) {
784
793
  const version = config.version || 'unknown';
785
794
 
786
795
  let watchdogInterval = null;
796
+ let _resetNativeMemBaseline = false; // Set by resetNativeMemBaseline(), read by watchdog
787
797
  let lastTick = Date.now();
788
798
  const inFlight = new Set();
789
799
 
@@ -953,7 +963,31 @@ export function createReporter(config) {
953
963
 
954
964
  const checkMs = 1000;
955
965
  lastTick = Date.now();
966
+ let lastCpuUsage = process.cpuUsage();
967
+ let lastHrtime = process.hrtime.bigint();
956
968
  let lastHeapUsed = process.memoryUsage().heapUsed;
969
+
970
+ // --- Native memory leak tracking ---
971
+ // Track RSS minus JS heap over time to detect native/external memory leaks.
972
+ // Sample every 30s, alert if native memory grows by >200MB from baseline.
973
+ let nativeMemBaseline = null; // RSS - heapUsed at first measurement
974
+ let nativeMemHighWater = 0;
975
+ let lastNativeMemCheck = 0;
976
+ const NATIVE_MEM_CHECK_INTERVAL_MS = 30_000;
977
+ const NATIVE_MEM_LEAK_THRESHOLD_MB = 200; // alert if native mem exceeds baseline by this much
978
+ let nativeMemAlertFired = false;
979
+
980
+ // SIGCONT detection — macOS sends SIGCONT on wake from sleep/suspend
981
+ let lastSigcont = 0;
982
+ try { process.on('SIGCONT', () => { lastSigcont = Date.now(); }); } catch { /* unavailable */ }
983
+
984
+ // Event loop delay histogram (perf_hooks) — correlating evidence
985
+ let elHistogram = null;
986
+ try {
987
+ elHistogram = monitorEventLoopDelay({ resolution: 20 });
988
+ elHistogram.enable();
989
+ } catch { /* unavailable */ }
990
+
957
991
  // Suppress false positives from OS sleep/suspend (laptop lid close, VM pause).
958
992
  // Stalls > 120s are almost certainly not event-loop bugs.
959
993
  const MAX_REPORTABLE_DRIFT_MS = 120_000;
@@ -963,7 +997,13 @@ export function createReporter(config) {
963
997
  watchdogInterval = setInterval(() => {
964
998
  const now = Date.now();
965
999
  const drift = now - lastTick - checkMs;
1000
+ const cpuDelta = process.cpuUsage(lastCpuUsage);
1001
+ const hrtimeNow = process.hrtime.bigint();
1002
+ const hrtimeDeltaMs = Number(hrtimeNow - lastHrtime) / 1e6;
1003
+
966
1004
  lastTick = now;
1005
+ lastCpuUsage = process.cpuUsage();
1006
+ lastHrtime = hrtimeNow;
967
1007
 
968
1008
  // After a long sleep/suspend, suppress the next few ticks (post-wake jitter)
969
1009
  if (drift > MAX_REPORTABLE_DRIFT_MS) {
@@ -977,7 +1017,77 @@ export function createReporter(config) {
977
1017
  return;
978
1018
  }
979
1019
 
1020
+ // --- Native memory leak detection (runs every ~30s) ---
1021
+ if (now - lastNativeMemCheck >= NATIVE_MEM_CHECK_INTERVAL_MS) {
1022
+ lastNativeMemCheck = now;
1023
+ try {
1024
+ // Check if baseline should be reset (e.g. after browser close)
1025
+ if (_resetNativeMemBaseline) {
1026
+ nativeMemBaseline = null;
1027
+ nativeMemHighWater = 0;
1028
+ nativeMemAlertFired = false;
1029
+ _resetNativeMemBaseline = false;
1030
+ }
1031
+ const mem = process.memoryUsage();
1032
+ const nativeMemMb = Math.round((mem.rss - mem.heapUsed) / 1048576);
1033
+ if (nativeMemBaseline === null) {
1034
+ nativeMemBaseline = nativeMemMb;
1035
+ }
1036
+ nativeMemHighWater = Math.max(nativeMemHighWater, nativeMemMb);
1037
+ const growth = nativeMemMb - nativeMemBaseline;
1038
+
1039
+ if (growth > NATIVE_MEM_LEAK_THRESHOLD_MB && !nativeMemAlertFired) {
1040
+ nativeMemAlertFired = true;
1041
+ let extra = {};
1042
+ try { if (getContext) extra = getContext(); } catch { /* swallow */ }
1043
+ const resources = collectResourceSnapshot(extra.resourceOpts || {});
1044
+ delete extra.resourceOpts;
1045
+
1046
+ fileReport('leak:native-memory', ['auto-report', 'memory-leak'], {
1047
+ message: `Native memory grew by ${growth}MB (baseline: ${nativeMemBaseline}MB, current: ${nativeMemMb}MB, high-water: ${nativeMemHighWater}MB)`,
1048
+ uptimeMinutes: Math.round(process.uptime() / 60),
1049
+ resources,
1050
+ nativeMemory: {
1051
+ baselineMb: nativeMemBaseline,
1052
+ currentMb: nativeMemMb,
1053
+ highWaterMb: nativeMemHighWater,
1054
+ growthMb: growth,
1055
+ rssMb: Math.round(mem.rss / 1048576),
1056
+ heapUsedMb: Math.round(mem.heapUsed / 1048576),
1057
+ externalMb: Math.round(mem.external / 1048576),
1058
+ },
1059
+ context: extra,
1060
+ });
1061
+ }
1062
+ } catch { /* swallow */ }
1063
+ }
1064
+
980
1065
  if (drift > thresholdMs) {
1066
+ // CPU time consumed during the stall interval (user + system, in seconds)
1067
+ const cpuElapsedS = (cpuDelta.user + cpuDelta.system) / 1e6;
1068
+ const wallElapsedS = drift / 1000;
1069
+ const cpuRatio = wallElapsedS > 0 ? cpuElapsedS / wallElapsedS : 0;
1070
+
1071
+ // SIGCONT within the stall window = OS sleep/resume
1072
+ const sigcontInWindow = lastSigcont > 0 && (now - lastSigcont) < drift + 2000;
1073
+
1074
+ // hrtime vs wall clock drift (macOS: hrtime doesn't advance during sleep)
1075
+ const hrtimeWallDriftS = Math.abs((drift - (hrtimeDeltaMs - checkMs))) / 1000;
1076
+
1077
+ // Classify: sleep vs real stall
1078
+ let classification;
1079
+ if (cpuRatio < 0.01 && sigcontInWindow) classification = 'sleep';
1080
+ else if (cpuRatio < 0.001) classification = 'likely_sleep';
1081
+ else if (cpuRatio < 0.01) classification = 'likely_sleep';
1082
+ else if (cpuRatio > 0.1) classification = 'real_stall';
1083
+ else classification = 'ambiguous';
1084
+
1085
+ // Don't file reports for definitive sleep
1086
+ if (classification === 'sleep') {
1087
+ lastHeapUsed = process.memoryUsage().heapUsed;
1088
+ return;
1089
+ }
1090
+
981
1091
  // Capture heap delta during stall (GC indicator)
982
1092
  const currentHeap = process.memoryUsage().heapUsed;
983
1093
  const heapDeltaMb = Math.round((currentHeap - lastHeapUsed) / 1048576);
@@ -990,7 +1100,23 @@ export function createReporter(config) {
990
1100
  // Remove resourceOpts from extra so it doesn't end up in context
991
1101
  delete extra.resourceOpts;
992
1102
 
993
- fileReport('stuck:event-loop', ['stuck', 'auto-report'], {
1103
+ // Event loop delay histogram snapshot
1104
+ let elDelay = null;
1105
+ if (elHistogram) {
1106
+ try {
1107
+ elDelay = {
1108
+ p50Ms: Math.round(elHistogram.percentile(50) / 1e6),
1109
+ p99Ms: Math.round(elHistogram.percentile(99) / 1e6),
1110
+ maxMs: Math.round(elHistogram.max / 1e6),
1111
+ };
1112
+ elHistogram.reset();
1113
+ } catch { /* unavailable */ }
1114
+ }
1115
+
1116
+ const labels = ['stuck', 'auto-report'];
1117
+ if (classification === 'likely_sleep') labels.push('likely-sleep');
1118
+
1119
+ fileReport('stuck:event-loop', labels, {
994
1120
  message: `Event loop stalled for ${Math.round(drift / 1000)}s (threshold: ${Math.round(thresholdMs / 1000)}s)`,
995
1121
  uptimeMinutes: typeof process !== 'undefined'
996
1122
  ? Math.round(process.uptime() / 60) : undefined,
@@ -998,10 +1124,20 @@ export function createReporter(config) {
998
1124
  stall: {
999
1125
  driftMs: drift,
1000
1126
  thresholdMs,
1127
+ classification,
1128
+ cpuElapsedS: Math.round(cpuElapsedS * 1000) / 1000,
1129
+ cpuRatio: Math.round(cpuRatio * 10000) / 10000,
1130
+ sigcontInWindow,
1131
+ hrtimeWallDriftS: Math.round(hrtimeWallDriftS * 100) / 100,
1132
+ eventLoopDelay: elDelay,
1001
1133
  lastRoute: _lastRoute,
1002
1134
  activeHandles: resources.activeHandles,
1003
1135
  activeRequests: resources.activeRequests,
1004
1136
  heapDeltaMb,
1137
+ nativeMemGrowthMb: nativeMemBaseline !== null
1138
+ ? Math.round((resources.nodeRssMb - resources.nodeHeapUsedMb) - nativeMemBaseline)
1139
+ : null,
1140
+ nativeMemBaselineMb: nativeMemBaseline,
1005
1141
  },
1006
1142
  context: extra,
1007
1143
  });
@@ -1021,6 +1157,16 @@ export function createReporter(config) {
1021
1157
  return Promise.allSettled([...inFlight]);
1022
1158
  }
1023
1159
 
1160
+ /**
1161
+ * Reset native memory baseline. Call after browser close so the next
1162
+ * browser session measures from a fresh baseline, not the old one.
1163
+ */
1164
+ function resetNativeMemBaseline() {
1165
+ // These are closure vars in startWatchdog — we need to reach them.
1166
+ // Since this runs in the same module, we set a flag the watchdog reads.
1167
+ _resetNativeMemBaseline = true;
1168
+ }
1169
+
1024
1170
  return {
1025
1171
  reportCrash,
1026
1172
  reportHang,
@@ -1028,6 +1174,7 @@ export function createReporter(config) {
1028
1174
  startWatchdog,
1029
1175
  trackRoute,
1030
1176
  stop,
1177
+ resetNativeMemBaseline,
1031
1178
  _anonymize: anonymize,
1032
1179
  _stackSignature: stackSignature,
1033
1180
  _rateLimiter: rateLimiter,
@@ -1,11 +1,17 @@
1
1
  import fs from 'fs';
2
2
  import path from 'path';
3
+ import os from 'os';
3
4
 
4
5
  const ORPHAN_PATTERNS = [
5
6
  /^\.fea5[a-f0-9]+\.so$/,
6
7
  /^\.5ef7[a-f0-9]+\.node$/,
7
8
  ];
8
9
 
10
+ // Firefox temp profile directories created by Playwright/Camoufox
11
+ const FIREFOX_PROFILE_PATTERN = /^playwright_firefoxdev_profile-/;
12
+ // Camoufox also creates these
13
+ const CAMOUFOX_TMP_PATTERN = /^camoufox[-_]/;
14
+
9
15
  export function cleanupOrphanedTempFiles({ tmpDir, minAgeMs = 5 * 60 * 1000, now = Date.now() } = {}) {
10
16
  const result = { scanned: 0, removed: 0, bytes: 0, skipped: 0 };
11
17
  if (!tmpDir) return result;
@@ -38,3 +44,65 @@ export function cleanupOrphanedTempFiles({ tmpDir, minAgeMs = 5 * 60 * 1000, now
38
44
 
39
45
  return result;
40
46
  }
47
+
48
+ /**
49
+ * Clean up stale Firefox/Camoufox temp profile directories.
50
+ * These accumulate when browser.close() doesn't fully clean up
51
+ * (especially with enable_cache: true). Each profile can be 10-100MB+.
52
+ *
53
+ * Only removes profiles older than minAgeMs (default 2 minutes)
54
+ * to avoid killing profiles belonging to an actively launching browser.
55
+ */
56
+ export function cleanupStaleFirefoxProfiles({ tmpDir, minAgeMs = 2 * 60 * 1000, now = Date.now() } = {}) {
57
+ const dir = tmpDir || os.tmpdir();
58
+ const result = { scanned: 0, removed: 0, bytes: 0, skipped: 0 };
59
+
60
+ let entries;
61
+ try {
62
+ entries = fs.readdirSync(dir);
63
+ } catch {
64
+ return result;
65
+ }
66
+
67
+ for (const name of entries) {
68
+ if (!FIREFOX_PROFILE_PATTERN.test(name) && !CAMOUFOX_TMP_PATTERN.test(name)) continue;
69
+ result.scanned++;
70
+ const full = path.join(dir, name);
71
+ try {
72
+ const st = fs.statSync(full);
73
+ if (!st.isDirectory()) continue;
74
+ if (now - st.mtimeMs < minAgeMs) {
75
+ result.skipped++;
76
+ continue;
77
+ }
78
+ // Calculate directory size before removing
79
+ const dirBytes = _dirSizeSync(full);
80
+ fs.rmSync(full, { recursive: true, force: true, maxRetries: 3 });
81
+ result.removed++;
82
+ result.bytes += dirBytes;
83
+ } catch {
84
+ // directory vanished, permission denied, or in-use — skip
85
+ }
86
+ }
87
+
88
+ return result;
89
+ }
90
+
91
+ /** Recursively calculate directory size (best effort, fast). */
92
+ function _dirSizeSync(dirPath) {
93
+ let total = 0;
94
+ try {
95
+ const entries = fs.readdirSync(dirPath, { withFileTypes: true });
96
+ for (const entry of entries) {
97
+ const full = path.join(dirPath, entry.name);
98
+ try {
99
+ if (entry.isDirectory()) {
100
+ total += _dirSizeSync(full);
101
+ } else {
102
+ total += fs.statSync(full).size;
103
+ }
104
+ } catch { /* skip */ }
105
+ }
106
+ } catch { /* skip */ }
107
+ return total;
108
+ }
@@ -2,7 +2,7 @@
2
2
  "id": "camofox-browser",
3
3
  "name": "Camofox Browser",
4
4
  "description": "Anti-detection browser automation for AI agents using Camoufox (Firefox-based)",
5
- "version": "1.7.2",
5
+ "version": "1.7.3",
6
6
  "configSchema": {
7
7
  "type": "object",
8
8
  "properties": {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@askjo/camofox-browser",
3
- "version": "1.7.2",
3
+ "version": "1.7.3",
4
4
  "description": "Headless browser automation server and OpenClaw plugin for AI agents - anti-detection, element refs, and session isolation",
5
5
  "type": "module",
6
6
  "main": "server.js",
@@ -62,7 +62,7 @@
62
62
  "plugin": "node scripts/plugin.js",
63
63
  "generate-openapi": "node scripts/generate-openapi.js",
64
64
  "version:sync": "node scripts/sync-version.js",
65
- "version": "node scripts/sync-version.js && git add openclaw.plugin.json",
65
+ "version": "node scripts/sync-version.js && node scripts/generate-openapi.js && git add openclaw.plugin.json openapi.json",
66
66
  "postinstall": "npx camoufox-js fetch || true"
67
67
  },
68
68
  "dependencies": {
package/server.js CHANGED
@@ -31,7 +31,7 @@ import {
31
31
  startMemoryReporter, stopMemoryReporter,
32
32
  } from './lib/metrics.js';
33
33
  import { actionFromReq, classifyError } from './lib/request-utils.js';
34
- import { cleanupOrphanedTempFiles } from './lib/tmp-cleanup.js';
34
+ import { cleanupOrphanedTempFiles, cleanupStaleFirefoxProfiles } from './lib/tmp-cleanup.js';
35
35
  import { coalesceInflight } from './lib/inflight.js';
36
36
  import { createReporter, createTabHealthTracker, collectResourceSnapshot, classifyProxyError } from './lib/reporter.js';
37
37
  import { mountDocs } from './lib/openapi.js';
@@ -56,7 +56,18 @@ function _resourceOpts() {
56
56
  return { sessionCount: sessions.size, tabCount: _countTabs(), browserPid: _browserPid() };
57
57
  }
58
58
  reporter.startWatchdog(5000, () => {
59
- return { resourceOpts: _resourceOpts() };
59
+ const summary = [];
60
+ for (const [sid, session] of sessions) {
61
+ const tabUrls = [];
62
+ for (const [tid, tab] of session.tabs) {
63
+ try {
64
+ const url = tab.page?.url?.() || 'unknown';
65
+ tabUrls.push(url);
66
+ } catch { tabUrls.push('error'); }
67
+ }
68
+ if (tabUrls.length > 0) summary.push({ session: sid, tabs: tabUrls.length, urls: tabUrls });
69
+ }
70
+ return { resourceOpts: _resourceOpts(), sessions: summary.length, summary };
60
71
  });
61
72
 
62
73
  // --- Plugin event bus ---
@@ -349,6 +360,8 @@ app.post('/sessions/:userId/cookies', express.json({ limit: '512kb' }), async (r
349
360
  });
350
361
 
351
362
  let browser = null;
363
+ let _lastBrowserPid = null; // Track PID independently for force-kill after close
364
+ let _browserClosePromise = null; // Shared promise for concurrent close serialization
352
365
  // userId -> { context, tabGroups: Map<sessionKey, Map<tabId, TabState>>, lastAccess }
353
366
  // TabState = { page, refs: Map<refId, {role, name, nth}>, visitedUrls: Set, downloads: Array, toolCalls: number }
354
367
  // Note: sessionKey was previously called listItemId - both are accepted for backward compatibility
@@ -535,9 +548,7 @@ function scheduleBrowserIdleShutdown() {
535
548
  browserIdleTimer = setTimeout(async () => {
536
549
  if (sessions.size === 0 && browser) {
537
550
  log('info', 'browser idle shutdown (no sessions)');
538
- const b = browser;
539
- browser = null;
540
- await b.close().catch(() => {});
551
+ await closeBrowserFully('idle_shutdown');
541
552
  }
542
553
  }, BROWSER_IDLE_TIMEOUT_MS);
543
554
  }
@@ -591,10 +602,7 @@ async function restartBrowser(reason) {
591
602
  pluginEvents.emit('browser:restart', { reason });
592
603
  try {
593
604
  await closeAllSessions(`browser_restart:${reason}`, { clearDownloads: true, clearLocks: true });
594
- if (browser) {
595
- await browser.close().catch(() => {});
596
- browser = null;
597
- }
605
+ await closeBrowserFully(`browser_restart:${reason}`);
598
606
  pluginEvents.emit('browser:closed', { reason });
599
607
  browserLaunchPromise = null;
600
608
  await ensureBrowser();
@@ -660,6 +668,167 @@ function attachBrowserCleanup(candidateBrowser, localVirtualDisplay) {
660
668
  };
661
669
  }
662
670
 
671
+ /**
672
+ * Close browser with full process-tree cleanup. Handles the race where
673
+ * browser.close() fails/hangs but process tree survives.
674
+ *
675
+ * Serialized: concurrent callers await the same promise (no double-close).
676
+ *
677
+ * Order: capture PID → close browser → force-kill survivors →
678
+ * clean temp profiles → verify FD/handle drop.
679
+ */
680
+ async function closeBrowserFully(reason) {
681
+ if (_browserClosePromise) return _browserClosePromise;
682
+ _browserClosePromise = _closeBrowserFullyImpl(reason);
683
+ try {
684
+ return await _browserClosePromise;
685
+ } finally {
686
+ _browserClosePromise = null;
687
+ }
688
+ }
689
+
690
+ async function _closeBrowserFullyImpl(reason) {
691
+ const b = browser;
692
+ if (!b) return;
693
+
694
+ // Capture PID before nulling browser ref — we need it for force-kill
695
+ const pid = _lastBrowserPid;
696
+ const preCloseFds = _countOpenFds();
697
+ const preCloseHandles = _countActiveHandles();
698
+
699
+ // Null the ref so new requests don't use a dying browser
700
+ browser = null;
701
+ _lastBrowserPid = null;
702
+
703
+ // Close through Playwright (sends CDP Browser.close, then SIGKILL process group)
704
+ let closeTimer;
705
+ try {
706
+ await Promise.race([
707
+ b.close(),
708
+ new Promise((_, reject) => { closeTimer = setTimeout(() => reject(new Error('browser.close() timeout')), 10000); }),
709
+ ]);
710
+ } catch (err) {
711
+ log('warn', 'browser.close() failed or timed out', { reason, error: err.message, pid });
712
+ } finally {
713
+ clearTimeout(closeTimer);
714
+ }
715
+
716
+ // Force-kill the entire process tree if any survivors
717
+ if (pid) {
718
+ await _forceKillProcessTree(pid, reason);
719
+ }
720
+
721
+ // Clean up stale Firefox temp profiles (enable_cache: true accumulates data)
722
+ try {
723
+ const cleaned = cleanupStaleFirefoxProfiles();
724
+ if (cleaned.removed > 0) {
725
+ log('info', 'cleaned stale firefox profiles after browser close', cleaned);
726
+ }
727
+ } catch { /* best effort */ }
728
+
729
+ // Reset native memory baseline so next browser measures from fresh
730
+ reporter.resetNativeMemBaseline();
731
+
732
+ // Verify cleanup: check FD/handle counts dropped (after force-kill completes)
733
+ const postCloseFds = _countOpenFds();
734
+ const postCloseHandles = _countActiveHandles();
735
+ if (postCloseFds !== null && preCloseFds !== null) {
736
+ const fdDelta = postCloseFds - preCloseFds;
737
+ // After close we expect fewer FDs. If more leaked, warn.
738
+ if (fdDelta > 10) {
739
+ log('warn', 'FD leak detected after browser close', {
740
+ reason, preCloseFds, postCloseFds, delta: fdDelta,
741
+ preCloseHandles, postCloseHandles,
742
+ });
743
+ }
744
+ }
745
+ log('info', 'browser closed fully', {
746
+ reason, pid, preCloseFds, postCloseFds, preCloseHandles, postCloseHandles,
747
+ });
748
+ }
749
+
750
+ /**
751
+ * Force-kill a browser process tree by PID. On Linux, kills the process group
752
+ * (SIGKILL -pid) then scans /proc for any orphaned children.
753
+ */
754
+ async function _forceKillProcessTree(pid, reason) {
755
+ if (!pid || pid <= 1) return;
756
+
757
+ // Kill the specific browser process first (positive PID = single process)
758
+ try {
759
+ process.kill(pid, 'SIGKILL');
760
+ log('info', 'sent SIGKILL to browser process', { pid, reason });
761
+ } catch (err) {
762
+ if (err.code !== 'ESRCH') {
763
+ log('warn', 'failed to kill browser process', { pid, error: err.message });
764
+ }
765
+ }
766
+
767
+ // Then try the process group (Playwright launches with detached:true on Linux,
768
+ // making the browser a process group leader)
769
+ try {
770
+ process.kill(-pid, 'SIGKILL');
771
+ } catch {
772
+ // ESRCH = group doesn't exist (browser wasn't a group leader), which is fine
773
+ }
774
+
775
+ // Wait for kernel to reparent children to PID 1 before scanning
776
+ await new Promise(r => setTimeout(r, 200));
777
+
778
+ // On Linux: scan /proc for orphaned children that escaped the process group
779
+ // (reparented to PID 1 by init/systemd, common with Firefox content processes).
780
+ // Also checks PPid === Node PID for containerized environments without init.
781
+ if (process.platform === 'linux') {
782
+ const myPid = process.pid;
783
+ // Snapshot the current browser PID to avoid killing a newly launched browser
784
+ const currentBrowserPid = _lastBrowserPid;
785
+ try {
786
+ const procDirs = fs.readdirSync('/proc').filter(d => /^\d+$/.test(d));
787
+ const orphans = [];
788
+ for (const procPid of procDirs) {
789
+ const numPid = parseInt(procPid);
790
+ // Never kill ourselves, the old PID (already killed), or the new browser
791
+ if (numPid === myPid || numPid === pid || numPid === currentBrowserPid) continue;
792
+ try {
793
+ const status = fs.readFileSync(`/proc/${procPid}/status`, 'utf8');
794
+ const ppidMatch = status.match(/PPid:\s+(\d+)/);
795
+ const ppid = ppidMatch ? parseInt(ppidMatch[1]) : -1;
796
+ // Orphaned to init (PID 1) or reparented to us (Node is PID 1 in containers)
797
+ if (ppid === 1 || ppid === myPid) {
798
+ const cmdline = fs.readFileSync(`/proc/${procPid}/cmdline`, 'utf8');
799
+ // Firefox-specific: binary name or Gecko child process marker
800
+ if (/firefox-esr|firefox|camoufox|libxul\.so|GeckoChildProcess/i.test(cmdline)) {
801
+ orphans.push(numPid);
802
+ }
803
+ }
804
+ } catch { /* process vanished or permission denied */ }
805
+ }
806
+ if (orphans.length > 0) {
807
+ log('warn', 'killing orphaned browser child processes', { orphans, reason });
808
+ for (const orphanPid of orphans) {
809
+ try { process.kill(orphanPid, 'SIGKILL'); } catch { /* already dead */ }
810
+ }
811
+ }
812
+ } catch (err) {
813
+ log('warn', 'failed to scan for orphaned browser processes', { error: err.message });
814
+ }
815
+ }
816
+
817
+ // Give the OS a moment to reclaim resources
818
+ await new Promise(r => setTimeout(r, 300));
819
+ }
820
+
821
+ function _countOpenFds() {
822
+ try {
823
+ if (process.platform === 'linux') return fs.readdirSync('/proc/self/fd').length;
824
+ } catch { /* unavailable */ }
825
+ return null;
826
+ }
827
+
828
+ function _countActiveHandles() {
829
+ try { return process._getActiveHandles().length; } catch { return null; }
830
+ }
831
+
663
832
  async function launchBrowserInstance() {
664
833
  const hostOS = getHostOS();
665
834
  const maxAttempts = proxyPool?.launchRetries ?? 1;
@@ -738,7 +907,8 @@ async function launchBrowserInstance() {
738
907
 
739
908
  virtualDisplay = localVirtualDisplay;
740
909
  browserLaunchProxy = launchProxy;
741
- browser = candidateBrowser;
910
+ _lastBrowserPid = candidateBrowser.process?.()?.pid ?? null;
911
+ browser = candidateBrowser; // publish AFTER PID is captured
742
912
  attachBrowserCleanup(browser, localVirtualDisplay);
743
913
  pluginEvents.emit('browser:launched', { browser, display: vdDisplay });
744
914
 
@@ -775,13 +945,7 @@ async function ensureBrowser() {
775
945
  deadSessions: sessions.size,
776
946
  });
777
947
  await closeAllSessions('browser_disconnected', { clearDownloads: true, clearLocks: true });
778
- // Clean up virtual display from dead browser before relaunching
779
- if (virtualDisplay) {
780
- virtualDisplay.kill();
781
- virtualDisplay = null;
782
- }
783
- browserLaunchProxy = null;
784
- browser = null;
948
+ await closeBrowserFully('browser_disconnected');
785
949
  }
786
950
  if (browser) return browser;
787
951
  if (browserLaunchPromise) return browserLaunchPromise;
@@ -1730,6 +1894,10 @@ app.get('/health', (req, res) => {
1730
1894
  ...(FLY_MACHINE_ID ? { machineId: FLY_MACHINE_ID } : {}),
1731
1895
  });
1732
1896
  }
1897
+ const mem = process.memoryUsage();
1898
+ const rssMb = Math.round(mem.rss / 1048576);
1899
+ const heapUsedMb = Math.round(mem.heapUsed / 1048576);
1900
+ const nativeMemMb = rssMb - heapUsedMb;
1733
1901
  res.json({
1734
1902
  ok: true,
1735
1903
  engine: 'camoufox',
@@ -1738,6 +1906,7 @@ app.get('/health', (req, res) => {
1738
1906
  activeTabs: getTotalTabCount(),
1739
1907
  activeSessions: sessions.size,
1740
1908
  consecutiveFailures: healthState.consecutiveNavFailures,
1909
+ memory: { rssMb, heapUsedMb, nativeMemMb },
1741
1910
  ...(FLY_MACHINE_ID ? { machineId: FLY_MACHINE_ID } : {}),
1742
1911
  });
1743
1912
  });
@@ -4390,11 +4559,8 @@ app.post('/stop', async (req, res) => {
4390
4559
  if (!adminKey || !timingSafeCompare(adminKey, CONFIG.adminKey)) {
4391
4560
  return res.status(403).json({ error: 'Forbidden' });
4392
4561
  }
4393
- if (browser) {
4394
- await browser.close().catch(() => {});
4395
- browser = null;
4396
- }
4397
4562
  await closeAllSessions('admin_stop', { clearDownloads: true, clearLocks: true });
4563
+ await closeBrowserFully('admin_stop');
4398
4564
  res.json({ ok: true, stopped: true, profile: 'camoufox' });
4399
4565
  } catch (err) {
4400
4566
  res.status(500).json({ ok: false, error: safeError(err) });
@@ -4971,7 +5137,7 @@ async function gracefulShutdown(signal) {
4971
5137
  clearLocks: false,
4972
5138
  });
4973
5139
 
4974
- if (browser) await browser.close().catch(() => {});
5140
+ await closeBrowserFully(`shutdown:${signal}`);
4975
5141
  process.exit(0);
4976
5142
  }
4977
5143
 
@@ -5031,6 +5197,20 @@ const server = app.listen(PORT, async () => {
5031
5197
  if (tmpCleanup.removed > 0) {
5032
5198
  log('info', 'cleaned up orphaned camoufox temp files', tmpCleanup);
5033
5199
  }
5200
+ const profileCleanup = cleanupStaleFirefoxProfiles();
5201
+ if (profileCleanup.removed > 0) {
5202
+ log('info', 'cleaned up stale firefox profiles on startup', profileCleanup);
5203
+ }
5204
+
5205
+ // Periodic temp profile cleanup every 10 minutes
5206
+ setInterval(() => {
5207
+ try {
5208
+ const cleaned = cleanupStaleFirefoxProfiles();
5209
+ if (cleaned.removed > 0) {
5210
+ log('info', 'periodic firefox profile cleanup', cleaned);
5211
+ }
5212
+ } catch { /* best effort */ }
5213
+ }, 10 * 60 * 1000).unref();
5034
5214
  const traceSweep = sweepOldTraces({
5035
5215
  baseDir: CONFIG.tracesDir,
5036
5216
  ttlMs: CONFIG.tracesTtlHours * 3600 * 1000,