@askjo/camofox-browser 1.7.2 → 1.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/reporter.js +149 -2
- package/lib/tmp-cleanup.js +68 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +2 -2
- package/server.js +202 -22
package/lib/reporter.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import crypto from 'crypto';
|
|
6
6
|
import fs from 'fs';
|
|
7
|
-
import {
|
|
7
|
+
import { monitorEventLoopDelay } from 'perf_hooks';
|
|
8
8
|
|
|
9
9
|
// ============================================================================
|
|
10
10
|
// Anonymization
|
|
@@ -720,6 +720,15 @@ function formatIssueBody(type, detail) {
|
|
|
720
720
|
const s = detail.stall;
|
|
721
721
|
sections.push('', '## Stall Details');
|
|
722
722
|
sections.push(`- **stall duration:** ${Math.round(s.driftMs / 1000)}s`);
|
|
723
|
+
if (s.classification) sections.push(`- **classification:** ${s.classification}`);
|
|
724
|
+
if (s.cpuElapsedS != null) sections.push(`- **CPU time during stall:** ${s.cpuElapsedS}s`);
|
|
725
|
+
if (s.cpuRatio != null) sections.push(`- **CPU/wall ratio:** ${s.cpuRatio}`);
|
|
726
|
+
if (s.sigcontInWindow != null) sections.push(`- **SIGCONT in window:** ${s.sigcontInWindow}`);
|
|
727
|
+
if (s.hrtimeWallDriftS != null) sections.push(`- **hrtime↔wall drift:** ${s.hrtimeWallDriftS}s`);
|
|
728
|
+
if (s.eventLoopDelay) {
|
|
729
|
+
const eld = s.eventLoopDelay;
|
|
730
|
+
sections.push(`- **event loop delay:** p50=${eld.p50Ms}ms p99=${eld.p99Ms}ms max=${eld.maxMs}ms`);
|
|
731
|
+
}
|
|
723
732
|
if (s.lastRoute) sections.push(`- **last route:** ${s.lastRoute}`);
|
|
724
733
|
if (s.activeHandles != null) sections.push(`- **active handles:** ${s.activeHandles}`);
|
|
725
734
|
if (s.activeRequests != null) sections.push(`- **active requests:** ${s.activeRequests}`);
|
|
@@ -784,6 +793,7 @@ export function createReporter(config) {
|
|
|
784
793
|
const version = config.version || 'unknown';
|
|
785
794
|
|
|
786
795
|
let watchdogInterval = null;
|
|
796
|
+
let _resetNativeMemBaseline = false; // Set by resetNativeMemBaseline(), read by watchdog
|
|
787
797
|
let lastTick = Date.now();
|
|
788
798
|
const inFlight = new Set();
|
|
789
799
|
|
|
@@ -953,7 +963,31 @@ export function createReporter(config) {
|
|
|
953
963
|
|
|
954
964
|
const checkMs = 1000;
|
|
955
965
|
lastTick = Date.now();
|
|
966
|
+
let lastCpuUsage = process.cpuUsage();
|
|
967
|
+
let lastHrtime = process.hrtime.bigint();
|
|
956
968
|
let lastHeapUsed = process.memoryUsage().heapUsed;
|
|
969
|
+
|
|
970
|
+
// --- Native memory leak tracking ---
|
|
971
|
+
// Track RSS minus JS heap over time to detect native/external memory leaks.
|
|
972
|
+
// Sample every 30s, alert if native memory grows by >200MB from baseline.
|
|
973
|
+
let nativeMemBaseline = null; // RSS - heapUsed at first measurement
|
|
974
|
+
let nativeMemHighWater = 0;
|
|
975
|
+
let lastNativeMemCheck = 0;
|
|
976
|
+
const NATIVE_MEM_CHECK_INTERVAL_MS = 30_000;
|
|
977
|
+
const NATIVE_MEM_LEAK_THRESHOLD_MB = 200; // alert if native mem exceeds baseline by this much
|
|
978
|
+
let nativeMemAlertFired = false;
|
|
979
|
+
|
|
980
|
+
// SIGCONT detection — macOS sends SIGCONT on wake from sleep/suspend
|
|
981
|
+
let lastSigcont = 0;
|
|
982
|
+
try { process.on('SIGCONT', () => { lastSigcont = Date.now(); }); } catch { /* unavailable */ }
|
|
983
|
+
|
|
984
|
+
// Event loop delay histogram (perf_hooks) — correlating evidence
|
|
985
|
+
let elHistogram = null;
|
|
986
|
+
try {
|
|
987
|
+
elHistogram = monitorEventLoopDelay({ resolution: 20 });
|
|
988
|
+
elHistogram.enable();
|
|
989
|
+
} catch { /* unavailable */ }
|
|
990
|
+
|
|
957
991
|
// Suppress false positives from OS sleep/suspend (laptop lid close, VM pause).
|
|
958
992
|
// Stalls > 120s are almost certainly not event-loop bugs.
|
|
959
993
|
const MAX_REPORTABLE_DRIFT_MS = 120_000;
|
|
@@ -963,7 +997,13 @@ export function createReporter(config) {
|
|
|
963
997
|
watchdogInterval = setInterval(() => {
|
|
964
998
|
const now = Date.now();
|
|
965
999
|
const drift = now - lastTick - checkMs;
|
|
1000
|
+
const cpuDelta = process.cpuUsage(lastCpuUsage);
|
|
1001
|
+
const hrtimeNow = process.hrtime.bigint();
|
|
1002
|
+
const hrtimeDeltaMs = Number(hrtimeNow - lastHrtime) / 1e6;
|
|
1003
|
+
|
|
966
1004
|
lastTick = now;
|
|
1005
|
+
lastCpuUsage = process.cpuUsage();
|
|
1006
|
+
lastHrtime = hrtimeNow;
|
|
967
1007
|
|
|
968
1008
|
// After a long sleep/suspend, suppress the next few ticks (post-wake jitter)
|
|
969
1009
|
if (drift > MAX_REPORTABLE_DRIFT_MS) {
|
|
@@ -977,7 +1017,77 @@ export function createReporter(config) {
|
|
|
977
1017
|
return;
|
|
978
1018
|
}
|
|
979
1019
|
|
|
1020
|
+
// --- Native memory leak detection (runs every ~30s) ---
|
|
1021
|
+
if (now - lastNativeMemCheck >= NATIVE_MEM_CHECK_INTERVAL_MS) {
|
|
1022
|
+
lastNativeMemCheck = now;
|
|
1023
|
+
try {
|
|
1024
|
+
// Check if baseline should be reset (e.g. after browser close)
|
|
1025
|
+
if (_resetNativeMemBaseline) {
|
|
1026
|
+
nativeMemBaseline = null;
|
|
1027
|
+
nativeMemHighWater = 0;
|
|
1028
|
+
nativeMemAlertFired = false;
|
|
1029
|
+
_resetNativeMemBaseline = false;
|
|
1030
|
+
}
|
|
1031
|
+
const mem = process.memoryUsage();
|
|
1032
|
+
const nativeMemMb = Math.round((mem.rss - mem.heapUsed) / 1048576);
|
|
1033
|
+
if (nativeMemBaseline === null) {
|
|
1034
|
+
nativeMemBaseline = nativeMemMb;
|
|
1035
|
+
}
|
|
1036
|
+
nativeMemHighWater = Math.max(nativeMemHighWater, nativeMemMb);
|
|
1037
|
+
const growth = nativeMemMb - nativeMemBaseline;
|
|
1038
|
+
|
|
1039
|
+
if (growth > NATIVE_MEM_LEAK_THRESHOLD_MB && !nativeMemAlertFired) {
|
|
1040
|
+
nativeMemAlertFired = true;
|
|
1041
|
+
let extra = {};
|
|
1042
|
+
try { if (getContext) extra = getContext(); } catch { /* swallow */ }
|
|
1043
|
+
const resources = collectResourceSnapshot(extra.resourceOpts || {});
|
|
1044
|
+
delete extra.resourceOpts;
|
|
1045
|
+
|
|
1046
|
+
fileReport('leak:native-memory', ['auto-report', 'memory-leak'], {
|
|
1047
|
+
message: `Native memory grew by ${growth}MB (baseline: ${nativeMemBaseline}MB, current: ${nativeMemMb}MB, high-water: ${nativeMemHighWater}MB)`,
|
|
1048
|
+
uptimeMinutes: Math.round(process.uptime() / 60),
|
|
1049
|
+
resources,
|
|
1050
|
+
nativeMemory: {
|
|
1051
|
+
baselineMb: nativeMemBaseline,
|
|
1052
|
+
currentMb: nativeMemMb,
|
|
1053
|
+
highWaterMb: nativeMemHighWater,
|
|
1054
|
+
growthMb: growth,
|
|
1055
|
+
rssMb: Math.round(mem.rss / 1048576),
|
|
1056
|
+
heapUsedMb: Math.round(mem.heapUsed / 1048576),
|
|
1057
|
+
externalMb: Math.round(mem.external / 1048576),
|
|
1058
|
+
},
|
|
1059
|
+
context: extra,
|
|
1060
|
+
});
|
|
1061
|
+
}
|
|
1062
|
+
} catch { /* swallow */ }
|
|
1063
|
+
}
|
|
1064
|
+
|
|
980
1065
|
if (drift > thresholdMs) {
|
|
1066
|
+
// CPU time consumed during the stall interval (user + system, in seconds)
|
|
1067
|
+
const cpuElapsedS = (cpuDelta.user + cpuDelta.system) / 1e6;
|
|
1068
|
+
const wallElapsedS = drift / 1000;
|
|
1069
|
+
const cpuRatio = wallElapsedS > 0 ? cpuElapsedS / wallElapsedS : 0;
|
|
1070
|
+
|
|
1071
|
+
// SIGCONT within the stall window = OS sleep/resume
|
|
1072
|
+
const sigcontInWindow = lastSigcont > 0 && (now - lastSigcont) < drift + 2000;
|
|
1073
|
+
|
|
1074
|
+
// hrtime vs wall clock drift (macOS: hrtime doesn't advance during sleep)
|
|
1075
|
+
const hrtimeWallDriftS = Math.abs((drift - (hrtimeDeltaMs - checkMs))) / 1000;
|
|
1076
|
+
|
|
1077
|
+
// Classify: sleep vs real stall
|
|
1078
|
+
let classification;
|
|
1079
|
+
if (cpuRatio < 0.01 && sigcontInWindow) classification = 'sleep';
|
|
1080
|
+
else if (cpuRatio < 0.001) classification = 'likely_sleep';
|
|
1081
|
+
else if (cpuRatio < 0.01) classification = 'likely_sleep';
|
|
1082
|
+
else if (cpuRatio > 0.1) classification = 'real_stall';
|
|
1083
|
+
else classification = 'ambiguous';
|
|
1084
|
+
|
|
1085
|
+
// Don't file reports for definitive sleep
|
|
1086
|
+
if (classification === 'sleep') {
|
|
1087
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
1088
|
+
return;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
981
1091
|
// Capture heap delta during stall (GC indicator)
|
|
982
1092
|
const currentHeap = process.memoryUsage().heapUsed;
|
|
983
1093
|
const heapDeltaMb = Math.round((currentHeap - lastHeapUsed) / 1048576);
|
|
@@ -990,7 +1100,23 @@ export function createReporter(config) {
|
|
|
990
1100
|
// Remove resourceOpts from extra so it doesn't end up in context
|
|
991
1101
|
delete extra.resourceOpts;
|
|
992
1102
|
|
|
993
|
-
|
|
1103
|
+
// Event loop delay histogram snapshot
|
|
1104
|
+
let elDelay = null;
|
|
1105
|
+
if (elHistogram) {
|
|
1106
|
+
try {
|
|
1107
|
+
elDelay = {
|
|
1108
|
+
p50Ms: Math.round(elHistogram.percentile(50) / 1e6),
|
|
1109
|
+
p99Ms: Math.round(elHistogram.percentile(99) / 1e6),
|
|
1110
|
+
maxMs: Math.round(elHistogram.max / 1e6),
|
|
1111
|
+
};
|
|
1112
|
+
elHistogram.reset();
|
|
1113
|
+
} catch { /* unavailable */ }
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
const labels = ['stuck', 'auto-report'];
|
|
1117
|
+
if (classification === 'likely_sleep') labels.push('likely-sleep');
|
|
1118
|
+
|
|
1119
|
+
fileReport('stuck:event-loop', labels, {
|
|
994
1120
|
message: `Event loop stalled for ${Math.round(drift / 1000)}s (threshold: ${Math.round(thresholdMs / 1000)}s)`,
|
|
995
1121
|
uptimeMinutes: typeof process !== 'undefined'
|
|
996
1122
|
? Math.round(process.uptime() / 60) : undefined,
|
|
@@ -998,10 +1124,20 @@ export function createReporter(config) {
|
|
|
998
1124
|
stall: {
|
|
999
1125
|
driftMs: drift,
|
|
1000
1126
|
thresholdMs,
|
|
1127
|
+
classification,
|
|
1128
|
+
cpuElapsedS: Math.round(cpuElapsedS * 1000) / 1000,
|
|
1129
|
+
cpuRatio: Math.round(cpuRatio * 10000) / 10000,
|
|
1130
|
+
sigcontInWindow,
|
|
1131
|
+
hrtimeWallDriftS: Math.round(hrtimeWallDriftS * 100) / 100,
|
|
1132
|
+
eventLoopDelay: elDelay,
|
|
1001
1133
|
lastRoute: _lastRoute,
|
|
1002
1134
|
activeHandles: resources.activeHandles,
|
|
1003
1135
|
activeRequests: resources.activeRequests,
|
|
1004
1136
|
heapDeltaMb,
|
|
1137
|
+
nativeMemGrowthMb: nativeMemBaseline !== null
|
|
1138
|
+
? Math.round((resources.nodeRssMb - resources.nodeHeapUsedMb) - nativeMemBaseline)
|
|
1139
|
+
: null,
|
|
1140
|
+
nativeMemBaselineMb: nativeMemBaseline,
|
|
1005
1141
|
},
|
|
1006
1142
|
context: extra,
|
|
1007
1143
|
});
|
|
@@ -1021,6 +1157,16 @@ export function createReporter(config) {
|
|
|
1021
1157
|
return Promise.allSettled([...inFlight]);
|
|
1022
1158
|
}
|
|
1023
1159
|
|
|
1160
|
+
/**
|
|
1161
|
+
* Reset native memory baseline. Call after browser close so the next
|
|
1162
|
+
* browser session measures from a fresh baseline, not the old one.
|
|
1163
|
+
*/
|
|
1164
|
+
function resetNativeMemBaseline() {
|
|
1165
|
+
// These are closure vars in startWatchdog — we need to reach them.
|
|
1166
|
+
// Since this runs in the same module, we set a flag the watchdog reads.
|
|
1167
|
+
_resetNativeMemBaseline = true;
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1024
1170
|
return {
|
|
1025
1171
|
reportCrash,
|
|
1026
1172
|
reportHang,
|
|
@@ -1028,6 +1174,7 @@ export function createReporter(config) {
|
|
|
1028
1174
|
startWatchdog,
|
|
1029
1175
|
trackRoute,
|
|
1030
1176
|
stop,
|
|
1177
|
+
resetNativeMemBaseline,
|
|
1031
1178
|
_anonymize: anonymize,
|
|
1032
1179
|
_stackSignature: stackSignature,
|
|
1033
1180
|
_rateLimiter: rateLimiter,
|
package/lib/tmp-cleanup.js
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
|
+
import os from 'os';
|
|
3
4
|
|
|
4
5
|
const ORPHAN_PATTERNS = [
|
|
5
6
|
/^\.fea5[a-f0-9]+\.so$/,
|
|
6
7
|
/^\.5ef7[a-f0-9]+\.node$/,
|
|
7
8
|
];
|
|
8
9
|
|
|
10
|
+
// Firefox temp profile directories created by Playwright/Camoufox
|
|
11
|
+
const FIREFOX_PROFILE_PATTERN = /^playwright_firefoxdev_profile-/;
|
|
12
|
+
// Camoufox also creates these
|
|
13
|
+
const CAMOUFOX_TMP_PATTERN = /^camoufox[-_]/;
|
|
14
|
+
|
|
9
15
|
export function cleanupOrphanedTempFiles({ tmpDir, minAgeMs = 5 * 60 * 1000, now = Date.now() } = {}) {
|
|
10
16
|
const result = { scanned: 0, removed: 0, bytes: 0, skipped: 0 };
|
|
11
17
|
if (!tmpDir) return result;
|
|
@@ -38,3 +44,65 @@ export function cleanupOrphanedTempFiles({ tmpDir, minAgeMs = 5 * 60 * 1000, now
|
|
|
38
44
|
|
|
39
45
|
return result;
|
|
40
46
|
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Clean up stale Firefox/Camoufox temp profile directories.
|
|
50
|
+
* These accumulate when browser.close() doesn't fully clean up
|
|
51
|
+
* (especially with enable_cache: true). Each profile can be 10-100MB+.
|
|
52
|
+
*
|
|
53
|
+
* Only removes profiles older than minAgeMs (default 2 minutes)
|
|
54
|
+
* to avoid killing profiles belonging to an actively launching browser.
|
|
55
|
+
*/
|
|
56
|
+
export function cleanupStaleFirefoxProfiles({ tmpDir, minAgeMs = 2 * 60 * 1000, now = Date.now() } = {}) {
|
|
57
|
+
const dir = tmpDir || os.tmpdir();
|
|
58
|
+
const result = { scanned: 0, removed: 0, bytes: 0, skipped: 0 };
|
|
59
|
+
|
|
60
|
+
let entries;
|
|
61
|
+
try {
|
|
62
|
+
entries = fs.readdirSync(dir);
|
|
63
|
+
} catch {
|
|
64
|
+
return result;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
for (const name of entries) {
|
|
68
|
+
if (!FIREFOX_PROFILE_PATTERN.test(name) && !CAMOUFOX_TMP_PATTERN.test(name)) continue;
|
|
69
|
+
result.scanned++;
|
|
70
|
+
const full = path.join(dir, name);
|
|
71
|
+
try {
|
|
72
|
+
const st = fs.statSync(full);
|
|
73
|
+
if (!st.isDirectory()) continue;
|
|
74
|
+
if (now - st.mtimeMs < minAgeMs) {
|
|
75
|
+
result.skipped++;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
// Calculate directory size before removing
|
|
79
|
+
const dirBytes = _dirSizeSync(full);
|
|
80
|
+
fs.rmSync(full, { recursive: true, force: true, maxRetries: 3 });
|
|
81
|
+
result.removed++;
|
|
82
|
+
result.bytes += dirBytes;
|
|
83
|
+
} catch {
|
|
84
|
+
// directory vanished, permission denied, or in-use — skip
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return result;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Recursively calculate directory size (best effort, fast). */
|
|
92
|
+
function _dirSizeSync(dirPath) {
|
|
93
|
+
let total = 0;
|
|
94
|
+
try {
|
|
95
|
+
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
|
96
|
+
for (const entry of entries) {
|
|
97
|
+
const full = path.join(dirPath, entry.name);
|
|
98
|
+
try {
|
|
99
|
+
if (entry.isDirectory()) {
|
|
100
|
+
total += _dirSizeSync(full);
|
|
101
|
+
} else {
|
|
102
|
+
total += fs.statSync(full).size;
|
|
103
|
+
}
|
|
104
|
+
} catch { /* skip */ }
|
|
105
|
+
}
|
|
106
|
+
} catch { /* skip */ }
|
|
107
|
+
return total;
|
|
108
|
+
}
|
package/openclaw.plugin.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@askjo/camofox-browser",
|
|
3
|
-
"version": "1.7.
|
|
3
|
+
"version": "1.7.3",
|
|
4
4
|
"description": "Headless browser automation server and OpenClaw plugin for AI agents - anti-detection, element refs, and session isolation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "server.js",
|
|
@@ -62,7 +62,7 @@
|
|
|
62
62
|
"plugin": "node scripts/plugin.js",
|
|
63
63
|
"generate-openapi": "node scripts/generate-openapi.js",
|
|
64
64
|
"version:sync": "node scripts/sync-version.js",
|
|
65
|
-
"version": "node scripts/sync-version.js && git add openclaw.plugin.json",
|
|
65
|
+
"version": "node scripts/sync-version.js && node scripts/generate-openapi.js && git add openclaw.plugin.json openapi.json",
|
|
66
66
|
"postinstall": "npx camoufox-js fetch || true"
|
|
67
67
|
},
|
|
68
68
|
"dependencies": {
|
package/server.js
CHANGED
|
@@ -31,7 +31,7 @@ import {
|
|
|
31
31
|
startMemoryReporter, stopMemoryReporter,
|
|
32
32
|
} from './lib/metrics.js';
|
|
33
33
|
import { actionFromReq, classifyError } from './lib/request-utils.js';
|
|
34
|
-
import { cleanupOrphanedTempFiles } from './lib/tmp-cleanup.js';
|
|
34
|
+
import { cleanupOrphanedTempFiles, cleanupStaleFirefoxProfiles } from './lib/tmp-cleanup.js';
|
|
35
35
|
import { coalesceInflight } from './lib/inflight.js';
|
|
36
36
|
import { createReporter, createTabHealthTracker, collectResourceSnapshot, classifyProxyError } from './lib/reporter.js';
|
|
37
37
|
import { mountDocs } from './lib/openapi.js';
|
|
@@ -56,7 +56,18 @@ function _resourceOpts() {
|
|
|
56
56
|
return { sessionCount: sessions.size, tabCount: _countTabs(), browserPid: _browserPid() };
|
|
57
57
|
}
|
|
58
58
|
reporter.startWatchdog(5000, () => {
|
|
59
|
-
|
|
59
|
+
const summary = [];
|
|
60
|
+
for (const [sid, session] of sessions) {
|
|
61
|
+
const tabUrls = [];
|
|
62
|
+
for (const [tid, tab] of session.tabs) {
|
|
63
|
+
try {
|
|
64
|
+
const url = tab.page?.url?.() || 'unknown';
|
|
65
|
+
tabUrls.push(url);
|
|
66
|
+
} catch { tabUrls.push('error'); }
|
|
67
|
+
}
|
|
68
|
+
if (tabUrls.length > 0) summary.push({ session: sid, tabs: tabUrls.length, urls: tabUrls });
|
|
69
|
+
}
|
|
70
|
+
return { resourceOpts: _resourceOpts(), sessions: summary.length, summary };
|
|
60
71
|
});
|
|
61
72
|
|
|
62
73
|
// --- Plugin event bus ---
|
|
@@ -349,6 +360,8 @@ app.post('/sessions/:userId/cookies', express.json({ limit: '512kb' }), async (r
|
|
|
349
360
|
});
|
|
350
361
|
|
|
351
362
|
let browser = null;
|
|
363
|
+
let _lastBrowserPid = null; // Track PID independently for force-kill after close
|
|
364
|
+
let _browserClosePromise = null; // Shared promise for concurrent close serialization
|
|
352
365
|
// userId -> { context, tabGroups: Map<sessionKey, Map<tabId, TabState>>, lastAccess }
|
|
353
366
|
// TabState = { page, refs: Map<refId, {role, name, nth}>, visitedUrls: Set, downloads: Array, toolCalls: number }
|
|
354
367
|
// Note: sessionKey was previously called listItemId - both are accepted for backward compatibility
|
|
@@ -535,9 +548,7 @@ function scheduleBrowserIdleShutdown() {
|
|
|
535
548
|
browserIdleTimer = setTimeout(async () => {
|
|
536
549
|
if (sessions.size === 0 && browser) {
|
|
537
550
|
log('info', 'browser idle shutdown (no sessions)');
|
|
538
|
-
|
|
539
|
-
browser = null;
|
|
540
|
-
await b.close().catch(() => {});
|
|
551
|
+
await closeBrowserFully('idle_shutdown');
|
|
541
552
|
}
|
|
542
553
|
}, BROWSER_IDLE_TIMEOUT_MS);
|
|
543
554
|
}
|
|
@@ -591,10 +602,7 @@ async function restartBrowser(reason) {
|
|
|
591
602
|
pluginEvents.emit('browser:restart', { reason });
|
|
592
603
|
try {
|
|
593
604
|
await closeAllSessions(`browser_restart:${reason}`, { clearDownloads: true, clearLocks: true });
|
|
594
|
-
|
|
595
|
-
await browser.close().catch(() => {});
|
|
596
|
-
browser = null;
|
|
597
|
-
}
|
|
605
|
+
await closeBrowserFully(`browser_restart:${reason}`);
|
|
598
606
|
pluginEvents.emit('browser:closed', { reason });
|
|
599
607
|
browserLaunchPromise = null;
|
|
600
608
|
await ensureBrowser();
|
|
@@ -660,6 +668,167 @@ function attachBrowserCleanup(candidateBrowser, localVirtualDisplay) {
|
|
|
660
668
|
};
|
|
661
669
|
}
|
|
662
670
|
|
|
671
|
+
/**
|
|
672
|
+
* Close browser with full process-tree cleanup. Handles the race where
|
|
673
|
+
* browser.close() fails/hangs but process tree survives.
|
|
674
|
+
*
|
|
675
|
+
* Serialized: concurrent callers await the same promise (no double-close).
|
|
676
|
+
*
|
|
677
|
+
* Order: capture PID → close browser → force-kill survivors →
|
|
678
|
+
* clean temp profiles → verify FD/handle drop.
|
|
679
|
+
*/
|
|
680
|
+
async function closeBrowserFully(reason) {
|
|
681
|
+
if (_browserClosePromise) return _browserClosePromise;
|
|
682
|
+
_browserClosePromise = _closeBrowserFullyImpl(reason);
|
|
683
|
+
try {
|
|
684
|
+
return await _browserClosePromise;
|
|
685
|
+
} finally {
|
|
686
|
+
_browserClosePromise = null;
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
async function _closeBrowserFullyImpl(reason) {
|
|
691
|
+
const b = browser;
|
|
692
|
+
if (!b) return;
|
|
693
|
+
|
|
694
|
+
// Capture PID before nulling browser ref — we need it for force-kill
|
|
695
|
+
const pid = _lastBrowserPid;
|
|
696
|
+
const preCloseFds = _countOpenFds();
|
|
697
|
+
const preCloseHandles = _countActiveHandles();
|
|
698
|
+
|
|
699
|
+
// Null the ref so new requests don't use a dying browser
|
|
700
|
+
browser = null;
|
|
701
|
+
_lastBrowserPid = null;
|
|
702
|
+
|
|
703
|
+
// Close through Playwright (sends CDP Browser.close, then SIGKILL process group)
|
|
704
|
+
let closeTimer;
|
|
705
|
+
try {
|
|
706
|
+
await Promise.race([
|
|
707
|
+
b.close(),
|
|
708
|
+
new Promise((_, reject) => { closeTimer = setTimeout(() => reject(new Error('browser.close() timeout')), 10000); }),
|
|
709
|
+
]);
|
|
710
|
+
} catch (err) {
|
|
711
|
+
log('warn', 'browser.close() failed or timed out', { reason, error: err.message, pid });
|
|
712
|
+
} finally {
|
|
713
|
+
clearTimeout(closeTimer);
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
// Force-kill the entire process tree if any survivors
|
|
717
|
+
if (pid) {
|
|
718
|
+
await _forceKillProcessTree(pid, reason);
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// Clean up stale Firefox temp profiles (enable_cache: true accumulates data)
|
|
722
|
+
try {
|
|
723
|
+
const cleaned = cleanupStaleFirefoxProfiles();
|
|
724
|
+
if (cleaned.removed > 0) {
|
|
725
|
+
log('info', 'cleaned stale firefox profiles after browser close', cleaned);
|
|
726
|
+
}
|
|
727
|
+
} catch { /* best effort */ }
|
|
728
|
+
|
|
729
|
+
// Reset native memory baseline so next browser measures from fresh
|
|
730
|
+
reporter.resetNativeMemBaseline();
|
|
731
|
+
|
|
732
|
+
// Verify cleanup: check FD/handle counts dropped (after force-kill completes)
|
|
733
|
+
const postCloseFds = _countOpenFds();
|
|
734
|
+
const postCloseHandles = _countActiveHandles();
|
|
735
|
+
if (postCloseFds !== null && preCloseFds !== null) {
|
|
736
|
+
const fdDelta = postCloseFds - preCloseFds;
|
|
737
|
+
// After close we expect fewer FDs. If more leaked, warn.
|
|
738
|
+
if (fdDelta > 10) {
|
|
739
|
+
log('warn', 'FD leak detected after browser close', {
|
|
740
|
+
reason, preCloseFds, postCloseFds, delta: fdDelta,
|
|
741
|
+
preCloseHandles, postCloseHandles,
|
|
742
|
+
});
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
log('info', 'browser closed fully', {
|
|
746
|
+
reason, pid, preCloseFds, postCloseFds, preCloseHandles, postCloseHandles,
|
|
747
|
+
});
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
/**
|
|
751
|
+
* Force-kill a browser process tree by PID. On Linux, kills the process group
|
|
752
|
+
* (SIGKILL -pid) then scans /proc for any orphaned children.
|
|
753
|
+
*/
|
|
754
|
+
async function _forceKillProcessTree(pid, reason) {
|
|
755
|
+
if (!pid || pid <= 1) return;
|
|
756
|
+
|
|
757
|
+
// Kill the specific browser process first (positive PID = single process)
|
|
758
|
+
try {
|
|
759
|
+
process.kill(pid, 'SIGKILL');
|
|
760
|
+
log('info', 'sent SIGKILL to browser process', { pid, reason });
|
|
761
|
+
} catch (err) {
|
|
762
|
+
if (err.code !== 'ESRCH') {
|
|
763
|
+
log('warn', 'failed to kill browser process', { pid, error: err.message });
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
// Then try the process group (Playwright launches with detached:true on Linux,
|
|
768
|
+
// making the browser a process group leader)
|
|
769
|
+
try {
|
|
770
|
+
process.kill(-pid, 'SIGKILL');
|
|
771
|
+
} catch {
|
|
772
|
+
// ESRCH = group doesn't exist (browser wasn't a group leader), which is fine
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
// Wait for kernel to reparent children to PID 1 before scanning
|
|
776
|
+
await new Promise(r => setTimeout(r, 200));
|
|
777
|
+
|
|
778
|
+
// On Linux: scan /proc for orphaned children that escaped the process group
|
|
779
|
+
// (reparented to PID 1 by init/systemd, common with Firefox content processes).
|
|
780
|
+
// Also checks PPid === Node PID for containerized environments without init.
|
|
781
|
+
if (process.platform === 'linux') {
|
|
782
|
+
const myPid = process.pid;
|
|
783
|
+
// Snapshot the current browser PID to avoid killing a newly launched browser
|
|
784
|
+
const currentBrowserPid = _lastBrowserPid;
|
|
785
|
+
try {
|
|
786
|
+
const procDirs = fs.readdirSync('/proc').filter(d => /^\d+$/.test(d));
|
|
787
|
+
const orphans = [];
|
|
788
|
+
for (const procPid of procDirs) {
|
|
789
|
+
const numPid = parseInt(procPid);
|
|
790
|
+
// Never kill ourselves, the old PID (already killed), or the new browser
|
|
791
|
+
if (numPid === myPid || numPid === pid || numPid === currentBrowserPid) continue;
|
|
792
|
+
try {
|
|
793
|
+
const status = fs.readFileSync(`/proc/${procPid}/status`, 'utf8');
|
|
794
|
+
const ppidMatch = status.match(/PPid:\s+(\d+)/);
|
|
795
|
+
const ppid = ppidMatch ? parseInt(ppidMatch[1]) : -1;
|
|
796
|
+
// Orphaned to init (PID 1) or reparented to us (Node is PID 1 in containers)
|
|
797
|
+
if (ppid === 1 || ppid === myPid) {
|
|
798
|
+
const cmdline = fs.readFileSync(`/proc/${procPid}/cmdline`, 'utf8');
|
|
799
|
+
// Firefox-specific: binary name or Gecko child process marker
|
|
800
|
+
if (/firefox-esr|firefox|camoufox|libxul\.so|GeckoChildProcess/i.test(cmdline)) {
|
|
801
|
+
orphans.push(numPid);
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
} catch { /* process vanished or permission denied */ }
|
|
805
|
+
}
|
|
806
|
+
if (orphans.length > 0) {
|
|
807
|
+
log('warn', 'killing orphaned browser child processes', { orphans, reason });
|
|
808
|
+
for (const orphanPid of orphans) {
|
|
809
|
+
try { process.kill(orphanPid, 'SIGKILL'); } catch { /* already dead */ }
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
} catch (err) {
|
|
813
|
+
log('warn', 'failed to scan for orphaned browser processes', { error: err.message });
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
// Give the OS a moment to reclaim resources
|
|
818
|
+
await new Promise(r => setTimeout(r, 300));
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
function _countOpenFds() {
|
|
822
|
+
try {
|
|
823
|
+
if (process.platform === 'linux') return fs.readdirSync('/proc/self/fd').length;
|
|
824
|
+
} catch { /* unavailable */ }
|
|
825
|
+
return null;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
function _countActiveHandles() {
|
|
829
|
+
try { return process._getActiveHandles().length; } catch { return null; }
|
|
830
|
+
}
|
|
831
|
+
|
|
663
832
|
async function launchBrowserInstance() {
|
|
664
833
|
const hostOS = getHostOS();
|
|
665
834
|
const maxAttempts = proxyPool?.launchRetries ?? 1;
|
|
@@ -738,7 +907,8 @@ async function launchBrowserInstance() {
|
|
|
738
907
|
|
|
739
908
|
virtualDisplay = localVirtualDisplay;
|
|
740
909
|
browserLaunchProxy = launchProxy;
|
|
741
|
-
|
|
910
|
+
_lastBrowserPid = candidateBrowser.process?.()?.pid ?? null;
|
|
911
|
+
browser = candidateBrowser; // publish AFTER PID is captured
|
|
742
912
|
attachBrowserCleanup(browser, localVirtualDisplay);
|
|
743
913
|
pluginEvents.emit('browser:launched', { browser, display: vdDisplay });
|
|
744
914
|
|
|
@@ -775,13 +945,7 @@ async function ensureBrowser() {
|
|
|
775
945
|
deadSessions: sessions.size,
|
|
776
946
|
});
|
|
777
947
|
await closeAllSessions('browser_disconnected', { clearDownloads: true, clearLocks: true });
|
|
778
|
-
|
|
779
|
-
if (virtualDisplay) {
|
|
780
|
-
virtualDisplay.kill();
|
|
781
|
-
virtualDisplay = null;
|
|
782
|
-
}
|
|
783
|
-
browserLaunchProxy = null;
|
|
784
|
-
browser = null;
|
|
948
|
+
await closeBrowserFully('browser_disconnected');
|
|
785
949
|
}
|
|
786
950
|
if (browser) return browser;
|
|
787
951
|
if (browserLaunchPromise) return browserLaunchPromise;
|
|
@@ -1730,6 +1894,10 @@ app.get('/health', (req, res) => {
|
|
|
1730
1894
|
...(FLY_MACHINE_ID ? { machineId: FLY_MACHINE_ID } : {}),
|
|
1731
1895
|
});
|
|
1732
1896
|
}
|
|
1897
|
+
const mem = process.memoryUsage();
|
|
1898
|
+
const rssMb = Math.round(mem.rss / 1048576);
|
|
1899
|
+
const heapUsedMb = Math.round(mem.heapUsed / 1048576);
|
|
1900
|
+
const nativeMemMb = rssMb - heapUsedMb;
|
|
1733
1901
|
res.json({
|
|
1734
1902
|
ok: true,
|
|
1735
1903
|
engine: 'camoufox',
|
|
@@ -1738,6 +1906,7 @@ app.get('/health', (req, res) => {
|
|
|
1738
1906
|
activeTabs: getTotalTabCount(),
|
|
1739
1907
|
activeSessions: sessions.size,
|
|
1740
1908
|
consecutiveFailures: healthState.consecutiveNavFailures,
|
|
1909
|
+
memory: { rssMb, heapUsedMb, nativeMemMb },
|
|
1741
1910
|
...(FLY_MACHINE_ID ? { machineId: FLY_MACHINE_ID } : {}),
|
|
1742
1911
|
});
|
|
1743
1912
|
});
|
|
@@ -4390,11 +4559,8 @@ app.post('/stop', async (req, res) => {
|
|
|
4390
4559
|
if (!adminKey || !timingSafeCompare(adminKey, CONFIG.adminKey)) {
|
|
4391
4560
|
return res.status(403).json({ error: 'Forbidden' });
|
|
4392
4561
|
}
|
|
4393
|
-
if (browser) {
|
|
4394
|
-
await browser.close().catch(() => {});
|
|
4395
|
-
browser = null;
|
|
4396
|
-
}
|
|
4397
4562
|
await closeAllSessions('admin_stop', { clearDownloads: true, clearLocks: true });
|
|
4563
|
+
await closeBrowserFully('admin_stop');
|
|
4398
4564
|
res.json({ ok: true, stopped: true, profile: 'camoufox' });
|
|
4399
4565
|
} catch (err) {
|
|
4400
4566
|
res.status(500).json({ ok: false, error: safeError(err) });
|
|
@@ -4971,7 +5137,7 @@ async function gracefulShutdown(signal) {
|
|
|
4971
5137
|
clearLocks: false,
|
|
4972
5138
|
});
|
|
4973
5139
|
|
|
4974
|
-
|
|
5140
|
+
await closeBrowserFully(`shutdown:${signal}`);
|
|
4975
5141
|
process.exit(0);
|
|
4976
5142
|
}
|
|
4977
5143
|
|
|
@@ -5031,6 +5197,20 @@ const server = app.listen(PORT, async () => {
|
|
|
5031
5197
|
if (tmpCleanup.removed > 0) {
|
|
5032
5198
|
log('info', 'cleaned up orphaned camoufox temp files', tmpCleanup);
|
|
5033
5199
|
}
|
|
5200
|
+
const profileCleanup = cleanupStaleFirefoxProfiles();
|
|
5201
|
+
if (profileCleanup.removed > 0) {
|
|
5202
|
+
log('info', 'cleaned up stale firefox profiles on startup', profileCleanup);
|
|
5203
|
+
}
|
|
5204
|
+
|
|
5205
|
+
// Periodic temp profile cleanup every 10 minutes
|
|
5206
|
+
setInterval(() => {
|
|
5207
|
+
try {
|
|
5208
|
+
const cleaned = cleanupStaleFirefoxProfiles();
|
|
5209
|
+
if (cleaned.removed > 0) {
|
|
5210
|
+
log('info', 'periodic firefox profile cleanup', cleaned);
|
|
5211
|
+
}
|
|
5212
|
+
} catch { /* best effort */ }
|
|
5213
|
+
}, 10 * 60 * 1000).unref();
|
|
5034
5214
|
const traceSweep = sweepOldTraces({
|
|
5035
5215
|
baseDir: CONFIG.tracesDir,
|
|
5036
5216
|
ttlMs: CONFIG.tracesTtlHours * 3600 * 1000,
|