@askjo/camofox-browser 1.7.2 → 1.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/reporter.js +168 -6
- package/lib/tmp-cleanup.js +68 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +2 -2
- package/server.js +203 -23
package/lib/reporter.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import crypto from 'crypto';
|
|
6
6
|
import fs from 'fs';
|
|
7
|
-
import {
|
|
7
|
+
import { monitorEventLoopDelay } from 'perf_hooks';
|
|
8
8
|
|
|
9
9
|
// ============================================================================
|
|
10
10
|
// Anonymization
|
|
@@ -720,6 +720,15 @@ function formatIssueBody(type, detail) {
|
|
|
720
720
|
const s = detail.stall;
|
|
721
721
|
sections.push('', '## Stall Details');
|
|
722
722
|
sections.push(`- **stall duration:** ${Math.round(s.driftMs / 1000)}s`);
|
|
723
|
+
if (s.classification) sections.push(`- **classification:** ${s.classification}`);
|
|
724
|
+
if (s.cpuElapsedS != null) sections.push(`- **CPU time during stall:** ${s.cpuElapsedS}s`);
|
|
725
|
+
if (s.cpuRatio != null) sections.push(`- **CPU/wall ratio:** ${s.cpuRatio}`);
|
|
726
|
+
if (s.sigcontInWindow != null) sections.push(`- **SIGCONT in window:** ${s.sigcontInWindow}`);
|
|
727
|
+
if (s.hrtimeWallDriftS != null) sections.push(`- **hrtime↔wall drift:** ${s.hrtimeWallDriftS}s`);
|
|
728
|
+
if (s.eventLoopDelay) {
|
|
729
|
+
const eld = s.eventLoopDelay;
|
|
730
|
+
sections.push(`- **event loop delay:** p50=${eld.p50Ms}ms p99=${eld.p99Ms}ms max=${eld.maxMs}ms`);
|
|
731
|
+
}
|
|
723
732
|
if (s.lastRoute) sections.push(`- **last route:** ${s.lastRoute}`);
|
|
724
733
|
if (s.activeHandles != null) sections.push(`- **active handles:** ${s.activeHandles}`);
|
|
725
734
|
if (s.activeRequests != null) sections.push(`- **active requests:** ${s.activeRequests}`);
|
|
@@ -780,10 +789,17 @@ export function createReporter(config) {
|
|
|
780
789
|
|
|
781
790
|
const enabled = config.crashReportEnabled !== false && !!_GH_APP_ID;
|
|
782
791
|
const repo = config.crashReportRepo || cr.repo || 'jo-inc/camofox-browser';
|
|
783
|
-
const
|
|
792
|
+
const rateLimiters = {
|
|
793
|
+
crash: new RateLimiter(5), // 5 crashes/hr
|
|
794
|
+
hang: new RateLimiter(5), // 5 hangs/hr
|
|
795
|
+
stuck: new RateLimiter(2), // 2 stalls/hr (with active tabs only)
|
|
796
|
+
leak: new RateLimiter(2), // 2 leak alerts/hr
|
|
797
|
+
_default: new RateLimiter(config.crashReportRateLimit || 10),
|
|
798
|
+
};
|
|
784
799
|
const version = config.version || 'unknown';
|
|
785
800
|
|
|
786
801
|
let watchdogInterval = null;
|
|
802
|
+
let _resetNativeMemBaseline = false; // Set by resetNativeMemBaseline(), read by watchdog
|
|
787
803
|
let lastTick = Date.now();
|
|
788
804
|
const inFlight = new Set();
|
|
789
805
|
|
|
@@ -806,7 +822,9 @@ export function createReporter(config) {
|
|
|
806
822
|
|
|
807
823
|
/** Core: file or deduplicate a report. NEVER throws. */
|
|
808
824
|
async function fileReport(type, labels, detail) {
|
|
809
|
-
|
|
825
|
+
const bucket = type.startsWith('stuck:') ? 'stuck' : type.startsWith('hang:') ? 'hang' : type.startsWith('leak:') ? 'leak' : 'crash';
|
|
826
|
+
const limiter = rateLimiters[bucket] || rateLimiters._default;
|
|
827
|
+
if (!limiter.tryAcquire()) return;
|
|
810
828
|
|
|
811
829
|
const reportPromise = (async () => {
|
|
812
830
|
try {
|
|
@@ -953,17 +971,47 @@ export function createReporter(config) {
|
|
|
953
971
|
|
|
954
972
|
const checkMs = 1000;
|
|
955
973
|
lastTick = Date.now();
|
|
974
|
+
let lastCpuUsage = process.cpuUsage();
|
|
975
|
+
let lastHrtime = process.hrtime.bigint();
|
|
956
976
|
let lastHeapUsed = process.memoryUsage().heapUsed;
|
|
977
|
+
|
|
978
|
+
// --- Native memory leak tracking ---
|
|
979
|
+
// Track RSS minus JS heap over time to detect native/external memory leaks.
|
|
980
|
+
// Sample every 30s, alert if native memory grows by >200MB from baseline.
|
|
981
|
+
let nativeMemBaseline = null; // RSS - heapUsed at first measurement
|
|
982
|
+
let nativeMemHighWater = 0;
|
|
983
|
+
let lastNativeMemCheck = 0;
|
|
984
|
+
const NATIVE_MEM_CHECK_INTERVAL_MS = 30_000;
|
|
985
|
+
const NATIVE_MEM_LEAK_THRESHOLD_MB = 200; // alert if native mem exceeds baseline by this much
|
|
986
|
+
let nativeMemAlertFired = false;
|
|
987
|
+
|
|
988
|
+
// SIGCONT detection — macOS sends SIGCONT on wake from sleep/suspend
|
|
989
|
+
let lastSigcont = 0;
|
|
990
|
+
try { process.on('SIGCONT', () => { lastSigcont = Date.now(); }); } catch { /* unavailable */ }
|
|
991
|
+
|
|
992
|
+
// Event loop delay histogram (perf_hooks) — correlating evidence
|
|
993
|
+
let elHistogram = null;
|
|
994
|
+
try {
|
|
995
|
+
elHistogram = monitorEventLoopDelay({ resolution: 20 });
|
|
996
|
+
elHistogram.enable();
|
|
997
|
+
} catch { /* unavailable */ }
|
|
998
|
+
|
|
957
999
|
// Suppress false positives from OS sleep/suspend (laptop lid close, VM pause).
|
|
958
1000
|
// Stalls > 120s are almost certainly not event-loop bugs.
|
|
959
|
-
const MAX_REPORTABLE_DRIFT_MS =
|
|
1001
|
+
const MAX_REPORTABLE_DRIFT_MS = 60_000;
|
|
960
1002
|
let suppressTicksRemaining = 0;
|
|
961
1003
|
const SUPPRESS_TICKS_AFTER_WAKE = 5;
|
|
962
1004
|
|
|
963
1005
|
watchdogInterval = setInterval(() => {
|
|
964
1006
|
const now = Date.now();
|
|
965
1007
|
const drift = now - lastTick - checkMs;
|
|
1008
|
+
const cpuDelta = process.cpuUsage(lastCpuUsage);
|
|
1009
|
+
const hrtimeNow = process.hrtime.bigint();
|
|
1010
|
+
const hrtimeDeltaMs = Number(hrtimeNow - lastHrtime) / 1e6;
|
|
1011
|
+
|
|
966
1012
|
lastTick = now;
|
|
1013
|
+
lastCpuUsage = process.cpuUsage();
|
|
1014
|
+
lastHrtime = hrtimeNow;
|
|
967
1015
|
|
|
968
1016
|
// After a long sleep/suspend, suppress the next few ticks (post-wake jitter)
|
|
969
1017
|
if (drift > MAX_REPORTABLE_DRIFT_MS) {
|
|
@@ -977,7 +1025,77 @@ export function createReporter(config) {
|
|
|
977
1025
|
return;
|
|
978
1026
|
}
|
|
979
1027
|
|
|
1028
|
+
// --- Native memory leak detection (runs every ~30s) ---
|
|
1029
|
+
if (now - lastNativeMemCheck >= NATIVE_MEM_CHECK_INTERVAL_MS) {
|
|
1030
|
+
lastNativeMemCheck = now;
|
|
1031
|
+
try {
|
|
1032
|
+
// Check if baseline should be reset (e.g. after browser close)
|
|
1033
|
+
if (_resetNativeMemBaseline) {
|
|
1034
|
+
nativeMemBaseline = null;
|
|
1035
|
+
nativeMemHighWater = 0;
|
|
1036
|
+
nativeMemAlertFired = false;
|
|
1037
|
+
_resetNativeMemBaseline = false;
|
|
1038
|
+
}
|
|
1039
|
+
const mem = process.memoryUsage();
|
|
1040
|
+
const nativeMemMb = Math.round((mem.rss - mem.heapUsed) / 1048576);
|
|
1041
|
+
if (nativeMemBaseline === null) {
|
|
1042
|
+
nativeMemBaseline = nativeMemMb;
|
|
1043
|
+
}
|
|
1044
|
+
nativeMemHighWater = Math.max(nativeMemHighWater, nativeMemMb);
|
|
1045
|
+
const growth = nativeMemMb - nativeMemBaseline;
|
|
1046
|
+
|
|
1047
|
+
if (growth > NATIVE_MEM_LEAK_THRESHOLD_MB && !nativeMemAlertFired) {
|
|
1048
|
+
nativeMemAlertFired = true;
|
|
1049
|
+
let extra = {};
|
|
1050
|
+
try { if (getContext) extra = getContext(); } catch { /* swallow */ }
|
|
1051
|
+
const resources = collectResourceSnapshot(extra.resourceOpts || {});
|
|
1052
|
+
delete extra.resourceOpts;
|
|
1053
|
+
|
|
1054
|
+
fileReport('leak:native-memory', ['auto-report', 'memory-leak'], {
|
|
1055
|
+
message: `Native memory grew by ${growth}MB (baseline: ${nativeMemBaseline}MB, current: ${nativeMemMb}MB, high-water: ${nativeMemHighWater}MB)`,
|
|
1056
|
+
uptimeMinutes: Math.round(process.uptime() / 60),
|
|
1057
|
+
resources,
|
|
1058
|
+
nativeMemory: {
|
|
1059
|
+
baselineMb: nativeMemBaseline,
|
|
1060
|
+
currentMb: nativeMemMb,
|
|
1061
|
+
highWaterMb: nativeMemHighWater,
|
|
1062
|
+
growthMb: growth,
|
|
1063
|
+
rssMb: Math.round(mem.rss / 1048576),
|
|
1064
|
+
heapUsedMb: Math.round(mem.heapUsed / 1048576),
|
|
1065
|
+
externalMb: Math.round(mem.external / 1048576),
|
|
1066
|
+
},
|
|
1067
|
+
context: extra,
|
|
1068
|
+
});
|
|
1069
|
+
}
|
|
1070
|
+
} catch { /* swallow */ }
|
|
1071
|
+
}
|
|
1072
|
+
|
|
980
1073
|
if (drift > thresholdMs) {
|
|
1074
|
+
// CPU time consumed during the stall interval (user + system, in seconds)
|
|
1075
|
+
const cpuElapsedS = (cpuDelta.user + cpuDelta.system) / 1e6;
|
|
1076
|
+
const wallElapsedS = drift / 1000;
|
|
1077
|
+
const cpuRatio = wallElapsedS > 0 ? cpuElapsedS / wallElapsedS : 0;
|
|
1078
|
+
|
|
1079
|
+
// SIGCONT within the stall window = OS sleep/resume
|
|
1080
|
+
const sigcontInWindow = lastSigcont > 0 && (now - lastSigcont) < drift + 2000;
|
|
1081
|
+
|
|
1082
|
+
// hrtime vs wall clock drift (macOS: hrtime doesn't advance during sleep)
|
|
1083
|
+
const hrtimeWallDriftS = Math.abs((drift - (hrtimeDeltaMs - checkMs))) / 1000;
|
|
1084
|
+
|
|
1085
|
+
// Classify: sleep vs real stall
|
|
1086
|
+
let classification;
|
|
1087
|
+
if (cpuRatio < 0.01 && sigcontInWindow) classification = 'sleep';
|
|
1088
|
+
else if (cpuRatio < 0.001) classification = 'likely_sleep';
|
|
1089
|
+
else if (cpuRatio < 0.01) classification = 'likely_sleep';
|
|
1090
|
+
else if (cpuRatio > 0.1) classification = 'real_stall';
|
|
1091
|
+
else classification = 'ambiguous';
|
|
1092
|
+
|
|
1093
|
+
// Don't file reports for definitive sleep
|
|
1094
|
+
if (classification === 'sleep') {
|
|
1095
|
+
lastHeapUsed = process.memoryUsage().heapUsed;
|
|
1096
|
+
return;
|
|
1097
|
+
}
|
|
1098
|
+
|
|
981
1099
|
// Capture heap delta during stall (GC indicator)
|
|
982
1100
|
const currentHeap = process.memoryUsage().heapUsed;
|
|
983
1101
|
const heapDeltaMb = Math.round((currentHeap - lastHeapUsed) / 1048576);
|
|
@@ -990,18 +1108,51 @@ export function createReporter(config) {
|
|
|
990
1108
|
// Remove resourceOpts from extra so it doesn't end up in context
|
|
991
1109
|
delete extra.resourceOpts;
|
|
992
1110
|
|
|
993
|
-
|
|
1111
|
+
// Don't report idle-server stalls — no user impact
|
|
1112
|
+
if ((resources.activeTabs || 0) === 0 && (resources.browserContexts || 0) === 0) {
|
|
1113
|
+
return;
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
// Event loop delay histogram snapshot
|
|
1117
|
+
let elDelay = null;
|
|
1118
|
+
if (elHistogram) {
|
|
1119
|
+
try {
|
|
1120
|
+
elDelay = {
|
|
1121
|
+
p50Ms: Math.round(elHistogram.percentile(50) / 1e6),
|
|
1122
|
+
p99Ms: Math.round(elHistogram.percentile(99) / 1e6),
|
|
1123
|
+
maxMs: Math.round(elHistogram.max / 1e6),
|
|
1124
|
+
};
|
|
1125
|
+
elHistogram.reset();
|
|
1126
|
+
} catch { /* unavailable */ }
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
const labels = ['stuck', 'auto-report'];
|
|
1130
|
+
if (classification === 'likely_sleep') labels.push('likely-sleep');
|
|
1131
|
+
|
|
1132
|
+
fileReport('stuck:event-loop', labels, {
|
|
994
1133
|
message: `Event loop stalled for ${Math.round(drift / 1000)}s (threshold: ${Math.round(thresholdMs / 1000)}s)`,
|
|
1134
|
+
// Stable signature: duration is NOT included — all stalls on the same route dedup
|
|
1135
|
+
error: { name: 'EventLoopStall', message: _lastRoute || 'idle', stack: '' },
|
|
995
1136
|
uptimeMinutes: typeof process !== 'undefined'
|
|
996
1137
|
? Math.round(process.uptime() / 60) : undefined,
|
|
997
1138
|
resources,
|
|
998
1139
|
stall: {
|
|
999
1140
|
driftMs: drift,
|
|
1000
1141
|
thresholdMs,
|
|
1142
|
+
classification,
|
|
1143
|
+
cpuElapsedS: Math.round(cpuElapsedS * 1000) / 1000,
|
|
1144
|
+
cpuRatio: Math.round(cpuRatio * 10000) / 10000,
|
|
1145
|
+
sigcontInWindow,
|
|
1146
|
+
hrtimeWallDriftS: Math.round(hrtimeWallDriftS * 100) / 100,
|
|
1147
|
+
eventLoopDelay: elDelay,
|
|
1001
1148
|
lastRoute: _lastRoute,
|
|
1002
1149
|
activeHandles: resources.activeHandles,
|
|
1003
1150
|
activeRequests: resources.activeRequests,
|
|
1004
1151
|
heapDeltaMb,
|
|
1152
|
+
nativeMemGrowthMb: nativeMemBaseline !== null
|
|
1153
|
+
? Math.round((resources.nodeRssMb - resources.nodeHeapUsedMb) - nativeMemBaseline)
|
|
1154
|
+
: null,
|
|
1155
|
+
nativeMemBaselineMb: nativeMemBaseline,
|
|
1005
1156
|
},
|
|
1006
1157
|
context: extra,
|
|
1007
1158
|
});
|
|
@@ -1021,6 +1172,16 @@ export function createReporter(config) {
|
|
|
1021
1172
|
return Promise.allSettled([...inFlight]);
|
|
1022
1173
|
}
|
|
1023
1174
|
|
|
1175
|
+
/**
|
|
1176
|
+
* Reset native memory baseline. Call after browser close so the next
|
|
1177
|
+
* browser session measures from a fresh baseline, not the old one.
|
|
1178
|
+
*/
|
|
1179
|
+
function resetNativeMemBaseline() {
|
|
1180
|
+
// These are closure vars in startWatchdog — we need to reach them.
|
|
1181
|
+
// Since this runs in the same module, we set a flag the watchdog reads.
|
|
1182
|
+
_resetNativeMemBaseline = true;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1024
1185
|
return {
|
|
1025
1186
|
reportCrash,
|
|
1026
1187
|
reportHang,
|
|
@@ -1028,8 +1189,9 @@ export function createReporter(config) {
|
|
|
1028
1189
|
startWatchdog,
|
|
1029
1190
|
trackRoute,
|
|
1030
1191
|
stop,
|
|
1192
|
+
resetNativeMemBaseline,
|
|
1031
1193
|
_anonymize: anonymize,
|
|
1032
1194
|
_stackSignature: stackSignature,
|
|
1033
|
-
_rateLimiter:
|
|
1195
|
+
_rateLimiter: rateLimiters,
|
|
1034
1196
|
};
|
|
1035
1197
|
}
|
package/lib/tmp-cleanup.js
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
|
+
import os from 'os';
|
|
3
4
|
|
|
4
5
|
const ORPHAN_PATTERNS = [
|
|
5
6
|
/^\.fea5[a-f0-9]+\.so$/,
|
|
6
7
|
/^\.5ef7[a-f0-9]+\.node$/,
|
|
7
8
|
];
|
|
8
9
|
|
|
10
|
+
// Firefox temp profile directories created by Playwright/Camoufox
|
|
11
|
+
const FIREFOX_PROFILE_PATTERN = /^playwright_firefoxdev_profile-/;
|
|
12
|
+
// Camoufox also creates these
|
|
13
|
+
const CAMOUFOX_TMP_PATTERN = /^camoufox[-_]/;
|
|
14
|
+
|
|
9
15
|
export function cleanupOrphanedTempFiles({ tmpDir, minAgeMs = 5 * 60 * 1000, now = Date.now() } = {}) {
|
|
10
16
|
const result = { scanned: 0, removed: 0, bytes: 0, skipped: 0 };
|
|
11
17
|
if (!tmpDir) return result;
|
|
@@ -38,3 +44,65 @@ export function cleanupOrphanedTempFiles({ tmpDir, minAgeMs = 5 * 60 * 1000, now
|
|
|
38
44
|
|
|
39
45
|
return result;
|
|
40
46
|
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Clean up stale Firefox/Camoufox temp profile directories.
|
|
50
|
+
* These accumulate when browser.close() doesn't fully clean up
|
|
51
|
+
* (especially with enable_cache: true). Each profile can be 10-100MB+.
|
|
52
|
+
*
|
|
53
|
+
* Only removes profiles older than minAgeMs (default 2 minutes)
|
|
54
|
+
* to avoid killing profiles belonging to an actively launching browser.
|
|
55
|
+
*/
|
|
56
|
+
export function cleanupStaleFirefoxProfiles({ tmpDir, minAgeMs = 2 * 60 * 1000, now = Date.now() } = {}) {
|
|
57
|
+
const dir = tmpDir || os.tmpdir();
|
|
58
|
+
const result = { scanned: 0, removed: 0, bytes: 0, skipped: 0 };
|
|
59
|
+
|
|
60
|
+
let entries;
|
|
61
|
+
try {
|
|
62
|
+
entries = fs.readdirSync(dir);
|
|
63
|
+
} catch {
|
|
64
|
+
return result;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
for (const name of entries) {
|
|
68
|
+
if (!FIREFOX_PROFILE_PATTERN.test(name) && !CAMOUFOX_TMP_PATTERN.test(name)) continue;
|
|
69
|
+
result.scanned++;
|
|
70
|
+
const full = path.join(dir, name);
|
|
71
|
+
try {
|
|
72
|
+
const st = fs.statSync(full);
|
|
73
|
+
if (!st.isDirectory()) continue;
|
|
74
|
+
if (now - st.mtimeMs < minAgeMs) {
|
|
75
|
+
result.skipped++;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
// Calculate directory size before removing
|
|
79
|
+
const dirBytes = _dirSizeSync(full);
|
|
80
|
+
fs.rmSync(full, { recursive: true, force: true, maxRetries: 3 });
|
|
81
|
+
result.removed++;
|
|
82
|
+
result.bytes += dirBytes;
|
|
83
|
+
} catch {
|
|
84
|
+
// directory vanished, permission denied, or in-use — skip
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return result;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Recursively calculate directory size (best effort, fast). */
|
|
92
|
+
function _dirSizeSync(dirPath) {
|
|
93
|
+
let total = 0;
|
|
94
|
+
try {
|
|
95
|
+
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
|
96
|
+
for (const entry of entries) {
|
|
97
|
+
const full = path.join(dirPath, entry.name);
|
|
98
|
+
try {
|
|
99
|
+
if (entry.isDirectory()) {
|
|
100
|
+
total += _dirSizeSync(full);
|
|
101
|
+
} else {
|
|
102
|
+
total += fs.statSync(full).size;
|
|
103
|
+
}
|
|
104
|
+
} catch { /* skip */ }
|
|
105
|
+
}
|
|
106
|
+
} catch { /* skip */ }
|
|
107
|
+
return total;
|
|
108
|
+
}
|
package/openclaw.plugin.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@askjo/camofox-browser",
|
|
3
|
-
"version": "1.7.
|
|
3
|
+
"version": "1.7.4",
|
|
4
4
|
"description": "Headless browser automation server and OpenClaw plugin for AI agents - anti-detection, element refs, and session isolation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "server.js",
|
|
@@ -62,7 +62,7 @@
|
|
|
62
62
|
"plugin": "node scripts/plugin.js",
|
|
63
63
|
"generate-openapi": "node scripts/generate-openapi.js",
|
|
64
64
|
"version:sync": "node scripts/sync-version.js",
|
|
65
|
-
"version": "node scripts/sync-version.js && git add openclaw.plugin.json",
|
|
65
|
+
"version": "node scripts/sync-version.js && node scripts/generate-openapi.js && git add openclaw.plugin.json openapi.json",
|
|
66
66
|
"postinstall": "npx camoufox-js fetch || true"
|
|
67
67
|
},
|
|
68
68
|
"dependencies": {
|
package/server.js
CHANGED
|
@@ -31,7 +31,7 @@ import {
|
|
|
31
31
|
startMemoryReporter, stopMemoryReporter,
|
|
32
32
|
} from './lib/metrics.js';
|
|
33
33
|
import { actionFromReq, classifyError } from './lib/request-utils.js';
|
|
34
|
-
import { cleanupOrphanedTempFiles } from './lib/tmp-cleanup.js';
|
|
34
|
+
import { cleanupOrphanedTempFiles, cleanupStaleFirefoxProfiles } from './lib/tmp-cleanup.js';
|
|
35
35
|
import { coalesceInflight } from './lib/inflight.js';
|
|
36
36
|
import { createReporter, createTabHealthTracker, collectResourceSnapshot, classifyProxyError } from './lib/reporter.js';
|
|
37
37
|
import { mountDocs } from './lib/openapi.js';
|
|
@@ -55,8 +55,19 @@ function _browserPid() {
|
|
|
55
55
|
function _resourceOpts() {
|
|
56
56
|
return { sessionCount: sessions.size, tabCount: _countTabs(), browserPid: _browserPid() };
|
|
57
57
|
}
|
|
58
|
-
reporter.startWatchdog(
|
|
59
|
-
|
|
58
|
+
reporter.startWatchdog(30_000, () => {
|
|
59
|
+
const summary = [];
|
|
60
|
+
for (const [sid, session] of sessions) {
|
|
61
|
+
const tabUrls = [];
|
|
62
|
+
for (const [tid, tab] of session.tabs) {
|
|
63
|
+
try {
|
|
64
|
+
const url = tab.page?.url?.() || 'unknown';
|
|
65
|
+
tabUrls.push(url);
|
|
66
|
+
} catch { tabUrls.push('error'); }
|
|
67
|
+
}
|
|
68
|
+
if (tabUrls.length > 0) summary.push({ session: sid, tabs: tabUrls.length, urls: tabUrls });
|
|
69
|
+
}
|
|
70
|
+
return { resourceOpts: _resourceOpts(), sessions: summary.length, summary };
|
|
60
71
|
});
|
|
61
72
|
|
|
62
73
|
// --- Plugin event bus ---
|
|
@@ -349,6 +360,8 @@ app.post('/sessions/:userId/cookies', express.json({ limit: '512kb' }), async (r
|
|
|
349
360
|
});
|
|
350
361
|
|
|
351
362
|
let browser = null;
|
|
363
|
+
let _lastBrowserPid = null; // Track PID independently for force-kill after close
|
|
364
|
+
let _browserClosePromise = null; // Shared promise for concurrent close serialization
|
|
352
365
|
// userId -> { context, tabGroups: Map<sessionKey, Map<tabId, TabState>>, lastAccess }
|
|
353
366
|
// TabState = { page, refs: Map<refId, {role, name, nth}>, visitedUrls: Set, downloads: Array, toolCalls: number }
|
|
354
367
|
// Note: sessionKey was previously called listItemId - both are accepted for backward compatibility
|
|
@@ -535,9 +548,7 @@ function scheduleBrowserIdleShutdown() {
|
|
|
535
548
|
browserIdleTimer = setTimeout(async () => {
|
|
536
549
|
if (sessions.size === 0 && browser) {
|
|
537
550
|
log('info', 'browser idle shutdown (no sessions)');
|
|
538
|
-
|
|
539
|
-
browser = null;
|
|
540
|
-
await b.close().catch(() => {});
|
|
551
|
+
await closeBrowserFully('idle_shutdown');
|
|
541
552
|
}
|
|
542
553
|
}, BROWSER_IDLE_TIMEOUT_MS);
|
|
543
554
|
}
|
|
@@ -591,10 +602,7 @@ async function restartBrowser(reason) {
|
|
|
591
602
|
pluginEvents.emit('browser:restart', { reason });
|
|
592
603
|
try {
|
|
593
604
|
await closeAllSessions(`browser_restart:${reason}`, { clearDownloads: true, clearLocks: true });
|
|
594
|
-
|
|
595
|
-
await browser.close().catch(() => {});
|
|
596
|
-
browser = null;
|
|
597
|
-
}
|
|
605
|
+
await closeBrowserFully(`browser_restart:${reason}`);
|
|
598
606
|
pluginEvents.emit('browser:closed', { reason });
|
|
599
607
|
browserLaunchPromise = null;
|
|
600
608
|
await ensureBrowser();
|
|
@@ -660,6 +668,167 @@ function attachBrowserCleanup(candidateBrowser, localVirtualDisplay) {
|
|
|
660
668
|
};
|
|
661
669
|
}
|
|
662
670
|
|
|
671
|
+
/**
|
|
672
|
+
* Close browser with full process-tree cleanup. Handles the race where
|
|
673
|
+
* browser.close() fails/hangs but process tree survives.
|
|
674
|
+
*
|
|
675
|
+
* Serialized: concurrent callers await the same promise (no double-close).
|
|
676
|
+
*
|
|
677
|
+
* Order: capture PID → close browser → force-kill survivors →
|
|
678
|
+
* clean temp profiles → verify FD/handle drop.
|
|
679
|
+
*/
|
|
680
|
+
async function closeBrowserFully(reason) {
|
|
681
|
+
if (_browserClosePromise) return _browserClosePromise;
|
|
682
|
+
_browserClosePromise = _closeBrowserFullyImpl(reason);
|
|
683
|
+
try {
|
|
684
|
+
return await _browserClosePromise;
|
|
685
|
+
} finally {
|
|
686
|
+
_browserClosePromise = null;
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
async function _closeBrowserFullyImpl(reason) {
|
|
691
|
+
const b = browser;
|
|
692
|
+
if (!b) return;
|
|
693
|
+
|
|
694
|
+
// Capture PID before nulling browser ref — we need it for force-kill
|
|
695
|
+
const pid = _lastBrowserPid;
|
|
696
|
+
const preCloseFds = _countOpenFds();
|
|
697
|
+
const preCloseHandles = _countActiveHandles();
|
|
698
|
+
|
|
699
|
+
// Null the ref so new requests don't use a dying browser
|
|
700
|
+
browser = null;
|
|
701
|
+
_lastBrowserPid = null;
|
|
702
|
+
|
|
703
|
+
// Close through Playwright (sends CDP Browser.close, then SIGKILL process group)
|
|
704
|
+
let closeTimer;
|
|
705
|
+
try {
|
|
706
|
+
await Promise.race([
|
|
707
|
+
b.close(),
|
|
708
|
+
new Promise((_, reject) => { closeTimer = setTimeout(() => reject(new Error('browser.close() timeout')), 10000); }),
|
|
709
|
+
]);
|
|
710
|
+
} catch (err) {
|
|
711
|
+
log('warn', 'browser.close() failed or timed out', { reason, error: err.message, pid });
|
|
712
|
+
} finally {
|
|
713
|
+
clearTimeout(closeTimer);
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
// Force-kill the entire process tree if any survivors
|
|
717
|
+
if (pid) {
|
|
718
|
+
await _forceKillProcessTree(pid, reason);
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// Clean up stale Firefox temp profiles (enable_cache: true accumulates data)
|
|
722
|
+
try {
|
|
723
|
+
const cleaned = cleanupStaleFirefoxProfiles();
|
|
724
|
+
if (cleaned.removed > 0) {
|
|
725
|
+
log('info', 'cleaned stale firefox profiles after browser close', cleaned);
|
|
726
|
+
}
|
|
727
|
+
} catch { /* best effort */ }
|
|
728
|
+
|
|
729
|
+
// Reset native memory baseline so next browser measures from fresh
|
|
730
|
+
reporter.resetNativeMemBaseline();
|
|
731
|
+
|
|
732
|
+
// Verify cleanup: check FD/handle counts dropped (after force-kill completes)
|
|
733
|
+
const postCloseFds = _countOpenFds();
|
|
734
|
+
const postCloseHandles = _countActiveHandles();
|
|
735
|
+
if (postCloseFds !== null && preCloseFds !== null) {
|
|
736
|
+
const fdDelta = postCloseFds - preCloseFds;
|
|
737
|
+
// After close we expect fewer FDs. If more leaked, warn.
|
|
738
|
+
if (fdDelta > 10) {
|
|
739
|
+
log('warn', 'FD leak detected after browser close', {
|
|
740
|
+
reason, preCloseFds, postCloseFds, delta: fdDelta,
|
|
741
|
+
preCloseHandles, postCloseHandles,
|
|
742
|
+
});
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
log('info', 'browser closed fully', {
|
|
746
|
+
reason, pid, preCloseFds, postCloseFds, preCloseHandles, postCloseHandles,
|
|
747
|
+
});
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
/**
|
|
751
|
+
* Force-kill a browser process tree by PID. On Linux, kills the process group
|
|
752
|
+
* (SIGKILL -pid) then scans /proc for any orphaned children.
|
|
753
|
+
*/
|
|
754
|
+
async function _forceKillProcessTree(pid, reason) {
|
|
755
|
+
if (!pid || pid <= 1) return;
|
|
756
|
+
|
|
757
|
+
// Kill the specific browser process first (positive PID = single process)
|
|
758
|
+
try {
|
|
759
|
+
process.kill(pid, 'SIGKILL');
|
|
760
|
+
log('info', 'sent SIGKILL to browser process', { pid, reason });
|
|
761
|
+
} catch (err) {
|
|
762
|
+
if (err.code !== 'ESRCH') {
|
|
763
|
+
log('warn', 'failed to kill browser process', { pid, error: err.message });
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
// Then try the process group (Playwright launches with detached:true on Linux,
|
|
768
|
+
// making the browser a process group leader)
|
|
769
|
+
try {
|
|
770
|
+
process.kill(-pid, 'SIGKILL');
|
|
771
|
+
} catch {
|
|
772
|
+
// ESRCH = group doesn't exist (browser wasn't a group leader), which is fine
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
// Wait for kernel to reparent children to PID 1 before scanning
|
|
776
|
+
await new Promise(r => setTimeout(r, 200));
|
|
777
|
+
|
|
778
|
+
// On Linux: scan /proc for orphaned children that escaped the process group
|
|
779
|
+
// (reparented to PID 1 by init/systemd, common with Firefox content processes).
|
|
780
|
+
// Also checks PPid === Node PID for containerized environments without init.
|
|
781
|
+
if (process.platform === 'linux') {
|
|
782
|
+
const myPid = process.pid;
|
|
783
|
+
// Snapshot the current browser PID to avoid killing a newly launched browser
|
|
784
|
+
const currentBrowserPid = _lastBrowserPid;
|
|
785
|
+
try {
|
|
786
|
+
const procDirs = fs.readdirSync('/proc').filter(d => /^\d+$/.test(d));
|
|
787
|
+
const orphans = [];
|
|
788
|
+
for (const procPid of procDirs) {
|
|
789
|
+
const numPid = parseInt(procPid);
|
|
790
|
+
// Never kill ourselves, the old PID (already killed), or the new browser
|
|
791
|
+
if (numPid === myPid || numPid === pid || numPid === currentBrowserPid) continue;
|
|
792
|
+
try {
|
|
793
|
+
const status = fs.readFileSync(`/proc/${procPid}/status`, 'utf8');
|
|
794
|
+
const ppidMatch = status.match(/PPid:\s+(\d+)/);
|
|
795
|
+
const ppid = ppidMatch ? parseInt(ppidMatch[1]) : -1;
|
|
796
|
+
// Orphaned to init (PID 1) or reparented to us (Node is PID 1 in containers)
|
|
797
|
+
if (ppid === 1 || ppid === myPid) {
|
|
798
|
+
const cmdline = fs.readFileSync(`/proc/${procPid}/cmdline`, 'utf8');
|
|
799
|
+
// Firefox-specific: binary name or Gecko child process marker
|
|
800
|
+
if (/firefox-esr|firefox|camoufox|libxul\.so|GeckoChildProcess/i.test(cmdline)) {
|
|
801
|
+
orphans.push(numPid);
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
} catch { /* process vanished or permission denied */ }
|
|
805
|
+
}
|
|
806
|
+
if (orphans.length > 0) {
|
|
807
|
+
log('warn', 'killing orphaned browser child processes', { orphans, reason });
|
|
808
|
+
for (const orphanPid of orphans) {
|
|
809
|
+
try { process.kill(orphanPid, 'SIGKILL'); } catch { /* already dead */ }
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
} catch (err) {
|
|
813
|
+
log('warn', 'failed to scan for orphaned browser processes', { error: err.message });
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
// Give the OS a moment to reclaim resources
|
|
818
|
+
await new Promise(r => setTimeout(r, 300));
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
function _countOpenFds() {
|
|
822
|
+
try {
|
|
823
|
+
if (process.platform === 'linux') return fs.readdirSync('/proc/self/fd').length;
|
|
824
|
+
} catch { /* unavailable */ }
|
|
825
|
+
return null;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
function _countActiveHandles() {
|
|
829
|
+
try { return process._getActiveHandles().length; } catch { return null; }
|
|
830
|
+
}
|
|
831
|
+
|
|
663
832
|
async function launchBrowserInstance() {
|
|
664
833
|
const hostOS = getHostOS();
|
|
665
834
|
const maxAttempts = proxyPool?.launchRetries ?? 1;
|
|
@@ -738,7 +907,8 @@ async function launchBrowserInstance() {
|
|
|
738
907
|
|
|
739
908
|
virtualDisplay = localVirtualDisplay;
|
|
740
909
|
browserLaunchProxy = launchProxy;
|
|
741
|
-
|
|
910
|
+
_lastBrowserPid = candidateBrowser.process?.()?.pid ?? null;
|
|
911
|
+
browser = candidateBrowser; // publish AFTER PID is captured
|
|
742
912
|
attachBrowserCleanup(browser, localVirtualDisplay);
|
|
743
913
|
pluginEvents.emit('browser:launched', { browser, display: vdDisplay });
|
|
744
914
|
|
|
@@ -775,13 +945,7 @@ async function ensureBrowser() {
|
|
|
775
945
|
deadSessions: sessions.size,
|
|
776
946
|
});
|
|
777
947
|
await closeAllSessions('browser_disconnected', { clearDownloads: true, clearLocks: true });
|
|
778
|
-
|
|
779
|
-
if (virtualDisplay) {
|
|
780
|
-
virtualDisplay.kill();
|
|
781
|
-
virtualDisplay = null;
|
|
782
|
-
}
|
|
783
|
-
browserLaunchProxy = null;
|
|
784
|
-
browser = null;
|
|
948
|
+
await closeBrowserFully('browser_disconnected');
|
|
785
949
|
}
|
|
786
950
|
if (browser) return browser;
|
|
787
951
|
if (browserLaunchPromise) return browserLaunchPromise;
|
|
@@ -1730,6 +1894,10 @@ app.get('/health', (req, res) => {
|
|
|
1730
1894
|
...(FLY_MACHINE_ID ? { machineId: FLY_MACHINE_ID } : {}),
|
|
1731
1895
|
});
|
|
1732
1896
|
}
|
|
1897
|
+
const mem = process.memoryUsage();
|
|
1898
|
+
const rssMb = Math.round(mem.rss / 1048576);
|
|
1899
|
+
const heapUsedMb = Math.round(mem.heapUsed / 1048576);
|
|
1900
|
+
const nativeMemMb = rssMb - heapUsedMb;
|
|
1733
1901
|
res.json({
|
|
1734
1902
|
ok: true,
|
|
1735
1903
|
engine: 'camoufox',
|
|
@@ -1738,6 +1906,7 @@ app.get('/health', (req, res) => {
|
|
|
1738
1906
|
activeTabs: getTotalTabCount(),
|
|
1739
1907
|
activeSessions: sessions.size,
|
|
1740
1908
|
consecutiveFailures: healthState.consecutiveNavFailures,
|
|
1909
|
+
memory: { rssMb, heapUsedMb, nativeMemMb },
|
|
1741
1910
|
...(FLY_MACHINE_ID ? { machineId: FLY_MACHINE_ID } : {}),
|
|
1742
1911
|
});
|
|
1743
1912
|
});
|
|
@@ -4390,11 +4559,8 @@ app.post('/stop', async (req, res) => {
|
|
|
4390
4559
|
if (!adminKey || !timingSafeCompare(adminKey, CONFIG.adminKey)) {
|
|
4391
4560
|
return res.status(403).json({ error: 'Forbidden' });
|
|
4392
4561
|
}
|
|
4393
|
-
if (browser) {
|
|
4394
|
-
await browser.close().catch(() => {});
|
|
4395
|
-
browser = null;
|
|
4396
|
-
}
|
|
4397
4562
|
await closeAllSessions('admin_stop', { clearDownloads: true, clearLocks: true });
|
|
4563
|
+
await closeBrowserFully('admin_stop');
|
|
4398
4564
|
res.json({ ok: true, stopped: true, profile: 'camoufox' });
|
|
4399
4565
|
} catch (err) {
|
|
4400
4566
|
res.status(500).json({ ok: false, error: safeError(err) });
|
|
@@ -4971,7 +5137,7 @@ async function gracefulShutdown(signal) {
|
|
|
4971
5137
|
clearLocks: false,
|
|
4972
5138
|
});
|
|
4973
5139
|
|
|
4974
|
-
|
|
5140
|
+
await closeBrowserFully(`shutdown:${signal}`);
|
|
4975
5141
|
process.exit(0);
|
|
4976
5142
|
}
|
|
4977
5143
|
|
|
@@ -5031,6 +5197,20 @@ const server = app.listen(PORT, async () => {
|
|
|
5031
5197
|
if (tmpCleanup.removed > 0) {
|
|
5032
5198
|
log('info', 'cleaned up orphaned camoufox temp files', tmpCleanup);
|
|
5033
5199
|
}
|
|
5200
|
+
const profileCleanup = cleanupStaleFirefoxProfiles();
|
|
5201
|
+
if (profileCleanup.removed > 0) {
|
|
5202
|
+
log('info', 'cleaned up stale firefox profiles on startup', profileCleanup);
|
|
5203
|
+
}
|
|
5204
|
+
|
|
5205
|
+
// Periodic temp profile cleanup every 10 minutes
|
|
5206
|
+
setInterval(() => {
|
|
5207
|
+
try {
|
|
5208
|
+
const cleaned = cleanupStaleFirefoxProfiles();
|
|
5209
|
+
if (cleaned.removed > 0) {
|
|
5210
|
+
log('info', 'periodic firefox profile cleanup', cleaned);
|
|
5211
|
+
}
|
|
5212
|
+
} catch { /* best effort */ }
|
|
5213
|
+
}, 10 * 60 * 1000).unref();
|
|
5034
5214
|
const traceSweep = sweepOldTraces({
|
|
5035
5215
|
baseDir: CONFIG.tracesDir,
|
|
5036
5216
|
ttlMs: CONFIG.tracesTtlHours * 3600 * 1000,
|