@askjo/camofox-browser 1.5.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +17 -2
- package/README.md +20 -0
- package/camofox.config.json +10 -0
- package/lib/auth.js +71 -0
- package/lib/config.js +1 -0
- package/lib/cookies.js +38 -1
- package/lib/downloads.js +10 -2
- package/lib/inflight.js +16 -0
- package/lib/metrics.js +29 -0
- package/lib/persistence.js +89 -0
- package/lib/plugins.js +174 -0
- package/lib/tmp-cleanup.js +40 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +5 -1
- package/plugin.ts +8 -1
- package/plugins/persistence/AGENTS.md +37 -0
- package/plugins/persistence/README.md +48 -0
- package/plugins/persistence/index.js +120 -0
- package/plugins/persistence/persistence.test.js +117 -0
- package/plugins/persistence/plugin.test.js +98 -0
- package/plugins/vnc/AGENTS.md +42 -0
- package/plugins/vnc/README.md +165 -0
- package/plugins/vnc/apt.txt +7 -0
- package/plugins/vnc/index.js +142 -0
- package/plugins/vnc/vnc-launcher.js +64 -0
- package/plugins/vnc/vnc-watcher.sh +82 -0
- package/plugins/vnc/vnc.test.js +204 -0
- package/plugins/youtube/AGENTS.md +25 -0
- package/plugins/youtube/apt.txt +1 -0
- package/plugins/youtube/index.js +206 -0
- package/plugins/youtube/post-install.sh +5 -0
- package/plugins/youtube/youtube.test.js +41 -0
- package/scripts/install-plugin-deps.sh +63 -0
- package/scripts/plugin.js +342 -0
- package/scripts/plugin.test.js +117 -0
- package/server.js +286 -328
- /package/{lib → plugins/youtube}/youtube.js +0 -0
package/Dockerfile
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
FROM node:20-slim
|
|
1
|
+
FROM node:20-slim AS camofox-browser
|
|
2
2
|
|
|
3
3
|
# Pinned Camoufox version for reproducible builds
|
|
4
4
|
# Update these when upgrading Camoufox
|
|
@@ -36,6 +36,7 @@ RUN apt-get update && apt-get install -y \
|
|
|
36
36
|
fontconfig \
|
|
37
37
|
# Utils
|
|
38
38
|
ca-certificates \
|
|
39
|
+
curl \
|
|
39
40
|
unzip \
|
|
40
41
|
# yt-dlp runtime dependency
|
|
41
42
|
python3-minimal \
|
|
@@ -60,11 +61,25 @@ COPY package.json ./
|
|
|
60
61
|
RUN npm install --production
|
|
61
62
|
|
|
62
63
|
COPY server.js ./
|
|
64
|
+
COPY camofox.config.json ./
|
|
63
65
|
COPY lib/ ./lib/
|
|
66
|
+
COPY plugins/ ./plugins/
|
|
67
|
+
COPY scripts/ ./scripts/
|
|
68
|
+
|
|
69
|
+
# Install default plugin dependencies (apt packages + post-install hooks)
|
|
70
|
+
RUN scripts/install-plugin-deps.sh
|
|
64
71
|
|
|
65
72
|
ENV NODE_ENV=production
|
|
66
|
-
ENV CAMOFOX_PORT=
|
|
73
|
+
ENV CAMOFOX_PORT=9377
|
|
67
74
|
|
|
68
75
|
EXPOSE 9377
|
|
69
76
|
|
|
70
77
|
CMD ["sh", "-c", "node --max-old-space-size=${MAX_OLD_SPACE_SIZE:-128} server.js"]
|
|
78
|
+
|
|
79
|
+
# Optional: rebuild plugin deps after adding third-party plugins
|
|
80
|
+
# Usage: docker build --target with-plugins -t camofox-browser .
|
|
81
|
+
FROM camofox-browser AS with-plugins
|
|
82
|
+
COPY plugins/ ./plugins/
|
|
83
|
+
COPY camofox.config.json ./
|
|
84
|
+
COPY scripts/install-plugin-deps.sh /tmp/install-plugin-deps.sh
|
|
85
|
+
RUN /tmp/install-plugin-deps.sh && rm /tmp/install-plugin-deps.sh
|
package/README.md
CHANGED
|
@@ -50,6 +50,7 @@ This project wraps that engine in a REST API built for agents: accessibility sna
|
|
|
50
50
|
- **Download Capture** - capture browser downloads and fetch them via API (optional inline base64)
|
|
51
51
|
- **DOM Image Extraction** - list `<img>` src/alt and optionally return inline data URLs
|
|
52
52
|
- **Deploy Anywhere** - Docker, Fly.io, Railway
|
|
53
|
+
- **VNC Interactive Login** - log into sites visually via noVNC, export storage state for agent reuse
|
|
53
54
|
|
|
54
55
|
## Optional Dependencies
|
|
55
56
|
|
|
@@ -178,6 +179,20 @@ Camoufox browser session (authenticated browsing)
|
|
|
178
179
|
- Max 500 cookies per request, 5MB file size limit
|
|
179
180
|
- Cookie objects are sanitized to an allowlist of Playwright fields
|
|
180
181
|
|
|
182
|
+
### Session Persistence
|
|
183
|
+
|
|
184
|
+
By default, camofox persists each user's cookies and localStorage to `~/.camofox/profiles/`. Sessions survive browser restarts — log in once (via cookies or VNC), and subsequent sessions restore the authenticated state automatically.
|
|
185
|
+
|
|
186
|
+
```
|
|
187
|
+
~/.camofox/
|
|
188
|
+
├── cookies/ # Bootstrap cookie files (Netscape format)
|
|
189
|
+
└── profiles/ # Persisted session state (auto-managed)
|
|
190
|
+
└── <hashed-userId>/
|
|
191
|
+
└── storage_state.json
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Override the directory with `CAMOFOX_PROFILE_DIR` or set `"profileDir"` in the persistence plugin config. To disable persistence, set `"persistence": { "enabled": false }` in `camofox.config.json`.
|
|
195
|
+
|
|
181
196
|
#### Standalone server usage
|
|
182
197
|
|
|
183
198
|
```bash
|
|
@@ -346,6 +361,7 @@ Uses [yt-dlp](https://github.com/yt-dlp/yt-dlp) when available (fast, no browser
|
|
|
346
361
|
| Method | Endpoint | Description |
|
|
347
362
|
|--------|----------|-------------|
|
|
348
363
|
| `POST` | `/sessions/:userId/cookies` | Add cookies to a user session (Playwright cookie objects) |
|
|
364
|
+
| `GET` | `/sessions/:userId/storage_state` | Export cookies + localStorage ([VNC plugin](plugins/vnc/)) |
|
|
349
365
|
|
|
350
366
|
## Search Macros
|
|
351
367
|
|
|
@@ -364,6 +380,7 @@ Reddit macros return JSON directly (no HTML parsing needed):
|
|
|
364
380
|
| `CAMOFOX_API_KEY` | Enable cookie import endpoint (disabled if unset) | - |
|
|
365
381
|
| `CAMOFOX_ADMIN_KEY` | Required for `POST /stop` | - |
|
|
366
382
|
| `CAMOFOX_COOKIES_DIR` | Directory for cookie files | `~/.camofox/cookies` |
|
|
383
|
+
| `CAMOFOX_PROFILE_DIR` | Directory for persisted session profiles | `~/.camofox/profiles` |
|
|
367
384
|
| `MAX_SESSIONS` | Max concurrent browser sessions | `50` |
|
|
368
385
|
| `MAX_TABS_PER_SESSION` | Max tabs per session | `10` |
|
|
369
386
|
| `SESSION_TIMEOUT_MS` | Session inactivity timeout | `1800000` (30min) |
|
|
@@ -382,6 +399,9 @@ Reddit macros return JSON directly (no HTML parsing needed):
|
|
|
382
399
|
| `PROXY_COUNTRY` | Target country for proxy geo-targeting | - |
|
|
383
400
|
| `PROXY_STATE` | Target state/region for proxy geo-targeting | - |
|
|
384
401
|
| `TAB_INACTIVITY_MS` | Close tabs idle longer than this | `300000` (5min) |
|
|
402
|
+
| `ENABLE_VNC` | Enable VNC plugin for interactive browser access (`1`) | - |
|
|
403
|
+
| `VNC_PASSWORD` | Password for VNC access (recommended in production) | - |
|
|
404
|
+
| `NOVNC_PORT` | noVNC web UI port | `6080` |
|
|
385
405
|
|
|
386
406
|
## Architecture
|
|
387
407
|
|
package/lib/auth.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared auth middleware for camofox-browser.
|
|
3
|
+
*
|
|
4
|
+
* Extracts the duplicated auth pattern from cookie/storage_state endpoints
|
|
5
|
+
* into a reusable Express middleware factory.
|
|
6
|
+
*
|
|
7
|
+
* Policy:
|
|
8
|
+
* - If CAMOFOX_API_KEY is set, require Bearer token match (timing-safe).
|
|
9
|
+
* - If not set and NODE_ENV !== production, allow loopback (127.0.0.1 / ::1).
|
|
10
|
+
* - Otherwise, reject.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import crypto from 'crypto';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Timing-safe string comparison.
|
|
17
|
+
*/
|
|
18
|
+
function timingSafeCompare(a, b) {
|
|
19
|
+
if (typeof a !== 'string' || typeof b !== 'string') return false;
|
|
20
|
+
const bufA = Buffer.from(a);
|
|
21
|
+
const bufB = Buffer.from(b);
|
|
22
|
+
if (bufA.length !== bufB.length) {
|
|
23
|
+
// Compare against self to burn constant time, then return false
|
|
24
|
+
crypto.timingSafeEqual(bufA, bufA);
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
return crypto.timingSafeEqual(bufA, bufB);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Check if an address is loopback.
|
|
32
|
+
*/
|
|
33
|
+
function isLoopbackAddress(address) {
|
|
34
|
+
if (!address) return false;
|
|
35
|
+
return address === '127.0.0.1' || address === '::1' || address === '::ffff:127.0.0.1';
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Create an Express middleware that enforces API key auth.
|
|
40
|
+
*
|
|
41
|
+
* @param {object} config - Must have { apiKey, nodeEnv }
|
|
42
|
+
* @param {object} [options]
|
|
43
|
+
* @param {string} [options.errorMessage] - Custom error message when rejecting unauthenticated requests
|
|
44
|
+
* @returns {function} Express middleware (req, res, next)
|
|
45
|
+
*/
|
|
46
|
+
export function requireAuth(config, options = {}) {
|
|
47
|
+
const errorMessage = options.errorMessage ||
|
|
48
|
+
'This endpoint requires CAMOFOX_API_KEY except for loopback requests in non-production environments.';
|
|
49
|
+
|
|
50
|
+
return (req, res, next) => {
|
|
51
|
+
if (config.apiKey) {
|
|
52
|
+
const auth = String(req.headers['authorization'] || '');
|
|
53
|
+
const match = auth.match(/^Bearer\s+(.+)$/i);
|
|
54
|
+
if (!match || !timingSafeCompare(match[1], config.apiKey)) {
|
|
55
|
+
return res.status(403).json({ error: 'Forbidden' });
|
|
56
|
+
}
|
|
57
|
+
return next();
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const remoteAddress = req.socket?.remoteAddress || '';
|
|
61
|
+
const allowUnauthedLocal = config.nodeEnv !== 'production' && isLoopbackAddress(remoteAddress);
|
|
62
|
+
if (!allowUnauthedLocal) {
|
|
63
|
+
return res.status(403).json({ error: errorMessage });
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
next();
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Re-export utilities so server.js can still use them directly
|
|
71
|
+
export { timingSafeCompare, isLoopbackAddress };
|
package/lib/config.js
CHANGED
|
@@ -46,6 +46,7 @@ function loadConfig() {
|
|
|
46
46
|
adminKey: process.env.CAMOFOX_ADMIN_KEY || '',
|
|
47
47
|
apiKey: process.env.CAMOFOX_API_KEY || '',
|
|
48
48
|
cookiesDir: process.env.CAMOFOX_COOKIES_DIR || join(os.homedir(), '.camofox', 'cookies'),
|
|
49
|
+
profileDir: process.env.CAMOFOX_PROFILE_DIR || join(os.homedir(), '.camofox', 'profiles'),
|
|
49
50
|
handlerTimeoutMs: parseInt(process.env.HANDLER_TIMEOUT_MS) || 30000,
|
|
50
51
|
maxConcurrentPerUser: parseInt(process.env.MAX_CONCURRENT_PER_USER) || 3,
|
|
51
52
|
sessionTimeoutMs: parseInt(process.env.SESSION_TIMEOUT_MS) || 600000,
|
package/lib/cookies.js
CHANGED
|
@@ -79,4 +79,41 @@ async function readCookieFile({ cookiesDir, cookiesPath, domainSuffix, maxBytes
|
|
|
79
79
|
}));
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
/**
|
|
83
|
+
* Import all cookies from the default bootstrap cookie file into a Playwright context.
|
|
84
|
+
* Intended for first-run session seeding before any persistent storage state exists.
|
|
85
|
+
* Missing file is treated as a no-op.
|
|
86
|
+
* @param {object} opts
|
|
87
|
+
* @param {string} opts.cookiesDir - Base directory for cookie files
|
|
88
|
+
* @param {object} opts.context - Playwright BrowserContext
|
|
89
|
+
* @param {string} [opts.cookiesPath='cookies.txt'] - Relative cookie file path within cookiesDir
|
|
90
|
+
* @param {object} [opts.logger=console] - Logger with warn()
|
|
91
|
+
* @returns {Promise<{imported: number, source: string|null}>}
|
|
92
|
+
*/
|
|
93
|
+
async function importBootstrapCookies({ cookiesDir, context, cookiesPath = 'cookies.txt', logger = console }) {
|
|
94
|
+
if (!cookiesDir || !context) {
|
|
95
|
+
return { imported: 0, source: null };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const resolved = path.resolve(cookiesDir, cookiesPath);
|
|
99
|
+
|
|
100
|
+
try {
|
|
101
|
+
const cookies = await readCookieFile({ cookiesDir, cookiesPath });
|
|
102
|
+
if (cookies.length === 0) {
|
|
103
|
+
return { imported: 0, source: resolved };
|
|
104
|
+
}
|
|
105
|
+
await context.addCookies(cookies);
|
|
106
|
+
return { imported: cookies.length, source: resolved };
|
|
107
|
+
} catch (err) {
|
|
108
|
+
if (err?.code === 'ENOENT') {
|
|
109
|
+
return { imported: 0, source: null };
|
|
110
|
+
}
|
|
111
|
+
logger?.warn?.('failed to import bootstrap cookies', {
|
|
112
|
+
cookiesPath: resolved,
|
|
113
|
+
error: err?.message || String(err),
|
|
114
|
+
});
|
|
115
|
+
return { imported: 0, source: resolved };
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export { parseNetscapeCookieFile, readCookieFile, importBootstrapCookies };
|
package/lib/downloads.js
CHANGED
|
@@ -60,7 +60,7 @@ async function clearSessionDownloads(session) {
|
|
|
60
60
|
await Promise.all(tasks);
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
function attachDownloadListener(tabState, tabId, log) {
|
|
63
|
+
function attachDownloadListener(tabState, tabId, log, pluginEvents, userId) {
|
|
64
64
|
if (tabState.downloadListenerAttached) return;
|
|
65
65
|
tabState.downloadListenerAttached = true;
|
|
66
66
|
|
|
@@ -69,6 +69,11 @@ function attachDownloadListener(tabState, tabId, log) {
|
|
|
69
69
|
const suggestedFilename = sanitizeFilename(download.suggestedFilename?.() || `download-${downloadId}.bin`);
|
|
70
70
|
const filePath = path.join(os.tmpdir(), `camofox-download-${downloadId}-${suggestedFilename}`);
|
|
71
71
|
|
|
72
|
+
const url = String(download.url?.() || '').trim();
|
|
73
|
+
if (pluginEvents) {
|
|
74
|
+
pluginEvents.emit('tab:download:start', { userId: userId || null, tabId, filename: suggestedFilename, url });
|
|
75
|
+
}
|
|
76
|
+
|
|
72
77
|
let failure = null;
|
|
73
78
|
let bytes = null;
|
|
74
79
|
|
|
@@ -86,7 +91,6 @@ function attachDownloadListener(tabState, tabId, log) {
|
|
|
86
91
|
failure = reportedFailure;
|
|
87
92
|
}
|
|
88
93
|
|
|
89
|
-
const url = String(download.url?.() || '').trim();
|
|
90
94
|
if (url) {
|
|
91
95
|
tabState.visitedUrls.add(url);
|
|
92
96
|
}
|
|
@@ -104,6 +108,10 @@ function attachDownloadListener(tabState, tabId, log) {
|
|
|
104
108
|
failure,
|
|
105
109
|
});
|
|
106
110
|
|
|
111
|
+
if (pluginEvents && !failure) {
|
|
112
|
+
pluginEvents.emit('tab:download:complete', { userId: userId || null, tabId, filename: suggestedFilename, path: filePath, size: bytes });
|
|
113
|
+
}
|
|
114
|
+
|
|
107
115
|
await trimTabDownloads(tabState);
|
|
108
116
|
log('info', 'download captured', {
|
|
109
117
|
tabId, downloadId, suggestedFilename, mimeType, bytes,
|
package/lib/inflight.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
async function coalesceInflight(map, key, factory) {
|
|
2
|
+
const existing = map.get(key);
|
|
3
|
+
if (existing) return existing;
|
|
4
|
+
|
|
5
|
+
const promise = (async () => {
|
|
6
|
+
try {
|
|
7
|
+
return await factory();
|
|
8
|
+
} finally {
|
|
9
|
+
map.delete(key);
|
|
10
|
+
}
|
|
11
|
+
})();
|
|
12
|
+
map.set(key, promise);
|
|
13
|
+
return promise;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export { coalesceInflight };
|
package/lib/metrics.js
CHANGED
|
@@ -12,6 +12,27 @@ const noopCounter = { inc() {}, labels() { return this; } };
|
|
|
12
12
|
const noopHistogram = { observe() {}, startTimer() { return () => {}; }, labels() { return this; } };
|
|
13
13
|
const noopGauge = { set() {}, inc() {}, dec() {}, labels() { return this; } };
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* Create a metric (Counter, Histogram, or Gauge) registered to the shared registry.
|
|
17
|
+
* Returns a no-op stub when Prometheus is disabled — plugins never need to check.
|
|
18
|
+
*
|
|
19
|
+
* @param {'counter'|'histogram'|'gauge'} type
|
|
20
|
+
* @param {object} opts - prom-client options: { name, help, labelNames, buckets, ... }
|
|
21
|
+
* @returns {object} The metric instance or a no-op stub
|
|
22
|
+
*/
|
|
23
|
+
export async function createMetric(type, opts) {
|
|
24
|
+
if (!_register) {
|
|
25
|
+
if (type === 'histogram') return noopHistogram;
|
|
26
|
+
if (type === 'gauge') return noopGauge;
|
|
27
|
+
return noopCounter;
|
|
28
|
+
}
|
|
29
|
+
const client = (await import('prom-client')).default;
|
|
30
|
+
const MetricClass = type === 'histogram' ? client.Histogram
|
|
31
|
+
: type === 'gauge' ? client.Gauge
|
|
32
|
+
: client.Counter;
|
|
33
|
+
return new MetricClass({ ...opts, registers: [_register] });
|
|
34
|
+
}
|
|
35
|
+
|
|
15
36
|
function buildNoopMetrics() {
|
|
16
37
|
return {
|
|
17
38
|
requestsTotal: noopCounter,
|
|
@@ -24,6 +45,7 @@ function buildNoopMetrics() {
|
|
|
24
45
|
tabsRecycledTotal: noopCounter,
|
|
25
46
|
requestDuration: noopHistogram,
|
|
26
47
|
pageLoadDuration: noopHistogram,
|
|
48
|
+
snapshotBytes: noopHistogram,
|
|
27
49
|
activeTabsGauge: noopGauge,
|
|
28
50
|
tabLockQueueDepth: noopGauge,
|
|
29
51
|
memoryUsageBytes: noopGauge,
|
|
@@ -93,6 +115,13 @@ async function buildRealMetrics() {
|
|
|
93
115
|
buckets: [0.5, 1, 2, 5, 10, 20, 30, 60],
|
|
94
116
|
registers: [_register],
|
|
95
117
|
}),
|
|
118
|
+
snapshotBytes: new client.Histogram({
|
|
119
|
+
name: 'camofox_snapshot_bytes',
|
|
120
|
+
help: 'Size of accessibility tree snapshots in bytes (before windowing)',
|
|
121
|
+
labelNames: ['type'],
|
|
122
|
+
buckets: [1000, 5000, 10000, 25000, 50000, 80000, 120000, 200000, 500000],
|
|
123
|
+
registers: [_register],
|
|
124
|
+
}),
|
|
96
125
|
activeTabsGauge: new client.Gauge({
|
|
97
126
|
name: 'camofox_active_tabs',
|
|
98
127
|
help: 'Current number of open browser tabs',
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
function getUserPersistencePaths(profileDir, userId) {
|
|
6
|
+
const rootDir = path.resolve(profileDir);
|
|
7
|
+
const safeUserDir = crypto
|
|
8
|
+
.createHash('sha256')
|
|
9
|
+
.update(String(userId))
|
|
10
|
+
.digest('hex')
|
|
11
|
+
.slice(0, 32);
|
|
12
|
+
|
|
13
|
+
const userDir = path.join(rootDir, safeUserDir);
|
|
14
|
+
return {
|
|
15
|
+
rootDir,
|
|
16
|
+
userDir,
|
|
17
|
+
storageStatePath: path.join(userDir, 'storage-state.json'),
|
|
18
|
+
metaPath: path.join(userDir, 'meta.json'),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function loadPersistedStorageState(profileDir, userId, logger = console) {
|
|
23
|
+
if (!profileDir) return undefined;
|
|
24
|
+
|
|
25
|
+
const { storageStatePath } = getUserPersistencePaths(profileDir, userId);
|
|
26
|
+
|
|
27
|
+
try {
|
|
28
|
+
const raw = await fs.readFile(storageStatePath, 'utf8');
|
|
29
|
+
const parsed = JSON.parse(raw);
|
|
30
|
+
if (!parsed || typeof parsed !== 'object') return undefined;
|
|
31
|
+
if (!Array.isArray(parsed.cookies)) return undefined;
|
|
32
|
+
if (parsed.origins !== undefined && !Array.isArray(parsed.origins)) return undefined;
|
|
33
|
+
return storageStatePath;
|
|
34
|
+
} catch (err) {
|
|
35
|
+
if (err?.code === 'ENOENT') return undefined;
|
|
36
|
+
logger?.warn?.('failed to load persisted storage state', {
|
|
37
|
+
userId: String(userId),
|
|
38
|
+
storageStatePath,
|
|
39
|
+
error: err?.message || String(err),
|
|
40
|
+
});
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function persistStorageState({ profileDir, userId, context, logger = console }) {
|
|
46
|
+
if (!profileDir || !context) {
|
|
47
|
+
return { persisted: false, reason: 'disabled' };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const { userDir, storageStatePath, metaPath } = getUserPersistencePaths(profileDir, userId);
|
|
51
|
+
const suffix = `.tmp-${process.pid}-${Date.now()}`;
|
|
52
|
+
const tmpStoragePath = `${storageStatePath}${suffix}`;
|
|
53
|
+
const tmpMetaPath = `${metaPath}${suffix}`;
|
|
54
|
+
|
|
55
|
+
try {
|
|
56
|
+
await fs.mkdir(userDir, { recursive: true });
|
|
57
|
+
await context.storageState({ path: tmpStoragePath });
|
|
58
|
+
await fs.rename(tmpStoragePath, storageStatePath);
|
|
59
|
+
await fs.writeFile(
|
|
60
|
+
tmpMetaPath,
|
|
61
|
+
JSON.stringify(
|
|
62
|
+
{
|
|
63
|
+
userId: String(userId),
|
|
64
|
+
updatedAt: new Date().toISOString(),
|
|
65
|
+
storageStatePath,
|
|
66
|
+
},
|
|
67
|
+
null,
|
|
68
|
+
2
|
|
69
|
+
)
|
|
70
|
+
);
|
|
71
|
+
await fs.rename(tmpMetaPath, metaPath);
|
|
72
|
+
return { persisted: true, userDir, storageStatePath, metaPath };
|
|
73
|
+
} catch (err) {
|
|
74
|
+
await fs.unlink(tmpStoragePath).catch(() => {});
|
|
75
|
+
await fs.unlink(tmpMetaPath).catch(() => {});
|
|
76
|
+
logger?.warn?.('failed to persist storage state', {
|
|
77
|
+
userId: String(userId),
|
|
78
|
+
storageStatePath,
|
|
79
|
+
error: err?.message || String(err),
|
|
80
|
+
});
|
|
81
|
+
return { persisted: false, reason: 'error', error: err };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export {
|
|
86
|
+
getUserPersistencePaths,
|
|
87
|
+
loadPersistedStorageState,
|
|
88
|
+
persistStorageState,
|
|
89
|
+
};
|
package/lib/plugins.js
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Camofox-browser plugin system.
|
|
3
|
+
*
|
|
4
|
+
* Plugins live in plugins/<name>/index.js and export a register(app, ctx) function.
|
|
5
|
+
* The ctx object provides access to sessions, config, logging, auth middleware,
|
|
6
|
+
* core functions, and an EventEmitter for lifecycle hooks.
|
|
7
|
+
*
|
|
8
|
+
* 29 events across 7 categories:
|
|
9
|
+
*
|
|
10
|
+
* BROWSER LIFECYCLE
|
|
11
|
+
* browser:launching { options } — mutate launch options
|
|
12
|
+
* browser:launched { browser, display } — after launch
|
|
13
|
+
* browser:restart { reason } — before restart cycle
|
|
14
|
+
* browser:closed { reason } — after browser closed
|
|
15
|
+
* browser:error { error } — uncaught browser error
|
|
16
|
+
*
|
|
17
|
+
* SESSION LIFECYCLE
|
|
18
|
+
* session:creating { userId, contextOptions } — mutate context options
|
|
19
|
+
* session:created { userId, context } — after context stored
|
|
20
|
+
* session:destroyed { userId, reason } — after cleanup
|
|
21
|
+
* session:expired { userId, idleMs } — reaper triggered
|
|
22
|
+
*
|
|
23
|
+
* TAB LIFECYCLE
|
|
24
|
+
* tab:created { userId, tabId, page, url }
|
|
25
|
+
* tab:navigated { userId, tabId, url, prevUrl }
|
|
26
|
+
* tab:destroyed { userId, tabId, reason }
|
|
27
|
+
* tab:recycled { userId, tabId }
|
|
28
|
+
* tab:error { userId, tabId, error }
|
|
29
|
+
*
|
|
30
|
+
* CONTENT
|
|
31
|
+
* tab:snapshot { userId, tabId, snapshot }
|
|
32
|
+
* tab:screenshot { userId, tabId, buffer }
|
|
33
|
+
* tab:evaluate { userId, tabId, expression }
|
|
34
|
+
* tab:evaluated { userId, tabId, result }
|
|
35
|
+
*
|
|
36
|
+
* INPUT
|
|
37
|
+
* tab:click { userId, tabId, ref, selector }
|
|
38
|
+
* tab:type { userId, tabId, text, ref, mode }
|
|
39
|
+
* tab:scroll { userId, tabId, direction, amount }
|
|
40
|
+
* tab:press { userId, tabId, key }
|
|
41
|
+
*
|
|
42
|
+
* DOWNLOADS
|
|
43
|
+
* tab:download:start { userId, tabId, filename, url }
|
|
44
|
+
* tab:download:complete { userId, tabId, filename, path, size }
|
|
45
|
+
*
|
|
46
|
+
* COOKIES / AUTH
|
|
47
|
+
* session:cookies:import { userId, count }
|
|
48
|
+
* session:storage:export { userId }
|
|
49
|
+
*
|
|
50
|
+
* SERVER
|
|
51
|
+
* server:starting { port }
|
|
52
|
+
* server:started { port, pid }
|
|
53
|
+
* server:shutdown { signal }
|
|
54
|
+
*
|
|
55
|
+
* Mutating hooks (browser:launching, session:creating) pass the options object
|
|
56
|
+
* by reference — plugins can modify it in place before core uses it.
|
|
57
|
+
*/
|
|
58
|
+
|
|
59
|
+
import { EventEmitter } from 'events';
|
|
60
|
+
import fs from 'fs';
|
|
61
|
+
import path from 'path';
|
|
62
|
+
import { fileURLToPath } from 'url';
|
|
63
|
+
|
|
64
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
65
|
+
const ROOT_DIR = path.join(__dirname, '..');
|
|
66
|
+
const PLUGINS_DIR = path.join(ROOT_DIR, 'plugins');
|
|
67
|
+
const CONFIG_PATH = path.join(ROOT_DIR, 'camofox.config.json');
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Read plugin configuration from camofox.config.json.
|
|
71
|
+
* Supports two formats:
|
|
72
|
+
* - Array of strings: ["youtube", "persistence"] (no per-plugin config)
|
|
73
|
+
* - Object with per-plugin config: { "youtube": { "enabled": true }, "persistence": { "enabled": true, "profileDir": "/data" } }
|
|
74
|
+
* Returns { list: string[] | null, configs: Map<string, object> }
|
|
75
|
+
*/
|
|
76
|
+
function readPluginConfig() {
|
|
77
|
+
const configs = new Map();
|
|
78
|
+
try {
|
|
79
|
+
const raw = fs.readFileSync(CONFIG_PATH, 'utf-8');
|
|
80
|
+
const config = JSON.parse(raw);
|
|
81
|
+
if (!config.plugins) return { list: null, configs };
|
|
82
|
+
if (Array.isArray(config.plugins)) {
|
|
83
|
+
return { list: config.plugins, configs };
|
|
84
|
+
}
|
|
85
|
+
if (typeof config.plugins === 'object') {
|
|
86
|
+
const list = [];
|
|
87
|
+
for (const [name, pluginConf] of Object.entries(config.plugins)) {
|
|
88
|
+
if (pluginConf === false || (typeof pluginConf === 'object' && pluginConf.enabled === false)) continue;
|
|
89
|
+
list.push(name);
|
|
90
|
+
if (typeof pluginConf === 'object') configs.set(name, pluginConf);
|
|
91
|
+
}
|
|
92
|
+
return { list, configs };
|
|
93
|
+
}
|
|
94
|
+
} catch {}
|
|
95
|
+
return { list: null, configs };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Create the plugin event bus.
|
|
100
|
+
*/
|
|
101
|
+
export function createPluginEvents() {
|
|
102
|
+
const events = new EventEmitter();
|
|
103
|
+
events.setMaxListeners(50); // generous for many plugins
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Emit an event and await all listeners (including async ones).
|
|
107
|
+
* Use for mutating hooks where plugins must finish before core continues.
|
|
108
|
+
* Regular emit() is still used for fire-and-forget observational events.
|
|
109
|
+
*/
|
|
110
|
+
events.emitAsync = async function emitAsync(eventName, payload) {
|
|
111
|
+
const listeners = this.listeners(eventName);
|
|
112
|
+
await Promise.all(listeners.map(fn => fn(payload)));
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
return events;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Load and register all plugins from plugins/<name>/index.js.
|
|
120
|
+
*
|
|
121
|
+
* @param {object} app - Express app
|
|
122
|
+
* @param {object} ctx - Plugin context: { sessions, config, log, events, auth, ensureBrowser, getSession, destroySession }
|
|
123
|
+
* Mutable — plugins can replace ctx.createVirtualDisplay etc.
|
|
124
|
+
* @returns {string[]} - Names of loaded plugins
|
|
125
|
+
*/
|
|
126
|
+
export async function loadPlugins(app, ctx) {
|
|
127
|
+
const loaded = [];
|
|
128
|
+
|
|
129
|
+
if (!fs.existsSync(PLUGINS_DIR)) {
|
|
130
|
+
ctx.log('info', 'no plugins directory found, skipping plugin load');
|
|
131
|
+
return loaded;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const { list: allowList, configs: pluginConfigs } = readPluginConfig();
|
|
135
|
+
const entries = fs.readdirSync(PLUGINS_DIR, { withFileTypes: true });
|
|
136
|
+
|
|
137
|
+
for (const entry of entries) {
|
|
138
|
+
if (!entry.isDirectory()) continue;
|
|
139
|
+
const name = entry.name;
|
|
140
|
+
|
|
141
|
+
// Skip directories starting with _ or .
|
|
142
|
+
if (name.startsWith('_') || name.startsWith('.')) continue;
|
|
143
|
+
|
|
144
|
+
// If camofox.config.json specifies a plugins list, only load those
|
|
145
|
+
if (allowList && !allowList.includes(name)) {
|
|
146
|
+
ctx.log('debug', `plugin "${name}" not in camofox.config.json plugins list, skipping`);
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const indexPath = path.join(PLUGINS_DIR, name, 'index.js');
|
|
151
|
+
if (!fs.existsSync(indexPath)) {
|
|
152
|
+
ctx.log('warn', `plugin "${name}" has no index.js, skipping`);
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
try {
|
|
157
|
+
const mod = await import(indexPath);
|
|
158
|
+
const register = mod.default || mod.register;
|
|
159
|
+
if (typeof register !== 'function') {
|
|
160
|
+
ctx.log('warn', `plugin "${name}" does not export a register function, skipping`);
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const pluginConfig = pluginConfigs.get(name) || {};
|
|
165
|
+
await register(app, ctx, pluginConfig);
|
|
166
|
+
loaded.push(name);
|
|
167
|
+
ctx.log('info', 'plugin loaded', { plugin: name });
|
|
168
|
+
} catch (err) {
|
|
169
|
+
ctx.log('error', 'plugin load failed', { plugin: name, error: err.message, stack: err.stack });
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return loaded;
|
|
174
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
|
|
4
|
+
const ORPHAN_PATTERNS = [
|
|
5
|
+
/^\.fea5[a-f0-9]+\.so$/,
|
|
6
|
+
/^\.5ef7[a-f0-9]+\.node$/,
|
|
7
|
+
];
|
|
8
|
+
|
|
9
|
+
export function cleanupOrphanedTempFiles({ tmpDir, minAgeMs = 5 * 60 * 1000, now = Date.now() } = {}) {
|
|
10
|
+
const result = { scanned: 0, removed: 0, bytes: 0, skipped: 0 };
|
|
11
|
+
if (!tmpDir) return result;
|
|
12
|
+
|
|
13
|
+
let entries;
|
|
14
|
+
try {
|
|
15
|
+
entries = fs.readdirSync(tmpDir);
|
|
16
|
+
} catch {
|
|
17
|
+
return result;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
for (const name of entries) {
|
|
21
|
+
if (!ORPHAN_PATTERNS.some((re) => re.test(name))) continue;
|
|
22
|
+
result.scanned++;
|
|
23
|
+
const full = path.join(tmpDir, name);
|
|
24
|
+
try {
|
|
25
|
+
const st = fs.statSync(full);
|
|
26
|
+
if (!st.isFile()) continue;
|
|
27
|
+
if (now - st.mtimeMs < minAgeMs) {
|
|
28
|
+
result.skipped++;
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
fs.unlinkSync(full);
|
|
32
|
+
result.removed++;
|
|
33
|
+
result.bytes += st.size;
|
|
34
|
+
} catch {
|
|
35
|
+
// file vanished, permission denied, or race with another process - skip silently
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return result;
|
|
40
|
+
}
|
package/openclaw.plugin.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@askjo/camofox-browser",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.0",
|
|
4
4
|
"description": "Headless browser automation server and OpenClaw plugin for AI agents - anti-detection, element refs, and session isolation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "server.js",
|
|
@@ -37,6 +37,8 @@
|
|
|
37
37
|
"files": [
|
|
38
38
|
"server.js",
|
|
39
39
|
"lib/",
|
|
40
|
+
"plugins/",
|
|
41
|
+
"camofox.config.json",
|
|
40
42
|
"plugin.ts",
|
|
41
43
|
"openclaw.plugin.json",
|
|
42
44
|
"scripts/",
|
|
@@ -54,8 +56,10 @@
|
|
|
54
56
|
"start": "node server.js",
|
|
55
57
|
"test": "NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit",
|
|
56
58
|
"test:e2e": "NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit tests/e2e",
|
|
59
|
+
"test:plugins": "NODE_OPTIONS='--experimental-vm-modules' jest --forceExit plugins/",
|
|
57
60
|
"test:live": "RUN_LIVE_TESTS=1 NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit tests/live",
|
|
58
61
|
"test:debug": "DEBUG_SERVER=1 NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit",
|
|
62
|
+
"plugin": "node scripts/plugin.js",
|
|
59
63
|
"version:sync": "node scripts/sync-version.js",
|
|
60
64
|
"version": "node scripts/sync-version.js && git add openclaw.plugin.json",
|
|
61
65
|
"postinstall": "npx camoufox-js fetch || true"
|