@askjo/camofox-browser 1.5.2 → 1.7.0

package/lib/tmp-cleanup.js

@@ -0,0 +1,40 @@
+import fs from 'fs';
+import path from 'path';
+
+const ORPHAN_PATTERNS = [
+  /^\.fea5[a-f0-9]+\.so$/,
+  /^\.5ef7[a-f0-9]+\.node$/,
+];
+
+export function cleanupOrphanedTempFiles({ tmpDir, minAgeMs = 5 * 60 * 1000, now = Date.now() } = {}) {
+  const result = { scanned: 0, removed: 0, bytes: 0, skipped: 0 };
+  if (!tmpDir) return result;
+
+  let entries;
+  try {
+    entries = fs.readdirSync(tmpDir);
+  } catch {
+    return result;
+  }
+
+  for (const name of entries) {
+    if (!ORPHAN_PATTERNS.some((re) => re.test(name))) continue;
+    result.scanned++;
+    const full = path.join(tmpDir, name);
+    try {
+      const st = fs.statSync(full);
+      if (!st.isFile()) continue;
+      if (now - st.mtimeMs < minAgeMs) {
+        result.skipped++;
+        continue;
+      }
+      fs.unlinkSync(full);
+      result.removed++;
+      result.bytes += st.size;
+    } catch {
+      // file vanished, permission denied, or race with another process - skip silently
+    }
+  }
+
+  return result;
+}

package/lib/tracing.js

@@ -0,0 +1,137 @@
+import fs from 'fs';
+import fsp from 'fs/promises';
+import path from 'path';
+import crypto from 'crypto';
+
+function hashUserId(userId) {
+  return crypto.createHash('sha256').update(String(userId)).digest('hex').slice(0, 16);
+}
+
+export function userTracesDir(baseDir, userId) {
+  return path.join(baseDir, hashUserId(userId));
+}
+
+export function ensureTracesDir(baseDir, userId) {
+  const dir = userTracesDir(baseDir, userId);
+  fs.mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+export function makeTraceFilename() {
+  const ts = new Date().toISOString().replace(/[:.]/g, '-');
+  const suffix = crypto.randomBytes(3).toString('hex');
+  return `trace-${ts}-${suffix}.zip`;
+}
+
+export function tracePathFor(baseDir, userId, filename) {
+  return path.join(ensureTracesDir(baseDir, userId), filename);
+}
+
+export function resolveTracePath(baseDir, userId, filename) {
+  if (!filename || filename.includes('/') || filename.includes('\\') || filename.includes('..') || filename.startsWith('.')) {
+    return null;
+  }
+  const userDir = userTracesDir(baseDir, userId);
+  const full = path.join(userDir, filename);
+  const resolved = path.resolve(full);
+  if (!resolved.startsWith(path.resolve(userDir) + path.sep)) return null;
+  return resolved;
+}
+
+export async function listUserTraces(baseDir, userId) {
+  const dir = userTracesDir(baseDir, userId);
+  let names;
+  try {
+    names = await fsp.readdir(dir);
+  } catch {
+    return [];
+  }
+  const out = [];
+  for (const name of names) {
+    if (!name.endsWith('.zip')) continue;
+    const full = path.join(dir, name);
+    try {
+      const st = await fsp.stat(full);
+      if (!st.isFile()) continue;
+      out.push({
+        filename: name,
+        sizeBytes: st.size,
+        createdAt: st.birthtimeMs || st.ctimeMs,
+        modifiedAt: st.mtimeMs,
+      });
+    } catch {
+      // vanished mid-scan
+    }
+  }
+  out.sort((a, b) => b.modifiedAt - a.modifiedAt);
+  return out;
+}
+
+export async function statTrace(fullPath) {
+  try {
+    const st = await fsp.stat(fullPath);
+    if (!st.isFile()) return null;
+    return st;
+  } catch {
+    return null;
+  }
+}
+
+export async function deleteTrace(fullPath) {
+  await fsp.unlink(fullPath);
+}
+
+export function sweepOldTraces({ baseDir, ttlMs, maxBytesPerFile, now = Date.now() } = {}) {
+  const result = { scanned: 0, removedTtl: 0, removedOversized: 0, bytes: 0 };
+  if (!baseDir) return result;
+
+  let userDirs;
+  try {
+    userDirs = fs.readdirSync(baseDir);
+  } catch {
+    return result;
+  }
+
+  for (const userDir of userDirs) {
+    const dir = path.join(baseDir, userDir);
+    let st;
+    try {
+      st = fs.statSync(dir);
+    } catch {
+      continue;
+    }
+    if (!st.isDirectory()) continue;
+
+    let files;
+    try {
+      files = fs.readdirSync(dir);
+    } catch {
+      continue;
+    }
+
+    for (const name of files) {
+      if (!name.endsWith('.zip')) continue;
+      result.scanned++;
+      const full = path.join(dir, name);
+      try {
+        const fst = fs.statSync(full);
+        if (!fst.isFile()) continue;
+        const tooOld = ttlMs && (now - fst.mtimeMs) > ttlMs;
+        const tooBig = maxBytesPerFile && fst.size > maxBytesPerFile;
+        if (tooOld) {
+          fs.unlinkSync(full);
+          result.removedTtl++;
+          result.bytes += fst.size;
+        } else if (tooBig) {
+          fs.unlinkSync(full);
+          result.removedOversized++;
+          result.bytes += fst.size;
+        }
+      } catch {
+        // vanished or permission denied
+      }
+    }
+  }
+
+  return result;
+}

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "camofox-browser",
   "name": "Camofox Browser",
   "description": "Anti-detection browser automation for AI agents using Camoufox (Firefox-based)",
-  "version": "1.5.2",
+  "version": "1.7.0",
   "configSchema": {
     "type": "object",
     "properties": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@askjo/camofox-browser",
-  "version": "1.5.2",
+  "version": "1.7.0",
   "description": "Headless browser automation server and OpenClaw plugin for AI agents - anti-detection, element refs, and session isolation",
   "type": "module",
   "main": "server.js",
@@ -37,6 +37,8 @@
   "files": [
     "server.js",
     "lib/",
+    "plugins/",
+    "camofox.config.json",
     "plugin.ts",
     "openclaw.plugin.json",
     "scripts/",
@@ -54,8 +56,11 @@
     "start": "node server.js",
     "test": "NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit",
     "test:e2e": "NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit tests/e2e",
+    "test:plugins": "NODE_OPTIONS='--experimental-vm-modules' jest --forceExit plugins/",
     "test:live": "RUN_LIVE_TESTS=1 NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit tests/live",
     "test:debug": "DEBUG_SERVER=1 NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit",
+    "plugin": "node scripts/plugin.js",
+    "generate-openapi": "node scripts/generate-openapi.js",
     "version:sync": "node scripts/sync-version.js",
     "version": "node scripts/sync-version.js && git add openclaw.plugin.json",
     "postinstall": "npx camoufox-js fetch || true"
@@ -67,7 +72,8 @@
     "playwright-core": "^1.58.0",
     "playwright-extra": "^4.3.6",
     "prom-client": "^15.1.3",
-    "puppeteer-extra-plugin-stealth": "^2.11.2"
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "swagger-jsdoc": "^6.2.8"
   },
   "devDependencies": {
     "jest": "^29.7.0",

package/plugins/persistence/AGENTS.md

@@ -0,0 +1,37 @@
+# Persistence Plugin — Agent Guide
+
+Saves and restores per-user browser storage state (cookies + localStorage) across session restarts using Playwright's `storageState` API. Enabled by default — profiles persist to `~/.camofox/profiles/`.
+
+## How It Works
+
+- `session:creating` hook → loads saved `storage_state.json` into `contextOptions.storageState`
+- `session:created` hook → imports bootstrap cookies if no persisted state exists
+- `session:cookies:import` / `session:destroyed` / `server:shutdown` → checkpoints state to disk
+
+All hooks are async and awaited via `emitAsync()` — storage state is guaranteed loaded before the context is created.
+
+## Key Files
+
+- `index.js` — lifecycle hooks (no routes, no `child_process`)
+- `persistence.test.js` — unit tests for `lib/persistence.js` helpers
+- `plugin.test.js` — integration tests for plugin lifecycle hooks
+
+## Storage Layout
+
+```
+~/.camofox/profiles/
+└── <sha256(userId)>/
+    └── storage_state.json
+```
+
+## Configuration
+
+Enabled by default. Override profile directory with `CAMOFOX_PROFILE_DIR` env var or `"profileDir"` in plugin config. To disable: `"persistence": { "enabled": false }` in `camofox.config.json`.
+
+## Original Contributors
+
+- [@company8](https://github.com/company8) — original persistence concept ([PR #62](https://github.com/jo-inc/camofox-browser/pull/62))
+- [@eddieoz](https://github.com/eddieoz) — cookie auto-load on startup ([PR #55](https://github.com/jo-inc/camofox-browser/pull/55))
+- [@pradeepe](https://github.com/pradeepe) — plugin system integration, atomic writes, inflight coalescing
+
+For PRs touching this plugin, tag the contributors above for review.

package/plugins/persistence/README.md

@@ -0,0 +1,48 @@
+# persistence
+
+Optional per-user browser storage state persistence for camofox-browser.
+
+Saves and restores cookies + localStorage across session restarts, container deploys, and idle timeouts using Playwright's `storageState` API.
+
+## Configuration
+
+In `camofox.config.json`:
+
+```json
+{
+  "plugins": {
+    "persistence": {
+      "enabled": true,
+      "profileDir": "/data/profiles"
+    }
+  }
+}
+```
+
+Or override via environment variable:
+
+```
+CAMOFOX_PROFILE_DIR=/data/profiles
+```
+
+## How it works
+
+- **Session create**: If a persisted `storageState` exists for the `userId`, it's restored into the new Playwright context.
+- **First run**: If no persisted state exists, bootstrap cookies from `CAMOFOX_COOKIES_DIR/cookies.txt` are imported (if present).
+- **Cookie import / session close / shutdown**: Storage state is checkpointed to disk via atomic tmp-write + rename.
+- **User isolation**: Each `userId` maps to a deterministic SHA256-hashed subdirectory under `profileDir`, so arbitrary userIds are path-safe.
+
+## Docker
+
+When running with Docker, mount the profile directory as a volume:
+
+```bash
+docker run -d \
+  -p 9377:9377 \
+  -v /host/profiles:/data/profiles \
+  camofox-browser
+```
+
+## Credits
+
+Based on PR #62 by [company8](https://github.com/company8).

package/plugins/persistence/index.js

@@ -0,0 +1,124 @@
+/**
+ * Persistence plugin for camofox-browser.
+ *
+ * Saves and restores per-user browser storage state (cookies + localStorage)
+ * across session restarts using Playwright's storageState API.
+ *
+ * Configuration (camofox.config.json):
+ *   {
+ *     "plugins": {
+ *       "persistence": {
+ *         "enabled": true,
+ *         "profileDir": "/data/profiles"
+ *       }
+ *     }
+ *   }
+ *
+ * Or via environment variables (overrides config file):
+ *   CAMOFOX_PROFILE_DIR=/data/profiles
+ *
+ * Each userId gets a deterministic SHA256-hashed subdirectory under profileDir.
+ * Storage state is checkpointed on cookie import, session close, and shutdown.
+ * On session creation, saved state is restored into the new Playwright context
+ * via the session:creating hook (mutates contextOptions.storageState).
+ */
+
+import {
+  getUserPersistencePaths,
+  loadPersistedStorageState,
+  persistStorageState,
+} from '../../lib/persistence.js';
+import { importBootstrapCookies } from '../../lib/cookies.js';
+
+export async function register(app, ctx, pluginConfig = {}) {
+  const { events, config, log } = ctx;
+
+  // Resolve profileDir: env var > plugin config > global config default (~/.camofox/profiles)
+  const profileDir = process.env.CAMOFOX_PROFILE_DIR || pluginConfig.profileDir || config.profileDir;
+  if (!profileDir) {
+    log('warn', 'persistence plugin: no profileDir configured, plugin disabled');
+    return;
+  }
+
+  const logger = {
+    warn: (msg, fields = {}) => log('warn', msg, fields),
+  };
+
+  log('info', 'persistence plugin enabled', { profileDir });
+
+  // Track active sessions for checkpoint on close
+  const activeSessions = new Map(); // userId -> context
+
+  /**
+   * Checkpoint storage state to disk for a userId.
+   */
+  async function checkpoint(userId, context, reason) {
+    if (!context) return;
+    const result = await persistStorageState({ profileDir, userId, context, logger });
+    if (result.persisted) {
+      log('info', 'storage state persisted', { userId, reason, path: result.storageStatePath });
+    }
+    return result;
+  }
+
+  // --- Lifecycle hooks ---
+
+  // Before session context is created: inject storageState if we have one saved
+  events.on('session:creating', async ({ userId, contextOptions }) => {
+    const storageStatePath = await loadPersistedStorageState(profileDir, userId, logger);
+    if (storageStatePath) {
+      contextOptions.storageState = storageStatePath;
+      log('info', 'restoring persisted storage state', { userId, storageStatePath });
+    }
+  });
+
+  // After session is created: import bootstrap cookies if no persisted state,
+  // and track the context for later checkpointing
+  events.on('session:created', async ({ userId, context }) => {
+    activeSessions.set(userId, context);
+
+    // If no persisted state was restored, try bootstrap cookies
+    const existingState = await loadPersistedStorageState(profileDir, userId, logger);
+    if (!existingState) {
+      const result = await importBootstrapCookies({
+        cookiesDir: config.cookiesDir,
+        context,
+        logger,
+      });
+      if (result.imported > 0) {
+        log('info', 'bootstrap cookies imported', { userId, count: result.imported, source: result.source });
+        await checkpoint(userId, context, 'bootstrap_cookies');
+      }
+    }
+  });
+
+  // On cookie import: checkpoint
+  events.on('session:cookies:import', async ({ userId }) => {
+    const context = activeSessions.get(userId);
+    if (context) {
+      await checkpoint(userId, context, 'cookie_import');
+    }
+  });
+
+  // On session destroying (pre-close): checkpoint while context is still alive
+  events.on('session:destroying', async ({ userId, reason }) => {
+    const context = activeSessions.get(userId);
+    if (context) {
+      await checkpoint(userId, context, reason).catch(() => {});
+      activeSessions.delete(userId);
+    }
+  });
+
+  // On session destroyed (post-close): cleanup tracking if not already done
+  events.on('session:destroyed', async ({ userId }) => {
+    activeSessions.delete(userId);
+  });
+
+  // On shutdown: checkpoint all remaining sessions
+  events.on('server:shutdown', async () => {
+    for (const [userId, context] of activeSessions) {
+      await checkpoint(userId, context, 'shutdown').catch(() => {});
+    }
+    activeSessions.clear();
+  });
+}

package/plugins/persistence/persistence.test.js

@@ -0,0 +1,117 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { jest } from '@jest/globals';
+import {
+  getUserPersistencePaths,
+  loadPersistedStorageState,
+  persistStorageState,
+} from '../../lib/persistence.js';
+
+describe('profile persistence helpers', () => {
+  let tmpDir;
+
+  beforeEach(async () => {
+    tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'camofox-persistence-'));
+  });
+
+  afterEach(async () => {
+    if (tmpDir) {
+      await fs.rm(tmpDir, { recursive: true, force: true });
+    }
+  });
+
+  test('getUserPersistencePaths is deterministic and stays under root', () => {
+    const first = getUserPersistencePaths(tmpDir, 'agent/profile:default');
+    const second = getUserPersistencePaths(tmpDir, 'agent/profile:default');
+
+    expect(first).toEqual(second);
+    expect(first.userDir.startsWith(tmpDir)).toBe(true);
+    expect(first.storageStatePath.startsWith(first.userDir)).toBe(true);
+    expect(first.metaPath.startsWith(first.userDir)).toBe(true);
+    expect(path.basename(first.userDir)).not.toContain('/');
+    expect(path.basename(first.userDir)).not.toContain(':');
+  });
+
+  test('loadPersistedStorageState returns undefined when no state exists', async () => {
+    await expect(loadPersistedStorageState(tmpDir, 'user-1')).resolves.toBeUndefined();
+  });
+
+  test('persistStorageState writes storage state and metadata, then load returns the storage path', async () => {
+    const storageState = {
+      cookies: [{ name: 'session', value: 'abc', domain: '.example.com', path: '/' }],
+      origins: [{ origin: 'https://app.example.com', localStorage: [{ name: 'foo', value: 'bar' }] }],
+    };
+
+    const context = {
+      storageState: jest.fn(async ({ path: targetPath }) => {
+        await fs.writeFile(targetPath, JSON.stringify(storageState, null, 2));
+      }),
+    };
+
+    const result = await persistStorageState({
+      profileDir: tmpDir,
+      userId: 'user-1',
+      context,
+      logger: { warn: jest.fn() },
+    });
+
+    expect(result.persisted).toBe(true);
+    expect(context.storageState).toHaveBeenCalledTimes(1);
+
+    const loadedPath = await loadPersistedStorageState(tmpDir, 'user-1');
+    expect(loadedPath).toBe(result.storageStatePath);
+
+    const meta = JSON.parse(await fs.readFile(result.metaPath, 'utf8'));
+    expect(meta.userId).toBe('user-1');
+    expect(meta.storageStatePath).toBe(result.storageStatePath);
+  });
+
+  test('loadPersistedStorageState ignores invalid JSON files', async () => {
+    const { storageStatePath } = getUserPersistencePaths(tmpDir, 'user-2');
+    await fs.mkdir(path.dirname(storageStatePath), { recursive: true });
+    await fs.writeFile(storageStatePath, '{not-json');
+
+    await expect(loadPersistedStorageState(tmpDir, 'user-2', { warn: jest.fn() })).resolves.toBeUndefined();
+  });
+
+  test('a failed persist leaves the previous storage-state intact and cleans up tmp files', async () => {
+    const originalState = {
+      cookies: [{ name: 'orig', value: 'v1', domain: '.example.com', path: '/' }],
+    };
+    const goodContext = {
+      storageState: jest.fn(async ({ path: targetPath }) => {
+        await fs.writeFile(targetPath, JSON.stringify(originalState, null, 2));
+      }),
+    };
+    const first = await persistStorageState({
+      profileDir: tmpDir,
+      userId: 'user-3',
+      context: goodContext,
+      logger: { warn: jest.fn() },
+    });
+    expect(first.persisted).toBe(true);
+
+    const failingContext = {
+      storageState: jest.fn(async () => {
+        throw new Error('simulated crash mid-write');
+      }),
+    };
+    const second = await persistStorageState({
+      profileDir: tmpDir,
+      userId: 'user-3',
+      context: failingContext,
+      logger: { warn: jest.fn() },
+    });
+    expect(second.persisted).toBe(false);
+
+    const { userDir, storageStatePath } = getUserPersistencePaths(tmpDir, 'user-3');
+    const loaded = await loadPersistedStorageState(tmpDir, 'user-3');
+    expect(loaded).toBe(storageStatePath);
+    const parsed = JSON.parse(await fs.readFile(storageStatePath, 'utf8'));
+    expect(parsed).toEqual(originalState);
+
+    const leftovers = (await fs.readdir(userDir)).filter((name) => name.includes('.tmp-'));
+    expect(leftovers).toEqual([]);
+  });
+});

package/plugins/persistence/plugin.test.js

@@ -0,0 +1,98 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { jest } from '@jest/globals';
+import { createPluginEvents } from '../../lib/plugins.js';
+import { register } from './index.js';
+
+describe('persistence plugin', () => {
+  let tmpDir, events, ctx, mockApp;
+
+  beforeEach(async () => {
+    tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'camofox-persist-plugin-'));
+    events = createPluginEvents();
+    mockApp = {};
+    ctx = {
+      events,
+      config: { cookiesDir: path.join(tmpDir, 'cookies') },
+      log: jest.fn(),
+    };
+  });
+
+  afterEach(async () => {
+    if (tmpDir) await fs.rm(tmpDir, { recursive: true, force: true });
+  });
+
+  test('skips registration when no profileDir configured', async () => {
+    await register(mockApp, ctx, {});
+    expect(ctx.log).toHaveBeenCalledWith('warn', expect.stringContaining('no profileDir'));
+  });
+
+  test('restores persisted state on session:creating', async () => {
+    await register(mockApp, ctx, { profileDir: tmpDir });
+
+    // Simulate a prior persisted state
+    const { getUserPersistencePaths } = await import('../../lib/persistence.js');
+    const { userDir, storageStatePath } = getUserPersistencePaths(tmpDir, 'user-1');
+    await fs.mkdir(userDir, { recursive: true });
+    await fs.writeFile(storageStatePath, JSON.stringify({
+      cookies: [{ name: 'sid', value: 'abc', domain: '.example.com', path: '/' }],
+      origins: [],
+    }));
+
+    const contextOptions = { viewport: { width: 1280, height: 720 } };
+    await events.emitAsync('session:creating', { userId: 'user-1', contextOptions });
+
+    expect(contextOptions.storageState).toBe(storageStatePath);
+  });
+
+  test('checkpoints on session:cookies:import', async () => {
+    await register(mockApp, ctx, { profileDir: tmpDir });
+
+    const mockContext = {
+      storageState: jest.fn(async ({ path: p }) => {
+        await fs.writeFile(p, JSON.stringify({ cookies: [{ name: 'x', value: 'y', domain: '.test.com', path: '/' }] }));
+      }),
+    };
+
+    // Simulate session created then cookie import
+    await events.emitAsync('session:created', { userId: 'user-2', context: mockContext });
+    await events.emitAsync('session:cookies:import', { userId: 'user-2' });
+
+    expect(mockContext.storageState).toHaveBeenCalled();
+
+    // Verify file was written
+    const { getUserPersistencePaths } = await import('../../lib/persistence.js');
+    const { storageStatePath } = getUserPersistencePaths(tmpDir, 'user-2');
+    const saved = JSON.parse(await fs.readFile(storageStatePath, 'utf8'));
+    expect(saved.cookies[0].name).toBe('x');
+  });
+
+  test('checkpoints on session:destroying', async () => {
+    await register(mockApp, ctx, { profileDir: tmpDir });
+
+    const mockContext = {
+      storageState: jest.fn(async ({ path: p }) => {
+        await fs.writeFile(p, JSON.stringify({ cookies: [], origins: [] }));
+      }),
+    };
+
+    await events.emitAsync('session:created', { userId: 'user-3', context: mockContext });
+    await events.emitAsync('session:destroying', { userId: 'user-3', reason: 'test' });
+
+    expect(mockContext.storageState).toHaveBeenCalled();
+  });
+
+  test('env var CAMOFOX_PROFILE_DIR overrides pluginConfig', async () => {
+    const envDir = path.join(tmpDir, 'env-override');
+    const orig = process.env.CAMOFOX_PROFILE_DIR;
+    process.env.CAMOFOX_PROFILE_DIR = envDir;
+    try {
+      await register(mockApp, ctx, { profileDir: '/should/not/use' });
+      expect(ctx.log).toHaveBeenCalledWith('info', 'persistence plugin enabled', { profileDir: envDir });
+    } finally {
+      if (orig === undefined) delete process.env.CAMOFOX_PROFILE_DIR;
+      else process.env.CAMOFOX_PROFILE_DIR = orig;
+    }
+  });
+});