@nikx/dory-worker 1.0.0

package/dist/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":";;AAQA,oDAkCC;AA1CD,mDAAgD;AAEhD,qCAAkC;AAElC;;;GAGG;AACI,KAAK,UAAU,oBAAoB,CACxC,MAAoB,EACpB,KAAa;IAEb,MAAM,QAAQ,GAAG,GAAG,KAAK,8CAA8C,CAAC;IAExE,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAC/B,6CAA6C;YAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,mBAAmB,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,GAAG,IAAI,0BAA0B,MAAM,CAAC,SAAS,MAAM,kBAAkB,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC5G,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,eAAM,CAAC,IAAI,CACT,8CAA8C,KAAK,UAAU,GAAG,CAAC,MAAM,uBAAuB,CAC/F,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA4B,CAAC;QACvD,CAAC;QAED,4DAA4D;QAC5D,MAAM,OAAO,GAAG,IAAI,iBAAO,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,OAAO;aAC7B,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC;aACxB,IAAI,CAAC,QAAQ,CAAC;aACd,QAAQ,EAAE,CAAC;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAA4B,CAAC;IACpE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CACV,qCAAqC,KAAK,KAAM,GAAa,CAAC,OAAO,EAAE,CACxE,CAAC;QACF,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,61 @@
+import type { ObjectId } from "mongodb";
+export interface Handler {
+    label: string;
+    code: string;
+    description?: string;
+}
+export interface ActorInput {
+    startUrls?: Array<string | {
+        url: string;
+    }>;
+    crawlerType?: string;
+    handlers?: Handler[];
+    maxPagesPerCrawl?: number;
+    maxConcurrency?: number;
+    actorMemoryMbytes?: number;
+    actorTimeoutSecs?: number;
+    tieredProxyUrls?: string[][];
+    webhookUrl?: string;
+    webhookRequestMethod?: string;
+    webhookPayloadSchema?: string;
+    webhookHeaders?: Array<{
+        key: string;
+        value: string;
+    }>;
+    uniqueKeySelector?: string[];
+    [key: string]: unknown;
+}
+export interface ActorDocument {
+    _id: ObjectId;
+    name: string;
+    crawlerType: "playwright" | "cheerio";
+    handlers: Handler[];
+    defaultInput: ActorInput;
+    [key: string]: unknown;
+}
+export type RunStatus = "queued" | "pending" | "running" | "completed" | "failed" | "cancelled";
+export interface RunDocument {
+    _id: ObjectId;
+    actorId: ObjectId;
+    jobName: string;
+    executionId: string;
+    status: RunStatus;
+    containerId?: string;
+    startedAt?: Date;
+    finishedAt?: Date;
+    exitCode?: number;
+    error?: string;
+    errorCount?: number;
+    [key: string]: unknown;
+}
+/** Shape of the ACTOR_CONFIG JSON passed to the scraping container */
+export interface ActorConfig {
+    jobId: string;
+    runId: string;
+    actorInstanceId: string;
+    userInput: ActorInput & {
+        crawlerType: string;
+        handlers: Handler[];
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,cAAc,CAAC,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,QAAQ,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,YAAY,GAAG,SAAS,CAAC;IACtC,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,YAAY,EAAE,UAAU,CAAC;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,MAAM,SAAS,GACjB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,WAAW,GACX,QAAQ,GACR,WAAW,CAAC;AAEhB,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,QAAQ,CAAC;IACd,OAAO,EAAE,QAAQ,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,UAAU,CAAC,EAAE,IAAI,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,sEAAsE;AACtE,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,UAAU,GAAG;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAO,EAAE,CAAC;KACrB,CAAC;CACH"}

package/dist/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/dist/worker.d.ts

@@ -0,0 +1,7 @@
+import { Worker } from "bullmq";
+import IORedis from "ioredis";
+import type { WorkerConfig } from "./config";
+export declare const RUN_EXECUTION_QUEUE = "run-execution";
+export declare function createRedisConnection(config: WorkerConfig): IORedis;
+export declare function createWorker(config: WorkerConfig, redis: IORedis): Worker;
+//# sourceMappingURL=worker.d.ts.map

package/dist/worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAI7C,eAAO,MAAM,mBAAmB,kBAAkB,CAAC;AAEnD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAenE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAsCzE"}

package/dist/worker.js

@@ -0,0 +1,56 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RUN_EXECUTION_QUEUE = void 0;
+exports.createRedisConnection = createRedisConnection;
+exports.createWorker = createWorker;
+const bullmq_1 = require("bullmq");
+const ioredis_1 = __importDefault(require("ioredis"));
+const processor_1 = require("./processor");
+const logger_1 = require("./logger");
+exports.RUN_EXECUTION_QUEUE = "run-execution";
+function createRedisConnection(config) {
+    const conn = config.redisUrl
+        ? new ioredis_1.default(config.redisUrl, { maxRetriesPerRequest: null })
+        : new ioredis_1.default({
+            host: config.redisHost,
+            port: config.redisPort,
+            password: config.redisPassword,
+            maxRetriesPerRequest: null,
+        });
+    conn.on("connect", () => logger_1.logger.info("Redis connected"));
+    conn.on("error", (err) => logger_1.logger.error(`Redis error: ${err.message}`));
+    conn.on("reconnecting", () => logger_1.logger.warn("Redis reconnecting..."));
+    return conn;
+}
+function createWorker(config, redis) {
+    const worker = new bullmq_1.Worker(exports.RUN_EXECUTION_QUEUE, (job) => (0, processor_1.processRunJob)(config, job), {
+        connection: redis,
+        concurrency: config.maxConcurrentRuns,
+        lockDuration: 300_000, // 5 min — containers can be long-running
+        stalledInterval: 30_000, // check for stalled jobs every 30 s
+        maxStalledCount: 2, // re-queue once, then fail
+    });
+    worker.on("completed", (job) => {
+        logger_1.logger.info(`Job ${job.id} completed`);
+    });
+    worker.on("failed", (job, err) => {
+        logger_1.logger.error(`Job ${job?.id ?? "unknown"} failed: ${err.message}`);
+    });
+    worker.on("stalled", (jobId) => {
+        logger_1.logger.warn(`Job ${jobId} stalled — BullMQ will re-queue or fail it`);
+    });
+    worker.on("error", (err) => {
+        // Suppress the well-known "Missing lock" race condition — BullMQ stalled-job
+        // detection will recover the job automatically.
+        if (err.message?.includes("Missing lock")) {
+            logger_1.logger.warn(`Lock expired before retry could be scheduled — job will be recovered. (${err.message})`);
+            return;
+        }
+        logger_1.logger.error(`Worker error: ${err.message}`);
+    });
+    return worker;
+}
+//# sourceMappingURL=worker.js.map

package/dist/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":";;;;;;AAQA,sDAeC;AAED,oCAsCC;AA/DD,mCAAgC;AAChC,sDAA8B;AAE9B,2CAA4C;AAC5C,qCAAkC;AAErB,QAAA,mBAAmB,GAAG,eAAe,CAAC;AAEnD,SAAgB,qBAAqB,CAAC,MAAoB;IACxD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ;QAC1B,CAAC,CAAC,IAAI,iBAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC;QAC9D,CAAC,CAAC,IAAI,iBAAO,CAAC;YACV,IAAI,EAAE,MAAM,CAAC,SAAS;YACtB,IAAI,EAAE,MAAM,CAAC,SAAS;YACtB,QAAQ,EAAE,MAAM,CAAC,aAAa;YAC9B,oBAAoB,EAAE,IAAI;SAC3B,CAAC,CAAC;IAEP,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,eAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;IACzD,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,eAAM,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC9E,IAAI,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,eAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC;IAEpE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,YAAY,CAAC,MAAoB,EAAE,KAAc;IAC/D,MAAM,MAAM,GAAG,IAAI,eAAM,CACvB,2BAAmB,EACnB,CAAC,GAAG,EAAE,EAAE,CAAC,IAAA,yBAAa,EAAC,MAAM,EAAE,GAAG,CAAC,EACnC;QACE,UAAU,EAAE,KAAK;QACjB,WAAW,EAAE,MAAM,CAAC,iBAAiB;QACrC,YAAY,EAAE,OAAO,EAAE,yCAAyC;QAChE,eAAe,EAAE,MAAM,EAAE,oCAAoC;QAC7D,eAAe,EAAE,CAAC,EAAE,2BAA2B;KAChD,CACF,CAAC;IAEF,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE;QAC7B,eAAM,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,YAAY,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC/B,eAAM,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,EAAE,IAAI,SAAS,YAAY,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,KAAa,EAAE,EAAE;QACrC,eAAM,CAAC,IAAI,CAAC,OAAO,KAAK,4CAA4C,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;QAChC,6EAA6E;QAC7E,gDAAgD;QAChD,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;YAC1C,eAAM,CAAC,IAAI,CACT,0EAA0E,GAAG,CAAC,OAAO,GAAG,CACzF,CAAC;YACF,OAAO;QACT,CAAC;QACD,eAAM,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@nikx/dory-worker",
+  "version": "1.0.0",
+  "description": "Standalone BullMQ worker for Dory – runs on any machine with Docker (including Raspberry Pi)",
+  "main": "dist/cli.js",
+  "bin": {
+    "dory-worker": "dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/cli.js",
+    "dev": "npx ts-node --project tsconfig.json src/cli.ts",
+    "prepare": "npm run build"
+  },
+  "keywords": [
+    "dory",
+    "bullmq",
+    "worker",
+    "crawlee",
+    "scraping"
+  ],
+  "author": "",
+  "license": "ISC",
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "dependencies": {
+    "bullmq": "^5.70.1",
+    "ioredis": "^5.9.3"
+  },
+  "devDependencies": {
+    "@types/node": "^20.19.27",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.9.3"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+import { loadConfig } from "./config";
+import { createRedisConnection, createWorker } from "./worker";
+import { logger, setLogLevel } from "./logger";
+
+async function main(): Promise<void> {
+  // ── Load & validate config ────────────────────────────────────────────
+  let config;
+  try {
+    config = loadConfig();
+  } catch (err) {
+    console.error(
+      `[dory-worker] Configuration error: ${(err as Error).message}`,
+    );
+    console.error("Set the required environment variables and try again.");
+    process.exit(1);
+  }
+
+  setLogLevel(config.logLevel);
+
+  logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+  logger.info("  dory-worker starting");
+  logger.info(`  Worker ID   : ${config.workerId}`);
+  logger.info(`  Concurrency : ${config.maxConcurrentRuns}`);
+  logger.info(`  API URL     : ${config.apiBaseUrl}`);
+  const redisAddr =
+    config.redisUrl ?? `${config.redisHost}:${config.redisPort}`;
+  logger.info(`  Redis       : ${redisAddr}`);
+  if (config.storageEmulatorHost) {
+    logger.info(`  GCS emulator: ${config.storageEmulatorHost} (dev mode)`);
+  }
+  logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+
+  // ── Start BullMQ worker ───────────────────────────────────────────────
+  const redis = createRedisConnection(config);
+  const worker = createWorker(config, redis);
+
+  logger.info('Listening on queue "run-execution" — ready to process jobs');
+
+  // ── Graceful shutdown ─────────────────────────────────────────────────
+  let shuttingDown = false;
+
+  async function shutdown(signal: string): Promise<void> {
+    if (shuttingDown) return;
+    shuttingDown = true;
+
+    logger.info(`${signal} received — shutting down gracefully...`);
+    try {
+      await worker.close();
+      await redis.quit();
+      logger.info("Worker shut down cleanly");
+      process.exit(0);
+    } catch (err) {
+      logger.error(`Error during shutdown: ${(err as Error).message}`);
+      process.exit(1);
+    }
+  }
+
+  process.on("SIGTERM", () => shutdown("SIGTERM"));
+  process.on("SIGINT", () => shutdown("SIGINT"));
+}
+
+main().catch((err: unknown) => {
+  console.error("[dory-worker] Fatal error:", err);
+  process.exit(1);
+});

package/src/config.ts

@@ -0,0 +1,64 @@
+export interface WorkerConfig {
+  // Redis — use REDIS_URL (e.g. Railway) or host/port separately
+  redisUrl?: string;
+  redisHost: string;
+  redisPort: number;
+  redisPassword?: string;
+
+  // dory-api public URL — worker calls this for config + status updates,
+  // and containers use it for their own status callbacks
+  apiBaseUrl: string;
+
+  // GCS pass-through — only needed so the worker can inject these env vars
+  // into Docker containers (containers upload results directly to GCS).
+  // Not needed if your docker image bakes these in via CI.
+  gcsBucket?: string;
+  gcpProjectId?: string;
+  /** fake-gcs-server URL for local dev (e.g. http://localhost:9199) */
+  storageEmulatorHost?: string;
+
+  // Fallback docker image if the API doesn't return one in /config
+  dockerImage?: string;
+
+  // Worker behaviour
+  maxConcurrentRuns: number;
+  workerId: string;
+  logLevel: string;
+}
+
+const REQUIRED_VARS = ["API_BASE_URL"] as const;
+
+export function loadConfig(): WorkerConfig {
+  const missing = REQUIRED_VARS.filter((k) => !process.env[k]);
+  if (missing.length > 0) {
+    throw new Error(
+      `Missing required environment variables: ${missing.join(", ")}`,
+    );
+  }
+
+  // At least one Redis connection method must be provided
+  if (!process.env.REDIS_URL && !process.env.REDIS_HOST) {
+    throw new Error(
+      "Missing Redis config: set REDIS_URL or REDIS_HOST (+ REDIS_PORT)",
+    );
+  }
+
+  return {
+    redisUrl: process.env.REDIS_URL,
+    redisHost: process.env.REDIS_HOST ?? "localhost",
+    redisPort: parseInt(process.env.REDIS_PORT ?? "6379", 10),
+    redisPassword: process.env.REDIS_PASSWORD,
+
+    apiBaseUrl: process.env.API_BASE_URL!,
+
+    gcsBucket: process.env.GCS_BUCKET,
+    gcpProjectId: process.env.GCP_PROJECT_ID,
+    storageEmulatorHost: process.env.STORAGE_EMULATOR_HOST,
+
+    dockerImage: process.env.DOCKER_IMAGE,
+
+    maxConcurrentRuns: parseInt(process.env.MAX_CONCURRENT_RUNS ?? "2", 10),
+    workerId: process.env.WORKER_ID ?? `dory-worker-${process.pid}`,
+    logLevel: process.env.LOG_LEVEL ?? "info",
+  };
+}

package/src/docker.ts

@@ -0,0 +1,108 @@
+import { exec } from "child_process";
+import { promisify } from "util";
+import type { WorkerConfig } from "./config";
+import { logger } from "./logger";
+
+const execAsync = promisify(exec);
+
+interface ActorConfigEnvelope {
+  jobId: string;
+  runId: string;
+  actorInstanceId: string;
+  userInput: Record<string, unknown>;
+}
+
+/**
+ * Rewrite localhost URLs to host.docker.internal so containers can reach
+ * the host machine. Public URLs (Railway, GCS, etc.) are left unchanged.
+ */
+function toDockerHost(url: string): string {
+  return url
+    .replace("http://localhost", "http://host.docker.internal")
+    .replace("https://localhost", "https://host.docker.internal");
+}
+
+function buildDockerEnvArgs(
+  config: WorkerConfig,
+  actorConfig: ActorConfigEnvelope,
+  memoryLimit: number,
+): string[] {
+  const configBase64 = Buffer.from(JSON.stringify(actorConfig)).toString(
+    "base64",
+  );
+
+  const args = [
+    `-e ACTOR_CONFIG=${configBase64}`,
+    `-e API_BASE_URL=${toDockerHost(config.apiBaseUrl)}`,
+    `-e CRAWLEE_MEMORY_MBYTES=${memoryLimit}`,
+  ];
+
+  // Pass through GCS config if set (containers need it to upload results)
+  if (config.gcsBucket) args.push(`-e GCS_BUCKET=${config.gcsBucket}`);
+  if (config.gcpProjectId)
+    args.push(`-e GCP_PROJECT_ID=${config.gcpProjectId}`);
+  if (config.storageEmulatorHost) {
+    args.push(
+      `-e STORAGE_EMULATOR_HOST=${toDockerHost(config.storageEmulatorHost)}`,
+    );
+  }
+
+  return args;
+}
+
+/**
+ * Start the dory-core scraping container in detached mode.
+ * Returns the container ID.
+ *
+ * @param dockerImage  Image to run — sourced from GET /api/runs/:id/config
+ *                     so the API controls what version runs everywhere.
+ */
+export async function startContainer(
+  config: WorkerConfig,
+  actorConfig: ActorConfigEnvelope,
+  memoryLimit: number,
+  dockerImage: string,
+): Promise<string> {
+  const envArgs = buildDockerEnvArgs(config, actorConfig, memoryLimit).join(
+    " ",
+  );
+  const memoryStr = `${memoryLimit}m`;
+
+  const cmd = [
+    "docker run -d --rm",
+    `--memory=${memoryStr}`,
+    "--add-host=host.docker.internal:host-gateway",
+    envArgs,
+    dockerImage,
+  ].join(" ");
+
+  logger.debug(`docker run: ${cmd}`);
+  const { stdout } = await execAsync(cmd);
+  return stdout.trim();
+}
+
+/**
+ * Block until the container exits and return its exit code.
+ * If the container exceeds `timeoutSecs`, it is force-killed.
+ */
+export async function waitForContainer(
+  containerId: string,
+  timeoutSecs: number,
+): Promise<number> {
+  try {
+    const { stdout } = await execAsync(`docker wait ${containerId}`, {
+      timeout: (timeoutSecs + 30) * 1_000,
+    });
+    return parseInt(stdout.trim(), 10);
+  } catch (err) {
+    logger.warn(
+      `docker wait timed out for ${containerId.slice(0, 12)}: ${(err as Error).message} — force-killing`,
+    );
+    try {
+      await execAsync(`docker kill ${containerId}`);
+    } catch {
+      // Container already gone — ignore
+    }
+    return 137; // SIGKILL exit code
+  }
+}

package/src/logger.ts

@@ -0,0 +1,34 @@
+type LogLevel = "debug" | "info" | "warn" | "error";
+
+const LEVEL_WEIGHT: Record<LogLevel, number> = {
+  debug: 0,
+  info: 1,
+  warn: 2,
+  error: 3,
+};
+
+let currentLevel: LogLevel = "info";
+
+export function setLogLevel(level: string): void {
+  if (level in LEVEL_WEIGHT) {
+    currentLevel = level as LogLevel;
+  }
+}
+
+function emit(level: LogLevel, message: string): void {
+  if (LEVEL_WEIGHT[level] < LEVEL_WEIGHT[currentLevel]) return;
+  const ts = new Date().toISOString();
+  const line = `[${ts}] [${level.toUpperCase().padEnd(5)}] ${message}`;
+  if (level === "error") {
+    process.stderr.write(line + "\n");
+  } else {
+    process.stdout.write(line + "\n");
+  }
+}
+
+export const logger = {
+  debug: (msg: string) => emit("debug", msg),
+  info: (msg: string) => emit("info", msg),
+  warn: (msg: string) => emit("warn", msg),
+  error: (msg: string) => emit("error", msg),
+};

package/src/processor.ts

@@ -0,0 +1,160 @@
+import type { Job } from "bullmq";
+import type { WorkerConfig } from "./config";
+import { startContainer, waitForContainer } from "./docker";
+import { logger } from "./logger";
+
+interface RunConfigResponse {
+  runId: string;
+  actorConfig: {
+    jobId: string;
+    runId: string;
+    actorInstanceId: string;
+    userInput: Record<string, unknown>;
+  };
+  memoryLimitMb: number;
+  actorTimeoutSecs: number;
+  dockerImage: string;
+}
+
+async function apiGet<T>(apiBaseUrl: string, path: string): Promise<T> {
+  const url = `${apiBaseUrl.replace(/\/+$/, "")}${path}`;
+  const res = await fetch(url);
+  if (!res.ok) {
+    const body = await res.text().catch(() => "");
+    throw new Error(`GET ${path} → HTTP ${res.status}: ${body}`);
+  }
+  return res.json() as Promise<T>;
+}
+
+async function apiPatch(
+  apiBaseUrl: string,
+  path: string,
+  body: Record<string, unknown>,
+): Promise<void> {
+  const url = `${apiBaseUrl.replace(/\/+$/, "")}${path}`;
+  const res = await fetch(url, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+  });
+  if (!res.ok) {
+    const text = await res.text().catch(() => "");
+    logger.warn(`POST ${path} → HTTP ${res.status}: ${text}`);
+  }
+}
+
+/**
+ * Core job processor.
+ *
+ * Flow:
+ *   1. GET  /api/runs/:id/config  — fetch actorConfig from dory-api (no DB/GCS access needed)
+ *   2. POST /api/runs/:id/status  — mark run as "running"
+ *   3. docker run                 — start the scraping container
+ *   4. Extend BullMQ lock every 2 min while waiting
+ *   5. docker wait                — block until container exits
+ *   6. POST /api/runs/:id/status  — exit-code fallback if HTTP callback never arrived
+ */
+export async function processRunJob(
+  config: WorkerConfig,
+  job: Job<{ runId: string }>,
+): Promise<{ runId: string; status: string }> {
+  const { runId } = job.data;
+
+  if (!runId) throw new Error("Job data missing runId");
+
+  logger.info(
+    `[${runId}] Processing job ${job.id} (attempt ${job.attemptsMade + 1})`,
+  );
+
+  // ── Step 1: Fetch run config from API ──────────────────────────────────
+  let runConfig: RunConfigResponse;
+  try {
+    runConfig = await apiGet<RunConfigResponse>(
+      config.apiBaseUrl,
+      `/api/runs/${runId}/config`,
+    );
+  } catch (err) {
+    logger.error(
+      `[${runId}] Failed to fetch run config: ${(err as Error).message}`,
+    );
+    throw err; // BullMQ will retry
+  }
+
+  // ── Step 2: Mark run as running ────────────────────────────────────────
+  await apiPatch(config.apiBaseUrl, `/api/runs/${runId}/status`, {
+    runId,
+    status: "running",
+  });
+
+  // ── Step 3: Start container ────────────────────────────────────────────
+  const dockerImage = runConfig.dockerImage || config.dockerImage;
+  if (!dockerImage) {
+    throw new Error(
+      `No docker image: API did not return one and DOCKER_IMAGE env var is not set`,
+    );
+  }
+  let containerId: string;
+  try {
+    containerId = await startContainer(
+      config,
+      runConfig.actorConfig,
+      runConfig.memoryLimitMb,
+      dockerImage,
+    );
+    logger.info(`[${runId}] Container started: ${containerId.slice(0, 12)}`);
+  } catch (err) {
+    logger.error(
+      `[${runId}] Failed to start container: ${(err as Error).message}`,
+    );
+    await apiPatch(config.apiBaseUrl, `/api/runs/${runId}/status`, {
+      runId,
+      status: "failed",
+      error: `Failed to start container: ${(err as Error).message}`,
+    });
+    throw err;
+  }
+
+  // ── Step 4: Extend BullMQ lock while container runs ─────────────────────
+  let lockTimer: ReturnType<typeof setInterval> | undefined;
+  if (job.token) {
+    lockTimer = setInterval(async () => {
+      try {
+        await job.extendLock(job.token!, 300_000);
+      } catch (extErr) {
+        logger.warn(
+          `[${runId}] Failed to extend lock: ${(extErr as Error).message}`,
+        );
+      }
+    }, 120_000);
+  }
+
+  // ── Step 5: Wait for container ────────────────────────────────────────
+  let exitCode = -1;
+  try {
+    exitCode = await waitForContainer(containerId, runConfig.actorTimeoutSecs);
+    logger.info(
+      `[${runId}] Container ${containerId.slice(0, 12)} exited with code ${exitCode}`,
+    );
+  } finally {
+    if (lockTimer) clearInterval(lockTimer);
+  }
+
+  // ── Step 6: Exit-code fallback ────────────────────────────────────────
+  // The container calls POST /api/runs/:id/status itself when it finishes.
+  // We POST here only as a safety net if that callback never arrived.
+  // The API's handleStatusUpdate ignores updates that move a run backward
+  // (e.g. completed → running), so this is safe to always send.
+  const finalStatus = exitCode === 0 ? "completed" : "failed";
+  await apiPatch(config.apiBaseUrl, `/api/runs/${runId}/status`, {
+    runId,
+    status: finalStatus,
+    exitCode,
+    timestamp: new Date().toISOString(),
+    ...(exitCode !== 0 && {
+      error: `Container exited with code ${exitCode} (status callback not received)`,
+    }),
+  });
+
+  logger.info(`[${runId}] Job ${job.id} done — status: ${finalStatus}`);
+  return { runId, status: finalStatus };
+}