@nikx/dory-worker 1.0.0 → 1.0.2

package/.env

@@ -0,0 +1,10 @@
+REDIS_HOST=localhost
+REDIS_PORT=6379
+API_BASE_URL=http://localhost:4500
+GCS_BUCKET=dory-results
+GCP_PROJECT_ID=dory-scraper-local
+STORAGE_EMULATOR_HOST=http://localhost:4443
+DOCKER_IMAGE=dory-actor:latest
+MAX_CONCURRENT_RUNS=2
+LOG_LEVEL=debug
+WORKER_ID=test-worker-01

package/dist/config.d.ts

@@ -9,6 +9,8 @@ export interface WorkerConfig {
     /** fake-gcs-server URL for local dev (e.g. http://localhost:9199) */
     storageEmulatorHost?: string;
     dockerImage?: string;
+    crawlerRedisUrl?: string;
+    containerCount: number;
     maxConcurrentRuns: number;
     workerId: string;
     logLevel: string;

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAE3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IAIvB,UAAU,EAAE,MAAM,CAAC;IAKnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAID,wBAAgB,UAAU,IAAI,YAAY,CAiCzC"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAE3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IAIvB,UAAU,EAAE,MAAM,CAAC;IAKnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,WAAW,CAAC,EAAE,MAAM,CAAC;IAKrB,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,cAAc,EAAE,MAAM,CAAC;IAGvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAID,wBAAgB,UAAU,IAAI,YAAY,CAoCzC"}

package/dist/config.js

@@ -21,6 +21,8 @@ function loadConfig() {
         gcpProjectId: process.env.GCP_PROJECT_ID,
         storageEmulatorHost: process.env.STORAGE_EMULATOR_HOST,
         dockerImage: process.env.DOCKER_IMAGE,
+        crawlerRedisUrl: process.env.CRAWLER_REDIS_URL,
+        containerCount: parseInt(process.env.CONTAINER_COUNT ?? "1", 10),
         maxConcurrentRuns: parseInt(process.env.MAX_CONCURRENT_RUNS ?? "2", 10),
         workerId: process.env.WORKER_ID ?? `dory-worker-${process.pid}`,
         logLevel: process.env.LOG_LEVEL ?? "info",

package/dist/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;AA8BA,gCAiCC;AAnCD,MAAM,aAAa,GAAG,CAAC,cAAc,CAAU,CAAC;AAEhD,SAAgB,UAAU;IACxB,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChE,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS;QAC/B,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,WAAW;QAChD,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,CAAC;QACzD,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;QAEzC,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,YAAa;QAErC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;QACjC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;QACxC,mBAAmB,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;QAEtD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QAErC,iBAAiB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,GAAG,EAAE,EAAE,CAAC;QACvE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,eAAe,OAAO,CAAC,GAAG,EAAE;QAC/D,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;KAC1C,CAAC;AACJ,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;AAsCA,gCAoCC;AAtCD,MAAM,aAAa,GAAG,CAAC,cAAc,CAAU,CAAC;AAEhD,SAAgB,UAAU;IACxB,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChE,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS;QAC/B,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,WAAW;QAChD,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,CAAC;QACzD,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;QAEzC,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,YAAa;QAErC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;QACjC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;QACxC,mBAAmB,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;QAEtD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QAErC,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAC9C,cAAc,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,GAAG,EAAE,EAAE,CAAC;QAEhE,iBAAiB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,GAAG,EAAE,EAAE,CAAC;QACvE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,eAAe,OAAO,CAAC,GAAG,EAAE;QAC/D,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;KAC1C,CAAC;AACJ,CAAC"}

package/dist/docker.d.ts

@@ -5,6 +5,15 @@ interface ActorConfigEnvelope {
     actorInstanceId: string;
     userInput: Record<string, unknown>;
 }
+/** Extra env vars injected when running in distributed (multi-container) mode. */
+interface DistributedOpts {
+    /** Full Redis URL for the per-run crawler queue, e.g. redis://host:6379 */
+    crawlerRedisUrl: string;
+    /** Redis key namespace — typically the runId */
+    queueName: string;
+    /** Human-readable container label, e.g. "worker-1" */
+    workerId: string;
+}
 /**
  * Start the dory-core scraping container in detached mode.
  * Returns the container ID.
@@ -12,7 +21,15 @@ interface ActorConfigEnvelope {
  * @param dockerImage  Image to run — sourced from GET /api/runs/:id/config
  *                     so the API controls what version runs everywhere.
  */
-export declare function startContainer(config: WorkerConfig, actorConfig: ActorConfigEnvelope, memoryLimit: number, dockerImage: string): Promise<string>;
+export declare function startContainer(config: WorkerConfig, actorConfig: ActorConfigEnvelope, memoryLimit: number, dockerImage: string, distributed?: DistributedOpts): Promise<string>;
+/**
+ * Spawn `count` containers for a distributed run.
+ * All containers share the same Redis queue (`queueName`) so they
+ * collaborate on the same crawl without duplicating work.
+ *
+ * Returns the list of container IDs in order (worker-1 … worker-N).
+ */
+export declare function startContainersForRun(config: WorkerConfig, actorConfig: ActorConfigEnvelope, memoryLimit: number, dockerImage: string, crawlerRedisUrl: string, queueName: string, count: number): Promise<string[]>;
 /**
  * Block until the container exits and return its exit code.
  * If the container exceeds `timeoutSecs`, it is force-killed.

package/dist/docker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"docker.d.ts","sourceRoot":"","sources":["../src/docker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAK7C,UAAU,mBAAmB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAwCD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,YAAY,EACpB,WAAW,EAAE,mBAAmB,EAChC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,CAAC,CAiBjB;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,CAAC,CAiBjB"}
+{"version":3,"file":"docker.d.ts","sourceRoot":"","sources":["../src/docker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAK7C,UAAU,mBAAmB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,kFAAkF;AAClF,UAAU,eAAe;IACvB,2EAA2E;IAC3E,eAAe,EAAE,MAAM,CAAC;IACxB,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAgDD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,YAAY,EACpB,WAAW,EAAE,mBAAmB,EAChC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,eAAe,GAC5B,OAAO,CAAC,MAAM,CAAC,CAoBjB;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,YAAY,EACpB,WAAW,EAAE,mBAAmB,EAChC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,MAAM,EACvB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CAqBnB;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,CAAC,CAiBjB"}

package/dist/docker.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.startContainer = startContainer;
+exports.startContainersForRun = startContainersForRun;
 exports.waitForContainer = waitForContainer;
 const child_process_1 = require("child_process");
 const util_1 = require("util");
@@ -15,7 +16,7 @@ function toDockerHost(url) {
         .replace("http://localhost", "http://host.docker.internal")
         .replace("https://localhost", "https://host.docker.internal");
 }
-function buildDockerEnvArgs(config, actorConfig, memoryLimit) {
+function buildDockerEnvArgs(config, actorConfig, memoryLimit, distributed) {
     const configBase64 = Buffer.from(JSON.stringify(actorConfig)).toString("base64");
     const args = [
         `-e ACTOR_CONFIG=${configBase64}`,
@@ -30,6 +31,12 @@ function buildDockerEnvArgs(config, actorConfig, memoryLimit) {
     if (config.storageEmulatorHost) {
         args.push(`-e STORAGE_EMULATOR_HOST=${toDockerHost(config.storageEmulatorHost)}`);
     }
+    // Distributed mode — wire the container to the shared crawler Redis queue
+    if (distributed) {
+        args.push(`-e REDIS_URL=${distributed.crawlerRedisUrl}`);
+        args.push(`-e QUEUE_NAME=${distributed.queueName}`);
+        args.push(`-e WORKER_ID=${distributed.workerId}`);
+    }
     return args;
 }
 /**
@@ -39,8 +46,8 @@ function buildDockerEnvArgs(config, actorConfig, memoryLimit) {
  * @param dockerImage  Image to run — sourced from GET /api/runs/:id/config
  *                     so the API controls what version runs everywhere.
  */
-async function startContainer(config, actorConfig, memoryLimit, dockerImage) {
-    const envArgs = buildDockerEnvArgs(config, actorConfig, memoryLimit).join(" ");
+async function startContainer(config, actorConfig, memoryLimit, dockerImage, distributed) {
+    const envArgs = buildDockerEnvArgs(config, actorConfig, memoryLimit, distributed).join(" ");
     const memoryStr = `${memoryLimit}m`;
     const cmd = [
         "docker run -d --rm",
@@ -53,6 +60,27 @@ async function startContainer(config, actorConfig, memoryLimit, dockerImage) {
     const { stdout } = await execAsync(cmd);
     return stdout.trim();
 }
+/**
+ * Spawn `count` containers for a distributed run.
+ * All containers share the same Redis queue (`queueName`) so they
+ * collaborate on the same crawl without duplicating work.
+ *
+ * Returns the list of container IDs in order (worker-1 … worker-N).
+ */
+async function startContainersForRun(config, actorConfig, memoryLimit, dockerImage, crawlerRedisUrl, queueName, count) {
+    const containerIds = [];
+    for (let i = 1; i <= count; i++) {
+        const workerId = `worker-${i}`;
+        const id = await startContainer(config, actorConfig, memoryLimit, dockerImage, {
+            crawlerRedisUrl,
+            queueName,
+            workerId,
+        });
+        logger_1.logger.info(`[${queueName}] ${workerId} container started: ${id.slice(0, 12)}`);
+        containerIds.push(id);
+    }
+    return containerIds;
+}
 /**
  * Block until the container exits and return its exit code.
  * If the container exceeds `timeoutSecs`, it is force-killed.

package/dist/docker.js.map

@@ -1 +1 @@
-{"version":3,"file":"docker.js","sourceRoot":"","sources":["../src/docker.ts"],"names":[],"mappings":";;AA2DA,wCAsBC;AAMD,4CAoBC;AA3GD,iDAAqC;AACrC,+BAAiC;AAEjC,qCAAkC;AAElC,MAAM,SAAS,GAAG,IAAA,gBAAS,EAAC,oBAAI,CAAC,CAAC;AASlC;;;GAGG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,GAAG;SACP,OAAO,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;SAC1D,OAAO,CAAC,mBAAmB,EAAE,8BAA8B,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAoB,EACpB,WAAgC,EAChC,WAAmB;IAEnB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CACpE,QAAQ,CACT,CAAC;IAEF,MAAM,IAAI,GAAG;QACX,mBAAmB,YAAY,EAAE;QACjC,mBAAmB,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE;QACpD,4BAA4B,WAAW,EAAE;KAC1C,CAAC;IAEF,wEAAwE;IACxE,IAAI,MAAM,CAAC,SAAS;QAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACrE,IAAI,MAAM,CAAC,YAAY;QACrB,IAAI,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;IACxD,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CACP,4BAA4B,YAAY,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,CACvE,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc,CAClC,MAAoB,EACpB,WAAgC,EAChC,WAAmB,EACnB,WAAmB;IAEnB,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,IAAI,CACvE,GAAG,CACJ,CAAC;IACF,MAAM,SAAS,GAAG,GAAG,WAAW,GAAG,CAAC;IAEpC,MAAM,GAAG,GAAG;QACV,oBAAoB;QACpB,YAAY,SAAS,EAAE;QACvB,8CAA8C;QAC9C,OAAO;QACP,WAAW;KACZ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEZ,eAAM,CAAC,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;IACnC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,gBAAgB,CACpC,WAAmB,EACnB,WAAmB;IAEnB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,eAAe,WAAW,EAAE,EAAE;YAC/D,OAAO,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,GAAG,KAAK;SACpC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,IAAI,CACT,6BAA6B,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAM,GAAa,CAAC,OAAO,kBAAkB,CACnG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,eAAe,WAAW,EAAE,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;QACD,OAAO,GAAG,CAAC,CAAC,oBAAoB;IAClC,CAAC;AACH,CAAC"}
+{"version":3,"file":"docker.js","sourceRoot":"","sources":["../src/docker.ts"],"names":[],"mappings":";;AA6EA,wCA0BC;AASD,sDA6BC;AAMD,4CAoBC;AAvKD,iDAAqC;AACrC,+BAAiC;AAEjC,qCAAkC;AAElC,MAAM,SAAS,GAAG,IAAA,gBAAS,EAAC,oBAAI,CAAC,CAAC;AAmBlC;;;GAGG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,GAAG;SACP,OAAO,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;SAC1D,OAAO,CAAC,mBAAmB,EAAE,8BAA8B,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAoB,EACpB,WAAgC,EAChC,WAAmB,EACnB,WAA6B;IAE7B,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CACpE,QAAQ,CACT,CAAC;IAEF,MAAM,IAAI,GAAG;QACX,mBAAmB,YAAY,EAAE;QACjC,mBAAmB,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE;QACpD,4BAA4B,WAAW,EAAE;KAC1C,CAAC;IAEF,wEAAwE;IACxE,IAAI,MAAM,CAAC,SAAS;QAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACrE,IAAI,MAAM,CAAC,YAAY;QACrB,IAAI,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;IACxD,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CACP,4BAA4B,YAAY,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,CACvE,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,CAAC,IAAI,CAAC,gBAAgB,WAAW,CAAC,eAAe,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,CAAC,iBAAiB,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc,CAClC,MAAoB,EACpB,WAAgC,EAChC,WAAmB,EACnB,WAAmB,EACnB,WAA6B;IAE7B,MAAM,OAAO,GAAG,kBAAkB,CAChC,MAAM,EACN,WAAW,EACX,WAAW,EACX,WAAW,CACZ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACZ,MAAM,SAAS,GAAG,GAAG,WAAW,GAAG,CAAC;IAEpC,MAAM,GAAG,GAAG;QACV,oBAAoB;QACpB,YAAY,SAAS,EAAE;QACvB,8CAA8C;QAC9C,OAAO;QACP,WAAW;KACZ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEZ,eAAM,CAAC,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;IACnC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,qBAAqB,CACzC,MAAoB,EACpB,WAAgC,EAChC,WAAmB,EACnB,WAAmB,EACnB,eAAuB,EACvB,SAAiB,EACjB,KAAa;IAEb,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,UAAU,CAAC,EAAE,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,cAAc,CAC7B,MAAM,EACN,WAAW,EACX,WAAW,EACX,WAAW,EACX;YACE,eAAe;YACf,SAAS;YACT,QAAQ;SACT,CACF,CAAC;QACF,eAAM,CAAC,IAAI,CACT,IAAI,SAAS,KAAK,QAAQ,uBAAuB,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CACnE,CAAC;QACF,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,gBAAgB,CACpC,WAAmB,EACnB,WAAmB;IAEnB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,eAAe,WAAW,EAAE,EAAE;YAC/D,OAAO,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,GAAG,KAAK;SACpC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,IAAI,CACT,6BAA6B,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAM,GAAa,CAAC,OAAO,kBAAkB,CACnG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,eAAe,WAAW,EAAE,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;QACD,OAAO,GAAG,CAAC,CAAC,oBAAoB;IAClC,CAAC;AACH,CAAC"}

package/dist/processor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AA4C7C;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,GAAG,EAAE,GAAG,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,GAC1B,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAoG5C"}
+{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAkD7C;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,GAAG,EAAE,GAAG,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,GAC1B,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAuI5C"}

package/dist/processor.js

@@ -54,22 +54,36 @@ async function processRunJob(config, job) {
         runId,
         status: "running",
     });
-    // ── Step 3: Start container ────────────────────────────────────────────
+    // ── Step 3: Start container(s) ────────────────────────────────────────
     const dockerImage = runConfig.dockerImage || config.dockerImage;
     if (!dockerImage) {
         throw new Error(`No docker image: API did not return one and DOCKER_IMAGE env var is not set`);
     }
-    let containerId;
+    // API-returned containerCount takes precedence over the worker env var
+    const containerCount = runConfig.containerCount ?? config.containerCount;
+    const isDistributed = !!config.crawlerRedisUrl || containerCount > 1;
+    let containerIds;
     try {
-        containerId = await (0, docker_1.startContainer)(config, runConfig.actorConfig, runConfig.memoryLimitMb, dockerImage);
-        logger_1.logger.info(`[${runId}] Container started: ${containerId.slice(0, 12)}`);
+        if (isDistributed) {
+            if (!config.crawlerRedisUrl) {
+                throw new Error("CONTAINER_COUNT > 1 but CRAWLER_REDIS_URL is not set — containers cannot share a queue");
+            }
+            containerIds = await (0, docker_1.startContainersForRun)(config, runConfig.actorConfig, runConfig.memoryLimitMb, dockerImage, config.crawlerRedisUrl, runId, // QUEUE_NAME = runId for job isolation
+            containerCount);
+            logger_1.logger.info(`[${runId}] ${containerIds.length} containers started (distributed mode, queue: ${runId})`);
+        }
+        else {
+            const id = await (0, docker_1.startContainer)(config, runConfig.actorConfig, runConfig.memoryLimitMb, dockerImage);
+            containerIds = [id];
+            logger_1.logger.info(`[${runId}] Container started: ${id.slice(0, 12)}`);
+        }
     }
     catch (err) {
-        logger_1.logger.error(`[${runId}] Failed to start container: ${err.message}`);
+        logger_1.logger.error(`[${runId}] Failed to start container(s): ${err.message}`);
         await apiPatch(config.apiBaseUrl, `/api/runs/${runId}/status`, {
             runId,
             status: "failed",
-            error: `Failed to start container: ${err.message}`,
+            error: `Failed to start container(s): ${err.message}`,
         });
         throw err;
     }
@@ -85,11 +99,13 @@ async function processRunJob(config, job) {
             }
         }, 120_000);
     }
-    // ── Step 5: Wait for container ────────────────────────────────────────
+    // ── Step 5: Wait for all containers ──────────────────────────────────
     let exitCode = -1;
     try {
-        exitCode = await (0, docker_1.waitForContainer)(containerId, runConfig.actorTimeoutSecs);
-        logger_1.logger.info(`[${runId}] Container ${containerId.slice(0, 12)} exited with code ${exitCode}`);
+        const exitCodes = await Promise.all(containerIds.map((id) => (0, docker_1.waitForContainer)(id, runConfig.actorTimeoutSecs)));
+        // Use the worst exit code — if any container failed, the run failed
+        exitCode = exitCodes.reduce((worst, code) => (code !== 0 ? code : worst), 0);
+        logger_1.logger.info(`[${runId}] All ${containerIds.length} container(s) exited — codes: [${exitCodes.join(", ")}]`);
     }
     finally {
         if (lockTimer)

package/dist/processor.js.map

@@ -1 +1 @@
-{"version":3,"file":"processor.js","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":";;AAwDA,sCAuGC;AA7JD,qCAA4D;AAC5D,qCAAkC;AAelC,KAAK,UAAU,MAAM,CAAI,UAAkB,EAAE,IAAY;IACvD,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI,WAAW,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,EAAgB,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,UAAkB,EAClB,IAAY,EACZ,IAA6B;IAE7B,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAC3B,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;KAC3B,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,eAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,WAAW,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,aAAa,CACjC,MAAoB,EACpB,GAA2B;IAE3B,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;IAE3B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAEtD,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,oBAAoB,GAAG,CAAC,EAAE,aAAa,GAAG,CAAC,YAAY,GAAG,CAAC,GAAG,CACxE,CAAC;IAEF,0EAA0E;IAC1E,IAAI,SAA4B,CAAC;IACjC,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,MAAM,CACtB,MAAM,CAAC,UAAU,EACjB,aAAa,KAAK,SAAS,CAC5B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CACV,IAAI,KAAK,iCAAkC,GAAa,CAAC,OAAO,EAAE,CACnE,CAAC;QACF,MAAM,GAAG,CAAC,CAAC,oBAAoB;IACjC,CAAC;IAED,0EAA0E;IAC1E,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;QAC7D,KAAK;QACL,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IAEH,0EAA0E;IAC1E,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;IAChE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAC;IACJ,CAAC;IACD,IAAI,WAAmB,CAAC;IACxB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,IAAA,uBAAc,EAChC,MAAM,EACN,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,aAAa,EACvB,WAAW,CACZ,CAAC;QACF,eAAM,CAAC,IAAI,CAAC,IAAI,KAAK,wBAAwB,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CACV,IAAI,KAAK,gCAAiC,GAAa,CAAC,OAAO,EAAE,CAClE,CAAC;QACF,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;YAC7D,KAAK;YACL,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,8BAA+B,GAAa,CAAC,OAAO,EAAE;SAC9D,CAAC,CAAC;QACH,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,2EAA2E;IAC3E,IAAI,SAAqD,CAAC;IAC1D,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QACd,SAAS,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACjC,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,KAAM,EAAE,OAAO,CAAC,CAAC;YAC5C,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,4BAA6B,MAAgB,CAAC,OAAO,EAAE,CACjE,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,OAAO,CAAC,CAAC;IACd,CAAC;IAED,yEAAyE;IACzE,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,IAAA,yBAAgB,EAAC,WAAW,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC3E,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,eAAe,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,qBAAqB,QAAQ,EAAE,CAChF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,SAAS;YAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAC1C,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,oEAAoE;IACpE,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,WAAW,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC5D,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;QAC7D,KAAK;QACL,MAAM,EAAE,WAAW;QACnB,QAAQ;QACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,GAAG,CAAC,QAAQ,KAAK,CAAC,IAAI;YACpB,KAAK,EAAE,8BAA8B,QAAQ,iCAAiC;SAC/E,CAAC;KACH,CAAC,CAAC;IAEH,eAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,GAAG,CAAC,EAAE,mBAAmB,WAAW,EAAE,CAAC,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACxC,CAAC"}
+{"version":3,"file":"processor.js","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":";;AA8DA,sCA0IC;AAtMD,qCAIkB;AAClB,qCAAkC;AAiBlC,KAAK,UAAU,MAAM,CAAI,UAAkB,EAAE,IAAY;IACvD,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI,WAAW,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,EAAgB,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,UAAkB,EAClB,IAAY,EACZ,IAA6B;IAE7B,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAC3B,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;KAC3B,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,eAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,WAAW,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,aAAa,CACjC,MAAoB,EACpB,GAA2B;IAE3B,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;IAE3B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAEtD,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,oBAAoB,GAAG,CAAC,EAAE,aAAa,GAAG,CAAC,YAAY,GAAG,CAAC,GAAG,CACxE,CAAC;IAEF,0EAA0E;IAC1E,IAAI,SAA4B,CAAC;IACjC,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,MAAM,CACtB,MAAM,CAAC,UAAU,EACjB,aAAa,KAAK,SAAS,CAC5B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CACV,IAAI,KAAK,iCAAkC,GAAa,CAAC,OAAO,EAAE,CACnE,CAAC;QACF,MAAM,GAAG,CAAC,CAAC,oBAAoB;IACjC,CAAC;IAED,0EAA0E;IAC1E,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;QAC7D,KAAK;QACL,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IAEH,yEAAyE;IACzE,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;IAChE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAC;IACJ,CAAC;IAED,uEAAuE;IACvE,MAAM,cAAc,GAAG,SAAS,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,CAAC;IACzE,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC,eAAe,IAAI,cAAc,GAAG,CAAC,CAAC;IAErE,IAAI,YAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;YACJ,CAAC;YACD,YAAY,GAAG,MAAM,IAAA,8BAAqB,EACxC,MAAM,EACN,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,aAAa,EACvB,WAAW,EACX,MAAM,CAAC,eAAe,EACtB,KAAK,EAAE,uCAAuC;YAC9C,cAAc,CACf,CAAC;YACF,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,KAAK,YAAY,CAAC,MAAM,iDAAiD,KAAK,GAAG,CAC3F,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,EAAE,GAAG,MAAM,IAAA,uBAAc,EAC7B,MAAM,EACN,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,aAAa,EACvB,WAAW,CACZ,CAAC;YACF,YAAY,GAAG,CAAC,EAAE,CAAC,CAAC;YACpB,eAAM,CAAC,IAAI,CAAC,IAAI,KAAK,wBAAwB,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CACV,IAAI,KAAK,mCAAoC,GAAa,CAAC,OAAO,EAAE,CACrE,CAAC;QACF,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;YAC7D,KAAK;YACL,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,iCAAkC,GAAa,CAAC,OAAO,EAAE;SACjE,CAAC,CAAC;QACH,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,2EAA2E;IAC3E,IAAI,SAAqD,CAAC;IAC1D,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QACd,SAAS,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACjC,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,KAAM,EAAE,OAAO,CAAC,CAAC;YAC5C,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,4BAA6B,MAAgB,CAAC,OAAO,EAAE,CACjE,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,OAAO,CAAC,CAAC;IACd,CAAC;IAED,wEAAwE;IACxE,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CACtB,IAAA,yBAAgB,EAAC,EAAE,EAAE,SAAS,CAAC,gBAAgB,CAAC,CACjD,CACF,CAAC;QACF,oEAAoE;QACpE,QAAQ,GAAG,SAAS,CAAC,MAAM,CACzB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,EAC5C,CAAC,CACF,CAAC;QACF,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,SAAS,YAAY,CAAC,MAAM,kCAAkC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC/F,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,SAAS;YAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAC1C,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,oEAAoE;IACpE,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,WAAW,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC5D,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;QAC7D,KAAK;QACL,MAAM,EAAE,WAAW;QACnB,QAAQ;QACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,GAAG,CAAC,QAAQ,KAAK,CAAC,IAAI;YACpB,KAAK,EAAE,8BAA8B,QAAQ,iCAAiC;SAC/E,CAAC;KACH,CAAC,CAAC;IAEH,eAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,GAAG,CAAC,EAAE,mBAAmB,WAAW,EAAE,CAAC,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACxC,CAAC"}

package/dist/test-harness.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Standalone test harness — no dory-api needed.
+ *
+ * What it does:
+ *   1. Starts a mock HTTP server that emulates the two dory-api endpoints
+ *      the worker uses: GET /api/runs/:id/config and POST /api/runs/:id/status
+ *   2. Pushes a real BullMQ job into local Redis
+ *   3. Starts the real worker (same code that runs in production)
+ *   4. Waits for the job to finish and reports pass / fail
+ *
+ * Docker image used: "alpine" with command override via a tiny wrapper image
+ * built on-the-fly, or just "hello-world" which exits 0 immediately.
+ *
+ * Usage:
+ *   npx ts-node --project tsconfig.json src/test-harness.ts
+ */
+export {};
+//# sourceMappingURL=test-harness.d.ts.map

package/dist/test-harness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-harness.d.ts","sourceRoot":"","sources":["../src/test-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG"}

package/dist/test-harness.js

@@ -0,0 +1,211 @@
+"use strict";
+/**
+ * Standalone test harness — no dory-api needed.
+ *
+ * What it does:
+ *   1. Starts a mock HTTP server that emulates the two dory-api endpoints
+ *      the worker uses: GET /api/runs/:id/config and POST /api/runs/:id/status
+ *   2. Pushes a real BullMQ job into local Redis
+ *   3. Starts the real worker (same code that runs in production)
+ *   4. Waits for the job to finish and reports pass / fail
+ *
+ * Docker image used: "alpine" with command override via a tiny wrapper image
+ * built on-the-fly, or just "hello-world" which exits 0 immediately.
+ *
+ * Usage:
+ *   npx ts-node --project tsconfig.json src/test-harness.ts
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const http = __importStar(require("http"));
+const bullmq_1 = require("bullmq");
+const ioredis_1 = __importDefault(require("ioredis"));
+const processor_1 = require("./processor");
+const logger_1 = require("./logger");
+// ── Config ────────────────────────────────────────────────────────────────────
+const MOCK_API_PORT = 14500;
+const REDIS_HOST = process.env.REDIS_HOST ?? "localhost";
+const REDIS_PORT = parseInt(process.env.REDIS_PORT ?? "6379", 10);
+const RUN_ID = `test-run-${Date.now()}`;
+// Dedicated test queue — avoids interference from any running dory-api workers
+// which consume from the real "run-execution" queue.
+const QUEUE_NAME = "dory-worker-test";
+// "dory-actor:local" is already pulled on this machine and exits quickly
+// when given an empty/invalid config (no startUrls → no work → exits 0).
+// Override with TEST_DOCKER_IMAGE env var to test a different image.
+const TEST_DOCKER_IMAGE = process.env.TEST_DOCKER_IMAGE ?? "dory-worker-test:fake";
+const statusUpdates = [];
+let configRequests = 0;
+function startMockApi() {
+    const actorConfig = {
+        jobId: `job-${RUN_ID}`,
+        runId: RUN_ID,
+        actorInstanceId: "test-actor-001",
+        userInput: {
+            crawlerType: "cheerio",
+            handlers: [],
+            startUrls: ["https://example.com"],
+        },
+    };
+    const server = http.createServer((req, res) => {
+        const url = req.url ?? "";
+        logger_1.logger.debug(`[mock-api] ${req.method} ${url}`);
+        if (req.method === "GET" && url.includes("/config")) {
+            configRequests++;
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({
+                runId: RUN_ID,
+                actorConfig,
+                memoryLimitMb: 256,
+                actorTimeoutSecs: 60,
+                dockerImage: TEST_DOCKER_IMAGE,
+            }));
+        }
+        else if (req.method === "POST" && url.includes("/status")) {
+            let body = "";
+            req.on("data", (chunk) => (body += chunk));
+            req.on("end", () => {
+                const data = JSON.parse(body || "{}");
+                statusUpdates.push(data);
+                logger_1.logger.info(`[mock-api] status callback → status=${data.status}${data.exitCode !== undefined ? ` exitCode=${data.exitCode}` : ""}`);
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify({ message: "ok" }));
+            });
+        }
+        else {
+            res.writeHead(404);
+            res.end();
+        }
+    });
+    return new Promise((resolve) => {
+        server.listen(MOCK_API_PORT, "0.0.0.0", () => {
+            logger_1.logger.info(`[mock-api] Listening on :${MOCK_API_PORT}`);
+            resolve(server);
+        });
+    });
+}
+// ── Main ─────────────────────────────────────────────────────────────────────
+async function sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+async function main() {
+    (0, logger_1.setLogLevel)("debug");
+    logger_1.logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+    logger_1.logger.info("  dory-worker standalone test harness");
+    logger_1.logger.info(`  Run ID      : ${RUN_ID}`);
+    logger_1.logger.info(`  Test image  : ${TEST_DOCKER_IMAGE}`);
+    logger_1.logger.info(`  Redis       : ${REDIS_HOST}:${REDIS_PORT}`);
+    logger_1.logger.info(`  Mock API    : http://localhost:${MOCK_API_PORT}`);
+    logger_1.logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+    // ── 1. Start mock API ─────────────────────────────────────────────────
+    const server = await startMockApi();
+    // ── 2. Connect to Redis ───────────────────────────────────────────────
+    const redis = new ioredis_1.default({
+        host: REDIS_HOST,
+        port: REDIS_PORT,
+        maxRetriesPerRequest: null,
+    });
+    // ── 3. Push test job to BullMQ queue ──────────────────────────────────
+    const queue = new bullmq_1.Queue(QUEUE_NAME, { connection: redis });
+    const job = await queue.add("execute-run", { runId: RUN_ID });
+    logger_1.logger.info(`[test] Job enqueued → id=${job.id}`);
+    // ── 4. Build worker config pointing at mock API ───────────────────────
+    const config = {
+        redisHost: REDIS_HOST,
+        redisPort: REDIS_PORT,
+        apiBaseUrl: `http://localhost:${MOCK_API_PORT}`,
+        maxConcurrentRuns: 1,
+        workerId: "test-worker",
+        logLevel: "debug",
+        // GCS pass-through not needed for hello-world test
+        gcsBucket: undefined,
+        gcpProjectId: undefined,
+        dockerImage: TEST_DOCKER_IMAGE,
+        containerCount: 1,
+    };
+    // ── 5. Start worker ───────────────────────────────────────────────────
+    logger_1.logger.info("[test] Starting worker...");
+    let jobCompleted = false;
+    let jobFailed = false;
+    const worker = new bullmq_1.Worker(QUEUE_NAME, async (j) => (0, processor_1.processRunJob)(config, j), {
+        connection: redis,
+        concurrency: 1,
+        lockDuration: 300_000,
+    });
+    worker.on("completed", (j) => {
+        logger_1.logger.info(`[test] ✓ Worker completed job ${j.id}`);
+        jobCompleted = true;
+    });
+    worker.on("failed", (j, err) => {
+        logger_1.logger.error(`[test] ✗ Worker failed job ${j?.id}: ${err.message}`);
+        jobFailed = true;
+    });
+    // ── 6. Wait for result (max 90s) ──────────────────────────────────────
+    const deadline = Date.now() + 90_000;
+    while (!jobCompleted && !jobFailed && Date.now() < deadline) {
+        await sleep(500);
+    }
+    // ── 7. Report ─────────────────────────────────────────────────────────
+    logger_1.logger.info("");
+    logger_1.logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+    logger_1.logger.info("  TEST RESULTS");
+    logger_1.logger.info(`  Config requests    : ${configRequests} (expected 1)`);
+    logger_1.logger.info(`  Status callbacks   : ${statusUpdates.length} (expected ≥1)`);
+    statusUpdates.forEach((u, i) => logger_1.logger.info(`    [${i + 1}] status=${u.status}${u.exitCode !== undefined ? ` exitCode=${u.exitCode}` : ""}`));
+    logger_1.logger.info(`  Job completed      : ${jobCompleted}`);
+    logger_1.logger.info(`  Job failed         : ${jobFailed}`);
+    const passed = jobCompleted &&
+        !jobFailed &&
+        configRequests === 1 &&
+        statusUpdates.length >= 1 &&
+        statusUpdates.some((u) => ["running", "completed"].includes(u.status));
+    logger_1.logger.info("");
+    logger_1.logger.info(passed ? "  ✓ ALL CHECKS PASSED" : "  ✗ SOME CHECKS FAILED");
+    logger_1.logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+    // ── Cleanup ───────────────────────────────────────────────────────────
+    await worker.close();
+    await queue.close();
+    await redis.quit();
+    server.close();
+    process.exit(passed ? 0 : 1);
+}
+main().catch((err) => {
+    console.error("[test-harness] Fatal:", err);
+    process.exit(1);
+});
+//# sourceMappingURL=test-harness.js.map

package/dist/test-harness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-harness.js","sourceRoot":"","sources":["../src/test-harness.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,2CAA6B;AAC7B,mCAAuC;AACvC,sDAA8B;AAE9B,2CAA4C;AAC5C,qCAA+C;AAE/C,iFAAiF;AAEjF,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,WAAW,CAAC;AACzD,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;AAClE,MAAM,MAAM,GAAG,YAAY,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;AACxC,+EAA+E;AAC/E,qDAAqD;AACrD,MAAM,UAAU,GAAG,kBAAkB,CAAC;AAEtC,yEAAyE;AACzE,yEAAyE;AACzE,qEAAqE;AACrE,MAAM,iBAAiB,GACrB,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,uBAAuB,CAAC;AAU3D,MAAM,aAAa,GAAmB,EAAE,CAAC;AACzC,IAAI,cAAc,GAAG,CAAC,CAAC;AAEvB,SAAS,YAAY;IACnB,MAAM,WAAW,GAAG;QAClB,KAAK,EAAE,OAAO,MAAM,EAAE;QACtB,KAAK,EAAE,MAAM;QACb,eAAe,EAAE,gBAAgB;QACjC,SAAS,EAAE;YACT,WAAW,EAAE,SAAS;YACtB,QAAQ,EAAE,EAAE;YACZ,SAAS,EAAE,CAAC,qBAAqB,CAAC;SACnC;KACF,CAAC;IAEF,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC5C,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC;QAC1B,eAAM,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QAEhD,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACpD,cAAc,EAAE,CAAC;YACjB,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAC3D,GAAG,CAAC,GAAG,CACL,IAAI,CAAC,SAAS,CAAC;gBACb,KAAK,EAAE,MAAM;gBACb,WAAW;gBACX,aAAa,EAAE,GAAG;gBAClB,gBAAgB,EAAE,EAAE;gBACpB,WAAW,EAAE,iBAAiB;aAC/B,CAAC,CACH,CAAC;QACJ,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5D,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC;YAC3C,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACjB,MAAM,IAAI,GAAiB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;gBACpD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzB,eAAM,CAAC,IAAI,CACT,uCAAuC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACvH,CAAC;gBACF,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC;gBAC3D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACnB,GAAG,CAAC,GAAG,EAAE,CAAC;QACZ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,SAAS,EAAE,GAAG,EAAE;YAC3C,eAAM,CAAC,IAAI,CAAC,4BAA4B,aAAa,EAAE,CAAC,CAAC;YACzD,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,gFAAgF;AAEhF,KAAK,UAAU,KAAK,CAAC,EAAU;IAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,IAAA,oBAAW,EAAC,OAAO,CAAC,CAAC;IAErB,eAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC5D,eAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IACrD,eAAM,CAAC,IAAI,CAAC,mBAAmB,MAAM,EAAE,CAAC,CAAC;IACzC,eAAM,CAAC,IAAI,CAAC,mBAAmB,iBAAiB,EAAE,CAAC,CAAC;IACpD,eAAM,CAAC,IAAI,CAAC,mBAAmB,UAAU,IAAI,UAAU,EAAE,CAAC,CAAC;IAC3D,eAAM,CAAC,IAAI,CAAC,oCAAoC,aAAa,EAAE,CAAC,CAAC;IACjE,eAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAE5D,yEAAyE;IACzE,MAAM,MAAM,GAAG,MAAM,YAAY,EAAE,CAAC;IAEpC,yEAAyE;IACzE,MAAM,KAAK,GAAG,IAAI,iBAAO,CAAC;QACxB,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,UAAU;QAChB,oBAAoB,EAAE,IAAI;KAC3B,CAAC,CAAC;IAEH,yEAAyE;IACzE,MAAM,KAAK,GAAG,IAAI,cAAK,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9D,eAAM,CAAC,IAAI,CAAC,4BAA4B,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAElD,yEAAyE;IACzE,MAAM,MAAM,GAAiB;QAC3B,SAAS,EAAE,UAAU;QACrB,SAAS,EAAE,UAAU;QACrB,UAAU,EAAE,oBAAoB,aAAa,EAAE;QAC/C,iBAAiB,EAAE,CAAC;QACpB,QAAQ,EAAE,aAAa;QACvB,QAAQ,EAAE,OAAO;QACjB,mDAAmD;QACnD,SAAS,EAAE,SAAS;QACpB,YAAY,EAAE,SAAS;QACvB,WAAW,EAAE,iBAAiB;QAC9B,cAAc,EAAE,CAAC;KAClB,CAAC;IAEF,yEAAyE;IACzE,eAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAEzC,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,IAAA,yBAAa,EAAC,MAAM,EAAE,CAAC,CAAC,EAAE;QAC3E,UAAU,EAAE,KAAK;QACjB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,OAAO;KACtB,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE;QAC3B,eAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrD,YAAY,GAAG,IAAI,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QAC7B,eAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC;IACrC,OAAO,CAAC,YAAY,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC5D,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAED,yEAAyE;IACzE,eAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,eAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC5D,eAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9B,eAAM,CAAC,IAAI,CAAC,0BAA0B,cAAc,eAAe,CAAC,CAAC;IACrE,eAAM,CAAC,IAAI,CAAC,0BAA0B,aAAa,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAC5E,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC7B,eAAM,CAAC,IAAI,CACT,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAChG,CACF,CAAC;IACF,eAAM,CAAC,IAAI,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAC;IACtD,eAAM,CAAC,IAAI,CAAC,0BAA0B,SAAS,EAAE,CAAC,CAAC;IAEnD,MAAM,MAAM,GACV,YAAY;QACZ,CAAC,SAAS;QACV,cAAc,KAAK,CAAC;QACpB,aAAa,CAAC,MAAM,IAAI,CAAC;QACzB,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAEzE,eAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,eAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC;IACzE,eAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAE5D,yEAAyE;IACzE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACpB,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACnB,MAAM,CAAC,KAAK,EAAE,CAAC;IAEf,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nikx/dory-worker",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Standalone BullMQ worker for Dory – runs on any machine with Docker (including Raspberry Pi)",
   "main": "dist/cli.js",
   "bin": {
@@ -10,6 +10,7 @@
     "build": "tsc",
     "start": "node dist/cli.js",
     "dev": "npx ts-node --project tsconfig.json src/cli.ts",
+    "test": "npx ts-node --project tsconfig.json src/test-harness.ts",
     "prepare": "npm run build"
   },
   "keywords": [

package/src/config.ts

@@ -20,6 +20,14 @@ export interface WorkerConfig {
   // Fallback docker image if the API doesn't return one in /config
   dockerImage?: string;
 
+  // Distributed crawling — separate Redis for the per-run crawler queue.
+  // When set, the worker spawns `containerCount` containers per job and
+  // wires them all to the same Redis queue (QUEUE_NAME = runId).
+  crawlerRedisUrl?: string;
+
+  // How many containers to spawn per job (default 1 = single-container mode).
+  containerCount: number;
+
   // Worker behaviour
   maxConcurrentRuns: number;
   workerId: string;
@@ -57,6 +65,9 @@ export function loadConfig(): WorkerConfig {
 
     dockerImage: process.env.DOCKER_IMAGE,
 
+    crawlerRedisUrl: process.env.CRAWLER_REDIS_URL,
+    containerCount: parseInt(process.env.CONTAINER_COUNT ?? "1", 10),
+
     maxConcurrentRuns: parseInt(process.env.MAX_CONCURRENT_RUNS ?? "2", 10),
     workerId: process.env.WORKER_ID ?? `dory-worker-${process.pid}`,
     logLevel: process.env.LOG_LEVEL ?? "info",

package/src/docker.ts

@@ -12,6 +12,16 @@ interface ActorConfigEnvelope {
   userInput: Record<string, unknown>;
 }
 
+/** Extra env vars injected when running in distributed (multi-container) mode. */
+interface DistributedOpts {
+  /** Full Redis URL for the per-run crawler queue, e.g. redis://host:6379 */
+  crawlerRedisUrl: string;
+  /** Redis key namespace — typically the runId */
+  queueName: string;
+  /** Human-readable container label, e.g. "worker-1" */
+  workerId: string;
+}
+
 /**
  * Rewrite localhost URLs to host.docker.internal so containers can reach
  * the host machine. Public URLs (Railway, GCS, etc.) are left unchanged.
@@ -26,6 +36,7 @@ function buildDockerEnvArgs(
   config: WorkerConfig,
   actorConfig: ActorConfigEnvelope,
   memoryLimit: number,
+  distributed?: DistributedOpts,
 ): string[] {
   const configBase64 = Buffer.from(JSON.stringify(actorConfig)).toString(
     "base64",
@@ -47,6 +58,13 @@ function buildDockerEnvArgs(
     );
   }
 
+  // Distributed mode — wire the container to the shared crawler Redis queue
+  if (distributed) {
+    args.push(`-e REDIS_URL=${distributed.crawlerRedisUrl}`);
+    args.push(`-e QUEUE_NAME=${distributed.queueName}`);
+    args.push(`-e WORKER_ID=${distributed.workerId}`);
+  }
+
   return args;
 }
 
@@ -62,10 +80,14 @@ export async function startContainer(
   actorConfig: ActorConfigEnvelope,
   memoryLimit: number,
   dockerImage: string,
+  distributed?: DistributedOpts,
 ): Promise<string> {
-  const envArgs = buildDockerEnvArgs(config, actorConfig, memoryLimit).join(
-    " ",
-  );
+  const envArgs = buildDockerEnvArgs(
+    config,
+    actorConfig,
+    memoryLimit,
+    distributed,
+  ).join(" ");
   const memoryStr = `${memoryLimit}m`;
 
   const cmd = [
@@ -81,6 +103,44 @@ export async function startContainer(
   return stdout.trim();
 }
 
+/**
+ * Spawn `count` containers for a distributed run.
+ * All containers share the same Redis queue (`queueName`) so they
+ * collaborate on the same crawl without duplicating work.
+ *
+ * Returns the list of container IDs in order (worker-1 … worker-N).
+ */
+export async function startContainersForRun(
+  config: WorkerConfig,
+  actorConfig: ActorConfigEnvelope,
+  memoryLimit: number,
+  dockerImage: string,
+  crawlerRedisUrl: string,
+  queueName: string,
+  count: number,
+): Promise<string[]> {
+  const containerIds: string[] = [];
+  for (let i = 1; i <= count; i++) {
+    const workerId = `worker-${i}`;
+    const id = await startContainer(
+      config,
+      actorConfig,
+      memoryLimit,
+      dockerImage,
+      {
+        crawlerRedisUrl,
+        queueName,
+        workerId,
+      },
+    );
+    logger.info(
+      `[${queueName}] ${workerId} container started: ${id.slice(0, 12)}`,
+    );
+    containerIds.push(id);
+  }
+  return containerIds;
+}
+
 /**
  * Block until the container exits and return its exit code.
  * If the container exceeds `timeoutSecs`, it is force-killed.

package/src/processor.ts

@@ -1,6 +1,10 @@
 import type { Job } from "bullmq";
 import type { WorkerConfig } from "./config";
-import { startContainer, waitForContainer } from "./docker";
+import {
+  startContainer,
+  startContainersForRun,
+  waitForContainer,
+} from "./docker";
 import { logger } from "./logger";
 
 interface RunConfigResponse {
@@ -14,6 +18,8 @@ interface RunConfigResponse {
   memoryLimitMb: number;
   actorTimeoutSecs: number;
   dockerImage: string;
+  /** Per-actor parallelism set in dory-api. Falls back to CONTAINER_COUNT env var (default 1). */
+  containerCount?: number;
 }
 
 async function apiGet<T>(apiBaseUrl: string, path: string): Promise<T> {
@@ -86,30 +92,56 @@ export async function processRunJob(
     status: "running",
   });
 
-  // ── Step 3: Start container ────────────────────────────────────────────
+  // ── Step 3: Start container(s) ────────────────────────────────────────
   const dockerImage = runConfig.dockerImage || config.dockerImage;
   if (!dockerImage) {
     throw new Error(
       `No docker image: API did not return one and DOCKER_IMAGE env var is not set`,
     );
   }
-  let containerId: string;
+
+  // API-returned containerCount takes precedence over the worker env var
+  const containerCount = runConfig.containerCount ?? config.containerCount;
+  const isDistributed = !!config.crawlerRedisUrl || containerCount > 1;
+
+  let containerIds: string[];
   try {
-    containerId = await startContainer(
-      config,
-      runConfig.actorConfig,
-      runConfig.memoryLimitMb,
-      dockerImage,
-    );
-    logger.info(`[${runId}] Container started: ${containerId.slice(0, 12)}`);
+    if (isDistributed) {
+      if (!config.crawlerRedisUrl) {
+        throw new Error(
+          "CONTAINER_COUNT > 1 but CRAWLER_REDIS_URL is not set — containers cannot share a queue",
+        );
+      }
+      containerIds = await startContainersForRun(
+        config,
+        runConfig.actorConfig,
+        runConfig.memoryLimitMb,
+        dockerImage,
+        config.crawlerRedisUrl,
+        runId, // QUEUE_NAME = runId for job isolation
+        containerCount,
+      );
+      logger.info(
+        `[${runId}] ${containerIds.length} containers started (distributed mode, queue: ${runId})`,
+      );
+    } else {
+      const id = await startContainer(
+        config,
+        runConfig.actorConfig,
+        runConfig.memoryLimitMb,
+        dockerImage,
+      );
+      containerIds = [id];
+      logger.info(`[${runId}] Container started: ${id.slice(0, 12)}`);
+    }
   } catch (err) {
     logger.error(
-      `[${runId}] Failed to start container: ${(err as Error).message}`,
+      `[${runId}] Failed to start container(s): ${(err as Error).message}`,
     );
     await apiPatch(config.apiBaseUrl, `/api/runs/${runId}/status`, {
       runId,
       status: "failed",
-      error: `Failed to start container: ${(err as Error).message}`,
+      error: `Failed to start container(s): ${(err as Error).message}`,
     });
     throw err;
   }
@@ -128,12 +160,21 @@ export async function processRunJob(
     }, 120_000);
   }
 
-  // ── Step 5: Wait for container ────────────────────────────────────────
+  // ── Step 5: Wait for all containers ──────────────────────────────────
   let exitCode = -1;
   try {
-    exitCode = await waitForContainer(containerId, runConfig.actorTimeoutSecs);
+    const exitCodes = await Promise.all(
+      containerIds.map((id) =>
+        waitForContainer(id, runConfig.actorTimeoutSecs),
+      ),
+    );
+    // Use the worst exit code — if any container failed, the run failed
+    exitCode = exitCodes.reduce(
+      (worst, code) => (code !== 0 ? code : worst),
+      0,
+    );
     logger.info(
-      `[${runId}] Container ${containerId.slice(0, 12)} exited with code ${exitCode}`,
+      `[${runId}] All ${containerIds.length} container(s) exited — codes: [${exitCodes.join(", ")}]`,
     );
   } finally {
     if (lockTimer) clearInterval(lockTimer);

package/src/test-harness.ts

@@ -0,0 +1,218 @@
+/**
+ * Standalone test harness — no dory-api needed.
+ *
+ * What it does:
+ *   1. Starts a mock HTTP server that emulates the two dory-api endpoints
+ *      the worker uses: GET /api/runs/:id/config and POST /api/runs/:id/status
+ *   2. Pushes a real BullMQ job into local Redis
+ *   3. Starts the real worker (same code that runs in production)
+ *   4. Waits for the job to finish and reports pass / fail
+ *
+ * Docker image used: "alpine" with command override via a tiny wrapper image
+ * built on-the-fly, or just "hello-world" which exits 0 immediately.
+ *
+ * Usage:
+ *   npx ts-node --project tsconfig.json src/test-harness.ts
+ */
+
+import * as http from "http";
+import { Queue, Worker } from "bullmq";
+import IORedis from "ioredis";
+import type { WorkerConfig } from "./config";
+import { processRunJob } from "./processor";
+import { logger, setLogLevel } from "./logger";
+
+// ── Config ────────────────────────────────────────────────────────────────────
+
+const MOCK_API_PORT = 14500;
+const REDIS_HOST = process.env.REDIS_HOST ?? "localhost";
+const REDIS_PORT = parseInt(process.env.REDIS_PORT ?? "6379", 10);
+const RUN_ID = `test-run-${Date.now()}`;
+// Dedicated test queue — avoids interference from any running dory-api workers
+// which consume from the real "run-execution" queue.
+const QUEUE_NAME = "dory-worker-test";
+
+// "dory-actor:local" is already pulled on this machine and exits quickly
+// when given an empty/invalid config (no startUrls → no work → exits 0).
+// Override with TEST_DOCKER_IMAGE env var to test a different image.
+const TEST_DOCKER_IMAGE =
+  process.env.TEST_DOCKER_IMAGE ?? "dory-worker-test:fake";
+
+// ── Mock API server ───────────────────────────────────────────────────────────
+
+interface StatusUpdate {
+  status: string;
+  exitCode?: number;
+  error?: string;
+}
+
+const statusUpdates: StatusUpdate[] = [];
+let configRequests = 0;
+
+function startMockApi(): Promise<http.Server> {
+  const actorConfig = {
+    jobId: `job-${RUN_ID}`,
+    runId: RUN_ID,
+    actorInstanceId: "test-actor-001",
+    userInput: {
+      crawlerType: "cheerio",
+      handlers: [],
+      startUrls: ["https://example.com"],
+    },
+  };
+
+  const server = http.createServer((req, res) => {
+    const url = req.url ?? "";
+    logger.debug(`[mock-api] ${req.method} ${url}`);
+
+    if (req.method === "GET" && url.includes("/config")) {
+      configRequests++;
+      res.writeHead(200, { "Content-Type": "application/json" });
+      res.end(
+        JSON.stringify({
+          runId: RUN_ID,
+          actorConfig,
+          memoryLimitMb: 256,
+          actorTimeoutSecs: 60,
+          dockerImage: TEST_DOCKER_IMAGE,
+        }),
+      );
+    } else if (req.method === "POST" && url.includes("/status")) {
+      let body = "";
+      req.on("data", (chunk) => (body += chunk));
+      req.on("end", () => {
+        const data: StatusUpdate = JSON.parse(body || "{}");
+        statusUpdates.push(data);
+        logger.info(
+          `[mock-api] status callback → status=${data.status}${data.exitCode !== undefined ? ` exitCode=${data.exitCode}` : ""}`,
+        );
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(JSON.stringify({ message: "ok" }));
+      });
+    } else {
+      res.writeHead(404);
+      res.end();
+    }
+  });
+
+  return new Promise((resolve) => {
+    server.listen(MOCK_API_PORT, "0.0.0.0", () => {
+      logger.info(`[mock-api] Listening on :${MOCK_API_PORT}`);
+      resolve(server);
+    });
+  });
+}
+
+// ── Main ─────────────────────────────────────────────────────────────────────
+
+async function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+async function main(): Promise<void> {
+  setLogLevel("debug");
+
+  logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+  logger.info("  dory-worker standalone test harness");
+  logger.info(`  Run ID      : ${RUN_ID}`);
+  logger.info(`  Test image  : ${TEST_DOCKER_IMAGE}`);
+  logger.info(`  Redis       : ${REDIS_HOST}:${REDIS_PORT}`);
+  logger.info(`  Mock API    : http://localhost:${MOCK_API_PORT}`);
+  logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+
+  // ── 1. Start mock API ─────────────────────────────────────────────────
+  const server = await startMockApi();
+
+  // ── 2. Connect to Redis ───────────────────────────────────────────────
+  const redis = new IORedis({
+    host: REDIS_HOST,
+    port: REDIS_PORT,
+    maxRetriesPerRequest: null,
+  });
+
+  // ── 3. Push test job to BullMQ queue ──────────────────────────────────
+  const queue = new Queue(QUEUE_NAME, { connection: redis });
+  const job = await queue.add("execute-run", { runId: RUN_ID });
+  logger.info(`[test] Job enqueued → id=${job.id}`);
+
+  // ── 4. Build worker config pointing at mock API ───────────────────────
+  const config: WorkerConfig = {
+    redisHost: REDIS_HOST,
+    redisPort: REDIS_PORT,
+    apiBaseUrl: `http://localhost:${MOCK_API_PORT}`,
+    maxConcurrentRuns: 1,
+    workerId: "test-worker",
+    logLevel: "debug",
+    // GCS pass-through not needed for hello-world test
+    gcsBucket: undefined,
+    gcpProjectId: undefined,
+    dockerImage: TEST_DOCKER_IMAGE,
+    containerCount: 1,
+  };
+
+  // ── 5. Start worker ───────────────────────────────────────────────────
+  logger.info("[test] Starting worker...");
+
+  let jobCompleted = false;
+  let jobFailed = false;
+
+  const worker = new Worker(QUEUE_NAME, async (j) => processRunJob(config, j), {
+    connection: redis,
+    concurrency: 1,
+    lockDuration: 300_000,
+  });
+
+  worker.on("completed", (j) => {
+    logger.info(`[test] ✓ Worker completed job ${j.id}`);
+    jobCompleted = true;
+  });
+
+  worker.on("failed", (j, err) => {
+    logger.error(`[test] ✗ Worker failed job ${j?.id}: ${err.message}`);
+    jobFailed = true;
+  });
+
+  // ── 6. Wait for result (max 90s) ──────────────────────────────────────
+  const deadline = Date.now() + 90_000;
+  while (!jobCompleted && !jobFailed && Date.now() < deadline) {
+    await sleep(500);
+  }
+
+  // ── 7. Report ─────────────────────────────────────────────────────────
+  logger.info("");
+  logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+  logger.info("  TEST RESULTS");
+  logger.info(`  Config requests    : ${configRequests} (expected 1)`);
+  logger.info(`  Status callbacks   : ${statusUpdates.length} (expected ≥1)`);
+  statusUpdates.forEach((u, i) =>
+    logger.info(
+      `    [${i + 1}] status=${u.status}${u.exitCode !== undefined ? ` exitCode=${u.exitCode}` : ""}`,
+    ),
+  );
+  logger.info(`  Job completed      : ${jobCompleted}`);
+  logger.info(`  Job failed         : ${jobFailed}`);
+
+  const passed =
+    jobCompleted &&
+    !jobFailed &&
+    configRequests === 1 &&
+    statusUpdates.length >= 1 &&
+    statusUpdates.some((u) => ["running", "completed"].includes(u.status));
+
+  logger.info("");
+  logger.info(passed ? "  ✓ ALL CHECKS PASSED" : "  ✗ SOME CHECKS FAILED");
+  logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+
+  // ── Cleanup ───────────────────────────────────────────────────────────
+  await worker.close();
+  await queue.close();
+  await redis.quit();
+  server.close();
+
+  process.exit(passed ? 0 : 1);
+}
+
+main().catch((err) => {
+  console.error("[test-harness] Fatal:", err);
+  process.exit(1);
+});

package/test-image/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:latest
+# Fake actor: sleeps briefly to mimic processing, then exits 0
+CMD ["sh", "-c", "sleep 2 && echo 'fake actor done' && exit 0"]