@nikx/dory-worker 1.0.0

package/.env.example

@@ -0,0 +1,37 @@
+# ─── Required ────────────────────────────────────────────────────────────────
+
+# Public URL of dory-api.
+# Containers POST their status callbacks here, so it must be reachable
+# from inside Docker containers running on this machine.
+# Production example: https://your-api.railway.app
+API_BASE_URL=http://localhost:4500
+
+# ─── Redis ───────────────────────────────────────────────────────────────────
+
+# Option A: full Redis URL (recommended for Railway / cloud Redis)
+# REDIS_URL=redis://default:password@host.railway.internal:6379
+
+# Option B: host + port + optional password
+REDIS_HOST=localhost
+REDIS_PORT=6379
+# REDIS_PASSWORD=
+
+# ─── Worker behaviour ────────────────────────────────────────────────────────
+
+# Max scraping containers to run in parallel on this machine
+# Keep at 1-2 for Raspberry Pi, higher for more powerful machines
+MAX_CONCURRENT_RUNS=2
+
+# Optional identifier shown in logs (defaults to "dory-worker-{pid}")
+# WORKER_ID=pi-worker-01
+
+# Log level: debug | info | warn | error
+LOG_LEVEL=info
+
+# ─── Development only ────────────────────────────────────────────────────────
+
+# Set this to use the fake-gcs-server emulator instead of real GCS
+# STORAGE_EMULATOR_HOST=http://localhost:9199
+
+# GCP service account key file path (production; alternative to GOOGLE_APPLICATION_CREDENTIALS)
+# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account-key.json

package/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

package/README.md

@@ -0,0 +1,2 @@
+# dory-worker
+

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const config_1 = require("./config");
+const worker_1 = require("./worker");
+const logger_1 = require("./logger");
+async function main() {
+    // ── Load & validate config ────────────────────────────────────────────
+    let config;
+    try {
+        config = (0, config_1.loadConfig)();
+    }
+    catch (err) {
+        console.error(`[dory-worker] Configuration error: ${err.message}`);
+        console.error("Set the required environment variables and try again.");
+        process.exit(1);
+    }
+    (0, logger_1.setLogLevel)(config.logLevel);
+    logger_1.logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+    logger_1.logger.info("  dory-worker starting");
+    logger_1.logger.info(`  Worker ID   : ${config.workerId}`);
+    logger_1.logger.info(`  Concurrency : ${config.maxConcurrentRuns}`);
+    logger_1.logger.info(`  API URL     : ${config.apiBaseUrl}`);
+    const redisAddr = config.redisUrl ?? `${config.redisHost}:${config.redisPort}`;
+    logger_1.logger.info(`  Redis       : ${redisAddr}`);
+    if (config.storageEmulatorHost) {
+        logger_1.logger.info(`  GCS emulator: ${config.storageEmulatorHost} (dev mode)`);
+    }
+    logger_1.logger.info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
+    // ── Start BullMQ worker ───────────────────────────────────────────────
+    const redis = (0, worker_1.createRedisConnection)(config);
+    const worker = (0, worker_1.createWorker)(config, redis);
+    logger_1.logger.info('Listening on queue "run-execution" — ready to process jobs');
+    // ── Graceful shutdown ─────────────────────────────────────────────────
+    let shuttingDown = false;
+    async function shutdown(signal) {
+        if (shuttingDown)
+            return;
+        shuttingDown = true;
+        logger_1.logger.info(`${signal} received — shutting down gracefully...`);
+        try {
+            await worker.close();
+            await redis.quit();
+            logger_1.logger.info("Worker shut down cleanly");
+            process.exit(0);
+        }
+        catch (err) {
+            logger_1.logger.error(`Error during shutdown: ${err.message}`);
+            process.exit(1);
+        }
+    }
+    process.on("SIGTERM", () => shutdown("SIGTERM"));
+    process.on("SIGINT", () => shutdown("SIGINT"));
+}
+main().catch((err) => {
+    console.error("[dory-worker] Fatal error:", err);
+    process.exit(1);
+});
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";;;AACA,qCAAsC;AACtC,qCAA+D;AAC/D,qCAA+C;AAE/C,KAAK,UAAU,IAAI;IACjB,yEAAyE;IACzE,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,IAAA,mBAAU,GAAE,CAAC;IACxB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CACX,sCAAuC,GAAa,CAAC,OAAO,EAAE,CAC/D,CAAC;QACF,OAAO,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;QACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAA,oBAAW,EAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAE7B,eAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;IAC3D,eAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACtC,eAAM,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAClD,eAAM,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAC3D,eAAM,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,MAAM,SAAS,GACb,MAAM,CAAC,QAAQ,IAAI,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;IAC/D,eAAM,CAAC,IAAI,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;IAC5C,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,eAAM,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,mBAAmB,aAAa,CAAC,CAAC;IAC1E,CAAC;IACD,eAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;IAE3D,yEAAyE;IACzE,MAAM,KAAK,GAAG,IAAA,8BAAqB,EAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAA,qBAAY,EAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAE3C,eAAM,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;IAE1E,yEAAyE;IACzE,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,UAAU,QAAQ,CAAC,MAAc;QACpC,IAAI,YAAY;YAAE,OAAO;QACzB,YAAY,GAAG,IAAI,CAAC;QAEpB,eAAM,CAAC,IAAI,CAAC,GAAG,MAAM,yCAAyC,CAAC,CAAC;QAChE,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,eAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,eAAM,CAAC,KAAK,CAAC,0BAA2B,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;IAC5B,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,CAAC,CAAC;IACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/config.d.ts

@@ -0,0 +1,17 @@
+export interface WorkerConfig {
+    redisUrl?: string;
+    redisHost: string;
+    redisPort: number;
+    redisPassword?: string;
+    apiBaseUrl: string;
+    gcsBucket?: string;
+    gcpProjectId?: string;
+    /** fake-gcs-server URL for local dev (e.g. http://localhost:9199) */
+    storageEmulatorHost?: string;
+    dockerImage?: string;
+    maxConcurrentRuns: number;
+    workerId: string;
+    logLevel: string;
+}
+export declare function loadConfig(): WorkerConfig;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAE3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IAIvB,UAAU,EAAE,MAAM,CAAC;IAKnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAID,wBAAgB,UAAU,IAAI,YAAY,CAiCzC"}

package/dist/config.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadConfig = loadConfig;
+const REQUIRED_VARS = ["API_BASE_URL"];
+function loadConfig() {
+    const missing = REQUIRED_VARS.filter((k) => !process.env[k]);
+    if (missing.length > 0) {
+        throw new Error(`Missing required environment variables: ${missing.join(", ")}`);
+    }
+    // At least one Redis connection method must be provided
+    if (!process.env.REDIS_URL && !process.env.REDIS_HOST) {
+        throw new Error("Missing Redis config: set REDIS_URL or REDIS_HOST (+ REDIS_PORT)");
+    }
+    return {
+        redisUrl: process.env.REDIS_URL,
+        redisHost: process.env.REDIS_HOST ?? "localhost",
+        redisPort: parseInt(process.env.REDIS_PORT ?? "6379", 10),
+        redisPassword: process.env.REDIS_PASSWORD,
+        apiBaseUrl: process.env.API_BASE_URL,
+        gcsBucket: process.env.GCS_BUCKET,
+        gcpProjectId: process.env.GCP_PROJECT_ID,
+        storageEmulatorHost: process.env.STORAGE_EMULATOR_HOST,
+        dockerImage: process.env.DOCKER_IMAGE,
+        maxConcurrentRuns: parseInt(process.env.MAX_CONCURRENT_RUNS ?? "2", 10),
+        workerId: process.env.WORKER_ID ?? `dory-worker-${process.pid}`,
+        logLevel: process.env.LOG_LEVEL ?? "info",
+    };
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;AA8BA,gCAiCC;AAnCD,MAAM,aAAa,GAAG,CAAC,cAAc,CAAU,CAAC;AAEhD,SAAgB,UAAU;IACxB,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChE,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS;QAC/B,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,WAAW;QAChD,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,CAAC;QACzD,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;QAEzC,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,YAAa;QAErC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;QACjC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;QACxC,mBAAmB,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;QAEtD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;QAErC,iBAAiB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,GAAG,EAAE,EAAE,CAAC;QACvE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,eAAe,OAAO,CAAC,GAAG,EAAE;QAC/D,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;KAC1C,CAAC;AACJ,CAAC"}

package/dist/db.d.ts

@@ -0,0 +1,14 @@
+import type { WorkerConfig } from "./config";
+import type { RunDocument, ActorDocument } from "./types";
+export declare function connectDb(config: WorkerConfig): Promise<void>;
+export declare function disconnectDb(): Promise<void>;
+/**
+ * Atomically transition a run from "queued" → "running".
+ * Returns the run document as it was BEFORE the update (status="queued"),
+ * or null if another worker already claimed it.
+ */
+export declare function claimRun(runId: string): Promise<RunDocument | null>;
+export declare function loadRun(runId: string): Promise<RunDocument | null>;
+export declare function loadActor(actorId: string): Promise<ActorDocument | null>;
+export declare function updateRun(runId: string, update: Partial<RunDocument>): Promise<void>;
+//# sourceMappingURL=db.d.ts.map

package/dist/db.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAC7C,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAa,MAAM,SAAS,CAAC;AAOrE,wBAAsB,SAAS,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAOnE;AAED,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAKlD;AAED;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAazE;AAED,wBAAsB,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAMxE;AAED,wBAAsB,SAAS,CAC7B,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAM/B;AAED,wBAAsB,SAAS,CAC7B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,GAC3B,OAAO,CAAC,IAAI,CAAC,CAOf"}

package/dist/db.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.connectDb = connectDb;
+exports.disconnectDb = disconnectDb;
+exports.claimRun = claimRun;
+exports.loadRun = loadRun;
+exports.loadActor = loadActor;
+exports.updateRun = updateRun;
+const mongodb_1 = require("mongodb");
+const logger_1 = require("./logger");
+let client = null;
+let runsCol = null;
+let actorsCol = null;
+async function connectDb(config) {
+    client = new mongodb_1.MongoClient(config.mongoUri);
+    await client.connect();
+    const db = client.db(config.mongoDbName);
+    runsCol = db.collection("runs");
+    actorsCol = db.collection("actors");
+    logger_1.logger.info(`MongoDB connected (db: ${config.mongoDbName})`);
+}
+async function disconnectDb() {
+    if (client) {
+        await client.close();
+        logger_1.logger.info("MongoDB disconnected");
+    }
+}
+/**
+ * Atomically transition a run from "queued" → "running".
+ * Returns the run document as it was BEFORE the update (status="queued"),
+ * or null if another worker already claimed it.
+ */
+async function claimRun(runId) {
+    if (!runsCol)
+        throw new Error("DB not connected");
+    if (!mongodb_1.ObjectId.isValid(runId))
+        return null;
+    const result = await runsCol.findOneAndUpdate({
+        _id: new mongodb_1.ObjectId(runId),
+        status: "queued",
+    }, { $set: { status: "running" } }, { returnDocument: "before" });
+    return result ?? null;
+}
+async function loadRun(runId) {
+    if (!runsCol)
+        throw new Error("DB not connected");
+    if (!mongodb_1.ObjectId.isValid(runId))
+        return null;
+    return runsCol.findOne({
+        _id: new mongodb_1.ObjectId(runId),
+    });
+}
+async function loadActor(actorId) {
+    if (!actorsCol)
+        throw new Error("DB not connected");
+    if (!mongodb_1.ObjectId.isValid(actorId))
+        return null;
+    return actorsCol.findOne({
+        _id: new mongodb_1.ObjectId(actorId),
+    });
+}
+async function updateRun(runId, update) {
+    if (!runsCol)
+        throw new Error("DB not connected");
+    if (!mongodb_1.ObjectId.isValid(runId))
+        return;
+    await runsCol.updateOne({ _id: new mongodb_1.ObjectId(runId) }, { $set: update });
+}
+//# sourceMappingURL=db.js.map

package/dist/db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":";;AAUA,8BAOC;AAED,oCAKC;AAOD,4BAaC;AAED,0BAMC;AAED,8BAQC;AAED,8BAUC;AA1ED,qCAAgD;AAIhD,qCAAkC;AAElC,IAAI,MAAM,GAAuB,IAAI,CAAC;AACtC,IAAI,OAAO,GAAmC,IAAI,CAAC;AACnD,IAAI,SAAS,GAAqC,IAAI,CAAC;AAEhD,KAAK,UAAU,SAAS,CAAC,MAAoB;IAClD,MAAM,GAAG,IAAI,qBAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACvB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACzC,OAAO,GAAG,EAAE,CAAC,UAAU,CAAc,MAAM,CAAC,CAAC;IAC7C,SAAS,GAAG,EAAE,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;IACnD,eAAM,CAAC,IAAI,CAAC,0BAA0B,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;AAC/D,CAAC;AAEM,KAAK,UAAU,YAAY;IAChC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,eAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACtC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACI,KAAK,UAAU,QAAQ,CAAC,KAAa;IAC1C,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAClD,IAAI,CAAC,kBAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,CAC3C;QACE,GAAG,EAAE,IAAI,kBAAQ,CAAC,KAAK,CAAkC;QACzD,MAAM,EAAE,QAAQ;KACjB,EACD,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,SAAsB,EAAE,EAAE,EAC5C,EAAE,cAAc,EAAE,QAAQ,EAAE,CAC7B,CAAC;IACF,OAAO,MAAM,IAAI,IAAI,CAAC;AACxB,CAAC;AAEM,KAAK,UAAU,OAAO,CAAC,KAAa;IACzC,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAClD,IAAI,CAAC,kBAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,OAAO,CAAC,OAAO,CAAC;QACrB,GAAG,EAAE,IAAI,kBAAQ,CAAC,KAAK,CAAkC;KAC1D,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,SAAS,CAC7B,OAAe;IAEf,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACpD,IAAI,CAAC,kBAAQ,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,OAAO,SAAS,CAAC,OAAO,CAAC;QACvB,GAAG,EAAE,IAAI,kBAAQ,CAAC,OAAO,CAAoC;KAC9D,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,SAAS,CAC7B,KAAa,EACb,MAA4B;IAE5B,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAClD,IAAI,CAAC,kBAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO;IACrC,MAAM,OAAO,CAAC,SAAS,CACrB,EAAE,GAAG,EAAE,IAAI,kBAAQ,CAAC,KAAK,CAAkC,EAAE,EAC7D,EAAE,IAAI,EAAE,MAAM,EAAE,CACjB,CAAC;AACJ,CAAC"}

package/dist/docker.d.ts

@@ -0,0 +1,22 @@
+import type { WorkerConfig } from "./config";
+interface ActorConfigEnvelope {
+    jobId: string;
+    runId: string;
+    actorInstanceId: string;
+    userInput: Record<string, unknown>;
+}
+/**
+ * Start the dory-core scraping container in detached mode.
+ * Returns the container ID.
+ *
+ * @param dockerImage  Image to run — sourced from GET /api/runs/:id/config
+ *                     so the API controls what version runs everywhere.
+ */
+export declare function startContainer(config: WorkerConfig, actorConfig: ActorConfigEnvelope, memoryLimit: number, dockerImage: string): Promise<string>;
+/**
+ * Block until the container exits and return its exit code.
+ * If the container exceeds `timeoutSecs`, it is force-killed.
+ */
+export declare function waitForContainer(containerId: string, timeoutSecs: number): Promise<number>;
+export {};
+//# sourceMappingURL=docker.d.ts.map

package/dist/docker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker.d.ts","sourceRoot":"","sources":["../src/docker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAK7C,UAAU,mBAAmB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAwCD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,YAAY,EACpB,WAAW,EAAE,mBAAmB,EAChC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,CAAC,CAiBjB;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,CAAC,CAiBjB"}

package/dist/docker.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startContainer = startContainer;
+exports.waitForContainer = waitForContainer;
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const logger_1 = require("./logger");
+const execAsync = (0, util_1.promisify)(child_process_1.exec);
+/**
+ * Rewrite localhost URLs to host.docker.internal so containers can reach
+ * the host machine. Public URLs (Railway, GCS, etc.) are left unchanged.
+ */
+function toDockerHost(url) {
+    return url
+        .replace("http://localhost", "http://host.docker.internal")
+        .replace("https://localhost", "https://host.docker.internal");
+}
+function buildDockerEnvArgs(config, actorConfig, memoryLimit) {
+    const configBase64 = Buffer.from(JSON.stringify(actorConfig)).toString("base64");
+    const args = [
+        `-e ACTOR_CONFIG=${configBase64}`,
+        `-e API_BASE_URL=${toDockerHost(config.apiBaseUrl)}`,
+        `-e CRAWLEE_MEMORY_MBYTES=${memoryLimit}`,
+    ];
+    // Pass through GCS config if set (containers need it to upload results)
+    if (config.gcsBucket)
+        args.push(`-e GCS_BUCKET=${config.gcsBucket}`);
+    if (config.gcpProjectId)
+        args.push(`-e GCP_PROJECT_ID=${config.gcpProjectId}`);
+    if (config.storageEmulatorHost) {
+        args.push(`-e STORAGE_EMULATOR_HOST=${toDockerHost(config.storageEmulatorHost)}`);
+    }
+    return args;
+}
+/**
+ * Start the dory-core scraping container in detached mode.
+ * Returns the container ID.
+ *
+ * @param dockerImage  Image to run — sourced from GET /api/runs/:id/config
+ *                     so the API controls what version runs everywhere.
+ */
+async function startContainer(config, actorConfig, memoryLimit, dockerImage) {
+    const envArgs = buildDockerEnvArgs(config, actorConfig, memoryLimit).join(" ");
+    const memoryStr = `${memoryLimit}m`;
+    const cmd = [
+        "docker run -d --rm",
+        `--memory=${memoryStr}`,
+        "--add-host=host.docker.internal:host-gateway",
+        envArgs,
+        dockerImage,
+    ].join(" ");
+    logger_1.logger.debug(`docker run: ${cmd}`);
+    const { stdout } = await execAsync(cmd);
+    return stdout.trim();
+}
+/**
+ * Block until the container exits and return its exit code.
+ * If the container exceeds `timeoutSecs`, it is force-killed.
+ */
+async function waitForContainer(containerId, timeoutSecs) {
+    try {
+        const { stdout } = await execAsync(`docker wait ${containerId}`, {
+            timeout: (timeoutSecs + 30) * 1_000,
+        });
+        return parseInt(stdout.trim(), 10);
+    }
+    catch (err) {
+        logger_1.logger.warn(`docker wait timed out for ${containerId.slice(0, 12)}: ${err.message} — force-killing`);
+        try {
+            await execAsync(`docker kill ${containerId}`);
+        }
+        catch {
+            // Container already gone — ignore
+        }
+        return 137; // SIGKILL exit code
+    }
+}
+//# sourceMappingURL=docker.js.map

package/dist/docker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker.js","sourceRoot":"","sources":["../src/docker.ts"],"names":[],"mappings":";;AA2DA,wCAsBC;AAMD,4CAoBC;AA3GD,iDAAqC;AACrC,+BAAiC;AAEjC,qCAAkC;AAElC,MAAM,SAAS,GAAG,IAAA,gBAAS,EAAC,oBAAI,CAAC,CAAC;AASlC;;;GAGG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,GAAG;SACP,OAAO,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;SAC1D,OAAO,CAAC,mBAAmB,EAAE,8BAA8B,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAoB,EACpB,WAAgC,EAChC,WAAmB;IAEnB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CACpE,QAAQ,CACT,CAAC;IAEF,MAAM,IAAI,GAAG;QACX,mBAAmB,YAAY,EAAE;QACjC,mBAAmB,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE;QACpD,4BAA4B,WAAW,EAAE;KAC1C,CAAC;IAEF,wEAAwE;IACxE,IAAI,MAAM,CAAC,SAAS;QAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACrE,IAAI,MAAM,CAAC,YAAY;QACrB,IAAI,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;IACxD,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CACP,4BAA4B,YAAY,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,CACvE,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,cAAc,CAClC,MAAoB,EACpB,WAAgC,EAChC,WAAmB,EACnB,WAAmB;IAEnB,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,IAAI,CACvE,GAAG,CACJ,CAAC;IACF,MAAM,SAAS,GAAG,GAAG,WAAW,GAAG,CAAC;IAEpC,MAAM,GAAG,GAAG;QACV,oBAAoB;QACpB,YAAY,SAAS,EAAE;QACvB,8CAA8C;QAC9C,OAAO;QACP,WAAW;KACZ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEZ,eAAM,CAAC,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;IACnC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,gBAAgB,CACpC,WAAmB,EACnB,WAAmB;IAEnB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,eAAe,WAAW,EAAE,EAAE;YAC/D,OAAO,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,GAAG,KAAK;SACpC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,IAAI,CACT,6BAA6B,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAM,GAAa,CAAC,OAAO,kBAAkB,CACnG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,eAAe,WAAW,EAAE,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;QACD,OAAO,GAAG,CAAC,CAAC,oBAAoB;IAClC,CAAC;AACH,CAAC"}

package/dist/logger.d.ts

@@ -0,0 +1,8 @@
+export declare function setLogLevel(level: string): void;
+export declare const logger: {
+    debug: (msg: string) => void;
+    info: (msg: string) => void;
+    warn: (msg: string) => void;
+    error: (msg: string) => void;
+};
+//# sourceMappingURL=logger.d.ts.map

package/dist/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAWA,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAI/C;AAaD,eAAO,MAAM,MAAM;iBACJ,MAAM;gBACP,MAAM;gBACN,MAAM;iBACL,MAAM;CACpB,CAAC"}

package/dist/logger.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.logger = void 0;
+exports.setLogLevel = setLogLevel;
+const LEVEL_WEIGHT = {
+    debug: 0,
+    info: 1,
+    warn: 2,
+    error: 3,
+};
+let currentLevel = "info";
+function setLogLevel(level) {
+    if (level in LEVEL_WEIGHT) {
+        currentLevel = level;
+    }
+}
+function emit(level, message) {
+    if (LEVEL_WEIGHT[level] < LEVEL_WEIGHT[currentLevel])
+        return;
+    const ts = new Date().toISOString();
+    const line = `[${ts}] [${level.toUpperCase().padEnd(5)}] ${message}`;
+    if (level === "error") {
+        process.stderr.write(line + "\n");
+    }
+    else {
+        process.stdout.write(line + "\n");
+    }
+}
+exports.logger = {
+    debug: (msg) => emit("debug", msg),
+    info: (msg) => emit("info", msg),
+    warn: (msg) => emit("warn", msg),
+    error: (msg) => emit("error", msg),
+};
+//# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":";;;AAWA,kCAIC;AAbD,MAAM,YAAY,GAA6B;IAC7C,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;CACT,CAAC;AAEF,IAAI,YAAY,GAAa,MAAM,CAAC;AAEpC,SAAgB,WAAW,CAAC,KAAa;IACvC,IAAI,KAAK,IAAI,YAAY,EAAE,CAAC;QAC1B,YAAY,GAAG,KAAiB,CAAC;IACnC,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,KAAe,EAAE,OAAe;IAC5C,IAAI,YAAY,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC;QAAE,OAAO;IAC7D,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,EAAE,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;IACrE,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;AACH,CAAC;AAEY,QAAA,MAAM,GAAG;IACpB,KAAK,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC;IAC1C,IAAI,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC;IACxC,IAAI,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC;IACxC,KAAK,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC;CAC3C,CAAC"}

package/dist/processor.d.ts

@@ -0,0 +1,20 @@
+import type { Job } from "bullmq";
+import type { WorkerConfig } from "./config";
+/**
+ * Core job processor.
+ *
+ * Flow:
+ *   1. GET  /api/runs/:id/config  — fetch actorConfig from dory-api (no DB/GCS access needed)
+ *   2. POST /api/runs/:id/status  — mark run as "running"
+ *   3. docker run                 — start the scraping container
+ *   4. Extend BullMQ lock every 2 min while waiting
+ *   5. docker wait                — block until container exits
+ *   6. POST /api/runs/:id/status  — exit-code fallback if HTTP callback never arrived
+ */
+export declare function processRunJob(config: WorkerConfig, job: Job<{
+    runId: string;
+}>): Promise<{
+    runId: string;
+    status: string;
+}>;
+//# sourceMappingURL=processor.d.ts.map

package/dist/processor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AA4C7C;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,GAAG,EAAE,GAAG,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,GAC1B,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAoG5C"}

package/dist/processor.js

@@ -0,0 +1,116 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processRunJob = processRunJob;
+const docker_1 = require("./docker");
+const logger_1 = require("./logger");
+async function apiGet(apiBaseUrl, path) {
+    const url = `${apiBaseUrl.replace(/\/+$/, "")}${path}`;
+    const res = await fetch(url);
+    if (!res.ok) {
+        const body = await res.text().catch(() => "");
+        throw new Error(`GET ${path} → HTTP ${res.status}: ${body}`);
+    }
+    return res.json();
+}
+async function apiPatch(apiBaseUrl, path, body) {
+    const url = `${apiBaseUrl.replace(/\/+$/, "")}${path}`;
+    const res = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(body),
+    });
+    if (!res.ok) {
+        const text = await res.text().catch(() => "");
+        logger_1.logger.warn(`POST ${path} → HTTP ${res.status}: ${text}`);
+    }
+}
+/**
+ * Core job processor.
+ *
+ * Flow:
+ *   1. GET  /api/runs/:id/config  — fetch actorConfig from dory-api (no DB/GCS access needed)
+ *   2. POST /api/runs/:id/status  — mark run as "running"
+ *   3. docker run                 — start the scraping container
+ *   4. Extend BullMQ lock every 2 min while waiting
+ *   5. docker wait                — block until container exits
+ *   6. POST /api/runs/:id/status  — exit-code fallback if HTTP callback never arrived
+ */
+async function processRunJob(config, job) {
+    const { runId } = job.data;
+    if (!runId)
+        throw new Error("Job data missing runId");
+    logger_1.logger.info(`[${runId}] Processing job ${job.id} (attempt ${job.attemptsMade + 1})`);
+    // ── Step 1: Fetch run config from API ──────────────────────────────────
+    let runConfig;
+    try {
+        runConfig = await apiGet(config.apiBaseUrl, `/api/runs/${runId}/config`);
+    }
+    catch (err) {
+        logger_1.logger.error(`[${runId}] Failed to fetch run config: ${err.message}`);
+        throw err; // BullMQ will retry
+    }
+    // ── Step 2: Mark run as running ────────────────────────────────────────
+    await apiPatch(config.apiBaseUrl, `/api/runs/${runId}/status`, {
+        runId,
+        status: "running",
+    });
+    // ── Step 3: Start container ────────────────────────────────────────────
+    const dockerImage = runConfig.dockerImage || config.dockerImage;
+    if (!dockerImage) {
+        throw new Error(`No docker image: API did not return one and DOCKER_IMAGE env var is not set`);
+    }
+    let containerId;
+    try {
+        containerId = await (0, docker_1.startContainer)(config, runConfig.actorConfig, runConfig.memoryLimitMb, dockerImage);
+        logger_1.logger.info(`[${runId}] Container started: ${containerId.slice(0, 12)}`);
+    }
+    catch (err) {
+        logger_1.logger.error(`[${runId}] Failed to start container: ${err.message}`);
+        await apiPatch(config.apiBaseUrl, `/api/runs/${runId}/status`, {
+            runId,
+            status: "failed",
+            error: `Failed to start container: ${err.message}`,
+        });
+        throw err;
+    }
+    // ── Step 4: Extend BullMQ lock while container runs ─────────────────────
+    let lockTimer;
+    if (job.token) {
+        lockTimer = setInterval(async () => {
+            try {
+                await job.extendLock(job.token, 300_000);
+            }
+            catch (extErr) {
+                logger_1.logger.warn(`[${runId}] Failed to extend lock: ${extErr.message}`);
+            }
+        }, 120_000);
+    }
+    // ── Step 5: Wait for container ────────────────────────────────────────
+    let exitCode = -1;
+    try {
+        exitCode = await (0, docker_1.waitForContainer)(containerId, runConfig.actorTimeoutSecs);
+        logger_1.logger.info(`[${runId}] Container ${containerId.slice(0, 12)} exited with code ${exitCode}`);
+    }
+    finally {
+        if (lockTimer)
+            clearInterval(lockTimer);
+    }
+    // ── Step 6: Exit-code fallback ────────────────────────────────────────
+    // The container calls POST /api/runs/:id/status itself when it finishes.
+    // We POST here only as a safety net if that callback never arrived.
+    // The API's handleStatusUpdate ignores updates that move a run backward
+    // (e.g. completed → running), so this is safe to always send.
+    const finalStatus = exitCode === 0 ? "completed" : "failed";
+    await apiPatch(config.apiBaseUrl, `/api/runs/${runId}/status`, {
+        runId,
+        status: finalStatus,
+        exitCode,
+        timestamp: new Date().toISOString(),
+        ...(exitCode !== 0 && {
+            error: `Container exited with code ${exitCode} (status callback not received)`,
+        }),
+    });
+    logger_1.logger.info(`[${runId}] Job ${job.id} done — status: ${finalStatus}`);
+    return { runId, status: finalStatus };
+}
+//# sourceMappingURL=processor.js.map

package/dist/processor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"processor.js","sourceRoot":"","sources":["../src/processor.ts"],"names":[],"mappings":";;AAwDA,sCAuGC;AA7JD,qCAA4D;AAC5D,qCAAkC;AAelC,KAAK,UAAU,MAAM,CAAI,UAAkB,EAAE,IAAY;IACvD,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI,WAAW,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,EAAgB,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,UAAkB,EAClB,IAAY,EACZ,IAA6B;IAE7B,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IACvD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAC3B,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;KAC3B,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,eAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,WAAW,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,aAAa,CACjC,MAAoB,EACpB,GAA2B;IAE3B,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;IAE3B,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAEtD,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,oBAAoB,GAAG,CAAC,EAAE,aAAa,GAAG,CAAC,YAAY,GAAG,CAAC,GAAG,CACxE,CAAC;IAEF,0EAA0E;IAC1E,IAAI,SAA4B,CAAC;IACjC,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,MAAM,CACtB,MAAM,CAAC,UAAU,EACjB,aAAa,KAAK,SAAS,CAC5B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CACV,IAAI,KAAK,iCAAkC,GAAa,CAAC,OAAO,EAAE,CACnE,CAAC;QACF,MAAM,GAAG,CAAC,CAAC,oBAAoB;IACjC,CAAC;IAED,0EAA0E;IAC1E,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;QAC7D,KAAK;QACL,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IAEH,0EAA0E;IAC1E,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;IAChE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAC;IACJ,CAAC;IACD,IAAI,WAAmB,CAAC;IACxB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,IAAA,uBAAc,EAChC,MAAM,EACN,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,aAAa,EACvB,WAAW,CACZ,CAAC;QACF,eAAM,CAAC,IAAI,CAAC,IAAI,KAAK,wBAAwB,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAM,CAAC,KAAK,CACV,IAAI,KAAK,gCAAiC,GAAa,CAAC,OAAO,EAAE,CAClE,CAAC;QACF,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;YAC7D,KAAK;YACL,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,8BAA+B,GAAa,CAAC,OAAO,EAAE;SAC9D,CAAC,CAAC;QACH,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,2EAA2E;IAC3E,IAAI,SAAqD,CAAC;IAC1D,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QACd,SAAS,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACjC,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,KAAM,EAAE,OAAO,CAAC,CAAC;YAC5C,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,4BAA6B,MAAgB,CAAC,OAAO,EAAE,CACjE,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,OAAO,CAAC,CAAC;IACd,CAAC;IAED,yEAAyE;IACzE,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,IAAA,yBAAgB,EAAC,WAAW,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC3E,eAAM,CAAC,IAAI,CACT,IAAI,KAAK,eAAe,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,qBAAqB,QAAQ,EAAE,CAChF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,SAAS;YAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAC1C,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,oEAAoE;IACpE,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,WAAW,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC5D,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,KAAK,SAAS,EAAE;QAC7D,KAAK;QACL,MAAM,EAAE,WAAW;QACnB,QAAQ;QACR,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,GAAG,CAAC,QAAQ,KAAK,CAAC,IAAI;YACpB,KAAK,EAAE,8BAA8B,QAAQ,iCAAiC;SAC/E,CAAC;KACH,CAAC,CAAC;IAEH,eAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,GAAG,CAAC,EAAE,mBAAmB,WAAW,EAAE,CAAC,CAAC;IACtE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACxC,CAAC"}

package/dist/storage.d.ts

@@ -0,0 +1,7 @@
+import type { WorkerConfig } from "./config";
+/**
+ * Load the INPUT.json that dory-api wrote to GCS when the run was created.
+ * Supports both real GCS (production) and the fake-gcs-server emulator (dev).
+ */
+export declare function loadInputFromStorage(config: WorkerConfig, runId: string): Promise<Record<string, unknown>>;
+//# sourceMappingURL=storage.d.ts.map

package/dist/storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG7C;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CA+BlC"}

package/dist/storage.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadInputFromStorage = loadInputFromStorage;
+const storage_1 = require("@google-cloud/storage");
+const logger_1 = require("./logger");
+/**
+ * Load the INPUT.json that dory-api wrote to GCS when the run was created.
+ * Supports both real GCS (production) and the fake-gcs-server emulator (dev).
+ */
+async function loadInputFromStorage(config, runId) {
+    const filePath = `${runId}/storage/key_value_stores/default/INPUT.json`;
+    try {
+        if (config.storageEmulatorHost) {
+            // Dev: download via fake-gcs-server HTTP API
+            const host = config.storageEmulatorHost.replace(/\/+$/, "");
+            const url = `${host}/download/storage/v1/b/${config.gcsBucket}/o/${encodeURIComponent(filePath)}?alt=media`;
+            const res = await fetch(url);
+            if (!res.ok) {
+                logger_1.logger.warn(`GCS emulator: INPUT.json not found for run ${runId} (HTTP ${res.status}) — using empty input`);
+                return {};
+            }
+            return (await res.json());
+        }
+        // Production: real GCS with application default credentials
+        const storage = new storage_1.Storage({ projectId: config.gcpProjectId });
+        const [contents] = await storage
+            .bucket(config.gcsBucket)
+            .file(filePath)
+            .download();
+        return JSON.parse(contents.toString());
+    }
+    catch (err) {
+        logger_1.logger.error(`Failed to load INPUT.json for run ${runId}: ${err.message}`);
+        return {};
+    }
+}
+//# sourceMappingURL=storage.js.map