@detiq/crawler 1.1.23 → 1.1.25

package/dist/sqs-queue.js

@@ -1,17 +1,80 @@
-import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from "@aws-sdk/client-sqs";
+import {
+  SQSClient,
+  SendMessageCommand,
+  ReceiveMessageCommand,
+  DeleteMessageCommand,
+  GetQueueAttributesCommand
+} from "@aws-sdk/client-sqs";
 const sqs = new SQSClient({ region: process.env.AWS_REGION ?? "us-east-1" });
-async function sendJobToSqs(jobId, tenantId, config) {
-  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
-  if (!queueUrl) throw new Error("SQS_CRAWL_QUEUE_URL not configured");
+function getQueueUrl() {
+  const url = process.env.SQS_QUEUE_URL ?? process.env.SQS_CRAWL_QUEUE_URL;
+  if (!url) throw new Error("SQS_QUEUE_URL not configured");
+  return url;
+}
+function getDlqUrl() {
+  return process.env.SQS_DLQ_URL ?? null;
+}
+async function sendJobToSqs(jobId) {
   await sqs.send(new SendMessageCommand({
-    QueueUrl: queueUrl,
-    MessageBody: JSON.stringify({ jobId, tenantId, config })
+    QueueUrl: getQueueUrl(),
+    MessageBody: JSON.stringify({ jobId })
   }));
 }
+async function getDlqStats() {
+  const dlqUrl = getDlqUrl();
+  if (!dlqUrl) return { approximateMessageCount: 0, dlqUrl: null };
+  try {
+    const { Attributes = {} } = await sqs.send(new GetQueueAttributesCommand({
+      QueueUrl: dlqUrl,
+      AttributeNames: ["ApproximateNumberOfMessages"]
+    }));
+    return {
+      approximateMessageCount: parseInt(Attributes.ApproximateNumberOfMessages ?? "0", 10),
+      dlqUrl
+    };
+  } catch (err) {
+    console.error("[sqs-dlq] getDlqStats error:", err.message);
+    return { approximateMessageCount: 0, dlqUrl };
+  }
+}
+async function redriveFromDlq(maxMessages = 10) {
+  const dlqUrl = getDlqUrl();
+  if (!dlqUrl) {
+    console.warn("[sqs-dlq] SQS_DLQ_URL not configured \u2014 cannot redrive");
+    return 0;
+  }
+  const mainUrl = getQueueUrl();
+  let totalRedriven = 0;
+  while (totalRedriven < maxMessages) {
+    const batch = Math.min(10, maxMessages - totalRedriven);
+    const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
+      QueueUrl: dlqUrl,
+      MaxNumberOfMessages: batch,
+      WaitTimeSeconds: 1
+    }));
+    if (Messages.length === 0) break;
+    for (const msg of Messages) {
+      try {
+        await sqs.send(new SendMessageCommand({
+          QueueUrl: mainUrl,
+          MessageBody: msg.Body
+        }));
+        await sqs.send(new DeleteMessageCommand({
+          QueueUrl: dlqUrl,
+          ReceiptHandle: msg.ReceiptHandle
+        }));
+        totalRedriven++;
+        console.log("[sqs-dlq] redriven message:", msg.MessageId);
+      } catch (err) {
+        console.error("[sqs-dlq] redrive error for message", msg.MessageId, ":", err.message);
+      }
+    }
+  }
+  return totalRedriven;
+}
 async function startSqsWorker() {
-  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
-  if (!queueUrl) throw new Error("SQS_CRAWL_QUEUE_URL not configured");
-  const { processJob } = await import("./job-queue.js");
+  const queueUrl = getQueueUrl();
+  const { claimSpecificJob, dispatchClaimedJob } = await import("./job-queue.js");
   console.log("[sqs-worker] starting, queue:", queueUrl);
   while (true) {
     try {
@@ -25,7 +88,10 @@ async function startSqsWorker() {
         try {
           const data = JSON.parse(msg.Body);
           jobId = data.jobId;
-          await processJob(data.jobId, data.config);
+          const claimed = await claimSpecificJob(jobId);
+          if (claimed) {
+            await dispatchClaimedJob(claimed);
+          }
           await sqs.send(new DeleteMessageCommand({
             QueueUrl: queueUrl,
             ReceiptHandle: msg.ReceiptHandle
@@ -41,6 +107,8 @@ async function startSqsWorker() {
   }
 }
 export {
+  getDlqStats,
+  redriveFromDlq,
   sendJobToSqs,
   startSqsWorker
 };

package/dist/sqs-queue.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/sqs-queue.ts"],
-  "sourcesContent": ["import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';\n\nconst sqs = new SQSClient({ region: process.env.AWS_REGION ?? 'us-east-1' });\n\nexport async function sendJobToSqs(jobId: string, tenantId: string, config: any): Promise<void> {\n  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;\n  if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured');\n  await sqs.send(new SendMessageCommand({\n    QueueUrl: queueUrl,\n    MessageBody: JSON.stringify({ jobId, tenantId, config }),\n  }));\n}\n\nexport async function startSqsWorker(): Promise<void> {\n  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;\n  if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured');\n\n  const { processJob } = await import('./job-queue.js');\n  console.log('[sqs-worker] starting, queue:', queueUrl);\n\n  while (true) {\n    try {\n      const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({\n        QueueUrl: queueUrl,\n        WaitTimeSeconds: 20,\n        MaxNumberOfMessages: 2,\n      }));\n\n      await Promise.all(Messages.map(async (msg) => {\n        let jobId = '?';\n        try {\n          const data = JSON.parse(msg.Body!);\n          jobId = data.jobId;\n          await processJob(data.jobId, data.config);\n          await sqs.send(new DeleteMessageCommand({\n            QueueUrl: queueUrl,\n            ReceiptHandle: msg.ReceiptHandle!,\n          }));\n        } catch (err: any) {\n          console.error(`[sqs-worker] job ${jobId} failed \u2014 leaving for SQS redrive:`, err.message);\n          // Do NOT delete \u2014 SQS redrive policy + DLQ handles retry\n        }\n      }));\n    } catch (err: any) {\n      console.error('[sqs-worker] receive error:', err.message);\n      await new Promise((r) => setTimeout(r, 2000));\n    }\n  }\n}\n"],
-  "mappings": "AAAA,SAAS,WAAW,oBAAoB,uBAAuB,4BAA4B;AAE3F,MAAM,MAAM,IAAI,UAAU,EAAE,QAAQ,QAAQ,IAAI,cAAc,YAAY,CAAC;AAE3E,eAAsB,aAAa,OAAe,UAAkB,QAA4B;AAC9F,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,oCAAoC;AACnE,QAAM,IAAI,KAAK,IAAI,mBAAmB;AAAA,IACpC,UAAU;AAAA,IACV,aAAa,KAAK,UAAU,EAAE,OAAO,UAAU,OAAO,CAAC;AAAA,EACzD,CAAC,CAAC;AACJ;AAEA,eAAsB,iBAAgC;AACpD,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,oCAAoC;AAEnE,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,gBAAgB;AACpD,UAAQ,IAAI,iCAAiC,QAAQ;AAErD,SAAO,MAAM;AACX,QAAI;AACF,YAAM,EAAE,WAAW,CAAC,EAAE,IAAI,MAAM,IAAI,KAAK,IAAI,sBAAsB;AAAA,QACjE,UAAU;AAAA,QACV,iBAAiB;AAAA,QACjB,qBAAqB;AAAA,MACvB,CAAC,CAAC;AAEF,YAAM,QAAQ,IAAI,SAAS,IAAI,OAAO,QAAQ;AAC5C,YAAI,QAAQ;AACZ,YAAI;AACF,gBAAM,OAAO,KAAK,MAAM,IAAI,IAAK;AACjC,kBAAQ,KAAK;AACb,gBAAM,WAAW,KAAK,OAAO,KAAK,MAAM;AACxC,gBAAM,IAAI,KAAK,IAAI,qBAAqB;AAAA,YACtC,UAAU;AAAA,YACV,eAAe,IAAI;AAAA,UACrB,CAAC,CAAC;AAAA,QACJ,SAAS,KAAU;AACjB,kBAAQ,MAAM,oBAAoB,KAAK,2CAAsC,IAAI,OAAO;AAAA,QAE1F;AAAA,MACF,CAAC,CAAC;AAAA,IACJ,SAAS,KAAU;AACjB,cAAQ,MAAM,+BAA+B,IAAI,OAAO;AACxD,YAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAI,CAAC;AAAA,IAC9C;AAAA,EACF;AACF;",
+  "sourcesContent": ["import {\n  SQSClient,\n  SendMessageCommand,\n  ReceiveMessageCommand,\n  DeleteMessageCommand,\n  GetQueueAttributesCommand,\n} from '@aws-sdk/client-sqs';\n\nconst sqs = new SQSClient({ region: process.env.AWS_REGION ?? 'us-east-1' });\n\nfunction getQueueUrl(): string {\n  const url = process.env.SQS_QUEUE_URL ?? process.env.SQS_CRAWL_QUEUE_URL;\n  if (!url) throw new Error('SQS_QUEUE_URL not configured');\n  return url;\n}\n\nfunction getDlqUrl(): string | null {\n  return process.env.SQS_DLQ_URL ?? null;\n}\n\nexport async function sendJobToSqs(jobId: string): Promise<void> {\n  await sqs.send(new SendMessageCommand({\n    QueueUrl: getQueueUrl(),\n    MessageBody: JSON.stringify({ jobId }),\n  }));\n}\n\n/** Returns the approximate number of messages in the SQS DLQ. */\nexport async function getDlqStats(): Promise<{ approximateMessageCount: number; dlqUrl: string | null }> {\n  const dlqUrl = getDlqUrl();\n  if (!dlqUrl) return { approximateMessageCount: 0, dlqUrl: null };\n  try {\n    const { Attributes = {} } = await sqs.send(new GetQueueAttributesCommand({\n      QueueUrl: dlqUrl,\n      AttributeNames: ['ApproximateNumberOfMessages'],\n    }));\n    return {\n      approximateMessageCount: parseInt(Attributes.ApproximateNumberOfMessages ?? '0', 10),\n      dlqUrl,\n    };\n  } catch (err: any) {\n    console.error('[sqs-dlq] getDlqStats error:', err.message);\n    return { approximateMessageCount: 0, dlqUrl };\n  }\n}\n\n/**\n * Redrive up to `maxMessages` messages from the SQS DLQ back to the main queue.\n * Each message is received from the DLQ, re-sent to the main queue, then deleted\n * from the DLQ. Returns the number of messages successfully redriven.\n */\nexport async function redriveFromDlq(maxMessages = 10): Promise<number> {\n  const dlqUrl = getDlqUrl();\n  if (!dlqUrl) {\n    console.warn('[sqs-dlq] SQS_DLQ_URL not configured \u2014 cannot redrive');\n    return 0;\n  }\n  const mainUrl = getQueueUrl();\n  let totalRedriven = 0;\n\n  while (totalRedriven < maxMessages) {\n    const batch = Math.min(10, maxMessages - totalRedriven);\n    const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({\n      QueueUrl: dlqUrl,\n      MaxNumberOfMessages: batch,\n      WaitTimeSeconds: 1,\n    }));\n    if (Messages.length === 0) break;\n\n    for (const msg of Messages) {\n      try {\n        await sqs.send(new SendMessageCommand({\n          QueueUrl: mainUrl,\n          MessageBody: msg.Body!,\n        }));\n        await sqs.send(new DeleteMessageCommand({\n          QueueUrl: dlqUrl,\n          ReceiptHandle: msg.ReceiptHandle!,\n        }));\n        totalRedriven++;\n        console.log('[sqs-dlq] redriven message:', msg.MessageId);\n      } catch (err: any) {\n        console.error('[sqs-dlq] redrive error for message', msg.MessageId, ':', err.message);\n      }\n    }\n  }\n\n  return totalRedriven;\n}\n\nexport async function startSqsWorker(): Promise<void> {\n  const queueUrl = getQueueUrl();\n  const { claimSpecificJob, dispatchClaimedJob } = await import('./job-queue.js');\n  console.log('[sqs-worker] starting, queue:', queueUrl);\n\n  while (true) {\n    try {\n      const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({\n        QueueUrl: queueUrl,\n        WaitTimeSeconds: 20,\n        MaxNumberOfMessages: 2,\n      }));\n\n      await Promise.all(Messages.map(async (msg) => {\n        let jobId = '?';\n        try {\n          const data = JSON.parse(msg.Body!);\n          jobId = data.jobId;\n          // Atomically move QUEUED \u2192 RUNNING; returns null if another worker beat us.\n          const claimed = await claimSpecificJob(jobId);\n          if (claimed) {\n            await dispatchClaimedJob(claimed);\n          }\n          // Delete whether we ran it or another worker did \u2014 message is handled either way.\n          await sqs.send(new DeleteMessageCommand({\n            QueueUrl: queueUrl,\n            ReceiptHandle: msg.ReceiptHandle!,\n          }));\n        } catch (err: any) {\n          console.error(`[sqs-worker] job ${jobId} failed \u2014 leaving for SQS redrive:`, err.message);\n          // Do NOT delete \u2014 SQS visibility timeout expires and DLQ handles retry.\n        }\n      }));\n    } catch (err: any) {\n      console.error('[sqs-worker] receive error:', err.message);\n      await new Promise((r) => setTimeout(r, 2000));\n    }\n  }\n}\n"],
+  "mappings": "AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,MAAM,MAAM,IAAI,UAAU,EAAE,QAAQ,QAAQ,IAAI,cAAc,YAAY,CAAC;AAE3E,SAAS,cAAsB;AAC7B,QAAM,MAAM,QAAQ,IAAI,iBAAiB,QAAQ,IAAI;AACrD,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,8BAA8B;AACxD,SAAO;AACT;AAEA,SAAS,YAA2B;AAClC,SAAO,QAAQ,IAAI,eAAe;AACpC;AAEA,eAAsB,aAAa,OAA8B;AAC/D,QAAM,IAAI,KAAK,IAAI,mBAAmB;AAAA,IACpC,UAAU,YAAY;AAAA,IACtB,aAAa,KAAK,UAAU,EAAE,MAAM,CAAC;AAAA,EACvC,CAAC,CAAC;AACJ;AAGA,eAAsB,cAAmF;AACvG,QAAM,SAAS,UAAU;AACzB,MAAI,CAAC,OAAQ,QAAO,EAAE,yBAAyB,GAAG,QAAQ,KAAK;AAC/D,MAAI;AACF,UAAM,EAAE,aAAa,CAAC,EAAE,IAAI,MAAM,IAAI,KAAK,IAAI,0BAA0B;AAAA,MACvE,UAAU;AAAA,MACV,gBAAgB,CAAC,6BAA6B;AAAA,IAChD,CAAC,CAAC;AACF,WAAO;AAAA,MACL,yBAAyB,SAAS,WAAW,+BAA+B,KAAK,EAAE;AAAA,MACnF;AAAA,IACF;AAAA,EACF,SAAS,KAAU;AACjB,YAAQ,MAAM,gCAAgC,IAAI,OAAO;AACzD,WAAO,EAAE,yBAAyB,GAAG,OAAO;AAAA,EAC9C;AACF;AAOA,eAAsB,eAAe,cAAc,IAAqB;AACtE,QAAM,SAAS,UAAU;AACzB,MAAI,CAAC,QAAQ;AACX,YAAQ,KAAK,4DAAuD;AACpE,WAAO;AAAA,EACT;AACA,QAAM,UAAU,YAAY;AAC5B,MAAI,gBAAgB;AAEpB,SAAO,gBAAgB,aAAa;AAClC,UAAM,QAAQ,KAAK,IAAI,IAAI,cAAc,aAAa;AACtD,UAAM,EAAE,WAAW,CAAC,EAAE,IAAI,MAAM,IAAI,KAAK,IAAI,sBAAsB;AAAA,MACjE,UAAU;AAAA,MACV,qBAAqB;AAAA,MACrB,iBAAiB;AAAA,IACnB,CAAC,CAAC;AACF,QAAI,SAAS,WAAW,EAAG;AAE3B,eAAW,OAAO,UAAU;AAC1B,UAAI;AACF,cAAM,IAAI,KAAK,IAAI,mBAAmB;AAAA,UACpC,UAAU;AAAA,UACV,aAAa,IAAI;AAAA,QACnB,CAAC,CAAC;AACF,cAAM,IAAI,KAAK,IAAI,qBAAqB;AAAA,UACtC,UAAU;AAAA,UACV,eAAe,IAAI;AAAA,QACrB,CAAC,CAAC;AACF;AACA,gBAAQ,IAAI,+BAA+B,IAAI,SAAS;AAAA,MAC1D,SAAS,KAAU;AACjB,gBAAQ,MAAM,uCAAuC,IAAI,WAAW,KAAK,IAAI,OAAO;AAAA,MACtF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,eAAsB,iBAAgC;AACpD,QAAM,WAAW,YAAY;AAC7B,QAAM,EAAE,kBAAkB,mBAAmB,IAAI,MAAM,OAAO,gBAAgB;AAC9E,UAAQ,IAAI,iCAAiC,QAAQ;AAErD,SAAO,MAAM;AACX,QAAI;AACF,YAAM,EAAE,WAAW,CAAC,EAAE,IAAI,MAAM,IAAI,KAAK,IAAI,sBAAsB;AAAA,QACjE,UAAU;AAAA,QACV,iBAAiB;AAAA,QACjB,qBAAqB;AAAA,MACvB,CAAC,CAAC;AAEF,YAAM,QAAQ,IAAI,SAAS,IAAI,OAAO,QAAQ;AAC5C,YAAI,QAAQ;AACZ,YAAI;AACF,gBAAM,OAAO,KAAK,MAAM,IAAI,IAAK;AACjC,kBAAQ,KAAK;AAEb,gBAAM,UAAU,MAAM,iBAAiB,KAAK;AAC5C,cAAI,SAAS;AACX,kBAAM,mBAAmB,OAAO;AAAA,UAClC;AAEA,gBAAM,IAAI,KAAK,IAAI,qBAAqB;AAAA,YACtC,UAAU;AAAA,YACV,eAAe,IAAI;AAAA,UACrB,CAAC,CAAC;AAAA,QACJ,SAAS,KAAU;AACjB,kBAAQ,MAAM,oBAAoB,KAAK,2CAAsC,IAAI,OAAO;AAAA,QAE1F;AAAA,MACF,CAAC,CAAC;AAAA,IACJ,SAAS,KAAU;AACjB,cAAQ,MAAM,+BAA+B,IAAI,OAAO;AACxD,YAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAI,CAAC;AAAA,IAC9C;AAAA,EACF;AACF;",
   "names": []
 }

package/dist/worker.js

@@ -2,6 +2,13 @@ import { startJobWorkerLoop, processDataRetentionPurge } from "./job-queue.js";
 import { logger } from "./logger.js";
 async function main() {
   logger.info("[worker] starting standalone crawler worker process");
+  if (process.env.JOB_QUEUE_BACKEND === "sqs") {
+    const { startSqsWorker } = await import("./sqs-queue.js");
+    startSqsWorker().catch((err) => {
+      logger.error({ err: String(err) }, "[worker] SQS worker crashed");
+      process.exit(1);
+    });
+  }
   startJobWorkerLoop();
   const runPurge = () => {
     processDataRetentionPurge().catch((err) => {
@@ -10,13 +17,6 @@ async function main() {
   };
   runPurge();
   setInterval(runPurge, 24 * 60 * 60 * 1e3);
-  if (process.env.JOB_QUEUE_BACKEND === "sqs") {
-    const { startSqsWorker } = await import("./sqs-queue.js");
-    startSqsWorker().catch((err) => {
-      logger.error({ err: String(err) }, "[worker] SQS worker crashed");
-      process.exit(1);
-    });
-  }
 }
 process.on("SIGTERM", () => {
   logger.info("[worker] SIGTERM received, exiting");

package/dist/worker.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/worker.ts"],
-  "sourcesContent": ["/**\n * Standalone worker process entrypoint for a dedicated crawler/worker container,\n * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).\n *\n * index.ts is a side-effect-free library barrel imported by @detiq/api \u2014 it must NOT\n * start any background loop itself, or importing a single helper from @detiq/crawler\n * would silently spin up a second worker inside the API process too. This file is\n * the actual entrypoint that keeps a dedicated worker container's process alive and\n * consuming jobs: `node dist/worker.js`.\n */\nimport { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';\nimport { logger } from './logger.js';\n\nasync function main() {\n  logger.info('[worker] starting standalone crawler worker process');\n\n  // Durable DB-backed queue \u2014 claims every non-CRAWL job type, and CRAWL too when\n  // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once.\n  startJobWorkerLoop();\n\n  // Daily data-retention purge: runs immediately on startup then every 24 h.\n  // Only runs in the dedicated worker container (this file), not in the API\n  // process, so there is exactly one purge sweep regardless of API replica count.\n  const runPurge = () => {\n    processDataRetentionPurge().catch((err) => {\n      logger.error({ err: String(err) }, '[worker] data-retention purge failed');\n    });\n  };\n  runPurge();\n  setInterval(runPurge, 24 * 60 * 60 * 1000);\n\n  // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets.\n  if (process.env.JOB_QUEUE_BACKEND === 'sqs') {\n    const { startSqsWorker } = await import('./sqs-queue.js');\n    startSqsWorker().catch((err) => {\n      logger.error({ err: String(err) }, '[worker] SQS worker crashed');\n      process.exit(1);\n    });\n  }\n}\n\nprocess.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });\nprocess.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });\n\nmain().catch((err) => {\n  logger.error({ err: String(err) }, '[worker] fatal startup error');\n  process.exit(1);\n});\n"],
-  "mappings": "AAUA,SAAS,oBAAoB,iCAAiC;AAC9D,SAAS,cAAc;AAEvB,eAAe,OAAO;AACpB,SAAO,KAAK,qDAAqD;AAIjE,qBAAmB;AAKnB,QAAM,WAAW,MAAM;AACrB,8BAA0B,EAAE,MAAM,CAAC,QAAQ;AACzC,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,sCAAsC;AAAA,IAC3E,CAAC;AAAA,EACH;AACA,WAAS;AACT,cAAY,UAAU,KAAK,KAAK,KAAK,GAAI;AAGzC,MAAI,QAAQ,IAAI,sBAAsB,OAAO;AAC3C,UAAM,EAAE,eAAe,IAAI,MAAM,OAAO,gBAAgB;AACxD,mBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,6BAA6B;AAChE,cAAQ,KAAK,CAAC;AAAA,IAChB,CAAC;AAAA,EACH;AACF;AAEA,QAAQ,GAAG,WAAW,MAAM;AAAE,SAAO,KAAK,oCAAoC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AACnG,QAAQ,GAAG,UAAU,MAAM;AAAE,SAAO,KAAK,mCAAmC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AAEjG,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,SAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,8BAA8B;AACjE,UAAQ,KAAK,CAAC;AAChB,CAAC;",
+  "sourcesContent": ["/**\n * Standalone worker process entrypoint for a dedicated crawler/worker container,\n * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).\n *\n * index.ts is a side-effect-free library barrel imported by @detiq/api \u2014 it must NOT\n * start any background loop itself, or importing a single helper from @detiq/crawler\n * would silently spin up a second worker inside the API process too. This file is\n * the actual entrypoint that keeps a dedicated worker container's process alive and\n * consuming jobs: `node dist/worker.js`.\n */\nimport { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';\nimport { logger } from './logger.js';\n\nasync function main() {\n  logger.info('[worker] starting standalone crawler worker process');\n\n  if (process.env.JOB_QUEUE_BACKEND === 'sqs') {\n    // SQS mode: messages arrive via long-poll; claimSpecificJob atomically moves\n    // QUEUED\u2192RUNNING before dispatch, so multiple worker replicas are safe.\n    // DB poll loop also runs as a safety net for jobs whose SQS message was lost.\n    const { startSqsWorker } = await import('./sqs-queue.js');\n    startSqsWorker().catch((err) => {\n      logger.error({ err: String(err) }, '[worker] SQS worker crashed');\n      process.exit(1);\n    });\n  }\n\n  // DB-backed poll loop \u2014 primary in non-SQS mode, safety net in SQS mode.\n  // Claims all job types. Safe to run in many worker replicas (FOR UPDATE SKIP LOCKED).\n  startJobWorkerLoop();\n\n  // Daily data-retention purge: runs immediately on startup then every 24 h.\n  // Only runs in the dedicated worker container (this file), not in the API\n  // process, so there is exactly one purge sweep regardless of API replica count.\n  const runPurge = () => {\n    processDataRetentionPurge().catch((err) => {\n      logger.error({ err: String(err) }, '[worker] data-retention purge failed');\n    });\n  };\n  runPurge();\n  setInterval(runPurge, 24 * 60 * 60 * 1000);\n}\n\nprocess.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });\nprocess.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });\n\nmain().catch((err) => {\n  logger.error({ err: String(err) }, '[worker] fatal startup error');\n  process.exit(1);\n});\n"],
+  "mappings": "AAUA,SAAS,oBAAoB,iCAAiC;AAC9D,SAAS,cAAc;AAEvB,eAAe,OAAO;AACpB,SAAO,KAAK,qDAAqD;AAEjE,MAAI,QAAQ,IAAI,sBAAsB,OAAO;AAI3C,UAAM,EAAE,eAAe,IAAI,MAAM,OAAO,gBAAgB;AACxD,mBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,6BAA6B;AAChE,cAAQ,KAAK,CAAC;AAAA,IAChB,CAAC;AAAA,EACH;AAIA,qBAAmB;AAKnB,QAAM,WAAW,MAAM;AACrB,8BAA0B,EAAE,MAAM,CAAC,QAAQ;AACzC,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,sCAAsC;AAAA,IAC3E,CAAC;AAAA,EACH;AACA,WAAS;AACT,cAAY,UAAU,KAAK,KAAK,KAAK,GAAI;AAC3C;AAEA,QAAQ,GAAG,WAAW,MAAM;AAAE,SAAO,KAAK,oCAAoC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AACnG,QAAQ,GAAG,UAAU,MAAM;AAAE,SAAO,KAAK,mCAAmC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AAEjG,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,SAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,8BAA8B;AACjE,UAAQ,KAAK,CAAC;AAChB,CAAC;",
   "names": []
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.23",
+  "version": "1.1.25",
   "private": false,
   "publishConfig": {
     "access": "public",