@detiq/crawler 1.1.8 → 1.1.9

package/dist/websocket-detector.js

@@ -1,60 +1,72 @@
+/**
+ * WebSocket detection — intercepts WS connections opened during page crawl.
+ * Records connection URLs, message schemas, and message counts.
+ * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
+ * that would be invisible to HTTP-only crawlers.
+ */
 function inferProtocol(sample) {
-  try {
-    const obj = JSON.parse(sample);
-    if (obj?.jsonrpc) return "json-rpc";
-    if (obj?.type === "connection_init" || obj?.type === "subscribe") return "graphql-ws";
-    if (typeof obj?.sid === "string" || sample.startsWith("0{")) return "socket.io";
-    return "json";
-  } catch {
-    return "raw";
-  }
-}
-function attachWebSocketDetector(cdpSession) {
-  const connections = /* @__PURE__ */ new Map();
-  const onCreated = (evt) => {
-    const { requestId, url, timestamp } = evt ?? {};
-    if (!requestId || !url) return;
-    connections.set(requestId, {
-      url,
-      framesSent: 0,
-      framesReceived: 0,
-      samples: [],
-      protocol: "unknown",
-      openedAt: new Date(timestamp * 1e3).toISOString()
-    });
-  };
-  const onFrame = (evt, direction) => {
-    const { requestId, response } = evt ?? {};
-    if (!requestId) return;
-    const conn = connections.get(requestId);
-    if (!conn) return;
-    if (direction === "sent") conn.framesSent++;
-    else conn.framesReceived++;
-    if (conn.samples.length < 10 && response?.payloadData) {
-      const payload = String(response.payloadData).slice(0, 500);
-      conn.samples.push({ direction, payload, ts: (/* @__PURE__ */ new Date()).toISOString() });
-      if (conn.protocol === "unknown") conn.protocol = inferProtocol(payload);
+    try {
+        const obj = JSON.parse(sample);
+        if (obj?.jsonrpc)
+            return 'json-rpc';
+        if (obj?.type === 'connection_init' || obj?.type === 'subscribe')
+            return 'graphql-ws';
+        if (typeof obj?.sid === 'string' || sample.startsWith('0{'))
+            return 'socket.io';
+        return 'json';
+    }
+    catch {
+        return 'raw';
     }
-  };
-  const onSent = (evt) => onFrame(evt, "sent");
-  const onReceived = (evt) => onFrame(evt, "received");
-  if (cdpSession?.on) {
-    cdpSession.on("Network.webSocketCreated", onCreated);
-    cdpSession.on("Network.webSocketFrameSent", onSent);
-    cdpSession.on("Network.webSocketFrameReceived", onReceived);
-  }
-  return {
-    getConnections: () => [...connections.values()],
-    detach() {
-      if (cdpSession?.off) {
-        cdpSession.off("Network.webSocketCreated", onCreated);
-        cdpSession.off("Network.webSocketFrameSent", onSent);
-        cdpSession.off("Network.webSocketFrameReceived", onReceived);
-      }
+}
+export function attachWebSocketDetector(cdpSession) {
+    const connections = new Map();
+    const onCreated = (evt) => {
+        const { requestId, url, timestamp } = evt ?? {};
+        if (!requestId || !url)
+            return;
+        connections.set(requestId, {
+            url,
+            framesSent: 0,
+            framesReceived: 0,
+            samples: [],
+            protocol: 'unknown',
+            openedAt: new Date(timestamp * 1000).toISOString(),
+        });
+    };
+    const onFrame = (evt, direction) => {
+        const { requestId, response } = evt ?? {};
+        if (!requestId)
+            return;
+        const conn = connections.get(requestId);
+        if (!conn)
+            return;
+        if (direction === 'sent')
+            conn.framesSent++;
+        else
+            conn.framesReceived++;
+        if (conn.samples.length < 10 && response?.payloadData) {
+            const payload = String(response.payloadData).slice(0, 500);
+            conn.samples.push({ direction, payload, ts: new Date().toISOString() });
+            if (conn.protocol === 'unknown')
+                conn.protocol = inferProtocol(payload);
+        }
+    };
+    const onSent = (evt) => onFrame(evt, 'sent');
+    const onReceived = (evt) => onFrame(evt, 'received');
+    if (cdpSession?.on) {
+        cdpSession.on('Network.webSocketCreated', onCreated);
+        cdpSession.on('Network.webSocketFrameSent', onSent);
+        cdpSession.on('Network.webSocketFrameReceived', onReceived);
     }
-  };
+    return {
+        getConnections: () => [...connections.values()],
+        detach() {
+            if (cdpSession?.off) {
+                cdpSession.off('Network.webSocketCreated', onCreated);
+                cdpSession.off('Network.webSocketFrameSent', onSent);
+                cdpSession.off('Network.webSocketFrameReceived', onReceived);
+            }
+        },
+    };
 }
-export {
-  attachWebSocketDetector
-};
-//# sourceMappingURL=websocket-detector.js.map

package/dist/worker.js

@@ -1,33 +1,42 @@
-import { startJobWorkerLoop, processDataRetentionPurge } from "./job-queue.js";
-import { logger } from "./logger.js";
+/**
+ * Standalone worker process entrypoint for a dedicated crawler/worker container,
+ * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).
+ *
+ * index.ts is a side-effect-free library barrel imported by @detiq/api — it must NOT
+ * start any background loop itself, or importing a single helper from @detiq/crawler
+ * would silently spin up a second worker inside the API process too. This file is
+ * the actual entrypoint that keeps a dedicated worker container's process alive and
+ * consuming jobs: `node dist/worker.js`.
+ */
+import { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';
+import { logger } from './logger.js';
 async function main() {
-  logger.info("[worker] starting standalone crawler worker process");
-  startJobWorkerLoop();
-  const runPurge = () => {
-    processDataRetentionPurge().catch((err) => {
-      logger.error({ err: String(err) }, "[worker] data-retention purge failed");
-    });
-  };
-  runPurge();
-  setInterval(runPurge, 24 * 60 * 60 * 1e3);
-  if (process.env.JOB_QUEUE_BACKEND === "sqs") {
-    const { startSqsWorker } = await import("./sqs-queue.js");
-    startSqsWorker().catch((err) => {
-      logger.error({ err: String(err) }, "[worker] SQS worker crashed");
-      process.exit(1);
-    });
-  }
+    logger.info('[worker] starting standalone crawler worker process');
+    // Durable DB-backed queue — claims every non-CRAWL job type, and CRAWL too when
+    // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once.
+    startJobWorkerLoop();
+    // Daily data-retention purge: runs immediately on startup then every 24 h.
+    // Only runs in the dedicated worker container (this file), not in the API
+    // process, so there is exactly one purge sweep regardless of API replica count.
+    const runPurge = () => {
+        processDataRetentionPurge().catch((err) => {
+            logger.error({ err: String(err) }, '[worker] data-retention purge failed');
+        });
+    };
+    runPurge();
+    setInterval(runPurge, 24 * 60 * 60 * 1000);
+    // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets.
+    if (process.env.JOB_QUEUE_BACKEND === 'sqs') {
+        const { startSqsWorker } = await import('./sqs-queue.js');
+        startSqsWorker().catch((err) => {
+            logger.error({ err: String(err) }, '[worker] SQS worker crashed');
+            process.exit(1);
+        });
+    }
 }
-process.on("SIGTERM", () => {
-  logger.info("[worker] SIGTERM received, exiting");
-  process.exit(0);
-});
-process.on("SIGINT", () => {
-  logger.info("[worker] SIGINT received, exiting");
-  process.exit(0);
-});
+process.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });
+process.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });
 main().catch((err) => {
-  logger.error({ err: String(err) }, "[worker] fatal startup error");
-  process.exit(1);
+    logger.error({ err: String(err) }, '[worker] fatal startup error');
+    process.exit(1);
 });
-//# sourceMappingURL=worker.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.8",
+  "version": "1.1.9",
   "private": false,
   "publishConfig": {
     "access": "public",