@detiq/crawler 1.1.9 → 1.1.10

package/dist/websocket-detector.js

@@ -1,72 +1,60 @@
-/**
- * WebSocket detection — intercepts WS connections opened during page crawl.
- * Records connection URLs, message schemas, and message counts.
- * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
- * that would be invisible to HTTP-only crawlers.
- */
 function inferProtocol(sample) {
-    try {
-        const obj = JSON.parse(sample);
-        if (obj?.jsonrpc)
-            return 'json-rpc';
-        if (obj?.type === 'connection_init' || obj?.type === 'subscribe')
-            return 'graphql-ws';
-        if (typeof obj?.sid === 'string' || sample.startsWith('0{'))
-            return 'socket.io';
-        return 'json';
-    }
-    catch {
-        return 'raw';
-    }
+  try {
+    const obj = JSON.parse(sample);
+    if (obj?.jsonrpc) return "json-rpc";
+    if (obj?.type === "connection_init" || obj?.type === "subscribe") return "graphql-ws";
+    if (typeof obj?.sid === "string" || sample.startsWith("0{")) return "socket.io";
+    return "json";
+  } catch {
+    return "raw";
+  }
 }
-export function attachWebSocketDetector(cdpSession) {
-    const connections = new Map();
-    const onCreated = (evt) => {
-        const { requestId, url, timestamp } = evt ?? {};
-        if (!requestId || !url)
-            return;
-        connections.set(requestId, {
-            url,
-            framesSent: 0,
-            framesReceived: 0,
-            samples: [],
-            protocol: 'unknown',
-            openedAt: new Date(timestamp * 1000).toISOString(),
-        });
-    };
-    const onFrame = (evt, direction) => {
-        const { requestId, response } = evt ?? {};
-        if (!requestId)
-            return;
-        const conn = connections.get(requestId);
-        if (!conn)
-            return;
-        if (direction === 'sent')
-            conn.framesSent++;
-        else
-            conn.framesReceived++;
-        if (conn.samples.length < 10 && response?.payloadData) {
-            const payload = String(response.payloadData).slice(0, 500);
-            conn.samples.push({ direction, payload, ts: new Date().toISOString() });
-            if (conn.protocol === 'unknown')
-                conn.protocol = inferProtocol(payload);
-        }
-    };
-    const onSent = (evt) => onFrame(evt, 'sent');
-    const onReceived = (evt) => onFrame(evt, 'received');
-    if (cdpSession?.on) {
-        cdpSession.on('Network.webSocketCreated', onCreated);
-        cdpSession.on('Network.webSocketFrameSent', onSent);
-        cdpSession.on('Network.webSocketFrameReceived', onReceived);
+function attachWebSocketDetector(cdpSession) {
+  const connections = /* @__PURE__ */ new Map();
+  const onCreated = (evt) => {
+    const { requestId, url, timestamp } = evt ?? {};
+    if (!requestId || !url) return;
+    connections.set(requestId, {
+      url,
+      framesSent: 0,
+      framesReceived: 0,
+      samples: [],
+      protocol: "unknown",
+      openedAt: new Date(timestamp * 1e3).toISOString()
+    });
+  };
+  const onFrame = (evt, direction) => {
+    const { requestId, response } = evt ?? {};
+    if (!requestId) return;
+    const conn = connections.get(requestId);
+    if (!conn) return;
+    if (direction === "sent") conn.framesSent++;
+    else conn.framesReceived++;
+    if (conn.samples.length < 10 && response?.payloadData) {
+      const payload = String(response.payloadData).slice(0, 500);
+      conn.samples.push({ direction, payload, ts: (/* @__PURE__ */ new Date()).toISOString() });
+      if (conn.protocol === "unknown") conn.protocol = inferProtocol(payload);
+    }
+  };
+  const onSent = (evt) => onFrame(evt, "sent");
+  const onReceived = (evt) => onFrame(evt, "received");
+  if (cdpSession?.on) {
+    cdpSession.on("Network.webSocketCreated", onCreated);
+    cdpSession.on("Network.webSocketFrameSent", onSent);
+    cdpSession.on("Network.webSocketFrameReceived", onReceived);
+  }
+  return {
+    getConnections: () => [...connections.values()],
+    detach() {
+      if (cdpSession?.off) {
+        cdpSession.off("Network.webSocketCreated", onCreated);
+        cdpSession.off("Network.webSocketFrameSent", onSent);
+        cdpSession.off("Network.webSocketFrameReceived", onReceived);
+      }
     }
-    return {
-        getConnections: () => [...connections.values()],
-        detach() {
-            if (cdpSession?.off) {
-                cdpSession.off('Network.webSocketCreated', onCreated);
-                cdpSession.off('Network.webSocketFrameSent', onSent);
-                cdpSession.off('Network.webSocketFrameReceived', onReceived);
-            }
-        },
-    };
+  };
 }
+export {
+  attachWebSocketDetector
+};
+//# sourceMappingURL=websocket-detector.js.map

package/dist/worker.js

@@ -1,42 +1,33 @@
-/**
- * Standalone worker process entrypoint for a dedicated crawler/worker container,
- * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).
- *
- * index.ts is a side-effect-free library barrel imported by @detiq/api — it must NOT
- * start any background loop itself, or importing a single helper from @detiq/crawler
- * would silently spin up a second worker inside the API process too. This file is
- * the actual entrypoint that keeps a dedicated worker container's process alive and
- * consuming jobs: `node dist/worker.js`.
- */
-import { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';
-import { logger } from './logger.js';
+import { startJobWorkerLoop, processDataRetentionPurge } from "./job-queue.js";
+import { logger } from "./logger.js";
 async function main() {
-    logger.info('[worker] starting standalone crawler worker process');
-    // Durable DB-backed queue — claims every non-CRAWL job type, and CRAWL too when
-    // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once.
-    startJobWorkerLoop();
-    // Daily data-retention purge: runs immediately on startup then every 24 h.
-    // Only runs in the dedicated worker container (this file), not in the API
-    // process, so there is exactly one purge sweep regardless of API replica count.
-    const runPurge = () => {
-        processDataRetentionPurge().catch((err) => {
-            logger.error({ err: String(err) }, '[worker] data-retention purge failed');
-        });
-    };
-    runPurge();
-    setInterval(runPurge, 24 * 60 * 60 * 1000);
-    // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets.
-    if (process.env.JOB_QUEUE_BACKEND === 'sqs') {
-        const { startSqsWorker } = await import('./sqs-queue.js');
-        startSqsWorker().catch((err) => {
-            logger.error({ err: String(err) }, '[worker] SQS worker crashed');
-            process.exit(1);
-        });
-    }
+  logger.info("[worker] starting standalone crawler worker process");
+  startJobWorkerLoop();
+  const runPurge = () => {
+    processDataRetentionPurge().catch((err) => {
+      logger.error({ err: String(err) }, "[worker] data-retention purge failed");
+    });
+  };
+  runPurge();
+  setInterval(runPurge, 24 * 60 * 60 * 1e3);
+  if (process.env.JOB_QUEUE_BACKEND === "sqs") {
+    const { startSqsWorker } = await import("./sqs-queue.js");
+    startSqsWorker().catch((err) => {
+      logger.error({ err: String(err) }, "[worker] SQS worker crashed");
+      process.exit(1);
+    });
+  }
 }
-process.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });
-process.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });
+process.on("SIGTERM", () => {
+  logger.info("[worker] SIGTERM received, exiting");
+  process.exit(0);
+});
+process.on("SIGINT", () => {
+  logger.info("[worker] SIGINT received, exiting");
+  process.exit(0);
+});
 main().catch((err) => {
-    logger.error({ err: String(err) }, '[worker] fatal startup error');
-    process.exit(1);
+  logger.error({ err: String(err) }, "[worker] fatal startup error");
+  process.exit(1);
 });
+//# sourceMappingURL=worker.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.9",
+  "version": "1.1.10",
   "private": false,
   "publishConfig": {
     "access": "public",