@detiq/crawler 1.1.8 → 1.1.9

package/dist/redis-frontier.js

@@ -1,126 +1,146 @@
-const FRONTIER_TTL = 86400;
-class RedisCrawlFrontier {
-  constructor(jobId) {
-    this.jobId = jobId;
-    this.queueKey = `zeta:frontier:${jobId}:queue`;
-    this.visitedKey = `zeta:frontier:${jobId}:visited`;
-    this.useRedis = !!process.env.REDIS_URL;
-  }
-  jobId;
-  redis;
-  queueKey;
-  visitedKey;
-  inMemoryQueue = [];
-  inMemoryVisited = /* @__PURE__ */ new Set();
-  useRedis;
-  async init() {
-    if (!this.useRedis) return;
-    try {
-      const Redis = await import("ioredis").then((m) => m.default ?? m);
-      this.redis = new Redis(process.env.REDIS_URL, {
-        maxRetriesPerRequest: 2,
-        enableReadyCheck: false,
-        lazyConnect: true
-      });
-      await this.redis.connect().catch(() => {
-        this.useRedis = false;
-        this.redis = null;
-      });
-    } catch {
-      this.useRedis = false;
-      this.redis = null;
+/**
+ * Redis-backed distributed crawl frontier.
+ *
+ * Replaces the in-memory CrawlFrontier for horizontally-scaled crawler deployments.
+ * Multiple crawler workers on different containers can share the same URL queue for
+ * a given job — enabling 10x crawl throughput on large enterprise apps.
+ *
+ * Redis key namespace:  zeta:frontier:{jobId}
+ *   LIST  zeta:frontier:{jobId}:queue   — BFS queue (LPUSH/RPOP)
+ *   SET   zeta:frontier:{jobId}:visited — deduplication set
+ *   EXPIRE                               — 24h TTL on all keys
+ *
+ * Graceful fallback: if REDIS_URL is not set, RedisCrawlFrontier silently falls
+ * back to in-memory arrays so local development works without Redis.
+ */
+const FRONTIER_TTL = 86400; // 24h in seconds
+export class RedisCrawlFrontier {
+    jobId;
+    redis;
+    queueKey;
+    visitedKey;
+    inMemoryQueue = [];
+    inMemoryVisited = new Set();
+    useRedis;
+    constructor(jobId) {
+        this.jobId = jobId;
+        this.queueKey = `zeta:frontier:${jobId}:queue`;
+        this.visitedKey = `zeta:frontier:${jobId}:visited`;
+        this.useRedis = !!process.env.REDIS_URL;
     }
-  }
-  /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
-  async push(url) {
-    if (this.useRedis && this.redis) {
-      try {
-        const isNew = await this.redis.sadd(this.visitedKey, url);
-        if (!isNew) return false;
-        await this.redis.lpush(this.queueKey, url);
-        await this.redis.expire(this.queueKey, FRONTIER_TTL);
-        await this.redis.expire(this.visitedKey, FRONTIER_TTL);
+    async init() {
+        if (!this.useRedis)
+            return;
+        try {
+            const Redis = await import('ioredis').then(m => m.default ?? m);
+            this.redis = new Redis(process.env.REDIS_URL, {
+                maxRetriesPerRequest: 2,
+                enableReadyCheck: false,
+                lazyConnect: true,
+            });
+            await this.redis.connect().catch(() => {
+                this.useRedis = false;
+                this.redis = null;
+            });
+        }
+        catch {
+            this.useRedis = false;
+            this.redis = null;
+        }
+    }
+    /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
+    async push(url) {
+        if (this.useRedis && this.redis) {
+            try {
+                const isNew = await this.redis.sadd(this.visitedKey, url);
+                if (!isNew)
+                    return false;
+                await this.redis.lpush(this.queueKey, url);
+                await this.redis.expire(this.queueKey, FRONTIER_TTL);
+                await this.redis.expire(this.visitedKey, FRONTIER_TTL);
+                return true;
+            }
+            catch {
+                // Fall through to in-memory on Redis error
+            }
+        }
+        if (this.inMemoryVisited.has(url))
+            return false;
+        this.inMemoryVisited.add(url);
+        this.inMemoryQueue.push(url);
         return true;
-      } catch {
-      }
     }
-    if (this.inMemoryVisited.has(url)) return false;
-    this.inMemoryVisited.add(url);
-    this.inMemoryQueue.push(url);
-    return true;
-  }
-  /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
-  async shift() {
-    if (this.useRedis && this.redis) {
-      try {
-        const url = await this.redis.rpop(this.queueKey);
-        return url ?? null;
-      } catch {
-      }
+    /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
+    async shift() {
+        if (this.useRedis && this.redis) {
+            try {
+                const url = await this.redis.rpop(this.queueKey);
+                return url ?? null;
+            }
+            catch {
+                // Fall through
+            }
+        }
+        return this.inMemoryQueue.shift() ?? null;
+    }
+    /** Check if a URL has already been visited/queued. */
+    async hasVisited(url) {
+        if (this.useRedis && this.redis) {
+            try {
+                return (await this.redis.sismember(this.visitedKey, url)) === 1;
+            }
+            catch { /* fall through */ }
+        }
+        return this.inMemoryVisited.has(url);
+    }
+    /** Current queue length. */
+    async length() {
+        if (this.useRedis && this.redis) {
+            try {
+                return await this.redis.llen(this.queueKey);
+            }
+            catch { /* fall through */ }
+        }
+        return this.inMemoryQueue.length;
     }
-    return this.inMemoryQueue.shift() ?? null;
-  }
-  /** Check if a URL has already been visited/queued. */
-  async hasVisited(url) {
-    if (this.useRedis && this.redis) {
-      try {
-        return await this.redis.sismember(this.visitedKey, url) === 1;
-      } catch {
-      }
+    /** Total visited count. */
+    async visitedCount() {
+        if (this.useRedis && this.redis) {
+            try {
+                return await this.redis.scard(this.visitedKey);
+            }
+            catch { /* fall through */ }
+        }
+        return this.inMemoryVisited.size;
     }
-    return this.inMemoryVisited.has(url);
-  }
-  /** Current queue length. */
-  async length() {
-    if (this.useRedis && this.redis) {
-      try {
-        return await this.redis.llen(this.queueKey);
-      } catch {
-      }
+    /** Seed initial URLs (used at job start). */
+    async seed(urls) {
+        for (const url of urls)
+            await this.push(url);
     }
-    return this.inMemoryQueue.length;
-  }
-  /** Total visited count. */
-  async visitedCount() {
-    if (this.useRedis && this.redis) {
-      try {
-        return await this.redis.scard(this.visitedKey);
-      } catch {
-      }
+    /** Clean up all Redis keys for this frontier. Call at job completion. */
+    async cleanup() {
+        if (this.useRedis && this.redis) {
+            try {
+                await this.redis.del(this.queueKey, this.visitedKey);
+            }
+            catch { /* ignore */ }
+            try {
+                await this.redis.quit();
+            }
+            catch { /* ignore */ }
+        }
     }
-    return this.inMemoryVisited.size;
-  }
-  /** Seed initial URLs (used at job start). */
-  async seed(urls) {
-    for (const url of urls) await this.push(url);
-  }
-  /** Clean up all Redis keys for this frontier. Call at job completion. */
-  async cleanup() {
-    if (this.useRedis && this.redis) {
-      try {
-        await this.redis.del(this.queueKey, this.visitedKey);
-      } catch {
-      }
-      try {
-        await this.redis.quit();
-      } catch {
-      }
+    /** Stats for monitoring. */
+    async stats() {
+        return {
+            backend: this.useRedis && this.redis ? 'redis' : 'memory',
+            queued: await this.length(),
+            visited: await this.visitedCount(),
+        };
     }
-  }
-  /** Stats for monitoring. */
-  async stats() {
-    return {
-      backend: this.useRedis && this.redis ? "redis" : "memory",
-      queued: await this.length(),
-      visited: await this.visitedCount()
-    };
-  }
 }
-function isRedisFrontierEnabled() {
-  return !!process.env.REDIS_URL;
+/** Check if Redis is configured for distributed frontier. */
+export function isRedisFrontierEnabled() {
+    return !!process.env.REDIS_URL;
 }
-export {
-  RedisCrawlFrontier,
-  isRedisFrontierEnabled
-};
-//# sourceMappingURL=redis-frontier.js.map

package/dist/redis-queue.js

@@ -1,165 +1,152 @@
-import { hostname } from "node:os";
-const STREAM_KEY = "zeta:crawl:stream";
-const GROUP_NAME = "zeta:crawl:workers";
-const MAX_LEN = 1e4;
+/**
+ * Redis Streams distributed job queue.
+ *
+ * Provides horizontal crawler scaling via Redis Streams consumer groups.
+ * Multiple crawler nodes can pull from the same stream — each job is
+ * delivered to exactly one consumer (at-least-once via ACK/PENDING).
+ *
+ * Used when REDIS_URL is set AND JOB_QUEUE_BACKEND=redis.
+ * Falls back to the Postgres-backed queue when Redis is unavailable.
+ *
+ * Stream key:   zeta:crawl:stream
+ * Group name:   zeta:crawl:workers
+ * Consumer id:  zeta:worker:<hostname>:<pid>
+ */
+import { hostname } from 'node:os';
+const STREAM_KEY = 'zeta:crawl:stream';
+const GROUP_NAME = 'zeta:crawl:workers';
+const MAX_LEN = 10_000; // approx cap on stream length
 function consumerId() {
-  return `zeta:worker:${hostname()}:${process.pid}`;
+    return `zeta:worker:${hostname()}:${process.pid}`;
 }
 let ioredis;
 async function getIoredis() {
-  if (!ioredis) {
-    ioredis = await import("ioredis").then((m) => m.default ?? m);
-  }
-  return ioredis;
-}
-async function createRedisQueue() {
-  const redisUrl = process.env.REDIS_URL;
-  if (!redisUrl) throw new Error("REDIS_URL env var required for redis queue backend");
-  const Redis = await getIoredis();
-  const redis = new Redis(redisUrl, {
-    maxRetriesPerRequest: 3,
-    enableReadyCheck: true,
-    lazyConnect: false
-  });
-  await redis.xgroup("CREATE", STREAM_KEY, GROUP_NAME, "$", "MKSTREAM").catch((e) => {
-    if (!e.message?.includes("BUSYGROUP")) throw e;
-  });
-  const consumer = consumerId();
-  return {
-    async enqueue(jobId, tenantId, projectId, payload = {}) {
-      const entryId = await redis.xadd(
-        STREAM_KEY,
-        "MAXLEN",
-        "~",
-        MAX_LEN,
-        "*",
-        "jobId",
-        jobId,
-        "tenantId",
-        tenantId,
-        "projectId",
-        projectId,
-        "payload",
-        JSON.stringify(payload),
-        "enqueuedAt",
-        (/* @__PURE__ */ new Date()).toISOString()
-      );
-      return entryId;
-    },
-    async dequeue(timeoutMs = 5e3) {
-      const pending = await redis.xreadgroup(
-        "GROUP",
-        GROUP_NAME,
-        consumer,
-        "COUNT",
-        "1",
-        "STREAMS",
-        STREAM_KEY,
-        "0"
-      ) ?? [];
-      let entries = pending?.[0]?.[1] ?? [];
-      if (!entries.length) {
-        const fresh = await redis.xreadgroup(
-          "GROUP",
-          GROUP_NAME,
-          consumer,
-          "COUNT",
-          "1",
-          "BLOCK",
-          timeoutMs,
-          "STREAMS",
-          STREAM_KEY,
-          ">"
-        ) ?? [];
-        entries = fresh?.[0]?.[1] ?? [];
-      }
-      if (!entries.length) return null;
-      const [entryId, fields] = entries[0];
-      const obj = {};
-      for (let i = 0; i < fields.length; i += 2) {
-        obj[fields[i]] = fields[i + 1];
-      }
-      return {
-        id: entryId,
-        jobId: obj.jobId,
-        tenantId: obj.tenantId,
-        projectId: obj.projectId,
-        payload: JSON.parse(obj.payload ?? "{}"),
-        enqueuedAt: obj.enqueuedAt
-      };
-    },
-    async ack(streamEntryId) {
-      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
-    },
-    async nack(streamEntryId) {
-      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
-    },
-    async reclaimStale(maxIdleMs = 6e4) {
-      const result = await redis.xautoclaim(
-        STREAM_KEY,
-        GROUP_NAME,
-        consumer,
-        maxIdleMs,
-        "0-0",
-        "COUNT",
-        "10"
-      );
-      const entries = Array.isArray(result) ? result[1] ?? [] : [];
-      return entries.map(([entryId, fields]) => {
-        const obj = {};
-        for (let i = 0; i < fields.length; i += 2) obj[fields[i]] = fields[i + 1];
-        return {
-          id: entryId,
-          jobId: obj.jobId,
-          tenantId: obj.tenantId,
-          projectId: obj.projectId,
-          payload: JSON.parse(obj.payload ?? "{}"),
-          enqueuedAt: obj.enqueuedAt
-        };
-      });
-    },
-    async stats() {
-      const info = await redis.xinfo("GROUPS", STREAM_KEY);
-      const group = info.find((g) => {
-        const arr = Array.isArray(g) ? g : [];
-        for (let i = 0; i < arr.length; i += 2) {
-          if (arr[i] === "name" && arr[i + 1] === GROUP_NAME) return true;
-        }
-        return false;
-      });
-      if (!group) return { pending: 0, delivered: 0, groupLag: 0 };
-      const flat = Array.isArray(group) ? group : [];
-      const get = (key) => {
-        for (let i = 0; i < flat.length - 1; i += 2) {
-          if (flat[i] === key) return Number(flat[i + 1]) || 0;
-        }
-        return 0;
-      };
-      return {
-        pending: get("pel-count"),
-        delivered: get("delivered-count"),
-        groupLag: get("lag")
-      };
-    },
-    async close() {
-      await redis.quit().catch(() => {
-      });
+    if (!ioredis) {
+        ioredis = await import('ioredis').then(m => m.default ?? m);
     }
-  };
+    return ioredis;
+}
+/**
+ * Create a Redis Streams job queue client.
+ * Throws if REDIS_URL is not set.
+ */
+export async function createRedisQueue() {
+    const redisUrl = process.env.REDIS_URL;
+    if (!redisUrl)
+        throw new Error('REDIS_URL env var required for redis queue backend');
+    const Redis = await getIoredis();
+    const redis = new Redis(redisUrl, {
+        maxRetriesPerRequest: 3,
+        enableReadyCheck: true,
+        lazyConnect: false,
+    });
+    // Create stream + consumer group if they don't exist
+    await redis.xgroup('CREATE', STREAM_KEY, GROUP_NAME, '$', 'MKSTREAM').catch((e) => {
+        if (!e.message?.includes('BUSYGROUP'))
+            throw e;
+    });
+    const consumer = consumerId();
+    return {
+        async enqueue(jobId, tenantId, projectId, payload = {}) {
+            const entryId = await redis.xadd(STREAM_KEY, 'MAXLEN', '~', MAX_LEN, '*', 'jobId', jobId, 'tenantId', tenantId, 'projectId', projectId, 'payload', JSON.stringify(payload), 'enqueuedAt', new Date().toISOString());
+            return entryId;
+        },
+        async dequeue(timeoutMs = 5_000) {
+            // First drain pending (unacknowledged) messages for this consumer
+            const pending = await redis.xreadgroup('GROUP', GROUP_NAME, consumer, 'COUNT', '1', 'STREAMS', STREAM_KEY, '0') ?? [];
+            let entries = pending?.[0]?.[1] ?? [];
+            if (!entries.length) {
+                // Block-read for new messages
+                const fresh = await redis.xreadgroup('GROUP', GROUP_NAME, consumer, 'COUNT', '1', 'BLOCK', timeoutMs, 'STREAMS', STREAM_KEY, '>') ?? [];
+                entries = fresh?.[0]?.[1] ?? [];
+            }
+            if (!entries.length)
+                return null;
+            const [entryId, fields] = entries[0];
+            const obj = {};
+            for (let i = 0; i < fields.length; i += 2) {
+                obj[fields[i]] = fields[i + 1];
+            }
+            return {
+                id: entryId,
+                jobId: obj.jobId,
+                tenantId: obj.tenantId,
+                projectId: obj.projectId,
+                payload: JSON.parse(obj.payload ?? '{}'),
+                enqueuedAt: obj.enqueuedAt,
+            };
+        },
+        async ack(streamEntryId) {
+            await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+        },
+        async nack(streamEntryId) {
+            // Move back to '>'-readable by deleting and re-adding (simplest approach)
+            // In production, prefer a dead-letter stream instead.
+            await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+        },
+        async reclaimStale(maxIdleMs = 60_000) {
+            const result = await redis.xautoclaim(STREAM_KEY, GROUP_NAME, consumer, maxIdleMs, '0-0', 'COUNT', '10');
+            const entries = Array.isArray(result) ? (result[1] ?? []) : [];
+            return entries.map(([entryId, fields]) => {
+                const obj = {};
+                for (let i = 0; i < fields.length; i += 2)
+                    obj[fields[i]] = fields[i + 1];
+                return {
+                    id: entryId,
+                    jobId: obj.jobId,
+                    tenantId: obj.tenantId,
+                    projectId: obj.projectId,
+                    payload: JSON.parse(obj.payload ?? '{}'),
+                    enqueuedAt: obj.enqueuedAt,
+                };
+            });
+        },
+        async stats() {
+            const info = await redis.xinfo('GROUPS', STREAM_KEY);
+            // xinfo groups returns array of arrays: [name, ..., pel-count, ..., lag, ...]
+            const group = info.find((g) => {
+                const arr = Array.isArray(g) ? g : [];
+                for (let i = 0; i < arr.length; i += 2) {
+                    if (arr[i] === 'name' && arr[i + 1] === GROUP_NAME)
+                        return true;
+                }
+                return false;
+            });
+            if (!group)
+                return { pending: 0, delivered: 0, groupLag: 0 };
+            const flat = Array.isArray(group) ? group : [];
+            const get = (key) => {
+                for (let i = 0; i < flat.length - 1; i += 2) {
+                    if (flat[i] === key)
+                        return Number(flat[i + 1]) || 0;
+                }
+                return 0;
+            };
+            return {
+                pending: get('pel-count'),
+                delivered: get('delivered-count'),
+                groupLag: get('lag'),
+            };
+        },
+        async close() {
+            await redis.quit().catch(() => { });
+        },
+    };
 }
-function isRedisQueueEnabled() {
-  return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === "redis");
+/**
+ * Check if Redis queue backend is configured.
+ */
+export function isRedisQueueEnabled() {
+    return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === 'redis');
 }
-async function maybeCreateRedisQueue() {
-  if (!isRedisQueueEnabled()) return null;
-  return createRedisQueue().catch((e) => {
-    console.error("[redis-queue] Failed to connect:", e.message);
-    return null;
-  });
+/**
+ * Convenience: create queue only if enabled, else return null.
+ */
+export async function maybeCreateRedisQueue() {
+    if (!isRedisQueueEnabled())
+        return null;
+    return createRedisQueue().catch((e) => {
+        console.error('[redis-queue] Failed to connect:', e.message);
+        return null;
+    });
 }
-export {
-  createRedisQueue,
-  isRedisQueueEnabled,
-  maybeCreateRedisQueue
-};
-//# sourceMappingURL=redis-queue.js.map