@detiq/crawler 1.1.9 → 1.1.11

package/dist/redis-frontier.js

@@ -1,146 +1,126 @@
-/**
- * Redis-backed distributed crawl frontier.
- *
- * Replaces the in-memory CrawlFrontier for horizontally-scaled crawler deployments.
- * Multiple crawler workers on different containers can share the same URL queue for
- * a given job — enabling 10x crawl throughput on large enterprise apps.
- *
- * Redis key namespace:  zeta:frontier:{jobId}
- *   LIST  zeta:frontier:{jobId}:queue   — BFS queue (LPUSH/RPOP)
- *   SET   zeta:frontier:{jobId}:visited — deduplication set
- *   EXPIRE                               — 24h TTL on all keys
- *
- * Graceful fallback: if REDIS_URL is not set, RedisCrawlFrontier silently falls
- * back to in-memory arrays so local development works without Redis.
- */
-const FRONTIER_TTL = 86400; // 24h in seconds
-export class RedisCrawlFrontier {
-    jobId;
-    redis;
-    queueKey;
-    visitedKey;
-    inMemoryQueue = [];
-    inMemoryVisited = new Set();
-    useRedis;
-    constructor(jobId) {
-        this.jobId = jobId;
-        this.queueKey = `zeta:frontier:${jobId}:queue`;
-        this.visitedKey = `zeta:frontier:${jobId}:visited`;
-        this.useRedis = !!process.env.REDIS_URL;
+const FRONTIER_TTL = 86400;
+class RedisCrawlFrontier {
+  constructor(jobId) {
+    this.jobId = jobId;
+    this.queueKey = `zeta:frontier:${jobId}:queue`;
+    this.visitedKey = `zeta:frontier:${jobId}:visited`;
+    this.useRedis = !!process.env.REDIS_URL;
+  }
+  jobId;
+  redis;
+  queueKey;
+  visitedKey;
+  inMemoryQueue = [];
+  inMemoryVisited = /* @__PURE__ */ new Set();
+  useRedis;
+  async init() {
+    if (!this.useRedis) return;
+    try {
+      const Redis = await import("ioredis").then((m) => m.default ?? m);
+      this.redis = new Redis(process.env.REDIS_URL, {
+        maxRetriesPerRequest: 2,
+        enableReadyCheck: false,
+        lazyConnect: true
+      });
+      await this.redis.connect().catch(() => {
+        this.useRedis = false;
+        this.redis = null;
+      });
+    } catch {
+      this.useRedis = false;
+      this.redis = null;
     }
-    async init() {
-        if (!this.useRedis)
-            return;
-        try {
-            const Redis = await import('ioredis').then(m => m.default ?? m);
-            this.redis = new Redis(process.env.REDIS_URL, {
-                maxRetriesPerRequest: 2,
-                enableReadyCheck: false,
-                lazyConnect: true,
-            });
-            await this.redis.connect().catch(() => {
-                this.useRedis = false;
-                this.redis = null;
-            });
-        }
-        catch {
-            this.useRedis = false;
-            this.redis = null;
-        }
-    }
-    /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
-    async push(url) {
-        if (this.useRedis && this.redis) {
-            try {
-                const isNew = await this.redis.sadd(this.visitedKey, url);
-                if (!isNew)
-                    return false;
-                await this.redis.lpush(this.queueKey, url);
-                await this.redis.expire(this.queueKey, FRONTIER_TTL);
-                await this.redis.expire(this.visitedKey, FRONTIER_TTL);
-                return true;
-            }
-            catch {
-                // Fall through to in-memory on Redis error
-            }
-        }
-        if (this.inMemoryVisited.has(url))
-            return false;
-        this.inMemoryVisited.add(url);
-        this.inMemoryQueue.push(url);
+  }
+  /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
+  async push(url) {
+    if (this.useRedis && this.redis) {
+      try {
+        const isNew = await this.redis.sadd(this.visitedKey, url);
+        if (!isNew) return false;
+        await this.redis.lpush(this.queueKey, url);
+        await this.redis.expire(this.queueKey, FRONTIER_TTL);
+        await this.redis.expire(this.visitedKey, FRONTIER_TTL);
         return true;
+      } catch {
+      }
     }
-    /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
-    async shift() {
-        if (this.useRedis && this.redis) {
-            try {
-                const url = await this.redis.rpop(this.queueKey);
-                return url ?? null;
-            }
-            catch {
-                // Fall through
-            }
-        }
-        return this.inMemoryQueue.shift() ?? null;
-    }
-    /** Check if a URL has already been visited/queued. */
-    async hasVisited(url) {
-        if (this.useRedis && this.redis) {
-            try {
-                return (await this.redis.sismember(this.visitedKey, url)) === 1;
-            }
-            catch { /* fall through */ }
-        }
-        return this.inMemoryVisited.has(url);
-    }
-    /** Current queue length. */
-    async length() {
-        if (this.useRedis && this.redis) {
-            try {
-                return await this.redis.llen(this.queueKey);
-            }
-            catch { /* fall through */ }
-        }
-        return this.inMemoryQueue.length;
+    if (this.inMemoryVisited.has(url)) return false;
+    this.inMemoryVisited.add(url);
+    this.inMemoryQueue.push(url);
+    return true;
+  }
+  /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
+  async shift() {
+    if (this.useRedis && this.redis) {
+      try {
+        const url = await this.redis.rpop(this.queueKey);
+        return url ?? null;
+      } catch {
+      }
     }
-    /** Total visited count. */
-    async visitedCount() {
-        if (this.useRedis && this.redis) {
-            try {
-                return await this.redis.scard(this.visitedKey);
-            }
-            catch { /* fall through */ }
-        }
-        return this.inMemoryVisited.size;
+    return this.inMemoryQueue.shift() ?? null;
+  }
+  /** Check if a URL has already been visited/queued. */
+  async hasVisited(url) {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.sismember(this.visitedKey, url) === 1;
+      } catch {
+      }
     }
-    /** Seed initial URLs (used at job start). */
-    async seed(urls) {
-        for (const url of urls)
-            await this.push(url);
+    return this.inMemoryVisited.has(url);
+  }
+  /** Current queue length. */
+  async length() {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.llen(this.queueKey);
+      } catch {
+      }
     }
-    /** Clean up all Redis keys for this frontier. Call at job completion. */
-    async cleanup() {
-        if (this.useRedis && this.redis) {
-            try {
-                await this.redis.del(this.queueKey, this.visitedKey);
-            }
-            catch { /* ignore */ }
-            try {
-                await this.redis.quit();
-            }
-            catch { /* ignore */ }
-        }
+    return this.inMemoryQueue.length;
+  }
+  /** Total visited count. */
+  async visitedCount() {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.scard(this.visitedKey);
+      } catch {
+      }
     }
-    /** Stats for monitoring. */
-    async stats() {
-        return {
-            backend: this.useRedis && this.redis ? 'redis' : 'memory',
-            queued: await this.length(),
-            visited: await this.visitedCount(),
-        };
+    return this.inMemoryVisited.size;
+  }
+  /** Seed initial URLs (used at job start). */
+  async seed(urls) {
+    for (const url of urls) await this.push(url);
+  }
+  /** Clean up all Redis keys for this frontier. Call at job completion. */
+  async cleanup() {
+    if (this.useRedis && this.redis) {
+      try {
+        await this.redis.del(this.queueKey, this.visitedKey);
+      } catch {
+      }
+      try {
+        await this.redis.quit();
+      } catch {
+      }
     }
+  }
+  /** Stats for monitoring. */
+  async stats() {
+    return {
+      backend: this.useRedis && this.redis ? "redis" : "memory",
+      queued: await this.length(),
+      visited: await this.visitedCount()
+    };
+  }
 }
-/** Check if Redis is configured for distributed frontier. */
-export function isRedisFrontierEnabled() {
-    return !!process.env.REDIS_URL;
+function isRedisFrontierEnabled() {
+  return !!process.env.REDIS_URL;
 }
+export {
+  RedisCrawlFrontier,
+  isRedisFrontierEnabled
+};
+//# sourceMappingURL=redis-frontier.js.map

package/dist/redis-queue.js

@@ -1,152 +1,165 @@
-/**
- * Redis Streams distributed job queue.
- *
- * Provides horizontal crawler scaling via Redis Streams consumer groups.
- * Multiple crawler nodes can pull from the same stream — each job is
- * delivered to exactly one consumer (at-least-once via ACK/PENDING).
- *
- * Used when REDIS_URL is set AND JOB_QUEUE_BACKEND=redis.
- * Falls back to the Postgres-backed queue when Redis is unavailable.
- *
- * Stream key:   zeta:crawl:stream
- * Group name:   zeta:crawl:workers
- * Consumer id:  zeta:worker:<hostname>:<pid>
- */
-import { hostname } from 'node:os';
-const STREAM_KEY = 'zeta:crawl:stream';
-const GROUP_NAME = 'zeta:crawl:workers';
-const MAX_LEN = 10_000; // approx cap on stream length
+import { hostname } from "node:os";
+const STREAM_KEY = "zeta:crawl:stream";
+const GROUP_NAME = "zeta:crawl:workers";
+const MAX_LEN = 1e4;
 function consumerId() {
-    return `zeta:worker:${hostname()}:${process.pid}`;
+  return `zeta:worker:${hostname()}:${process.pid}`;
 }
 let ioredis;
 async function getIoredis() {
-    if (!ioredis) {
-        ioredis = await import('ioredis').then(m => m.default ?? m);
-    }
-    return ioredis;
+  if (!ioredis) {
+    ioredis = await import("ioredis").then((m) => m.default ?? m);
+  }
+  return ioredis;
 }
-/**
- * Create a Redis Streams job queue client.
- * Throws if REDIS_URL is not set.
- */
-export async function createRedisQueue() {
-    const redisUrl = process.env.REDIS_URL;
-    if (!redisUrl)
-        throw new Error('REDIS_URL env var required for redis queue backend');
-    const Redis = await getIoredis();
-    const redis = new Redis(redisUrl, {
-        maxRetriesPerRequest: 3,
-        enableReadyCheck: true,
-        lazyConnect: false,
-    });
-    // Create stream + consumer group if they don't exist
-    await redis.xgroup('CREATE', STREAM_KEY, GROUP_NAME, '$', 'MKSTREAM').catch((e) => {
-        if (!e.message?.includes('BUSYGROUP'))
-            throw e;
-    });
-    const consumer = consumerId();
-    return {
-        async enqueue(jobId, tenantId, projectId, payload = {}) {
-            const entryId = await redis.xadd(STREAM_KEY, 'MAXLEN', '~', MAX_LEN, '*', 'jobId', jobId, 'tenantId', tenantId, 'projectId', projectId, 'payload', JSON.stringify(payload), 'enqueuedAt', new Date().toISOString());
-            return entryId;
-        },
-        async dequeue(timeoutMs = 5_000) {
-            // First drain pending (unacknowledged) messages for this consumer
-            const pending = await redis.xreadgroup('GROUP', GROUP_NAME, consumer, 'COUNT', '1', 'STREAMS', STREAM_KEY, '0') ?? [];
-            let entries = pending?.[0]?.[1] ?? [];
-            if (!entries.length) {
-                // Block-read for new messages
-                const fresh = await redis.xreadgroup('GROUP', GROUP_NAME, consumer, 'COUNT', '1', 'BLOCK', timeoutMs, 'STREAMS', STREAM_KEY, '>') ?? [];
-                entries = fresh?.[0]?.[1] ?? [];
-            }
-            if (!entries.length)
-                return null;
-            const [entryId, fields] = entries[0];
-            const obj = {};
-            for (let i = 0; i < fields.length; i += 2) {
-                obj[fields[i]] = fields[i + 1];
-            }
-            return {
-                id: entryId,
-                jobId: obj.jobId,
-                tenantId: obj.tenantId,
-                projectId: obj.projectId,
-                payload: JSON.parse(obj.payload ?? '{}'),
-                enqueuedAt: obj.enqueuedAt,
-            };
-        },
-        async ack(streamEntryId) {
-            await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
-        },
-        async nack(streamEntryId) {
-            // Move back to '>'-readable by deleting and re-adding (simplest approach)
-            // In production, prefer a dead-letter stream instead.
-            await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
-        },
-        async reclaimStale(maxIdleMs = 60_000) {
-            const result = await redis.xautoclaim(STREAM_KEY, GROUP_NAME, consumer, maxIdleMs, '0-0', 'COUNT', '10');
-            const entries = Array.isArray(result) ? (result[1] ?? []) : [];
-            return entries.map(([entryId, fields]) => {
-                const obj = {};
-                for (let i = 0; i < fields.length; i += 2)
-                    obj[fields[i]] = fields[i + 1];
-                return {
-                    id: entryId,
-                    jobId: obj.jobId,
-                    tenantId: obj.tenantId,
-                    projectId: obj.projectId,
-                    payload: JSON.parse(obj.payload ?? '{}'),
-                    enqueuedAt: obj.enqueuedAt,
-                };
-            });
-        },
-        async stats() {
-            const info = await redis.xinfo('GROUPS', STREAM_KEY);
-            // xinfo groups returns array of arrays: [name, ..., pel-count, ..., lag, ...]
-            const group = info.find((g) => {
-                const arr = Array.isArray(g) ? g : [];
-                for (let i = 0; i < arr.length; i += 2) {
-                    if (arr[i] === 'name' && arr[i + 1] === GROUP_NAME)
-                        return true;
-                }
-                return false;
-            });
-            if (!group)
-                return { pending: 0, delivered: 0, groupLag: 0 };
-            const flat = Array.isArray(group) ? group : [];
-            const get = (key) => {
-                for (let i = 0; i < flat.length - 1; i += 2) {
-                    if (flat[i] === key)
-                        return Number(flat[i + 1]) || 0;
-                }
-                return 0;
-            };
-            return {
-                pending: get('pel-count'),
-                delivered: get('delivered-count'),
-                groupLag: get('lag'),
-            };
-        },
-        async close() {
-            await redis.quit().catch(() => { });
-        },
-    };
+async function createRedisQueue() {
+  const redisUrl = process.env.REDIS_URL;
+  if (!redisUrl) throw new Error("REDIS_URL env var required for redis queue backend");
+  const Redis = await getIoredis();
+  const redis = new Redis(redisUrl, {
+    maxRetriesPerRequest: 3,
+    enableReadyCheck: true,
+    lazyConnect: false
+  });
+  await redis.xgroup("CREATE", STREAM_KEY, GROUP_NAME, "$", "MKSTREAM").catch((e) => {
+    if (!e.message?.includes("BUSYGROUP")) throw e;
+  });
+  const consumer = consumerId();
+  return {
+    async enqueue(jobId, tenantId, projectId, payload = {}) {
+      const entryId = await redis.xadd(
+        STREAM_KEY,
+        "MAXLEN",
+        "~",
+        MAX_LEN,
+        "*",
+        "jobId",
+        jobId,
+        "tenantId",
+        tenantId,
+        "projectId",
+        projectId,
+        "payload",
+        JSON.stringify(payload),
+        "enqueuedAt",
+        (/* @__PURE__ */ new Date()).toISOString()
+      );
+      return entryId;
+    },
+    async dequeue(timeoutMs = 5e3) {
+      const pending = await redis.xreadgroup(
+        "GROUP",
+        GROUP_NAME,
+        consumer,
+        "COUNT",
+        "1",
+        "STREAMS",
+        STREAM_KEY,
+        "0"
+      ) ?? [];
+      let entries = pending?.[0]?.[1] ?? [];
+      if (!entries.length) {
+        const fresh = await redis.xreadgroup(
+          "GROUP",
+          GROUP_NAME,
+          consumer,
+          "COUNT",
+          "1",
+          "BLOCK",
+          timeoutMs,
+          "STREAMS",
+          STREAM_KEY,
+          ">"
+        ) ?? [];
+        entries = fresh?.[0]?.[1] ?? [];
+      }
+      if (!entries.length) return null;
+      const [entryId, fields] = entries[0];
+      const obj = {};
+      for (let i = 0; i < fields.length; i += 2) {
+        obj[fields[i]] = fields[i + 1];
+      }
+      return {
+        id: entryId,
+        jobId: obj.jobId,
+        tenantId: obj.tenantId,
+        projectId: obj.projectId,
+        payload: JSON.parse(obj.payload ?? "{}"),
+        enqueuedAt: obj.enqueuedAt
+      };
+    },
+    async ack(streamEntryId) {
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+    async nack(streamEntryId) {
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+    async reclaimStale(maxIdleMs = 6e4) {
+      const result = await redis.xautoclaim(
+        STREAM_KEY,
+        GROUP_NAME,
+        consumer,
+        maxIdleMs,
+        "0-0",
+        "COUNT",
+        "10"
+      );
+      const entries = Array.isArray(result) ? result[1] ?? [] : [];
+      return entries.map(([entryId, fields]) => {
+        const obj = {};
+        for (let i = 0; i < fields.length; i += 2) obj[fields[i]] = fields[i + 1];
+        return {
+          id: entryId,
+          jobId: obj.jobId,
+          tenantId: obj.tenantId,
+          projectId: obj.projectId,
+          payload: JSON.parse(obj.payload ?? "{}"),
+          enqueuedAt: obj.enqueuedAt
+        };
+      });
+    },
+    async stats() {
+      const info = await redis.xinfo("GROUPS", STREAM_KEY);
+      const group = info.find((g) => {
+        const arr = Array.isArray(g) ? g : [];
+        for (let i = 0; i < arr.length; i += 2) {
+          if (arr[i] === "name" && arr[i + 1] === GROUP_NAME) return true;
+        }
+        return false;
+      });
+      if (!group) return { pending: 0, delivered: 0, groupLag: 0 };
+      const flat = Array.isArray(group) ? group : [];
+      const get = (key) => {
+        for (let i = 0; i < flat.length - 1; i += 2) {
+          if (flat[i] === key) return Number(flat[i + 1]) || 0;
+        }
+        return 0;
+      };
+      return {
+        pending: get("pel-count"),
+        delivered: get("delivered-count"),
+        groupLag: get("lag")
+      };
+    },
+    async close() {
+      await redis.quit().catch(() => {
+      });
+    }
+  };
 }
-/**
- * Check if Redis queue backend is configured.
- */
-export function isRedisQueueEnabled() {
-    return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === 'redis');
+function isRedisQueueEnabled() {
+  return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === "redis");
 }
-/**
- * Convenience: create queue only if enabled, else return null.
- */
-export async function maybeCreateRedisQueue() {
-    if (!isRedisQueueEnabled())
-        return null;
-    return createRedisQueue().catch((e) => {
-        console.error('[redis-queue] Failed to connect:', e.message);
-        return null;
-    });
+async function maybeCreateRedisQueue() {
+  if (!isRedisQueueEnabled()) return null;
+  return createRedisQueue().catch((e) => {
+    console.error("[redis-queue] Failed to connect:", e.message);
+    return null;
+  });
 }
+export {
+  createRedisQueue,
+  isRedisQueueEnabled,
+  maybeCreateRedisQueue
+};
+//# sourceMappingURL=redis-queue.js.map