@taodigital/crawler 0.0.0-stage → 1.2.0

package/dist/redirect-tracker.js

@@ -0,0 +1,65 @@
+async function trackRedirects(page, url) {
+  const hops = [];
+  const seen = /* @__PURE__ */ new Set();
+  let lastResponseTime = Date.now();
+  const onResponse = (response) => {
+    const status = response.status();
+    if (status >= 300 && status < 400) {
+      const from = response.url();
+      const to = response.headers()["location"] ?? "";
+      const now = Date.now();
+      hops.push({ from, to, statusCode: status, durationMs: now - lastResponseTime });
+      lastResponseTime = now;
+    }
+  };
+  page.on("response", onResponse);
+  try {
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch(() => {
+    });
+  } finally {
+    page.off("response", onResponse);
+  }
+  const finalUrl = page.url();
+  const isCircular = hops.some((h) => {
+    if (seen.has(h.to)) return true;
+    seen.add(h.from);
+    return false;
+  });
+  return {
+    originalUrl: url,
+    finalUrl,
+    hops,
+    isCircular,
+    hopCount: hops.length,
+    totalDurationMs: hops.reduce((sum, h) => sum + h.durationMs, 0)
+  };
+}
+class RedirectRegistry {
+  chains = /* @__PURE__ */ new Map();
+  register(chain) {
+    this.chains.set(chain.originalUrl, chain);
+  }
+  get(url) {
+    return this.chains.get(url);
+  }
+  getAll() {
+    return [...this.chains.values()];
+  }
+  getProblematic() {
+    return [...this.chains.values()].filter((c) => c.isCircular || c.hopCount > 3);
+  }
+  stats() {
+    const all = [...this.chains.values()];
+    return {
+      total: all.length,
+      withRedirects: all.filter((c) => c.hopCount > 0).length,
+      circular: all.filter((c) => c.isCircular).length,
+      deepChains: all.filter((c) => c.hopCount > 3).length
+    };
+  }
+}
+export {
+  RedirectRegistry,
+  trackRedirects
+};
+//# sourceMappingURL=redirect-tracker.js.map

package/dist/redirect-tracker.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/redirect-tracker.ts"],
+  "sourcesContent": ["/**\n * Redirect chain tracker \u2014 records the full redirect chain for crawled URLs.\n *\n * Why: redirect chains (A\u2192B\u2192C) slow down pages; circular redirects crash crawlers;\n * redirect loops indicate config errors. ZeTa records chains for SEO/perf audit.\n *\n * Integration: hook into Playwright's page.on('response') to see each redirect hop.\n */\n\nexport interface RedirectHop {\n  from: string;\n  to: string;\n  statusCode: number;\n  durationMs: number;\n}\n\nexport interface RedirectChain {\n  originalUrl: string;\n  finalUrl: string;\n  hops: RedirectHop[];\n  isCircular: boolean;\n  hopCount: number;\n  totalDurationMs: number;\n}\n\nimport type { Page } from 'playwright';\n\nexport async function trackRedirects(page: Page, url: string): Promise<RedirectChain> {\n  const hops: RedirectHop[] = [];\n  const seen = new Set<string>();\n  let lastResponseTime = Date.now();\n\n  const onResponse = (response: any) => {\n    const status = response.status();\n    if (status >= 300 && status < 400) {\n      const from = response.url();\n      const to = response.headers()['location'] ?? '';\n      const now = Date.now();\n      hops.push({ from, to, statusCode: status, durationMs: now - lastResponseTime });\n      lastResponseTime = now;\n    }\n  };\n\n  page.on('response', onResponse);\n\n  try {\n    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});\n  } finally {\n    page.off('response', onResponse);\n  }\n\n  const finalUrl = page.url();\n  const isCircular = hops.some(h => {\n    if (seen.has(h.to)) return true;\n    seen.add(h.from);\n    return false;\n  });\n\n  return {\n    originalUrl: url,\n    finalUrl,\n    hops,\n    isCircular,\n    hopCount: hops.length,\n    totalDurationMs: hops.reduce((sum, h) => sum + h.durationMs, 0),\n  };\n}\n\nexport class RedirectRegistry {\n  private chains = new Map<string, RedirectChain>();\n\n  register(chain: RedirectChain): void {\n    this.chains.set(chain.originalUrl, chain);\n  }\n\n  get(url: string): RedirectChain | undefined {\n    return this.chains.get(url);\n  }\n\n  getAll(): RedirectChain[] {\n    return [...this.chains.values()];\n  }\n\n  getProblematic(): RedirectChain[] {\n    return [...this.chains.values()].filter(c => c.isCircular || c.hopCount > 3);\n  }\n\n  stats(): { total: number; withRedirects: number; circular: number; deepChains: number } {\n    const all = [...this.chains.values()];\n    return {\n      total: all.length,\n      withRedirects: all.filter(c => c.hopCount > 0).length,\n      circular: all.filter(c => c.isCircular).length,\n      deepChains: all.filter(c => c.hopCount > 3).length,\n    };\n  }\n}\n"],
+  "mappings": "AA2BA,eAAsB,eAAe,MAAY,KAAqC;AACpF,QAAM,OAAsB,CAAC;AAC7B,QAAM,OAAO,oBAAI,IAAY;AAC7B,MAAI,mBAAmB,KAAK,IAAI;AAEhC,QAAM,aAAa,CAAC,aAAkB;AACpC,UAAM,SAAS,SAAS,OAAO;AAC/B,QAAI,UAAU,OAAO,SAAS,KAAK;AACjC,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,KAAK,SAAS,QAAQ,EAAE,UAAU,KAAK;AAC7C,YAAM,MAAM,KAAK,IAAI;AACrB,WAAK,KAAK,EAAE,MAAM,IAAI,YAAY,QAAQ,YAAY,MAAM,iBAAiB,CAAC;AAC9E,yBAAmB;AAAA,IACrB;AAAA,EACF;AAEA,OAAK,GAAG,YAAY,UAAU;AAE9B,MAAI;AACF,UAAM,KAAK,KAAK,KAAK,EAAE,WAAW,oBAAoB,SAAS,IAAO,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzF,UAAE;AACA,SAAK,IAAI,YAAY,UAAU;AAAA,EACjC;AAEA,QAAM,WAAW,KAAK,IAAI;AAC1B,QAAM,aAAa,KAAK,KAAK,OAAK;AAChC,QAAI,KAAK,IAAI,EAAE,EAAE,EAAG,QAAO;AAC3B,SAAK,IAAI,EAAE,IAAI;AACf,WAAO;AAAA,EACT,CAAC;AAED,SAAO;AAAA,IACL,aAAa;AAAA,IACb;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,KAAK;AAAA,IACf,iBAAiB,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,YAAY,CAAC;AAAA,EAChE;AACF;AAEO,MAAM,iBAAiB;AAAA,EACpB,SAAS,oBAAI,IAA2B;AAAA,EAEhD,SAAS,OAA4B;AACnC,SAAK,OAAO,IAAI,MAAM,aAAa,KAAK;AAAA,EAC1C;AAAA,EAEA,IAAI,KAAwC;AAC1C,WAAO,KAAK,OAAO,IAAI,GAAG;AAAA,EAC5B;AAAA,EAEA,SAA0B;AACxB,WAAO,CAAC,GAAG,KAAK,OAAO,OAAO,CAAC;AAAA,EACjC;AAAA,EAEA,iBAAkC;AAChC,WAAO,CAAC,GAAG,KAAK,OAAO,OAAO,CAAC,EAAE,OAAO,OAAK,EAAE,cAAc,EAAE,WAAW,CAAC;AAAA,EAC7E;AAAA,EAEA,QAAwF;AACtF,UAAM,MAAM,CAAC,GAAG,KAAK,OAAO,OAAO,CAAC;AACpC,WAAO;AAAA,MACL,OAAO,IAAI;AAAA,MACX,eAAe,IAAI,OAAO,OAAK,EAAE,WAAW,CAAC,EAAE;AAAA,MAC/C,UAAU,IAAI,OAAO,OAAK,EAAE,UAAU,EAAE;AAAA,MACxC,YAAY,IAAI,OAAO,OAAK,EAAE,WAAW,CAAC,EAAE;AAAA,IAC9C;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/redis-frontier.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Redis-backed distributed crawl frontier.
+ *
+ * Replaces the in-memory CrawlFrontier for horizontally-scaled crawler deployments.
+ * Multiple crawler workers on different containers can share the same URL queue for
+ * a given job — enabling 10x crawl throughput on large enterprise apps.
+ *
+ * Redis key namespace:  zeta:frontier:{jobId}
+ *   LIST  zeta:frontier:{jobId}:queue   — BFS queue (LPUSH/RPOP)
+ *   SET   zeta:frontier:{jobId}:visited — deduplication set
+ *   EXPIRE                               — 24h TTL on all keys
+ *
+ * Graceful fallback: if REDIS_URL is not set, RedisCrawlFrontier silently falls
+ * back to in-memory arrays so local development works without Redis.
+ */
+export declare class RedisCrawlFrontier {
+    private readonly jobId;
+    private redis;
+    private queueKey;
+    private visitedKey;
+    private inMemoryQueue;
+    private inMemoryVisited;
+    private useRedis;
+    constructor(jobId: string);
+    init(): Promise<void>;
+    /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
+    push(url: string): Promise<boolean>;
+    /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
+    shift(): Promise<string | null>;
+    /** Check if a URL has already been visited/queued. */
+    hasVisited(url: string): Promise<boolean>;
+    /** Current queue length. */
+    length(): Promise<number>;
+    /** Total visited count. */
+    visitedCount(): Promise<number>;
+    /** Seed initial URLs (used at job start). */
+    seed(urls: string[]): Promise<void>;
+    /** Clean up all Redis keys for this frontier. Call at job completion. */
+    cleanup(): Promise<void>;
+    /** Stats for monitoring. */
+    stats(): Promise<{
+        backend: 'redis' | 'memory';
+        queued: number;
+        visited: number;
+    }>;
+}
+/** Check if Redis is configured for distributed frontier. */
+export declare function isRedisFrontierEnabled(): boolean;

package/dist/redis-frontier.js

@@ -0,0 +1,126 @@
+const FRONTIER_TTL = 86400;
+class RedisCrawlFrontier {
+  constructor(jobId) {
+    this.jobId = jobId;
+    this.queueKey = `zeta:frontier:${jobId}:queue`;
+    this.visitedKey = `zeta:frontier:${jobId}:visited`;
+    this.useRedis = !!process.env.REDIS_URL;
+  }
+  jobId;
+  redis;
+  queueKey;
+  visitedKey;
+  inMemoryQueue = [];
+  inMemoryVisited = /* @__PURE__ */ new Set();
+  useRedis;
+  async init() {
+    if (!this.useRedis) return;
+    try {
+      const Redis = await import("ioredis").then((m) => m.default ?? m);
+      this.redis = new Redis(process.env.REDIS_URL, {
+        maxRetriesPerRequest: 2,
+        enableReadyCheck: false,
+        lazyConnect: true
+      });
+      await this.redis.connect().catch(() => {
+        this.useRedis = false;
+        this.redis = null;
+      });
+    } catch {
+      this.useRedis = false;
+      this.redis = null;
+    }
+  }
+  /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
+  async push(url) {
+    if (this.useRedis && this.redis) {
+      try {
+        const isNew = await this.redis.sadd(this.visitedKey, url);
+        if (!isNew) return false;
+        await this.redis.lpush(this.queueKey, url);
+        await this.redis.expire(this.queueKey, FRONTIER_TTL);
+        await this.redis.expire(this.visitedKey, FRONTIER_TTL);
+        return true;
+      } catch {
+      }
+    }
+    if (this.inMemoryVisited.has(url)) return false;
+    this.inMemoryVisited.add(url);
+    this.inMemoryQueue.push(url);
+    return true;
+  }
+  /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
+  async shift() {
+    if (this.useRedis && this.redis) {
+      try {
+        const url = await this.redis.rpop(this.queueKey);
+        return url ?? null;
+      } catch {
+      }
+    }
+    return this.inMemoryQueue.shift() ?? null;
+  }
+  /** Check if a URL has already been visited/queued. */
+  async hasVisited(url) {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.sismember(this.visitedKey, url) === 1;
+      } catch {
+      }
+    }
+    return this.inMemoryVisited.has(url);
+  }
+  /** Current queue length. */
+  async length() {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.llen(this.queueKey);
+      } catch {
+      }
+    }
+    return this.inMemoryQueue.length;
+  }
+  /** Total visited count. */
+  async visitedCount() {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.scard(this.visitedKey);
+      } catch {
+      }
+    }
+    return this.inMemoryVisited.size;
+  }
+  /** Seed initial URLs (used at job start). */
+  async seed(urls) {
+    for (const url of urls) await this.push(url);
+  }
+  /** Clean up all Redis keys for this frontier. Call at job completion. */
+  async cleanup() {
+    if (this.useRedis && this.redis) {
+      try {
+        await this.redis.del(this.queueKey, this.visitedKey);
+      } catch {
+      }
+      try {
+        await this.redis.quit();
+      } catch {
+      }
+    }
+  }
+  /** Stats for monitoring. */
+  async stats() {
+    return {
+      backend: this.useRedis && this.redis ? "redis" : "memory",
+      queued: await this.length(),
+      visited: await this.visitedCount()
+    };
+  }
+}
+function isRedisFrontierEnabled() {
+  return !!process.env.REDIS_URL;
+}
+export {
+  RedisCrawlFrontier,
+  isRedisFrontierEnabled
+};
+//# sourceMappingURL=redis-frontier.js.map

package/dist/redis-frontier.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/redis-frontier.ts"],
+  "sourcesContent": ["/**\n * Redis-backed distributed crawl frontier.\n *\n * Replaces the in-memory CrawlFrontier for horizontally-scaled crawler deployments.\n * Multiple crawler workers on different containers can share the same URL queue for\n * a given job \u2014 enabling 10x crawl throughput on large enterprise apps.\n *\n * Redis key namespace:  zeta:frontier:{jobId}\n *   LIST  zeta:frontier:{jobId}:queue   \u2014 BFS queue (LPUSH/RPOP)\n *   SET   zeta:frontier:{jobId}:visited \u2014 deduplication set\n *   EXPIRE                               \u2014 24h TTL on all keys\n *\n * Graceful fallback: if REDIS_URL is not set, RedisCrawlFrontier silently falls\n * back to in-memory arrays so local development works without Redis.\n */\n\nconst FRONTIER_TTL = 86400; // 24h in seconds\n\nexport class RedisCrawlFrontier {\n  private redis: any;\n  private queueKey: string;\n  private visitedKey: string;\n  private inMemoryQueue: string[] = [];\n  private inMemoryVisited = new Set<string>();\n  private useRedis: boolean;\n\n  constructor(private readonly jobId: string) {\n    this.queueKey = `zeta:frontier:${jobId}:queue`;\n    this.visitedKey = `zeta:frontier:${jobId}:visited`;\n    this.useRedis = !!process.env.REDIS_URL;\n  }\n\n  async init(): Promise<void> {\n    if (!this.useRedis) return;\n    try {\n      const Redis: any = await import('ioredis').then(m => m.default ?? m);\n      this.redis = new Redis(process.env.REDIS_URL!, {\n        maxRetriesPerRequest: 2,\n        enableReadyCheck: false,\n        lazyConnect: true,\n      });\n      await this.redis.connect().catch(() => {\n        this.useRedis = false;\n        this.redis = null;\n      });\n    } catch {\n      this.useRedis = false;\n      this.redis = null;\n    }\n  }\n\n  /** Add a URL to the frontier. No-op if already visited. Returns true if added. */\n  async push(url: string): Promise<boolean> {\n    if (this.useRedis && this.redis) {\n      try {\n        const isNew = await this.redis.sadd(this.visitedKey, url);\n        if (!isNew) return false;\n        await this.redis.lpush(this.queueKey, url);\n        await this.redis.expire(this.queueKey, FRONTIER_TTL);\n        await this.redis.expire(this.visitedKey, FRONTIER_TTL);\n        return true;\n      } catch {\n        // Fall through to in-memory on Redis error\n      }\n    }\n    if (this.inMemoryVisited.has(url)) return false;\n    this.inMemoryVisited.add(url);\n    this.inMemoryQueue.push(url);\n    return true;\n  }\n\n  /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */\n  async shift(): Promise<string | null> {\n    if (this.useRedis && this.redis) {\n      try {\n        const url = await this.redis.rpop(this.queueKey);\n        return url ?? null;\n      } catch {\n        // Fall through\n      }\n    }\n    return this.inMemoryQueue.shift() ?? null;\n  }\n\n  /** Check if a URL has already been visited/queued. */\n  async hasVisited(url: string): Promise<boolean> {\n    if (this.useRedis && this.redis) {\n      try {\n        return (await this.redis.sismember(this.visitedKey, url)) === 1;\n      } catch { /* fall through */ }\n    }\n    return this.inMemoryVisited.has(url);\n  }\n\n  /** Current queue length. */\n  async length(): Promise<number> {\n    if (this.useRedis && this.redis) {\n      try {\n        return await this.redis.llen(this.queueKey);\n      } catch { /* fall through */ }\n    }\n    return this.inMemoryQueue.length;\n  }\n\n  /** Total visited count. */\n  async visitedCount(): Promise<number> {\n    if (this.useRedis && this.redis) {\n      try {\n        return await this.redis.scard(this.visitedKey);\n      } catch { /* fall through */ }\n    }\n    return this.inMemoryVisited.size;\n  }\n\n  /** Seed initial URLs (used at job start). */\n  async seed(urls: string[]): Promise<void> {\n    for (const url of urls) await this.push(url);\n  }\n\n  /** Clean up all Redis keys for this frontier. Call at job completion. */\n  async cleanup(): Promise<void> {\n    if (this.useRedis && this.redis) {\n      try {\n        await this.redis.del(this.queueKey, this.visitedKey);\n      } catch { /* ignore */ }\n      try { await this.redis.quit(); } catch { /* ignore */ }\n    }\n  }\n\n  /** Stats for monitoring. */\n  async stats(): Promise<{ backend: 'redis' | 'memory'; queued: number; visited: number }> {\n    return {\n      backend: this.useRedis && this.redis ? 'redis' : 'memory',\n      queued: await this.length(),\n      visited: await this.visitedCount(),\n    };\n  }\n}\n\n/** Check if Redis is configured for distributed frontier. */\nexport function isRedisFrontierEnabled(): boolean {\n  return !!process.env.REDIS_URL;\n}\n"],
+  "mappings": "AAgBA,MAAM,eAAe;AAEd,MAAM,mBAAmB;AAAA,EAQ9B,YAA6B,OAAe;AAAf;AAC3B,SAAK,WAAW,iBAAiB,KAAK;AACtC,SAAK,aAAa,iBAAiB,KAAK;AACxC,SAAK,WAAW,CAAC,CAAC,QAAQ,IAAI;AAAA,EAChC;AAAA,EAJ6B;AAAA,EAPrB;AAAA,EACA;AAAA,EACA;AAAA,EACA,gBAA0B,CAAC;AAAA,EAC3B,kBAAkB,oBAAI,IAAY;AAAA,EAClC;AAAA,EAQR,MAAM,OAAsB;AAC1B,QAAI,CAAC,KAAK,SAAU;AACpB,QAAI;AACF,YAAM,QAAa,MAAM,OAAO,SAAS,EAAE,KAAK,OAAK,EAAE,WAAW,CAAC;AACnE,WAAK,QAAQ,IAAI,MAAM,QAAQ,IAAI,WAAY;AAAA,QAC7C,sBAAsB;AAAA,QACtB,kBAAkB;AAAA,QAClB,aAAa;AAAA,MACf,CAAC;AACD,YAAM,KAAK,MAAM,QAAQ,EAAE,MAAM,MAAM;AACrC,aAAK,WAAW;AAChB,aAAK,QAAQ;AAAA,MACf,CAAC;AAAA,IACH,QAAQ;AACN,WAAK,WAAW;AAChB,WAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,KAAK,KAA+B;AACxC,QAAI,KAAK,YAAY,KAAK,OAAO;AAC/B,UAAI;AACF,cAAM,QAAQ,MAAM,KAAK,MAAM,KAAK,KAAK,YAAY,GAAG;AACxD,YAAI,CAAC,MAAO,QAAO;AACnB,cAAM,KAAK,MAAM,MAAM,KAAK,UAAU,GAAG;AACzC,cAAM,KAAK,MAAM,OAAO,KAAK,UAAU,YAAY;AACnD,cAAM,KAAK,MAAM,OAAO,KAAK,YAAY,YAAY;AACrD,eAAO;AAAA,MACT,QAAQ;AAAA,MAER;AAAA,IACF;AACA,QAAI,KAAK,gBAAgB,IAAI,GAAG,EAAG,QAAO;AAC1C,SAAK,gBAAgB,IAAI,GAAG;AAC5B,SAAK,cAAc,KAAK,GAAG;AAC3B,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAM,QAAgC;AACpC,QAAI,KAAK,YAAY,KAAK,OAAO;AAC/B,UAAI;AACF,cAAM,MAAM,MAAM,KAAK,MAAM,KAAK,KAAK,QAAQ;AAC/C,eAAO,OAAO;AAAA,MAChB,QAAQ;AAAA,MAER;AAAA,IACF;AACA,WAAO,KAAK,cAAc,MAAM,KAAK;AAAA,EACvC;AAAA;AAAA,EAGA,MAAM,WAAW,KAA+B;AAC9C,QAAI,KAAK,YAAY,KAAK,OAAO;AAC/B,UAAI;AACF,eAAQ,MAAM,KAAK,MAAM,UAAU,KAAK,YAAY,GAAG,MAAO;AAAA,MAChE,QAAQ;AAAA,MAAqB;AAAA,IAC/B;AACA,WAAO,KAAK,gBAAgB,IAAI,GAAG;AAAA,EACrC;AAAA;AAAA,EAGA,MAAM,SAA0B;AAC9B,QAAI,KAAK,YAAY,KAAK,OAAO;AAC/B,UAAI;AACF,eAAO,MAAM,KAAK,MAAM,KAAK,KAAK,QAAQ;AAAA,MAC5C,QAAQ;AAAA,MAAqB;AAAA,IAC/B;AACA,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA;AAAA,EAGA,MAAM,eAAgC;AACpC,QAAI,KAAK,YAAY,KAAK,OAAO;AAC/B,UAAI;AACF,eAAO,MAAM,KAAK,MAAM,MAAM,KAAK,UAAU;AAAA,MAC/C,QAAQ;AAAA,MAAqB;AAAA,IAC/B;AACA,WAAO,KAAK,gBAAgB;AAAA,EAC9B;AAAA;AAAA,EAGA,MAAM,KAAK,MAA+B;AACxC,eAAW,OAAO,KAAM,OAAM,KAAK,KAAK,GAAG;AAAA,EAC7C;AAAA;AAAA,EAGA,MAAM,UAAyB;AAC7B,QAAI,KAAK,YAAY,KAAK,OAAO;AAC/B,UAAI;AACF,cAAM,KAAK,MAAM,IAAI,KAAK,UAAU,KAAK,UAAU;AAAA,MACrD,QAAQ;AAAA,MAAe;AACvB,UAAI;AAAE,cAAM,KAAK,MAAM,KAAK;AAAA,MAAG,QAAQ;AAAA,MAAe;AAAA,IACxD;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,QAAmF;AACvF,WAAO;AAAA,MACL,SAAS,KAAK,YAAY,KAAK,QAAQ,UAAU;AAAA,MACjD,QAAQ,MAAM,KAAK,OAAO;AAAA,MAC1B,SAAS,MAAM,KAAK,aAAa;AAAA,IACnC;AAAA,EACF;AACF;AAGO,SAAS,yBAAkC;AAChD,SAAO,CAAC,CAAC,QAAQ,IAAI;AACvB;",
+  "names": []
+}

package/dist/redis-queue.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Redis Streams distributed job queue.
+ *
+ * Provides horizontal crawler scaling via Redis Streams consumer groups.
+ * Multiple crawler nodes can pull from the same stream — each job is
+ * delivered to exactly one consumer (at-least-once via ACK/PENDING).
+ *
+ * Used when REDIS_URL is set AND JOB_QUEUE_BACKEND=redis.
+ * Falls back to the Postgres-backed queue when Redis is unavailable.
+ *
+ * Stream key:   zeta:crawl:stream
+ * Group name:   zeta:crawl:workers
+ * Consumer id:  zeta:worker:<hostname>:<pid>
+ */
+export interface RedisQueueJob {
+    id: string;
+    jobId: string;
+    tenantId: string;
+    projectId: string;
+    payload: Record<string, unknown>;
+    enqueuedAt: string;
+}
+export interface RedisQueueClient {
+    enqueue(jobId: string, tenantId: string, projectId: string, payload?: Record<string, unknown>): Promise<string>;
+    dequeue(timeoutMs?: number): Promise<RedisQueueJob | null>;
+    ack(streamEntryId: string): Promise<void>;
+    nack(streamEntryId: string): Promise<void>;
+    reclaimStale(maxIdleMs?: number): Promise<RedisQueueJob[]>;
+    stats(): Promise<{
+        pending: number;
+        delivered: number;
+        groupLag: number;
+    }>;
+    close(): Promise<void>;
+}
+/**
+ * Create a Redis Streams job queue client.
+ * Throws if REDIS_URL is not set.
+ */
+export declare function createRedisQueue(): Promise<RedisQueueClient>;
+/**
+ * Check if Redis queue backend is configured.
+ */
+export declare function isRedisQueueEnabled(): boolean;
+/**
+ * Convenience: create queue only if enabled, else return null.
+ */
+export declare function maybeCreateRedisQueue(): Promise<RedisQueueClient | null>;

package/dist/redis-queue.js

@@ -0,0 +1,165 @@
+import { hostname } from "node:os";
+const STREAM_KEY = "zeta:crawl:stream";
+const GROUP_NAME = "zeta:crawl:workers";
+const MAX_LEN = 1e4;
+function consumerId() {
+  return `zeta:worker:${hostname()}:${process.pid}`;
+}
+let ioredis;
+async function getIoredis() {
+  if (!ioredis) {
+    ioredis = await import("ioredis").then((m) => m.default ?? m);
+  }
+  return ioredis;
+}
+async function createRedisQueue() {
+  const redisUrl = process.env.REDIS_URL;
+  if (!redisUrl) throw new Error("REDIS_URL env var required for redis queue backend");
+  const Redis = await getIoredis();
+  const redis = new Redis(redisUrl, {
+    maxRetriesPerRequest: 3,
+    enableReadyCheck: true,
+    lazyConnect: false
+  });
+  await redis.xgroup("CREATE", STREAM_KEY, GROUP_NAME, "$", "MKSTREAM").catch((e) => {
+    if (!e.message?.includes("BUSYGROUP")) throw e;
+  });
+  const consumer = consumerId();
+  return {
+    async enqueue(jobId, tenantId, projectId, payload = {}) {
+      const entryId = await redis.xadd(
+        STREAM_KEY,
+        "MAXLEN",
+        "~",
+        MAX_LEN,
+        "*",
+        "jobId",
+        jobId,
+        "tenantId",
+        tenantId,
+        "projectId",
+        projectId,
+        "payload",
+        JSON.stringify(payload),
+        "enqueuedAt",
+        (/* @__PURE__ */ new Date()).toISOString()
+      );
+      return entryId;
+    },
+    async dequeue(timeoutMs = 5e3) {
+      const pending = await redis.xreadgroup(
+        "GROUP",
+        GROUP_NAME,
+        consumer,
+        "COUNT",
+        "1",
+        "STREAMS",
+        STREAM_KEY,
+        "0"
+      ) ?? [];
+      let entries = pending?.[0]?.[1] ?? [];
+      if (!entries.length) {
+        const fresh = await redis.xreadgroup(
+          "GROUP",
+          GROUP_NAME,
+          consumer,
+          "COUNT",
+          "1",
+          "BLOCK",
+          timeoutMs,
+          "STREAMS",
+          STREAM_KEY,
+          ">"
+        ) ?? [];
+        entries = fresh?.[0]?.[1] ?? [];
+      }
+      if (!entries.length) return null;
+      const [entryId, fields] = entries[0];
+      const obj = {};
+      for (let i = 0; i < fields.length; i += 2) {
+        obj[fields[i]] = fields[i + 1];
+      }
+      return {
+        id: entryId,
+        jobId: obj.jobId,
+        tenantId: obj.tenantId,
+        projectId: obj.projectId,
+        payload: JSON.parse(obj.payload ?? "{}"),
+        enqueuedAt: obj.enqueuedAt
+      };
+    },
+    async ack(streamEntryId) {
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+    async nack(streamEntryId) {
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+    async reclaimStale(maxIdleMs = 6e4) {
+      const result = await redis.xautoclaim(
+        STREAM_KEY,
+        GROUP_NAME,
+        consumer,
+        maxIdleMs,
+        "0-0",
+        "COUNT",
+        "10"
+      );
+      const entries = Array.isArray(result) ? result[1] ?? [] : [];
+      return entries.map(([entryId, fields]) => {
+        const obj = {};
+        for (let i = 0; i < fields.length; i += 2) obj[fields[i]] = fields[i + 1];
+        return {
+          id: entryId,
+          jobId: obj.jobId,
+          tenantId: obj.tenantId,
+          projectId: obj.projectId,
+          payload: JSON.parse(obj.payload ?? "{}"),
+          enqueuedAt: obj.enqueuedAt
+        };
+      });
+    },
+    async stats() {
+      const info = await redis.xinfo("GROUPS", STREAM_KEY);
+      const group = info.find((g) => {
+        const arr = Array.isArray(g) ? g : [];
+        for (let i = 0; i < arr.length; i += 2) {
+          if (arr[i] === "name" && arr[i + 1] === GROUP_NAME) return true;
+        }
+        return false;
+      });
+      if (!group) return { pending: 0, delivered: 0, groupLag: 0 };
+      const flat = Array.isArray(group) ? group : [];
+      const get = (key) => {
+        for (let i = 0; i < flat.length - 1; i += 2) {
+          if (flat[i] === key) return Number(flat[i + 1]) || 0;
+        }
+        return 0;
+      };
+      return {
+        pending: get("pel-count"),
+        delivered: get("delivered-count"),
+        groupLag: get("lag")
+      };
+    },
+    async close() {
+      await redis.quit().catch(() => {
+      });
+    }
+  };
+}
+function isRedisQueueEnabled() {
+  return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === "redis");
+}
+async function maybeCreateRedisQueue() {
+  if (!isRedisQueueEnabled()) return null;
+  return createRedisQueue().catch((e) => {
+    console.error("[redis-queue] Failed to connect:", e.message);
+    return null;
+  });
+}
+export {
+  createRedisQueue,
+  isRedisQueueEnabled,
+  maybeCreateRedisQueue
+};
+//# sourceMappingURL=redis-queue.js.map

package/dist/redis-queue.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/redis-queue.ts"],
+  "sourcesContent": ["/**\n * Redis Streams distributed job queue.\n *\n * Provides horizontal crawler scaling via Redis Streams consumer groups.\n * Multiple crawler nodes can pull from the same stream \u2014 each job is\n * delivered to exactly one consumer (at-least-once via ACK/PENDING).\n *\n * Used when REDIS_URL is set AND JOB_QUEUE_BACKEND=redis.\n * Falls back to the Postgres-backed queue when Redis is unavailable.\n *\n * Stream key:   zeta:crawl:stream\n * Group name:   zeta:crawl:workers\n * Consumer id:  zeta:worker:<hostname>:<pid>\n */\n\nimport { createHash } from 'node:crypto';\nimport { hostname } from 'node:os';\n\nexport interface RedisQueueJob {\n  id: string;        // Redis stream entry ID (e.g. \"1720001234567-0\")\n  jobId: string;     // Postgres job row ID\n  tenantId: string;\n  projectId: string;\n  payload: Record<string, unknown>;\n  enqueuedAt: string;\n}\n\nexport interface RedisQueueClient {\n  enqueue(jobId: string, tenantId: string, projectId: string, payload?: Record<string, unknown>): Promise<string>;\n  dequeue(timeoutMs?: number): Promise<RedisQueueJob | null>;\n  ack(streamEntryId: string): Promise<void>;\n  nack(streamEntryId: string): Promise<void>;\n  reclaimStale(maxIdleMs?: number): Promise<RedisQueueJob[]>;\n  stats(): Promise<{ pending: number; delivered: number; groupLag: number }>;\n  close(): Promise<void>;\n}\n\nconst STREAM_KEY = 'zeta:crawl:stream';\nconst GROUP_NAME = 'zeta:crawl:workers';\nconst MAX_LEN = 10_000; // approx cap on stream length\n\nfunction consumerId(): string {\n  return `zeta:worker:${hostname()}:${process.pid}`;\n}\n\nlet ioredis: any;\nasync function getIoredis() {\n  if (!ioredis) {\n    ioredis = await import('ioredis').then(m => m.default ?? m);\n  }\n  return ioredis;\n}\n\n/**\n * Create a Redis Streams job queue client.\n * Throws if REDIS_URL is not set.\n */\nexport async function createRedisQueue(): Promise<RedisQueueClient> {\n  const redisUrl = process.env.REDIS_URL;\n  if (!redisUrl) throw new Error('REDIS_URL env var required for redis queue backend');\n\n  const Redis = await getIoredis();\n  const redis = new Redis(redisUrl, {\n    maxRetriesPerRequest: 3,\n    enableReadyCheck: true,\n    lazyConnect: false,\n  });\n\n  // Create stream + consumer group if they don't exist\n  await redis.xgroup('CREATE', STREAM_KEY, GROUP_NAME, '$', 'MKSTREAM').catch((e: any) => {\n    if (!e.message?.includes('BUSYGROUP')) throw e;\n  });\n\n  const consumer = consumerId();\n\n  return {\n    async enqueue(jobId, tenantId, projectId, payload = {}): Promise<string> {\n      const entryId = await redis.xadd(\n        STREAM_KEY, 'MAXLEN', '~', MAX_LEN, '*',\n        'jobId', jobId,\n        'tenantId', tenantId,\n        'projectId', projectId,\n        'payload', JSON.stringify(payload),\n        'enqueuedAt', new Date().toISOString(),\n      );\n      return entryId as string;\n    },\n\n    async dequeue(timeoutMs = 5_000): Promise<RedisQueueJob | null> {\n      // First drain pending (unacknowledged) messages for this consumer\n      const pending: any[][] = await redis.xreadgroup(\n        'GROUP', GROUP_NAME, consumer,\n        'COUNT', '1',\n        'STREAMS', STREAM_KEY, '0',\n      ) ?? [];\n\n      let entries = pending?.[0]?.[1] ?? [];\n\n      if (!entries.length) {\n        // Block-read for new messages\n        const fresh: any[][] = await redis.xreadgroup(\n          'GROUP', GROUP_NAME, consumer,\n          'COUNT', '1',\n          'BLOCK', timeoutMs,\n          'STREAMS', STREAM_KEY, '>',\n        ) ?? [];\n        entries = fresh?.[0]?.[1] ?? [];\n      }\n\n      if (!entries.length) return null;\n\n      const [entryId, fields] = entries[0] as [string, string[]];\n      const obj: Record<string, string> = {};\n      for (let i = 0; i < fields.length; i += 2) {\n        obj[fields[i]] = fields[i + 1];\n      }\n\n      return {\n        id: entryId,\n        jobId: obj.jobId,\n        tenantId: obj.tenantId,\n        projectId: obj.projectId,\n        payload: JSON.parse(obj.payload ?? '{}'),\n        enqueuedAt: obj.enqueuedAt,\n      };\n    },\n\n    async ack(streamEntryId: string): Promise<void> {\n      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);\n    },\n\n    async nack(streamEntryId: string): Promise<void> {\n      // Move back to '>'-readable by deleting and re-adding (simplest approach)\n      // In production, prefer a dead-letter stream instead.\n      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);\n    },\n\n    async reclaimStale(maxIdleMs = 60_000): Promise<RedisQueueJob[]> {\n      const result: any = await redis.xautoclaim(\n        STREAM_KEY, GROUP_NAME, consumer,\n        maxIdleMs, '0-0', 'COUNT', '10',\n      );\n      const entries: [string, string[]][] = Array.isArray(result) ? (result[1] ?? []) : [];\n      return entries.map(([entryId, fields]) => {\n        const obj: Record<string, string> = {};\n        for (let i = 0; i < fields.length; i += 2) obj[fields[i]] = fields[i + 1];\n        return {\n          id: entryId,\n          jobId: obj.jobId,\n          tenantId: obj.tenantId,\n          projectId: obj.projectId,\n          payload: JSON.parse(obj.payload ?? '{}'),\n          enqueuedAt: obj.enqueuedAt,\n        };\n      });\n    },\n\n    async stats(): Promise<{ pending: number; delivered: number; groupLag: number }> {\n      const info: any[] = await redis.xinfo('GROUPS', STREAM_KEY);\n      // xinfo groups returns array of arrays: [name, ..., pel-count, ..., lag, ...]\n      const group = info.find((g: any) => {\n        const arr = Array.isArray(g) ? g : [];\n        for (let i = 0; i < arr.length; i += 2) {\n          if (arr[i] === 'name' && arr[i + 1] === GROUP_NAME) return true;\n        }\n        return false;\n      });\n\n      if (!group) return { pending: 0, delivered: 0, groupLag: 0 };\n\n      const flat = Array.isArray(group) ? group : [];\n      const get = (key: string) => {\n        for (let i = 0; i < flat.length - 1; i += 2) {\n          if (flat[i] === key) return Number(flat[i + 1]) || 0;\n        }\n        return 0;\n      };\n\n      return {\n        pending: get('pel-count'),\n        delivered: get('delivered-count'),\n        groupLag: get('lag'),\n      };\n    },\n\n    async close(): Promise<void> {\n      await redis.quit().catch(() => {});\n    },\n  };\n}\n\n/**\n * Check if Redis queue backend is configured.\n */\nexport function isRedisQueueEnabled(): boolean {\n  return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === 'redis');\n}\n\n/**\n * Convenience: create queue only if enabled, else return null.\n */\nexport async function maybeCreateRedisQueue(): Promise<RedisQueueClient | null> {\n  if (!isRedisQueueEnabled()) return null;\n  return createRedisQueue().catch((e) => {\n    console.error('[redis-queue] Failed to connect:', e.message);\n    return null;\n  });\n}\n"],
+  "mappings": "AAgBA,SAAS,gBAAgB;AAqBzB,MAAM,aAAa;AACnB,MAAM,aAAa;AACnB,MAAM,UAAU;AAEhB,SAAS,aAAqB;AAC5B,SAAO,eAAe,SAAS,CAAC,IAAI,QAAQ,GAAG;AACjD;AAEA,IAAI;AACJ,eAAe,aAAa;AAC1B,MAAI,CAAC,SAAS;AACZ,cAAU,MAAM,OAAO,SAAS,EAAE,KAAK,OAAK,EAAE,WAAW,CAAC;AAAA,EAC5D;AACA,SAAO;AACT;AAMA,eAAsB,mBAA8C;AAClE,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,oDAAoD;AAEnF,QAAM,QAAQ,MAAM,WAAW;AAC/B,QAAM,QAAQ,IAAI,MAAM,UAAU;AAAA,IAChC,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,aAAa;AAAA,EACf,CAAC;AAGD,QAAM,MAAM,OAAO,UAAU,YAAY,YAAY,KAAK,UAAU,EAAE,MAAM,CAAC,MAAW;AACtF,QAAI,CAAC,EAAE,SAAS,SAAS,WAAW,EAAG,OAAM;AAAA,EAC/C,CAAC;AAED,QAAM,WAAW,WAAW;AAE5B,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO,UAAU,WAAW,UAAU,CAAC,GAAoB;AACvE,YAAM,UAAU,MAAM,MAAM;AAAA,QAC1B;AAAA,QAAY;AAAA,QAAU;AAAA,QAAK;AAAA,QAAS;AAAA,QACpC;AAAA,QAAS;AAAA,QACT;AAAA,QAAY;AAAA,QACZ;AAAA,QAAa;AAAA,QACb;AAAA,QAAW,KAAK,UAAU,OAAO;AAAA,QACjC;AAAA,SAAc,oBAAI,KAAK,GAAE,YAAY;AAAA,MACvC;AACA,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,QAAQ,YAAY,KAAsC;AAE9D,YAAM,UAAmB,MAAM,MAAM;AAAA,QACnC;AAAA,QAAS;AAAA,QAAY;AAAA,QACrB;AAAA,QAAS;AAAA,QACT;AAAA,QAAW;AAAA,QAAY;AAAA,MACzB,KAAK,CAAC;AAEN,UAAI,UAAU,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;AAEpC,UAAI,CAAC,QAAQ,QAAQ;AAEnB,cAAM,QAAiB,MAAM,MAAM;AAAA,UACjC;AAAA,UAAS;AAAA,UAAY;AAAA,UACrB;AAAA,UAAS;AAAA,UACT;AAAA,UAAS;AAAA,UACT;AAAA,UAAW;AAAA,UAAY;AAAA,QACzB,KAAK,CAAC;AACN,kBAAU,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;AAAA,MAChC;AAEA,UAAI,CAAC,QAAQ,OAAQ,QAAO;AAE5B,YAAM,CAAC,SAAS,MAAM,IAAI,QAAQ,CAAC;AACnC,YAAM,MAA8B,CAAC;AACrC,eAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,YAAI,OAAO,CAAC,CAAC,IAAI,OAAO,IAAI,CAAC;AAAA,MAC/B;AAEA,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO,IAAI;AAAA,QACX,UAAU,IAAI;AAAA,QACd,WAAW,IAAI;AAAA,QACf,SAAS,KAAK,MAAM,IAAI,WAAW,IAAI;AAAA,QACvC,YAAY,IAAI;AAAA,MAClB;AAAA,IACF;AAAA,IAEA,MAAM,IAAI,eAAsC;AAC9C,YAAM,MAAM,KAAK,YAAY,YAAY,aAAa;AAAA,IACxD;AAAA,IAEA,MAAM,KAAK,eAAsC;AAG/C,YAAM,MAAM,KAAK,YAAY,YAAY,aAAa;AAAA,IACxD;AAAA,IAEA,MAAM,aAAa,YAAY,KAAkC;AAC/D,YAAM,SAAc,MAAM,MAAM;AAAA,QAC9B;AAAA,QAAY;AAAA,QAAY;AAAA,QACxB;AAAA,QAAW;AAAA,QAAO;AAAA,QAAS;AAAA,MAC7B;AACA,YAAM,UAAgC,MAAM,QAAQ,MAAM,IAAK,OAAO,CAAC,KAAK,CAAC,IAAK,CAAC;AACnF,aAAO,QAAQ,IAAI,CAAC,CAAC,SAAS,MAAM,MAAM;AACxC,cAAM,MAA8B,CAAC;AACrC,iBAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,EAAG,KAAI,OAAO,CAAC,CAAC,IAAI,OAAO,IAAI,CAAC;AACxE,eAAO;AAAA,UACL,IAAI;AAAA,UACJ,OAAO,IAAI;AAAA,UACX,UAAU,IAAI;AAAA,UACd,WAAW,IAAI;AAAA,UACf,SAAS,KAAK,MAAM,IAAI,WAAW,IAAI;AAAA,UACvC,YAAY,IAAI;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QAA2E;AAC/E,YAAM,OAAc,MAAM,MAAM,MAAM,UAAU,UAAU;AAE1D,YAAM,QAAQ,KAAK,KAAK,CAAC,MAAW;AAClC,cAAM,MAAM,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC;AACpC,iBAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK,GAAG;AACtC,cAAI,IAAI,CAAC,MAAM,UAAU,IAAI,IAAI,CAAC,MAAM,WAAY,QAAO;AAAA,QAC7D;AACA,eAAO;AAAA,MACT,CAAC;AAED,UAAI,CAAC,MAAO,QAAO,EAAE,SAAS,GAAG,WAAW,GAAG,UAAU,EAAE;AAE3D,YAAM,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC;AAC7C,YAAM,MAAM,CAAC,QAAgB;AAC3B,iBAAS,IAAI,GAAG,IAAI,KAAK,SAAS,GAAG,KAAK,GAAG;AAC3C,cAAI,KAAK,CAAC,MAAM,IAAK,QAAO,OAAO,KAAK,IAAI,CAAC,CAAC,KAAK;AAAA,QACrD;AACA,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,QACL,SAAS,IAAI,WAAW;AAAA,QACxB,WAAW,IAAI,iBAAiB;AAAA,QAChC,UAAU,IAAI,KAAK;AAAA,MACrB;AAAA,IACF;AAAA,IAEA,MAAM,QAAuB;AAC3B,YAAM,MAAM,KAAK,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACnC;AAAA,EACF;AACF;AAKO,SAAS,sBAA+B;AAC7C,SAAO,CAAC,EAAE,QAAQ,IAAI,aAAa,QAAQ,IAAI,sBAAsB;AACvE;AAKA,eAAsB,wBAA0D;AAC9E,MAAI,CAAC,oBAAoB,EAAG,QAAO;AACnC,SAAO,iBAAiB,EAAE,MAAM,CAAC,MAAM;AACrC,YAAQ,MAAM,oCAAoC,EAAE,OAAO;AAC3D,WAAO;AAAA,EACT,CAAC;AACH;",
+  "names": []
+}

package/dist/response-cache.d.ts

@@ -0,0 +1,38 @@
+export interface ResponseCacheEntry {
+    url: string;
+    etag?: string;
+    lastModified?: string;
+    cacheControl?: string;
+    body: string;
+    contentType: string;
+    statusCode: number;
+    cachedAt: number;
+    hitCount: number;
+}
+export declare class ResponseCache {
+    private readonly dir;
+    private totalHits;
+    private totalRequests;
+    constructor(cacheDir: string);
+    private key;
+    private path;
+    get(url: string): ResponseCacheEntry | null;
+    private read;
+    isFresh(url: string): boolean;
+    staleIfError(url: string): boolean;
+    set(url: string, entry: Omit<ResponseCacheEntry, 'cachedAt' | 'hitCount'>): void;
+    getValidationHeaders(url: string): {
+        'If-None-Match'?: string;
+        'If-Modified-Since'?: string;
+    };
+    markHit(url: string): void;
+    invalidate(url: string): void;
+    clear(): void;
+    stats(): {
+        entries: number;
+        totalSizeBytes: number;
+        hitRate: number;
+        totalHits: number;
+    };
+    prune(maxAgeMs: number): number;
+}