@detiq/crawler 0.1.3

package/dist/redis-queue.js

@@ -0,0 +1,165 @@
+import { hostname } from "node:os";
+const STREAM_KEY = "zeta:crawl:stream";
+const GROUP_NAME = "zeta:crawl:workers";
+const MAX_LEN = 1e4;
+function consumerId() {
+  return `zeta:worker:${hostname()}:${process.pid}`;
+}
+let ioredis;
+async function getIoredis() {
+  if (!ioredis) {
+    ioredis = await import("ioredis").then((m) => m.default ?? m);
+  }
+  return ioredis;
+}
+async function createRedisQueue() {
+  const redisUrl = process.env.REDIS_URL;
+  if (!redisUrl) throw new Error("REDIS_URL env var required for redis queue backend");
+  const Redis = await getIoredis();
+  const redis = new Redis(redisUrl, {
+    maxRetriesPerRequest: 3,
+    enableReadyCheck: true,
+    lazyConnect: false
+  });
+  await redis.xgroup("CREATE", STREAM_KEY, GROUP_NAME, "$", "MKSTREAM").catch((e) => {
+    if (!e.message?.includes("BUSYGROUP")) throw e;
+  });
+  const consumer = consumerId();
+  return {
+    async enqueue(jobId, tenantId, projectId, payload = {}) {
+      const entryId = await redis.xadd(
+        STREAM_KEY,
+        "MAXLEN",
+        "~",
+        MAX_LEN,
+        "*",
+        "jobId",
+        jobId,
+        "tenantId",
+        tenantId,
+        "projectId",
+        projectId,
+        "payload",
+        JSON.stringify(payload),
+        "enqueuedAt",
+        (/* @__PURE__ */ new Date()).toISOString()
+      );
+      return entryId;
+    },
+    async dequeue(timeoutMs = 5e3) {
+      const pending = await redis.xreadgroup(
+        "GROUP",
+        GROUP_NAME,
+        consumer,
+        "COUNT",
+        "1",
+        "STREAMS",
+        STREAM_KEY,
+        "0"
+      ) ?? [];
+      let entries = pending?.[0]?.[1] ?? [];
+      if (!entries.length) {
+        const fresh = await redis.xreadgroup(
+          "GROUP",
+          GROUP_NAME,
+          consumer,
+          "COUNT",
+          "1",
+          "BLOCK",
+          timeoutMs,
+          "STREAMS",
+          STREAM_KEY,
+          ">"
+        ) ?? [];
+        entries = fresh?.[0]?.[1] ?? [];
+      }
+      if (!entries.length) return null;
+      const [entryId, fields] = entries[0];
+      const obj = {};
+      for (let i = 0; i < fields.length; i += 2) {
+        obj[fields[i]] = fields[i + 1];
+      }
+      return {
+        id: entryId,
+        jobId: obj.jobId,
+        tenantId: obj.tenantId,
+        projectId: obj.projectId,
+        payload: JSON.parse(obj.payload ?? "{}"),
+        enqueuedAt: obj.enqueuedAt
+      };
+    },
+    async ack(streamEntryId) {
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+    async nack(streamEntryId) {
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+    async reclaimStale(maxIdleMs = 6e4) {
+      const result = await redis.xautoclaim(
+        STREAM_KEY,
+        GROUP_NAME,
+        consumer,
+        maxIdleMs,
+        "0-0",
+        "COUNT",
+        "10"
+      );
+      const entries = Array.isArray(result) ? result[1] ?? [] : [];
+      return entries.map(([entryId, fields]) => {
+        const obj = {};
+        for (let i = 0; i < fields.length; i += 2) obj[fields[i]] = fields[i + 1];
+        return {
+          id: entryId,
+          jobId: obj.jobId,
+          tenantId: obj.tenantId,
+          projectId: obj.projectId,
+          payload: JSON.parse(obj.payload ?? "{}"),
+          enqueuedAt: obj.enqueuedAt
+        };
+      });
+    },
+    async stats() {
+      const info = await redis.xinfo("GROUPS", STREAM_KEY);
+      const group = info.find((g) => {
+        const arr = Array.isArray(g) ? g : [];
+        for (let i = 0; i < arr.length; i += 2) {
+          if (arr[i] === "name" && arr[i + 1] === GROUP_NAME) return true;
+        }
+        return false;
+      });
+      if (!group) return { pending: 0, delivered: 0, groupLag: 0 };
+      const flat = Array.isArray(group) ? group : [];
+      const get = (key) => {
+        for (let i = 0; i < flat.length - 1; i += 2) {
+          if (flat[i] === key) return Number(flat[i + 1]) || 0;
+        }
+        return 0;
+      };
+      return {
+        pending: get("pel-count"),
+        delivered: get("delivered-count"),
+        groupLag: get("lag")
+      };
+    },
+    async close() {
+      await redis.quit().catch(() => {
+      });
+    }
+  };
+}
+function isRedisQueueEnabled() {
+  return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === "redis");
+}
+async function maybeCreateRedisQueue() {
+  if (!isRedisQueueEnabled()) return null;
+  return createRedisQueue().catch((e) => {
+    console.error("[redis-queue] Failed to connect:", e.message);
+    return null;
+  });
+}
+export {
+  createRedisQueue,
+  isRedisQueueEnabled,
+  maybeCreateRedisQueue
+};
+//# sourceMappingURL=redis-queue.js.map

package/dist/redis-queue.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/redis-queue.ts"],
+  "sourcesContent": ["/**\n * Redis Streams distributed job queue.\n *\n * Provides horizontal crawler scaling via Redis Streams consumer groups.\n * Multiple crawler nodes can pull from the same stream \u2014 each job is\n * delivered to exactly one consumer (at-least-once via ACK/PENDING).\n *\n * Used when REDIS_URL is set AND JOB_QUEUE_BACKEND=redis.\n * Falls back to the Postgres-backed queue when Redis is unavailable.\n *\n * Stream key:   zeta:crawl:stream\n * Group name:   zeta:crawl:workers\n * Consumer id:  zeta:worker:<hostname>:<pid>\n */\n\nimport { createHash } from 'node:crypto';\nimport { hostname } from 'node:os';\n\nexport interface RedisQueueJob {\n  id: string;        // Redis stream entry ID (e.g. \"1720001234567-0\")\n  jobId: string;     // Postgres job row ID\n  tenantId: string;\n  projectId: string;\n  payload: Record<string, unknown>;\n  enqueuedAt: string;\n}\n\nexport interface RedisQueueClient {\n  enqueue(jobId: string, tenantId: string, projectId: string, payload?: Record<string, unknown>): Promise<string>;\n  dequeue(timeoutMs?: number): Promise<RedisQueueJob | null>;\n  ack(streamEntryId: string): Promise<void>;\n  nack(streamEntryId: string): Promise<void>;\n  reclaimStale(maxIdleMs?: number): Promise<RedisQueueJob[]>;\n  stats(): Promise<{ pending: number; delivered: number; groupLag: number }>;\n  close(): Promise<void>;\n}\n\nconst STREAM_KEY = 'zeta:crawl:stream';\nconst GROUP_NAME = 'zeta:crawl:workers';\nconst MAX_LEN = 10_000; // approx cap on stream length\n\nfunction consumerId(): string {\n  return `zeta:worker:${hostname()}:${process.pid}`;\n}\n\nlet ioredis: any;\nasync function getIoredis() {\n  if (!ioredis) {\n    ioredis = await import('ioredis').then(m => m.default ?? m);\n  }\n  return ioredis;\n}\n\n/**\n * Create a Redis Streams job queue client.\n * Throws if REDIS_URL is not set.\n */\nexport async function createRedisQueue(): Promise<RedisQueueClient> {\n  const redisUrl = process.env.REDIS_URL;\n  if (!redisUrl) throw new Error('REDIS_URL env var required for redis queue backend');\n\n  const Redis = await getIoredis();\n  const redis = new Redis(redisUrl, {\n    maxRetriesPerRequest: 3,\n    enableReadyCheck: true,\n    lazyConnect: false,\n  });\n\n  // Create stream + consumer group if they don't exist\n  await redis.xgroup('CREATE', STREAM_KEY, GROUP_NAME, '$', 'MKSTREAM').catch((e: any) => {\n    if (!e.message?.includes('BUSYGROUP')) throw e;\n  });\n\n  const consumer = consumerId();\n\n  return {\n    async enqueue(jobId, tenantId, projectId, payload = {}): Promise<string> {\n      const entryId = await redis.xadd(\n        STREAM_KEY, 'MAXLEN', '~', MAX_LEN, '*',\n        'jobId', jobId,\n        'tenantId', tenantId,\n        'projectId', projectId,\n        'payload', JSON.stringify(payload),\n        'enqueuedAt', new Date().toISOString(),\n      );\n      return entryId as string;\n    },\n\n    async dequeue(timeoutMs = 5_000): Promise<RedisQueueJob | null> {\n      // First drain pending (unacknowledged) messages for this consumer\n      const pending: any[][] = await redis.xreadgroup(\n        'GROUP', GROUP_NAME, consumer,\n        'COUNT', '1',\n        'STREAMS', STREAM_KEY, '0',\n      ) ?? [];\n\n      let entries = pending?.[0]?.[1] ?? [];\n\n      if (!entries.length) {\n        // Block-read for new messages\n        const fresh: any[][] = await redis.xreadgroup(\n          'GROUP', GROUP_NAME, consumer,\n          'COUNT', '1',\n          'BLOCK', timeoutMs,\n          'STREAMS', STREAM_KEY, '>',\n        ) ?? [];\n        entries = fresh?.[0]?.[1] ?? [];\n      }\n\n      if (!entries.length) return null;\n\n      const [entryId, fields] = entries[0] as [string, string[]];\n      const obj: Record<string, string> = {};\n      for (let i = 0; i < fields.length; i += 2) {\n        obj[fields[i]] = fields[i + 1];\n      }\n\n      return {\n        id: entryId,\n        jobId: obj.jobId,\n        tenantId: obj.tenantId,\n        projectId: obj.projectId,\n        payload: JSON.parse(obj.payload ?? '{}'),\n        enqueuedAt: obj.enqueuedAt,\n      };\n    },\n\n    async ack(streamEntryId: string): Promise<void> {\n      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);\n    },\n\n    async nack(streamEntryId: string): Promise<void> {\n      // Move back to '>'-readable by deleting and re-adding (simplest approach)\n      // In production, prefer a dead-letter stream instead.\n      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);\n    },\n\n    async reclaimStale(maxIdleMs = 60_000): Promise<RedisQueueJob[]> {\n      const result: any = await redis.xautoclaim(\n        STREAM_KEY, GROUP_NAME, consumer,\n        maxIdleMs, '0-0', 'COUNT', '10',\n      );\n      const entries: [string, string[]][] = Array.isArray(result) ? (result[1] ?? []) : [];\n      return entries.map(([entryId, fields]) => {\n        const obj: Record<string, string> = {};\n        for (let i = 0; i < fields.length; i += 2) obj[fields[i]] = fields[i + 1];\n        return {\n          id: entryId,\n          jobId: obj.jobId,\n          tenantId: obj.tenantId,\n          projectId: obj.projectId,\n          payload: JSON.parse(obj.payload ?? '{}'),\n          enqueuedAt: obj.enqueuedAt,\n        };\n      });\n    },\n\n    async stats(): Promise<{ pending: number; delivered: number; groupLag: number }> {\n      const info: any[] = await redis.xinfo('GROUPS', STREAM_KEY);\n      // xinfo groups returns array of arrays: [name, ..., pel-count, ..., lag, ...]\n      const group = info.find((g: any) => {\n        const arr = Array.isArray(g) ? g : [];\n        for (let i = 0; i < arr.length; i += 2) {\n          if (arr[i] === 'name' && arr[i + 1] === GROUP_NAME) return true;\n        }\n        return false;\n      });\n\n      if (!group) return { pending: 0, delivered: 0, groupLag: 0 };\n\n      const flat = Array.isArray(group) ? group : [];\n      const get = (key: string) => {\n        for (let i = 0; i < flat.length - 1; i += 2) {\n          if (flat[i] === key) return Number(flat[i + 1]) || 0;\n        }\n        return 0;\n      };\n\n      return {\n        pending: get('pel-count'),\n        delivered: get('delivered-count'),\n        groupLag: get('lag'),\n      };\n    },\n\n    async close(): Promise<void> {\n      await redis.quit().catch(() => {});\n    },\n  };\n}\n\n/**\n * Check if Redis queue backend is configured.\n */\nexport function isRedisQueueEnabled(): boolean {\n  return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === 'redis');\n}\n\n/**\n * Convenience: create queue only if enabled, else return null.\n */\nexport async function maybeCreateRedisQueue(): Promise<RedisQueueClient | null> {\n  if (!isRedisQueueEnabled()) return null;\n  return createRedisQueue().catch((e) => {\n    console.error('[redis-queue] Failed to connect:', e.message);\n    return null;\n  });\n}\n"],
+  "mappings": "AAgBA,SAAS,gBAAgB;AAqBzB,MAAM,aAAa;AACnB,MAAM,aAAa;AACnB,MAAM,UAAU;AAEhB,SAAS,aAAqB;AAC5B,SAAO,eAAe,SAAS,CAAC,IAAI,QAAQ,GAAG;AACjD;AAEA,IAAI;AACJ,eAAe,aAAa;AAC1B,MAAI,CAAC,SAAS;AACZ,cAAU,MAAM,OAAO,SAAS,EAAE,KAAK,OAAK,EAAE,WAAW,CAAC;AAAA,EAC5D;AACA,SAAO;AACT;AAMA,eAAsB,mBAA8C;AAClE,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,oDAAoD;AAEnF,QAAM,QAAQ,MAAM,WAAW;AAC/B,QAAM,QAAQ,IAAI,MAAM,UAAU;AAAA,IAChC,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,aAAa;AAAA,EACf,CAAC;AAGD,QAAM,MAAM,OAAO,UAAU,YAAY,YAAY,KAAK,UAAU,EAAE,MAAM,CAAC,MAAW;AACtF,QAAI,CAAC,EAAE,SAAS,SAAS,WAAW,EAAG,OAAM;AAAA,EAC/C,CAAC;AAED,QAAM,WAAW,WAAW;AAE5B,SAAO;AAAA,IACL,MAAM,QAAQ,OAAO,UAAU,WAAW,UAAU,CAAC,GAAoB;AACvE,YAAM,UAAU,MAAM,MAAM;AAAA,QAC1B;AAAA,QAAY;AAAA,QAAU;AAAA,QAAK;AAAA,QAAS;AAAA,QACpC;AAAA,QAAS;AAAA,QACT;AAAA,QAAY;AAAA,QACZ;AAAA,QAAa;AAAA,QACb;AAAA,QAAW,KAAK,UAAU,OAAO;AAAA,QACjC;AAAA,SAAc,oBAAI,KAAK,GAAE,YAAY;AAAA,MACvC;AACA,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,QAAQ,YAAY,KAAsC;AAE9D,YAAM,UAAmB,MAAM,MAAM;AAAA,QACnC;AAAA,QAAS;AAAA,QAAY;AAAA,QACrB;AAAA,QAAS;AAAA,QACT;AAAA,QAAW;AAAA,QAAY;AAAA,MACzB,KAAK,CAAC;AAEN,UAAI,UAAU,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;AAEpC,UAAI,CAAC,QAAQ,QAAQ;AAEnB,cAAM,QAAiB,MAAM,MAAM;AAAA,UACjC;AAAA,UAAS;AAAA,UAAY;AAAA,UACrB;AAAA,UAAS;AAAA,UACT;AAAA,UAAS;AAAA,UACT;AAAA,UAAW;AAAA,UAAY;AAAA,QACzB,KAAK,CAAC;AACN,kBAAU,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;AAAA,MAChC;AAEA,UAAI,CAAC,QAAQ,OAAQ,QAAO;AAE5B,YAAM,CAAC,SAAS,MAAM,IAAI,QAAQ,CAAC;AACnC,YAAM,MAA8B,CAAC;AACrC,eAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,GAAG;AACzC,YAAI,OAAO,CAAC,CAAC,IAAI,OAAO,IAAI,CAAC;AAAA,MAC/B;AAEA,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO,IAAI;AAAA,QACX,UAAU,IAAI;AAAA,QACd,WAAW,IAAI;AAAA,QACf,SAAS,KAAK,MAAM,IAAI,WAAW,IAAI;AAAA,QACvC,YAAY,IAAI;AAAA,MAClB;AAAA,IACF;AAAA,IAEA,MAAM,IAAI,eAAsC;AAC9C,YAAM,MAAM,KAAK,YAAY,YAAY,aAAa;AAAA,IACxD;AAAA,IAEA,MAAM,KAAK,eAAsC;AAG/C,YAAM,MAAM,KAAK,YAAY,YAAY,aAAa;AAAA,IACxD;AAAA,IAEA,MAAM,aAAa,YAAY,KAAkC;AAC/D,YAAM,SAAc,MAAM,MAAM;AAAA,QAC9B;AAAA,QAAY;AAAA,QAAY;AAAA,QACxB;AAAA,QAAW;AAAA,QAAO;AAAA,QAAS;AAAA,MAC7B;AACA,YAAM,UAAgC,MAAM,QAAQ,MAAM,IAAK,OAAO,CAAC,KAAK,CAAC,IAAK,CAAC;AACnF,aAAO,QAAQ,IAAI,CAAC,CAAC,SAAS,MAAM,MAAM;AACxC,cAAM,MAA8B,CAAC;AACrC,iBAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,EAAG,KAAI,OAAO,CAAC,CAAC,IAAI,OAAO,IAAI,CAAC;AACxE,eAAO;AAAA,UACL,IAAI;AAAA,UACJ,OAAO,IAAI;AAAA,UACX,UAAU,IAAI;AAAA,UACd,WAAW,IAAI;AAAA,UACf,SAAS,KAAK,MAAM,IAAI,WAAW,IAAI;AAAA,UACvC,YAAY,IAAI;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QAA2E;AAC/E,YAAM,OAAc,MAAM,MAAM,MAAM,UAAU,UAAU;AAE1D,YAAM,QAAQ,KAAK,KAAK,CAAC,MAAW;AAClC,cAAM,MAAM,MAAM,QAAQ,CAAC,IAAI,IAAI,CAAC;AACpC,iBAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK,GAAG;AACtC,cAAI,IAAI,CAAC,MAAM,UAAU,IAAI,IAAI,CAAC,MAAM,WAAY,QAAO;AAAA,QAC7D;AACA,eAAO;AAAA,MACT,CAAC;AAED,UAAI,CAAC,MAAO,QAAO,EAAE,SAAS,GAAG,WAAW,GAAG,UAAU,EAAE;AAE3D,YAAM,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC;AAC7C,YAAM,MAAM,CAAC,QAAgB;AAC3B,iBAAS,IAAI,GAAG,IAAI,KAAK,SAAS,GAAG,KAAK,GAAG;AAC3C,cAAI,KAAK,CAAC,MAAM,IAAK,QAAO,OAAO,KAAK,IAAI,CAAC,CAAC,KAAK;AAAA,QACrD;AACA,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,QACL,SAAS,IAAI,WAAW;AAAA,QACxB,WAAW,IAAI,iBAAiB;AAAA,QAChC,UAAU,IAAI,KAAK;AAAA,MACrB;AAAA,IACF;AAAA,IAEA,MAAM,QAAuB;AAC3B,YAAM,MAAM,KAAK,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACnC;AAAA,EACF;AACF;AAKO,SAAS,sBAA+B;AAC7C,SAAO,CAAC,EAAE,QAAQ,IAAI,aAAa,QAAQ,IAAI,sBAAsB;AACvE;AAKA,eAAsB,wBAA0D;AAC9E,MAAI,CAAC,oBAAoB,EAAG,QAAO;AACnC,SAAO,iBAAiB,EAAE,MAAM,CAAC,MAAM;AACrC,YAAQ,MAAM,oCAAoC,EAAE,OAAO;AAC3D,WAAO;AAAA,EACT,CAAC;AACH;",
+  "names": []
+}

package/dist/response-cache.d.ts

@@ -0,0 +1,38 @@
+export interface ResponseCacheEntry {
+    url: string;
+    etag?: string;
+    lastModified?: string;
+    cacheControl?: string;
+    body: string;
+    contentType: string;
+    statusCode: number;
+    cachedAt: number;
+    hitCount: number;
+}
+export declare class ResponseCache {
+    private readonly dir;
+    private totalHits;
+    private totalRequests;
+    constructor(cacheDir: string);
+    private key;
+    private path;
+    get(url: string): ResponseCacheEntry | null;
+    private read;
+    isFresh(url: string): boolean;
+    staleIfError(url: string): boolean;
+    set(url: string, entry: Omit<ResponseCacheEntry, 'cachedAt' | 'hitCount'>): void;
+    getValidationHeaders(url: string): {
+        'If-None-Match'?: string;
+        'If-Modified-Since'?: string;
+    };
+    markHit(url: string): void;
+    invalidate(url: string): void;
+    clear(): void;
+    stats(): {
+        entries: number;
+        totalSizeBytes: number;
+        hitRate: number;
+        totalHits: number;
+    };
+    prune(maxAgeMs: number): number;
+}

package/dist/response-cache.js

@@ -0,0 +1,140 @@
+import { createHash } from "node:crypto";
+import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+class ResponseCache {
+  dir;
+  totalHits = 0;
+  totalRequests = 0;
+  constructor(cacheDir) {
+    this.dir = cacheDir;
+    if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true });
+  }
+  key(url) {
+    return createHash("sha256").update(url).digest("hex").slice(0, 16);
+  }
+  path(url) {
+    return join(this.dir, `${this.key(url)}.json`);
+  }
+  get(url) {
+    this.totalRequests++;
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return null;
+      const entry = JSON.parse(readFileSync(p, "utf-8"));
+      this.totalHits++;
+      return entry;
+    } catch {
+      return null;
+    }
+  }
+  read(url) {
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return null;
+      return JSON.parse(readFileSync(p, "utf-8"));
+    } catch {
+      return null;
+    }
+  }
+  isFresh(url) {
+    const entry = this.read(url);
+    if (!entry?.cacheControl) return false;
+    const m = /max-age=(\d+)/i.exec(entry.cacheControl);
+    if (!m) return false;
+    const maxAgeMs = parseInt(m[1], 10) * 1e3;
+    return entry.cachedAt + maxAgeMs > Date.now();
+  }
+  staleIfError(url) {
+    const entry = this.read(url);
+    if (!entry?.cacheControl) return false;
+    const m = /stale-if-error=(\d+)/i.exec(entry.cacheControl);
+    if (!m) return false;
+    const staleIfErrorMs = parseInt(m[1], 10) * 1e3;
+    return entry.cachedAt + staleIfErrorMs >= Date.now();
+  }
+  set(url, entry) {
+    try {
+      const full = { ...entry, cachedAt: Date.now(), hitCount: 0 };
+      writeFileSync(this.path(url), JSON.stringify(full), "utf-8");
+    } catch {
+    }
+  }
+  getValidationHeaders(url) {
+    const entry = this.get(url);
+    if (!entry) return {};
+    const headers = {};
+    if (entry.etag) headers["If-None-Match"] = entry.etag;
+    if (entry.lastModified) headers["If-Modified-Since"] = entry.lastModified;
+    return headers;
+  }
+  markHit(url) {
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return;
+      const entry = JSON.parse(readFileSync(p, "utf-8"));
+      entry.hitCount++;
+      writeFileSync(p, JSON.stringify(entry), "utf-8");
+    } catch {
+    }
+  }
+  invalidate(url) {
+    try {
+      const p = this.path(url);
+      if (existsSync(p)) unlinkSync(p);
+    } catch {
+    }
+  }
+  clear() {
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (f.endsWith(".json")) unlinkSync(join(this.dir, f));
+      }
+    } catch {
+    }
+  }
+  stats() {
+    let entries = 0;
+    let totalSizeBytes = 0;
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (!f.endsWith(".json")) continue;
+        entries++;
+        try {
+          totalSizeBytes += statSync(join(this.dir, f)).size;
+        } catch {
+        }
+      }
+    } catch {
+    }
+    return {
+      entries,
+      totalSizeBytes,
+      hitRate: this.totalRequests > 0 ? this.totalHits / this.totalRequests : 0,
+      totalHits: this.totalHits
+    };
+  }
+  prune(maxAgeMs) {
+    let deleted = 0;
+    const cutoff = Date.now() - maxAgeMs;
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (!f.endsWith(".json")) continue;
+        try {
+          const p = join(this.dir, f);
+          const entry = JSON.parse(readFileSync(p, "utf-8"));
+          if (entry.cachedAt < cutoff) {
+            unlinkSync(p);
+            deleted++;
+          }
+        } catch {
+        }
+      }
+    } catch {
+    }
+    return deleted;
+  }
+}
+export {
+  ResponseCache
+};
+//# sourceMappingURL=response-cache.js.map

package/dist/response-cache.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/response-cache.ts"],
+  "sourcesContent": ["import { createHash } from 'node:crypto';\nimport { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync, unlinkSync } from 'node:fs';\nimport { join } from 'node:path';\n\nexport interface ResponseCacheEntry {\n  url: string;\n  etag?: string;\n  lastModified?: string;\n  cacheControl?: string;\n  body: string;\n  contentType: string;\n  statusCode: number;\n  cachedAt: number;\n  hitCount: number;\n}\n\nexport class ResponseCache {\n  private readonly dir: string;\n  private totalHits = 0;\n  private totalRequests = 0;\n\n  constructor(cacheDir: string) {\n    this.dir = cacheDir;\n    if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true });\n  }\n\n  private key(url: string): string {\n    return createHash('sha256').update(url).digest('hex').slice(0, 16);\n  }\n\n  private path(url: string): string {\n    return join(this.dir, `${this.key(url)}.json`);\n  }\n\n  get(url: string): ResponseCacheEntry | null {\n    this.totalRequests++;\n    try {\n      const p = this.path(url);\n      if (!existsSync(p)) return null;\n      const entry: ResponseCacheEntry = JSON.parse(readFileSync(p, 'utf-8'));\n      this.totalHits++;\n      return entry;\n    } catch {\n      return null;\n    }\n  }\n\n  private read(url: string): ResponseCacheEntry | null {\n    try {\n      const p = this.path(url);\n      if (!existsSync(p)) return null;\n      return JSON.parse(readFileSync(p, 'utf-8')) as ResponseCacheEntry;\n    } catch {\n      return null;\n    }\n  }\n\n  isFresh(url: string): boolean {\n    const entry = this.read(url);\n    if (!entry?.cacheControl) return false;\n    const m = /max-age=(\\d+)/i.exec(entry.cacheControl);\n    if (!m) return false;\n    const maxAgeMs = parseInt(m[1], 10) * 1000;\n    return entry.cachedAt + maxAgeMs > Date.now();\n  }\n\n  staleIfError(url: string): boolean {\n    const entry = this.read(url);\n    if (!entry?.cacheControl) return false;\n    const m = /stale-if-error=(\\d+)/i.exec(entry.cacheControl);\n    if (!m) return false;\n    const staleIfErrorMs = parseInt(m[1], 10) * 1000;\n    return entry.cachedAt + staleIfErrorMs >= Date.now();\n  }\n\n  set(url: string, entry: Omit<ResponseCacheEntry, 'cachedAt' | 'hitCount'>): void {\n    try {\n      const full: ResponseCacheEntry = { ...entry, cachedAt: Date.now(), hitCount: 0 };\n      writeFileSync(this.path(url), JSON.stringify(full), 'utf-8');\n    } catch { /* non-fatal */ }\n  }\n\n  getValidationHeaders(url: string): { 'If-None-Match'?: string; 'If-Modified-Since'?: string } {\n    const entry = this.get(url);\n    if (!entry) return {};\n    const headers: { 'If-None-Match'?: string; 'If-Modified-Since'?: string } = {};\n    if (entry.etag) headers['If-None-Match'] = entry.etag;\n    if (entry.lastModified) headers['If-Modified-Since'] = entry.lastModified;\n    return headers;\n  }\n\n  markHit(url: string): void {\n    try {\n      const p = this.path(url);\n      if (!existsSync(p)) return;\n      const entry: ResponseCacheEntry = JSON.parse(readFileSync(p, 'utf-8'));\n      entry.hitCount++;\n      writeFileSync(p, JSON.stringify(entry), 'utf-8');\n    } catch { /* non-fatal */ }\n  }\n\n  invalidate(url: string): void {\n    try {\n      const p = this.path(url);\n      if (existsSync(p)) unlinkSync(p);\n    } catch { /* non-fatal */ }\n  }\n\n  clear(): void {\n    try {\n      for (const f of readdirSync(this.dir)) {\n        if (f.endsWith('.json')) unlinkSync(join(this.dir, f));\n      }\n    } catch { /* non-fatal */ }\n  }\n\n  stats(): { entries: number; totalSizeBytes: number; hitRate: number; totalHits: number } {\n    let entries = 0;\n    let totalSizeBytes = 0;\n    try {\n      for (const f of readdirSync(this.dir)) {\n        if (!f.endsWith('.json')) continue;\n        entries++;\n        try { totalSizeBytes += statSync(join(this.dir, f)).size; } catch { /* skip */ }\n      }\n    } catch { /* non-fatal */ }\n    return {\n      entries,\n      totalSizeBytes,\n      hitRate: this.totalRequests > 0 ? this.totalHits / this.totalRequests : 0,\n      totalHits: this.totalHits,\n    };\n  }\n\n  prune(maxAgeMs: number): number {\n    let deleted = 0;\n    const cutoff = Date.now() - maxAgeMs;\n    try {\n      for (const f of readdirSync(this.dir)) {\n        if (!f.endsWith('.json')) continue;\n        try {\n          const p = join(this.dir, f);\n          const entry: ResponseCacheEntry = JSON.parse(readFileSync(p, 'utf-8'));\n          if (entry.cachedAt < cutoff) { unlinkSync(p); deleted++; }\n        } catch { /* skip */ }\n      }\n    } catch { /* non-fatal */ }\n    return deleted;\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,kBAAkB;AAC3B,SAAS,cAAc,eAAe,WAAW,YAAY,aAAa,UAAU,kBAAkB;AACtG,SAAS,YAAY;AAcd,MAAM,cAAc;AAAA,EACR;AAAA,EACT,YAAY;AAAA,EACZ,gBAAgB;AAAA,EAExB,YAAY,UAAkB;AAC5B,SAAK,MAAM;AACX,QAAI,CAAC,WAAW,QAAQ,EAAG,WAAU,UAAU,EAAE,WAAW,KAAK,CAAC;AAAA,EACpE;AAAA,EAEQ,IAAI,KAAqB;AAC/B,WAAO,WAAW,QAAQ,EAAE,OAAO,GAAG,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE;AAAA,EACnE;AAAA,EAEQ,KAAK,KAAqB;AAChC,WAAO,KAAK,KAAK,KAAK,GAAG,KAAK,IAAI,GAAG,CAAC,OAAO;AAAA,EAC/C;AAAA,EAEA,IAAI,KAAwC;AAC1C,SAAK;AACL,QAAI;AACF,YAAM,IAAI,KAAK,KAAK,GAAG;AACvB,UAAI,CAAC,WAAW,CAAC,EAAG,QAAO;AAC3B,YAAM,QAA4B,KAAK,MAAM,aAAa,GAAG,OAAO,CAAC;AACrE,WAAK;AACL,aAAO;AAAA,IACT,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEQ,KAAK,KAAwC;AACnD,QAAI;AACF,YAAM,IAAI,KAAK,KAAK,GAAG;AACvB,UAAI,CAAC,WAAW,CAAC,EAAG,QAAO;AAC3B,aAAO,KAAK,MAAM,aAAa,GAAG,OAAO,CAAC;AAAA,IAC5C,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,QAAQ,KAAsB;AAC5B,UAAM,QAAQ,KAAK,KAAK,GAAG;AAC3B,QAAI,CAAC,OAAO,aAAc,QAAO;AACjC,UAAM,IAAI,iBAAiB,KAAK,MAAM,YAAY;AAClD,QAAI,CAAC,EAAG,QAAO;AACf,UAAM,WAAW,SAAS,EAAE,CAAC,GAAG,EAAE,IAAI;AACtC,WAAO,MAAM,WAAW,WAAW,KAAK,IAAI;AAAA,EAC9C;AAAA,EAEA,aAAa,KAAsB;AACjC,UAAM,QAAQ,KAAK,KAAK,GAAG;AAC3B,QAAI,CAAC,OAAO,aAAc,QAAO;AACjC,UAAM,IAAI,wBAAwB,KAAK,MAAM,YAAY;AACzD,QAAI,CAAC,EAAG,QAAO;AACf,UAAM,iBAAiB,SAAS,EAAE,CAAC,GAAG,EAAE,IAAI;AAC5C,WAAO,MAAM,WAAW,kBAAkB,KAAK,IAAI;AAAA,EACrD;AAAA,EAEA,IAAI,KAAa,OAAgE;AAC/E,QAAI;AACF,YAAM,OAA2B,EAAE,GAAG,OAAO,UAAU,KAAK,IAAI,GAAG,UAAU,EAAE;AAC/E,oBAAc,KAAK,KAAK,GAAG,GAAG,KAAK,UAAU,IAAI,GAAG,OAAO;AAAA,IAC7D,QAAQ;AAAA,IAAkB;AAAA,EAC5B;AAAA,EAEA,qBAAqB,KAAyE;AAC5F,UAAM,QAAQ,KAAK,IAAI,GAAG;AAC1B,QAAI,CAAC,MAAO,QAAO,CAAC;AACpB,UAAM,UAAsE,CAAC;AAC7E,QAAI,MAAM,KAAM,SAAQ,eAAe,IAAI,MAAM;AACjD,QAAI,MAAM,aAAc,SAAQ,mBAAmB,IAAI,MAAM;AAC7D,WAAO;AAAA,EACT;AAAA,EAEA,QAAQ,KAAmB;AACzB,QAAI;AACF,YAAM,IAAI,KAAK,KAAK,GAAG;AACvB,UAAI,CAAC,WAAW,CAAC,EAAG;AACpB,YAAM,QAA4B,KAAK,MAAM,aAAa,GAAG,OAAO,CAAC;AACrE,YAAM;AACN,oBAAc,GAAG,KAAK,UAAU,KAAK,GAAG,OAAO;AAAA,IACjD,QAAQ;AAAA,IAAkB;AAAA,EAC5B;AAAA,EAEA,WAAW,KAAmB;AAC5B,QAAI;AACF,YAAM,IAAI,KAAK,KAAK,GAAG;AACvB,UAAI,WAAW,CAAC,EAAG,YAAW,CAAC;AAAA,IACjC,QAAQ;AAAA,IAAkB;AAAA,EAC5B;AAAA,EAEA,QAAc;AACZ,QAAI;AACF,iBAAW,KAAK,YAAY,KAAK,GAAG,GAAG;AACrC,YAAI,EAAE,SAAS,OAAO,EAAG,YAAW,KAAK,KAAK,KAAK,CAAC,CAAC;AAAA,MACvD;AAAA,IACF,QAAQ;AAAA,IAAkB;AAAA,EAC5B;AAAA,EAEA,QAAyF;AACvF,QAAI,UAAU;AACd,QAAI,iBAAiB;AACrB,QAAI;AACF,iBAAW,KAAK,YAAY,KAAK,GAAG,GAAG;AACrC,YAAI,CAAC,EAAE,SAAS,OAAO,EAAG;AAC1B;AACA,YAAI;AAAE,4BAAkB,SAAS,KAAK,KAAK,KAAK,CAAC,CAAC,EAAE;AAAA,QAAM,QAAQ;AAAA,QAAa;AAAA,MACjF;AAAA,IACF,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,SAAS,KAAK,gBAAgB,IAAI,KAAK,YAAY,KAAK,gBAAgB;AAAA,MACxE,WAAW,KAAK;AAAA,IAClB;AAAA,EACF;AAAA,EAEA,MAAM,UAA0B;AAC9B,QAAI,UAAU;AACd,UAAM,SAAS,KAAK,IAAI,IAAI;AAC5B,QAAI;AACF,iBAAW,KAAK,YAAY,KAAK,GAAG,GAAG;AACrC,YAAI,CAAC,EAAE,SAAS,OAAO,EAAG;AAC1B,YAAI;AACF,gBAAM,IAAI,KAAK,KAAK,KAAK,CAAC;AAC1B,gBAAM,QAA4B,KAAK,MAAM,aAAa,GAAG,OAAO,CAAC;AACrE,cAAI,MAAM,WAAW,QAAQ;AAAE,uBAAW,CAAC;AAAG;AAAA,UAAW;AAAA,QAC3D,QAAQ;AAAA,QAAa;AAAA,MACvB;AAAA,IACF,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/responsive-detector.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Responsive design detector — captures page at multiple breakpoints
+ * and detects layout differences between desktop and mobile viewports.
+ *
+ * Breakpoints: mobile(375), tablet(768), desktop(1280), wide(1920)
+ * Detects: viewport-specific content visibility, menu collapse state,
+ * font size changes, hidden/shown elements.
+ */
+import type { Page } from 'playwright';
+export interface ViewportCapture {
+    width: number;
+    height: number;
+    label: 'mobile' | 'tablet' | 'desktop' | 'wide';
+    screenshotBuffer?: Buffer;
+    visibleTextLength: number;
+    hasHamburgerMenu: boolean;
+    hiddenElements: number;
+    scrollHeight: number;
+}
+export interface ResponsiveReport {
+    url: string;
+    captures: ViewportCapture[];
+    isResponsive: boolean;
+    hasMobileMenu: boolean;
+    issues: string[];
+}
+export declare function detectResponsive(page: Page, url: string, opts?: {
+    captureScreenshots?: boolean;
+}): Promise<ResponsiveReport>;

package/dist/responsive-detector.js

@@ -0,0 +1,46 @@
+const BREAKPOINTS = [
+  { width: 375, height: 812, label: "mobile" },
+  { width: 768, height: 1024, label: "tablet" },
+  { width: 1280, height: 800, label: "desktop" }
+];
+async function detectResponsive(page, url, opts = {}) {
+  const captures = [];
+  const issues = [];
+  for (const bp of BREAKPOINTS) {
+    try {
+      await page.setViewportSize({ width: bp.width, height: bp.height });
+      await page.waitForTimeout(500);
+      const data = await page.evaluate(() => ({
+        visibleTextLength: (document.body.innerText ?? "").length,
+        hasHamburgerMenu: !!document.querySelector('[class*="hamburger"],[class*="menu-toggle"],[class*="nav-toggle"],[aria-label*="menu"],[aria-label*="Menu"]'),
+        hiddenElements: Array.from(document.querySelectorAll("*")).filter(
+          (el) => window.getComputedStyle(el).display === "none"
+        ).length,
+        scrollHeight: document.body.scrollHeight
+      })).catch(() => ({ visibleTextLength: 0, hasHamburgerMenu: false, hiddenElements: 0, scrollHeight: 0 }));
+      let screenshotBuffer;
+      if (opts.captureScreenshots) {
+        screenshotBuffer = await page.screenshot({ type: "jpeg", quality: 60 }).catch(() => void 0);
+      }
+      captures.push({ ...bp, ...data, screenshotBuffer });
+    } catch {
+    }
+  }
+  await page.setViewportSize({ width: 1280, height: 800 }).catch(() => {
+  });
+  const mobile = captures.find((c) => c.label === "mobile");
+  const desktop = captures.find((c) => c.label === "desktop");
+  const isResponsive = !!(mobile && desktop && (mobile.hiddenElements !== desktop.hiddenElements || mobile.scrollHeight !== desktop.scrollHeight || mobile.hasHamburgerMenu));
+  const hasMobileMenu = captures.some((c) => c.label === "mobile" && c.hasHamburgerMenu);
+  if (mobile && desktop && mobile.visibleTextLength > desktop.visibleTextLength * 1.3) {
+    issues.push("Mobile shows significantly more text than desktop \u2014 possible responsive issue");
+  }
+  if (!isResponsive && captures.length > 1) {
+    issues.push("Layout appears identical across breakpoints \u2014 may not be responsive");
+  }
+  return { url, captures, isResponsive, hasMobileMenu, issues };
+}
+export {
+  detectResponsive
+};
+//# sourceMappingURL=responsive-detector.js.map

package/dist/responsive-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/responsive-detector.ts"],
+  "sourcesContent": ["/**\n * Responsive design detector \u2014 captures page at multiple breakpoints\n * and detects layout differences between desktop and mobile viewports.\n *\n * Breakpoints: mobile(375), tablet(768), desktop(1280), wide(1920)\n * Detects: viewport-specific content visibility, menu collapse state,\n * font size changes, hidden/shown elements.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface ViewportCapture {\n  width: number;\n  height: number;\n  label: 'mobile' | 'tablet' | 'desktop' | 'wide';\n  screenshotBuffer?: Buffer;\n  visibleTextLength: number;\n  hasHamburgerMenu: boolean;\n  hiddenElements: number; // display:none element count\n  scrollHeight: number;\n}\n\nexport interface ResponsiveReport {\n  url: string;\n  captures: ViewportCapture[];\n  isResponsive: boolean; // true if layout changes between viewports\n  hasMobileMenu: boolean;\n  issues: string[];\n}\n\nconst BREAKPOINTS: Array<{ width: number; height: number; label: ViewportCapture['label'] }> = [\n  { width: 375, height: 812, label: 'mobile' },\n  { width: 768, height: 1024, label: 'tablet' },\n  { width: 1280, height: 800, label: 'desktop' },\n];\n\nexport async function detectResponsive(\n  page: Page,\n  url: string,\n  opts: { captureScreenshots?: boolean } = {},\n): Promise<ResponsiveReport> {\n  const captures: ViewportCapture[] = [];\n  const issues: string[] = [];\n\n  for (const bp of BREAKPOINTS) {\n    try {\n      await page.setViewportSize({ width: bp.width, height: bp.height });\n      await page.waitForTimeout(500); // allow CSS reflow\n\n      const data = await page.evaluate(() => ({\n        visibleTextLength: (document.body.innerText ?? '').length,\n        hasHamburgerMenu: !!(\n          document.querySelector('[class*=\"hamburger\"],[class*=\"menu-toggle\"],[class*=\"nav-toggle\"],[aria-label*=\"menu\"],[aria-label*=\"Menu\"]')\n        ),\n        hiddenElements: Array.from(document.querySelectorAll('*')).filter(\n          (el: any) => window.getComputedStyle(el).display === 'none'\n        ).length,\n        scrollHeight: document.body.scrollHeight,\n      })).catch(() => ({ visibleTextLength: 0, hasHamburgerMenu: false, hiddenElements: 0, scrollHeight: 0 }));\n\n      let screenshotBuffer: Buffer | undefined;\n      if (opts.captureScreenshots) {\n        screenshotBuffer = await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined);\n      }\n\n      captures.push({ ...bp, ...data, screenshotBuffer });\n    } catch { /* viewport failed \u2014 skip */ }\n  }\n\n  // Restore to desktop\n  await page.setViewportSize({ width: 1280, height: 800 }).catch(() => {});\n\n  const mobile = captures.find(c => c.label === 'mobile');\n  const desktop = captures.find(c => c.label === 'desktop');\n\n  const isResponsive = !!(mobile && desktop && (\n    mobile.hiddenElements !== desktop.hiddenElements ||\n    mobile.scrollHeight !== desktop.scrollHeight ||\n    mobile.hasHamburgerMenu\n  ));\n\n  const hasMobileMenu = captures.some(c => c.label === 'mobile' && c.hasHamburgerMenu);\n\n  if (mobile && desktop && mobile.visibleTextLength > desktop.visibleTextLength * 1.3) {\n    issues.push('Mobile shows significantly more text than desktop \u2014 possible responsive issue');\n  }\n  if (!isResponsive && captures.length > 1) {\n    issues.push('Layout appears identical across breakpoints \u2014 may not be responsive');\n  }\n\n  return { url, captures, isResponsive, hasMobileMenu, issues };\n}\n"],
+  "mappings": "AA8BA,MAAM,cAAyF;AAAA,EAC7F,EAAE,OAAO,KAAK,QAAQ,KAAK,OAAO,SAAS;AAAA,EAC3C,EAAE,OAAO,KAAK,QAAQ,MAAM,OAAO,SAAS;AAAA,EAC5C,EAAE,OAAO,MAAM,QAAQ,KAAK,OAAO,UAAU;AAC/C;AAEA,eAAsB,iBACpB,MACA,KACA,OAAyC,CAAC,GACf;AAC3B,QAAM,WAA8B,CAAC;AACrC,QAAM,SAAmB,CAAC;AAE1B,aAAW,MAAM,aAAa;AAC5B,QAAI;AACF,YAAM,KAAK,gBAAgB,EAAE,OAAO,GAAG,OAAO,QAAQ,GAAG,OAAO,CAAC;AACjE,YAAM,KAAK,eAAe,GAAG;AAE7B,YAAM,OAAO,MAAM,KAAK,SAAS,OAAO;AAAA,QACtC,oBAAoB,SAAS,KAAK,aAAa,IAAI;AAAA,QACnD,kBAAkB,CAAC,CACjB,SAAS,cAAc,6GAA6G;AAAA,QAEtI,gBAAgB,MAAM,KAAK,SAAS,iBAAiB,GAAG,CAAC,EAAE;AAAA,UACzD,CAAC,OAAY,OAAO,iBAAiB,EAAE,EAAE,YAAY;AAAA,QACvD,EAAE;AAAA,QACF,cAAc,SAAS,KAAK;AAAA,MAC9B,EAAE,EAAE,MAAM,OAAO,EAAE,mBAAmB,GAAG,kBAAkB,OAAO,gBAAgB,GAAG,cAAc,EAAE,EAAE;AAEvG,UAAI;AACJ,UAAI,KAAK,oBAAoB;AAC3B,2BAAmB,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,GAAG,CAAC,EAAE,MAAM,MAAM,MAAS;AAAA,MAC/F;AAEA,eAAS,KAAK,EAAE,GAAG,IAAI,GAAG,MAAM,iBAAiB,CAAC;AAAA,IACpD,QAAQ;AAAA,IAA+B;AAAA,EACzC;AAGA,QAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,QAAQ,IAAI,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAEvE,QAAM,SAAS,SAAS,KAAK,OAAK,EAAE,UAAU,QAAQ;AACtD,QAAM,UAAU,SAAS,KAAK,OAAK,EAAE,UAAU,SAAS;AAExD,QAAM,eAAe,CAAC,EAAE,UAAU,YAChC,OAAO,mBAAmB,QAAQ,kBAClC,OAAO,iBAAiB,QAAQ,gBAChC,OAAO;AAGT,QAAM,gBAAgB,SAAS,KAAK,OAAK,EAAE,UAAU,YAAY,EAAE,gBAAgB;AAEnF,MAAI,UAAU,WAAW,OAAO,oBAAoB,QAAQ,oBAAoB,KAAK;AACnF,WAAO,KAAK,oFAA+E;AAAA,EAC7F;AACA,MAAI,CAAC,gBAAgB,SAAS,SAAS,GAAG;AACxC,WAAO,KAAK,0EAAqE;AAAA,EACnF;AAEA,SAAO,EAAE,KAAK,UAAU,cAAc,eAAe,OAAO;AAC9D;",
+  "names": []
+}

package/dist/retry-policy.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Retry policy — configurable exponential backoff with jitter.
+ * Covers: 5xx server errors, network timeouts, ECONNREFUSED, ECONNRESET.
+ * NOT retried: 4xx client errors (except 429 — handled separately), redirects.
+ */
+export interface RetryPolicyOpts {
+    maxRetries?: number;
+    baseDelayMs?: number;
+    maxDelayMs?: number;
+    jitterMs?: number;
+    retryOn?: number[];
+}
+export interface RetryResult<T> {
+    value: T;
+    attempts: number;
+    totalDelayMs: number;
+}
+export declare function withRetry<T>(fn: (attempt: number) => Promise<T>, opts?: RetryPolicyOpts): Promise<RetryResult<T>>;
+/** Wraps fetch() with retry policy */
+export declare function fetchWithRetry(url: string, init?: RequestInit, retryOpts?: RetryPolicyOpts): Promise<Response>;

package/dist/retry-policy.js

@@ -0,0 +1,54 @@
+const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
+function shouldRetry(error, retryOn) {
+  if (error?.code && ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EPIPE"].includes(error.code)) return true;
+  if (error?.message && /timeout|net::ERR_/i.test(error.message)) return true;
+  if (typeof error?.status === "number") return retryOn.includes(error.status);
+  return false;
+}
+function delay(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+async function withRetry(fn, opts = {}) {
+  const {
+    maxRetries = 3,
+    baseDelayMs = 1e3,
+    maxDelayMs = 3e4,
+    jitterMs = 500,
+    retryOn = DEFAULT_RETRY_STATUSES
+  } = opts;
+  let lastError;
+  let totalDelayMs = 0;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const value = await fn(attempt);
+      return { value, attempts: attempt + 1, totalDelayMs };
+    } catch (err) {
+      lastError = err;
+      if (attempt === maxRetries) break;
+      if (!shouldRetry(err, retryOn)) throw err;
+      const backoff = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));
+      const jitter = Math.random() * jitterMs;
+      const waitMs = Math.round(backoff + jitter);
+      totalDelayMs += waitMs;
+      await delay(waitMs);
+    }
+  }
+  throw lastError;
+}
+async function fetchWithRetry(url, init = {}, retryOpts = {}) {
+  const result = await withRetry(async () => {
+    const res = await fetch(url, init);
+    if (retryOpts.retryOn?.includes(res.status) ?? DEFAULT_RETRY_STATUSES.includes(res.status)) {
+      const err = new Error(`HTTP ${res.status}`);
+      err.status = res.status;
+      throw err;
+    }
+    return res;
+  }, retryOpts);
+  return result.value;
+}
+export {
+  fetchWithRetry,
+  withRetry
+};
+//# sourceMappingURL=retry-policy.js.map

package/dist/retry-policy.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/retry-policy.ts"],
+  "sourcesContent": ["/**\n * Retry policy \u2014 configurable exponential backoff with jitter.\n * Covers: 5xx server errors, network timeouts, ECONNREFUSED, ECONNRESET.\n * NOT retried: 4xx client errors (except 429 \u2014 handled separately), redirects.\n */\n\nexport interface RetryPolicyOpts {\n  maxRetries?: number;       // default 3\n  baseDelayMs?: number;      // default 1000\n  maxDelayMs?: number;       // default 30000\n  jitterMs?: number;         // default 500\n  retryOn?: number[];        // HTTP status codes to retry (default: [429, 500, 502, 503, 504])\n}\n\nexport interface RetryResult<T> {\n  value: T;\n  attempts: number;\n  totalDelayMs: number;\n}\n\nconst DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];\n\nfunction shouldRetry(error: any, retryOn: number[]): boolean {\n  // Network-level errors\n  if (error?.code && ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EPIPE'].includes(error.code)) return true;\n  // Playwright navigation timeout\n  if (error?.message && /timeout|net::ERR_/i.test(error.message)) return true;\n  // HTTP status errors\n  if (typeof error?.status === 'number') return retryOn.includes(error.status);\n  return false;\n}\n\nfunction delay(ms: number): Promise<void> {\n  return new Promise(r => setTimeout(r, ms));\n}\n\nexport async function withRetry<T>(\n  fn: (attempt: number) => Promise<T>,\n  opts: RetryPolicyOpts = {},\n): Promise<RetryResult<T>> {\n  const {\n    maxRetries = 3,\n    baseDelayMs = 1_000,\n    maxDelayMs = 30_000,\n    jitterMs = 500,\n    retryOn = DEFAULT_RETRY_STATUSES,\n  } = opts;\n\n  let lastError: any;\n  let totalDelayMs = 0;\n\n  for (let attempt = 0; attempt <= maxRetries; attempt++) {\n    try {\n      const value = await fn(attempt);\n      return { value, attempts: attempt + 1, totalDelayMs };\n    } catch (err) {\n      lastError = err;\n      if (attempt === maxRetries) break;\n      if (!shouldRetry(err, retryOn)) throw err; // non-retryable \u2014 fail fast\n\n      const backoff = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));\n      const jitter = Math.random() * jitterMs;\n      const waitMs = Math.round(backoff + jitter);\n      totalDelayMs += waitMs;\n      await delay(waitMs);\n    }\n  }\n  throw lastError;\n}\n\n/** Wraps fetch() with retry policy */\nexport async function fetchWithRetry(\n  url: string,\n  init: RequestInit = {},\n  retryOpts: RetryPolicyOpts = {},\n): Promise<Response> {\n  const result = await withRetry(async () => {\n    const res = await fetch(url, init);\n    if (retryOpts.retryOn?.includes(res.status) ?? DEFAULT_RETRY_STATUSES.includes(res.status)) {\n      const err: any = new Error(`HTTP ${res.status}`);\n      err.status = res.status;\n      throw err;\n    }\n    return res;\n  }, retryOpts);\n  return result.value;\n}\n"],
+  "mappings": "AAoBA,MAAM,yBAAyB,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG;AAEvD,SAAS,YAAY,OAAY,SAA4B;AAE3D,MAAI,OAAO,QAAQ,CAAC,cAAc,gBAAgB,aAAa,aAAa,OAAO,EAAE,SAAS,MAAM,IAAI,EAAG,QAAO;AAElH,MAAI,OAAO,WAAW,qBAAqB,KAAK,MAAM,OAAO,EAAG,QAAO;AAEvE,MAAI,OAAO,OAAO,WAAW,SAAU,QAAO,QAAQ,SAAS,MAAM,MAAM;AAC3E,SAAO;AACT;AAEA,SAAS,MAAM,IAA2B;AACxC,SAAO,IAAI,QAAQ,OAAK,WAAW,GAAG,EAAE,CAAC;AAC3C;AAEA,eAAsB,UACpB,IACA,OAAwB,CAAC,GACA;AACzB,QAAM;AAAA,IACJ,aAAa;AAAA,IACb,cAAc;AAAA,IACd,aAAa;AAAA,IACb,WAAW;AAAA,IACX,UAAU;AAAA,EACZ,IAAI;AAEJ,MAAI;AACJ,MAAI,eAAe;AAEnB,WAAS,UAAU,GAAG,WAAW,YAAY,WAAW;AACtD,QAAI;AACF,YAAM,QAAQ,MAAM,GAAG,OAAO;AAC9B,aAAO,EAAE,OAAO,UAAU,UAAU,GAAG,aAAa;AAAA,IACtD,SAAS,KAAK;AACZ,kBAAY;AACZ,UAAI,YAAY,WAAY;AAC5B,UAAI,CAAC,YAAY,KAAK,OAAO,EAAG,OAAM;AAEtC,YAAM,UAAU,KAAK,IAAI,YAAY,cAAc,KAAK,IAAI,GAAG,OAAO,CAAC;AACvE,YAAM,SAAS,KAAK,OAAO,IAAI;AAC/B,YAAM,SAAS,KAAK,MAAM,UAAU,MAAM;AAC1C,sBAAgB;AAChB,YAAM,MAAM,MAAM;AAAA,IACpB;AAAA,EACF;AACA,QAAM;AACR;AAGA,eAAsB,eACpB,KACA,OAAoB,CAAC,GACrB,YAA6B,CAAC,GACX;AACnB,QAAM,SAAS,MAAM,UAAU,YAAY;AACzC,UAAM,MAAM,MAAM,MAAM,KAAK,IAAI;AACjC,QAAI,UAAU,SAAS,SAAS,IAAI,MAAM,KAAK,uBAAuB,SAAS,IAAI,MAAM,GAAG;AAC1F,YAAM,MAAW,IAAI,MAAM,QAAQ,IAAI,MAAM,EAAE;AAC/C,UAAI,SAAS,IAAI;AACjB,YAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT,GAAG,SAAS;AACZ,SAAO,OAAO;AAChB;",
+  "names": []
+}

package/dist/robots-txt.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Minimal robots.txt fetcher + parser. No external deps. Caches per origin.
+ * Implements the subset of robots.txt spec used in practice:
+ *   User-agent, Disallow, Allow — longest match wins (RFC 9309).
+ */
+export declare class RobotsCache {
+    private readonly cache;
+    isAllowed(url: string, userAgent?: string): Promise<boolean>;
+}

package/dist/robots-txt.js

@@ -0,0 +1,77 @@
+function parseRobots(text) {
+  const rules = [];
+  let current = null;
+  for (const rawLine of text.split("\n")) {
+    const line = rawLine.split("#")[0].trim();
+    if (!line) {
+      current = null;
+      continue;
+    }
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) continue;
+    const key = line.slice(0, colonIdx).trim().toLowerCase();
+    const value = line.slice(colonIdx + 1).trim();
+    if (key === "user-agent") {
+      if (!current || current.disallow.length > 0 || current.allow.length > 0) {
+        current = { userAgent: value.toLowerCase(), disallow: [], allow: [] };
+        rules.push(current);
+      } else {
+        current.userAgent = value.toLowerCase();
+      }
+    } else if (key === "disallow" && current && value) {
+      current.disallow.push(value);
+    } else if (key === "allow" && current && value) {
+      current.allow.push(value);
+    }
+  }
+  return rules;
+}
+function isAllowedByRules(rules, userAgent, pathname) {
+  const ua = userAgent.toLowerCase();
+  const exactRules = rules.filter((r) => r.userAgent === ua);
+  const wildcardRules = rules.filter((r) => r.userAgent === "*");
+  const applicable = exactRules.length > 0 ? exactRules : wildcardRules;
+  if (applicable.length === 0) return true;
+  const matched = [];
+  for (const rule of applicable) {
+    for (const p of rule.allow) {
+      if (pathname.startsWith(p)) matched.push({ path: p, allow: true });
+    }
+    for (const p of rule.disallow) {
+      if (pathname.startsWith(p)) matched.push({ path: p, allow: false });
+    }
+  }
+  if (matched.length === 0) return true;
+  matched.sort((a, b) => b.path.length - a.path.length);
+  return matched[0].allow;
+}
+class RobotsCache {
+  cache = /* @__PURE__ */ new Map();
+  async isAllowed(url, userAgent = "ZeTa-Crawler") {
+    let origin;
+    let pathname;
+    try {
+      const u = new URL(url);
+      origin = u.origin;
+      pathname = u.pathname || "/";
+    } catch {
+      return true;
+    }
+    if (!this.cache.has(origin)) {
+      try {
+        const res = await fetch(`${origin}/robots.txt`, {
+          headers: { "User-Agent": userAgent },
+          signal: AbortSignal.timeout(5e3)
+        });
+        this.cache.set(origin, res.ok ? parseRobots(await res.text()) : []);
+      } catch {
+        this.cache.set(origin, []);
+      }
+    }
+    return isAllowedByRules(this.cache.get(origin), userAgent, pathname);
+  }
+}
+export {
+  RobotsCache
+};
+//# sourceMappingURL=robots-txt.js.map