@detiq/crawler 0.1.3

package/dist/url-pattern-completer.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/url-pattern-completer.ts"],
+  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      // @ts-ignore: cache_control supported in beta\n      system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
+  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAoBnE,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,eAAe,SAAS,EAAG,QAAO,CAAC;AAE1D,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,SAAS;AAAA;AAAA;AAAA,EAGf,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA;AAAA,MAEZ,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,uEAAuE,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAC5I,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,OAAO,GAAG,yCAAyC;AACnG,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/visual-diff-verdict.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested
+ * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and
+ * transitively @qip/database, which instantiates a PrismaClient at module load and
+ * so requires a live DATABASE_URL just to import the module).
+ */
+export interface VisualDiffVerdict {
+    changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';
+    recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';
+    changes: Array<{
+        location: string;
+        severity: string;
+        description: string;
+    }>;
+    bugReportDraft?: {
+        title: string;
+        severity: string;
+        steps: string[];
+        expected: string;
+        actual: string;
+    };
+    aiSummary?: string;
+}
+/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */
+export declare function pHashVerdict(pHashDistance: number, changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT', changes: Array<{
+    location: string;
+    severity: string;
+    description: string;
+}>): VisualDiffVerdict;
+/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,
+ *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review
+ *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards
+ *  as NONE/MINOR — only what survives AI review is worth a draft). */
+export declare function mergeAiVisualVerdict(pHashResult: VisualDiffVerdict, aiResult: {
+    changeType: string;
+    changes: any[];
+    recommendation: string;
+    diffScore: number;
+    summary?: string;
+}, screenName: string): VisualDiffVerdict;

package/dist/visual-diff-verdict.js

@@ -0,0 +1,25 @@
+function pHashVerdict(pHashDistance, changeLevel, changes) {
+  const changeType = changeLevel === "IDENTICAL" ? "NONE" : changeLevel === "MINOR" ? "MINOR" : changeLevel === "MODERATE" ? "SIGNIFICANT" : "CRITICAL";
+  const recommendation = pHashDistance === 0 ? "APPROVE" : pHashDistance < 5 ? "APPROVE" : pHashDistance < 20 ? "REVIEW" : "FLAG";
+  return { changeType, recommendation, changes };
+}
+function mergeAiVisualVerdict(pHashResult, aiResult, screenName) {
+  if (!aiResult.changeType) return pHashResult;
+  const changeType = aiResult.changeType;
+  const recommendation = aiResult.recommendation;
+  const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;
+  const aiSummary = aiResult.summary ?? void 0;
+  const bugReportDraft = changeType === "CRITICAL" || changeType === "SIGNIFICANT" ? {
+    title: `Visual regression on "${screenName}"`,
+    severity: changeType === "CRITICAL" ? "High" : "Medium",
+    steps: ["Navigate to the screen", "Compare current view to baseline screenshot"],
+    expected: "Screen appearance should match the stored baseline.",
+    actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`
+  } : void 0;
+  return { changeType, recommendation, changes, bugReportDraft, aiSummary };
+}
+export {
+  mergeAiVisualVerdict,
+  pHashVerdict
+};
+//# sourceMappingURL=visual-diff-verdict.js.map

package/dist/visual-diff-verdict.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/visual-diff-verdict.ts"],
+  "sourcesContent": ["/**\n * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested\n * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and\n * transitively @qip/database, which instantiates a PrismaClient at module load and\n * so requires a live DATABASE_URL just to import the module).\n */\n\nexport interface VisualDiffVerdict {\n  changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';\n  recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';\n  changes: Array<{ location: string; severity: string; description: string }>;\n  bugReportDraft?: { title: string; severity: string; steps: string[]; expected: string; actual: string };\n  aiSummary?: string;\n}\n\n/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */\nexport function pHashVerdict(\n  pHashDistance: number,\n  changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT',\n  changes: Array<{ location: string; severity: string; description: string }>,\n): VisualDiffVerdict {\n  const changeType: VisualDiffVerdict['changeType'] = changeLevel === 'IDENTICAL' ? 'NONE'\n    : changeLevel === 'MINOR' ? 'MINOR'\n      : changeLevel === 'MODERATE' ? 'SIGNIFICANT'\n        : 'CRITICAL';\n  const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'\n    : pHashDistance < 5 ? 'APPROVE'\n      : pHashDistance < 20 ? 'REVIEW'\n        : 'FLAG';\n  return { changeType, recommendation, changes };\n}\n\n/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,\n *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review\n *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards\n *  as NONE/MINOR \u2014 only what survives AI review is worth a draft). */\nexport function mergeAiVisualVerdict(\n  pHashResult: VisualDiffVerdict,\n  aiResult: { changeType: string; changes: any[]; recommendation: string; diffScore: number; summary?: string },\n  screenName: string,\n): VisualDiffVerdict {\n  if (!aiResult.changeType) return pHashResult;\n  const changeType = aiResult.changeType as VisualDiffVerdict['changeType'];\n  const recommendation = aiResult.recommendation as VisualDiffVerdict['recommendation'];\n  const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;\n  const aiSummary = aiResult.summary ?? undefined;\n  const bugReportDraft = (changeType === 'CRITICAL' || changeType === 'SIGNIFICANT')\n    ? {\n        title: `Visual regression on \"${screenName}\"`,\n        severity: changeType === 'CRITICAL' ? 'High' : 'Medium',\n        steps: ['Navigate to the screen', 'Compare current view to baseline screenshot'],\n        expected: 'Screen appearance should match the stored baseline.',\n        actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`,\n      }\n    : undefined;\n  return { changeType, recommendation, changes, bugReportDraft, aiSummary };\n}\n"],
+  "mappings": "AAgBO,SAAS,aACd,eACA,aACA,SACmB;AACnB,QAAM,aAA8C,gBAAgB,cAAc,SAC9E,gBAAgB,UAAU,UACxB,gBAAgB,aAAa,gBAC3B;AACR,QAAM,iBAAsD,kBAAkB,IAAI,YAC9E,gBAAgB,IAAI,YAClB,gBAAgB,KAAK,WACnB;AACR,SAAO,EAAE,YAAY,gBAAgB,QAAQ;AAC/C;AAMO,SAAS,qBACd,aACA,UACA,YACmB;AACnB,MAAI,CAAC,SAAS,WAAY,QAAO;AACjC,QAAM,aAAa,SAAS;AAC5B,QAAM,iBAAiB,SAAS;AAChC,QAAM,UAAU,MAAM,QAAQ,SAAS,OAAO,KAAK,SAAS,QAAQ,SAAS,IAAI,SAAS,UAAU,YAAY;AAChH,QAAM,YAAY,SAAS,WAAW;AACtC,QAAM,iBAAkB,eAAe,cAAc,eAAe,gBAChE;AAAA,IACE,OAAO,yBAAyB,UAAU;AAAA,IAC1C,UAAU,eAAe,aAAa,SAAS;AAAA,IAC/C,OAAO,CAAC,0BAA0B,6CAA6C;AAAA,IAC/E,UAAU;AAAA,IACV,QAAQ,aAAa,mCAAmC,SAAS,UAAU,QAAQ,CAAC,CAAC;AAAA,EACvF,IACA;AACJ,SAAO,EAAE,YAAY,gBAAgB,SAAS,gBAAgB,UAAU;AAC1E;",
+  "names": []
+}

package/dist/webhook-notifier.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Webhook notifier — fire HTTP POST callbacks on crawl lifecycle events.
+ *
+ * Supports: crawl.started, crawl.progress, crawl.completed, crawl.failed
+ * Payload format compatible with Slack, Discord, and generic REST webhooks.
+ * Retries 3x on failure with exponential backoff.
+ */
+export type CrawlEventType = 'crawl.started' | 'crawl.progress' | 'crawl.completed' | 'crawl.failed';
+export interface WebhookPayload {
+    event: CrawlEventType;
+    jobId: string;
+    projectId: string;
+    tenantId: string;
+    timestamp: string;
+    data: Record<string, unknown>;
+}
+export interface WebhookNotifierOpts {
+    url: string;
+    secret?: string;
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+    maxRetries?: number;
+}
+export declare class WebhookNotifier {
+    private readonly opts;
+    constructor(opts: WebhookNotifierOpts);
+    notify(event: CrawlEventType, jobId: string, projectId: string, tenantId: string, data?: Record<string, unknown>): Promise<void>;
+    /** Slack-compatible message format */
+    static slackPayload(event: CrawlEventType, data: Record<string, unknown>): Record<string, unknown>;
+}

package/dist/webhook-notifier.js

@@ -0,0 +1,76 @@
+async function hmacSign(secret, payload) {
+  const key = await crypto.subtle.importKey(
+    "raw",
+    new TextEncoder().encode(secret),
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"]
+  );
+  const sig = await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(payload));
+  return Array.from(new Uint8Array(sig)).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+class WebhookNotifier {
+  opts;
+  constructor(opts) {
+    this.opts = {
+      url: opts.url,
+      secret: opts.secret,
+      headers: opts.headers ?? {},
+      timeoutMs: opts.timeoutMs ?? 1e4,
+      maxRetries: opts.maxRetries ?? 3
+    };
+  }
+  async notify(event, jobId, projectId, tenantId, data = {}) {
+    const payload = {
+      event,
+      jobId,
+      projectId,
+      tenantId,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      data
+    };
+    const body = JSON.stringify(payload);
+    const headers = {
+      "Content-Type": "application/json",
+      "User-Agent": "ZeTa-Crawler/1.0",
+      ...this.opts.headers
+    };
+    if (this.opts.secret) {
+      headers["X-ZeTa-Signature"] = `sha256=${await hmacSign(this.opts.secret, body)}`;
+    }
+    let lastErr;
+    for (let attempt = 0; attempt <= this.opts.maxRetries; attempt++) {
+      try {
+        const res = await fetch(this.opts.url, {
+          method: "POST",
+          headers,
+          body,
+          signal: AbortSignal.timeout(this.opts.timeoutMs)
+        });
+        if (res.ok) return;
+        lastErr = new Error(`Webhook HTTP ${res.status}`);
+      } catch (e) {
+        lastErr = e;
+      }
+      if (attempt < this.opts.maxRetries) {
+        await new Promise((r) => setTimeout(r, 1e3 * Math.pow(2, attempt)));
+      }
+    }
+    console.error(`[webhook] Failed to deliver ${event} after ${this.opts.maxRetries + 1} attempts:`, lastErr?.message);
+  }
+  /** Slack-compatible message format */
+  static slackPayload(event, data) {
+    const emoji = event === "crawl.completed" ? "\u2705" : event === "crawl.failed" ? "\u274C" : "\u{1F504}";
+    return {
+      text: `${emoji} *ZeTa Crawl Event: ${event}*`,
+      blocks: [
+        { type: "section", text: { type: "mrkdwn", text: `${emoji} *${event}*
+${Object.entries(data).map(([k, v]) => `\u2022 *${k}*: ${v}`).join("\n")}` } }
+      ]
+    };
+  }
+}
+export {
+  WebhookNotifier
+};
+//# sourceMappingURL=webhook-notifier.js.map

package/dist/webhook-notifier.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/webhook-notifier.ts"],
+  "sourcesContent": ["/**\n * Webhook notifier \u2014 fire HTTP POST callbacks on crawl lifecycle events.\n *\n * Supports: crawl.started, crawl.progress, crawl.completed, crawl.failed\n * Payload format compatible with Slack, Discord, and generic REST webhooks.\n * Retries 3x on failure with exponential backoff.\n */\n\nexport type CrawlEventType = 'crawl.started' | 'crawl.progress' | 'crawl.completed' | 'crawl.failed';\n\nexport interface WebhookPayload {\n  event: CrawlEventType;\n  jobId: string;\n  projectId: string;\n  tenantId: string;\n  timestamp: string;\n  data: Record<string, unknown>;\n}\n\nexport interface WebhookNotifierOpts {\n  url: string;\n  secret?: string;       // HMAC-SHA256 signing secret \u2014 adds X-ZeTa-Signature header\n  headers?: Record<string, string>;\n  timeoutMs?: number;    // default 10s\n  maxRetries?: number;   // default 3\n}\n\nasync function hmacSign(secret: string, payload: string): Promise<string> {\n  const key = await crypto.subtle.importKey(\n    'raw',\n    new TextEncoder().encode(secret),\n    { name: 'HMAC', hash: 'SHA-256' },\n    false,\n    ['sign'],\n  );\n  const sig = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(payload));\n  return Array.from(new Uint8Array(sig)).map(b => b.toString(16).padStart(2, '0')).join('');\n}\n\nexport class WebhookNotifier {\n  private readonly opts: Required<Omit<WebhookNotifierOpts, 'secret'>> & { secret?: string };\n\n  constructor(opts: WebhookNotifierOpts) {\n    this.opts = {\n      url: opts.url,\n      secret: opts.secret,\n      headers: opts.headers ?? {},\n      timeoutMs: opts.timeoutMs ?? 10_000,\n      maxRetries: opts.maxRetries ?? 3,\n    };\n  }\n\n  async notify(\n    event: CrawlEventType,\n    jobId: string,\n    projectId: string,\n    tenantId: string,\n    data: Record<string, unknown> = {},\n  ): Promise<void> {\n    const payload: WebhookPayload = {\n      event,\n      jobId,\n      projectId,\n      tenantId,\n      timestamp: new Date().toISOString(),\n      data,\n    };\n    const body = JSON.stringify(payload);\n    const headers: Record<string, string> = {\n      'Content-Type': 'application/json',\n      'User-Agent': 'ZeTa-Crawler/1.0',\n      ...this.opts.headers,\n    };\n    if (this.opts.secret) {\n      headers['X-ZeTa-Signature'] = `sha256=${await hmacSign(this.opts.secret, body)}`;\n    }\n\n    let lastErr: any;\n    for (let attempt = 0; attempt <= this.opts.maxRetries; attempt++) {\n      try {\n        const res = await fetch(this.opts.url, {\n          method: 'POST',\n          headers,\n          body,\n          signal: AbortSignal.timeout(this.opts.timeoutMs),\n        });\n        if (res.ok) return;\n        lastErr = new Error(`Webhook HTTP ${res.status}`);\n      } catch (e) {\n        lastErr = e;\n      }\n      if (attempt < this.opts.maxRetries) {\n        await new Promise(r => setTimeout(r, 1_000 * Math.pow(2, attempt)));\n      }\n    }\n    console.error(`[webhook] Failed to deliver ${event} after ${this.opts.maxRetries + 1} attempts:`, lastErr?.message);\n  }\n\n  /** Slack-compatible message format */\n  static slackPayload(event: CrawlEventType, data: Record<string, unknown>): Record<string, unknown> {\n    const emoji = event === 'crawl.completed' ? '\u2705' : event === 'crawl.failed' ? '\u274C' : '\uD83D\uDD04';\n    return {\n      text: `${emoji} *ZeTa Crawl Event: ${event}*`,\n      blocks: [\n        { type: 'section', text: { type: 'mrkdwn', text: `${emoji} *${event}*\\n${Object.entries(data).map(([k, v]) => `\u2022 *${k}*: ${v}`).join('\\n')}` } },\n      ],\n    };\n  }\n}\n"],
+  "mappings": "AA2BA,eAAe,SAAS,QAAgB,SAAkC;AACxE,QAAM,MAAM,MAAM,OAAO,OAAO;AAAA,IAC9B;AAAA,IACA,IAAI,YAAY,EAAE,OAAO,MAAM;AAAA,IAC/B,EAAE,MAAM,QAAQ,MAAM,UAAU;AAAA,IAChC;AAAA,IACA,CAAC,MAAM;AAAA,EACT;AACA,QAAM,MAAM,MAAM,OAAO,OAAO,KAAK,QAAQ,KAAK,IAAI,YAAY,EAAE,OAAO,OAAO,CAAC;AACnF,SAAO,MAAM,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,IAAI,OAAK,EAAE,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG,CAAC,EAAE,KAAK,EAAE;AAC1F;AAEO,MAAM,gBAAgB;AAAA,EACV;AAAA,EAEjB,YAAY,MAA2B;AACrC,SAAK,OAAO;AAAA,MACV,KAAK,KAAK;AAAA,MACV,QAAQ,KAAK;AAAA,MACb,SAAS,KAAK,WAAW,CAAC;AAAA,MAC1B,WAAW,KAAK,aAAa;AAAA,MAC7B,YAAY,KAAK,cAAc;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,OACJ,OACA,OACA,WACA,UACA,OAAgC,CAAC,GAClB;AACf,UAAM,UAA0B;AAAA,MAC9B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC;AAAA,IACF;AACA,UAAM,OAAO,KAAK,UAAU,OAAO;AACnC,UAAM,UAAkC;AAAA,MACtC,gBAAgB;AAAA,MAChB,cAAc;AAAA,MACd,GAAG,KAAK,KAAK;AAAA,IACf;AACA,QAAI,KAAK,KAAK,QAAQ;AACpB,cAAQ,kBAAkB,IAAI,UAAU,MAAM,SAAS,KAAK,KAAK,QAAQ,IAAI,CAAC;AAAA,IAChF;AAEA,QAAI;AACJ,aAAS,UAAU,GAAG,WAAW,KAAK,KAAK,YAAY,WAAW;AAChE,UAAI;AACF,cAAM,MAAM,MAAM,MAAM,KAAK,KAAK,KAAK;AAAA,UACrC,QAAQ;AAAA,UACR;AAAA,UACA;AAAA,UACA,QAAQ,YAAY,QAAQ,KAAK,KAAK,SAAS;AAAA,QACjD,CAAC;AACD,YAAI,IAAI,GAAI;AACZ,kBAAU,IAAI,MAAM,gBAAgB,IAAI,MAAM,EAAE;AAAA,MAClD,SAAS,GAAG;AACV,kBAAU;AAAA,MACZ;AACA,UAAI,UAAU,KAAK,KAAK,YAAY;AAClC,cAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,MAAQ,KAAK,IAAI,GAAG,OAAO,CAAC,CAAC;AAAA,MACpE;AAAA,IACF;AACA,YAAQ,MAAM,+BAA+B,KAAK,UAAU,KAAK,KAAK,aAAa,CAAC,cAAc,SAAS,OAAO;AAAA,EACpH;AAAA;AAAA,EAGA,OAAO,aAAa,OAAuB,MAAwD;AACjG,UAAM,QAAQ,UAAU,oBAAoB,WAAM,UAAU,iBAAiB,WAAM;AACnF,WAAO;AAAA,MACL,MAAM,GAAG,KAAK,uBAAuB,KAAK;AAAA,MAC1C,QAAQ;AAAA,QACN,EAAE,MAAM,WAAW,MAAM,EAAE,MAAM,UAAU,MAAM,GAAG,KAAK,KAAK,KAAK;AAAA,EAAM,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,WAAM,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,CAAC,GAAG,EAAE;AAAA,MACjJ;AAAA,IACF;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/websocket-detector.d.ts

@@ -0,0 +1,24 @@
+/**
+ * WebSocket detection — intercepts WS connections opened during page crawl.
+ * Records connection URLs, message schemas, and message counts.
+ * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
+ * that would be invisible to HTTP-only crawlers.
+ */
+export interface WsConnection {
+    url: string;
+    framesSent: number;
+    framesReceived: number;
+    /** Sampled message bodies (first 10 per connection, truncated to 500 chars) */
+    samples: Array<{
+        direction: 'sent' | 'received';
+        payload: string;
+        ts: string;
+    }>;
+    /** Inferred protocol: 'json-rpc' | 'socket.io' | 'graphql-ws' | 'raw' */
+    protocol: string;
+    openedAt: string;
+}
+export declare function attachWebSocketDetector(cdpSession: any): {
+    getConnections(): WsConnection[];
+    detach(): void;
+};

package/dist/websocket-detector.js

@@ -0,0 +1,60 @@
+function inferProtocol(sample) {
+  try {
+    const obj = JSON.parse(sample);
+    if (obj?.jsonrpc) return "json-rpc";
+    if (obj?.type === "connection_init" || obj?.type === "subscribe") return "graphql-ws";
+    if (typeof obj?.sid === "string" || sample.startsWith("0{")) return "socket.io";
+    return "json";
+  } catch {
+    return "raw";
+  }
+}
+function attachWebSocketDetector(cdpSession) {
+  const connections = /* @__PURE__ */ new Map();
+  const onCreated = (evt) => {
+    const { requestId, url, timestamp } = evt ?? {};
+    if (!requestId || !url) return;
+    connections.set(requestId, {
+      url,
+      framesSent: 0,
+      framesReceived: 0,
+      samples: [],
+      protocol: "unknown",
+      openedAt: new Date(timestamp * 1e3).toISOString()
+    });
+  };
+  const onFrame = (evt, direction) => {
+    const { requestId, response } = evt ?? {};
+    if (!requestId) return;
+    const conn = connections.get(requestId);
+    if (!conn) return;
+    if (direction === "sent") conn.framesSent++;
+    else conn.framesReceived++;
+    if (conn.samples.length < 10 && response?.payloadData) {
+      const payload = String(response.payloadData).slice(0, 500);
+      conn.samples.push({ direction, payload, ts: (/* @__PURE__ */ new Date()).toISOString() });
+      if (conn.protocol === "unknown") conn.protocol = inferProtocol(payload);
+    }
+  };
+  const onSent = (evt) => onFrame(evt, "sent");
+  const onReceived = (evt) => onFrame(evt, "received");
+  if (cdpSession?.on) {
+    cdpSession.on("Network.webSocketCreated", onCreated);
+    cdpSession.on("Network.webSocketFrameSent", onSent);
+    cdpSession.on("Network.webSocketFrameReceived", onReceived);
+  }
+  return {
+    getConnections: () => [...connections.values()],
+    detach() {
+      if (cdpSession?.off) {
+        cdpSession.off("Network.webSocketCreated", onCreated);
+        cdpSession.off("Network.webSocketFrameSent", onSent);
+        cdpSession.off("Network.webSocketFrameReceived", onReceived);
+      }
+    }
+  };
+}
+export {
+  attachWebSocketDetector
+};
+//# sourceMappingURL=websocket-detector.js.map

package/dist/websocket-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/websocket-detector.ts"],
+  "sourcesContent": ["/**\n * WebSocket detection \u2014 intercepts WS connections opened during page crawl.\n * Records connection URLs, message schemas, and message counts.\n * This lets ZeTa discover real-time API contracts (chat, notifications, presence)\n * that would be invisible to HTTP-only crawlers.\n */\n\nexport interface WsConnection {\n  url: string;\n  framesSent: number;\n  framesReceived: number;\n  /** Sampled message bodies (first 10 per connection, truncated to 500 chars) */\n  samples: Array<{ direction: 'sent' | 'received'; payload: string; ts: string }>;\n  /** Inferred protocol: 'json-rpc' | 'socket.io' | 'graphql-ws' | 'raw' */\n  protocol: string;\n  openedAt: string;\n}\n\nfunction inferProtocol(sample: string): string {\n  try {\n    const obj = JSON.parse(sample);\n    if (obj?.jsonrpc) return 'json-rpc';\n    if (obj?.type === 'connection_init' || obj?.type === 'subscribe') return 'graphql-ws';\n    if (typeof obj?.sid === 'string' || sample.startsWith('0{')) return 'socket.io';\n    return 'json';\n  } catch {\n    return 'raw';\n  }\n}\n\nexport function attachWebSocketDetector(cdpSession: any): {\n  getConnections(): WsConnection[];\n  detach(): void;\n} {\n  const connections = new Map<string, WsConnection>();\n\n  const onCreated = (evt: any) => {\n    const { requestId, url, timestamp } = evt ?? {};\n    if (!requestId || !url) return;\n    connections.set(requestId, {\n      url,\n      framesSent: 0,\n      framesReceived: 0,\n      samples: [],\n      protocol: 'unknown',\n      openedAt: new Date(timestamp * 1000).toISOString(),\n    });\n  };\n\n  const onFrame = (evt: any, direction: 'sent' | 'received') => {\n    const { requestId, response } = evt ?? {};\n    if (!requestId) return;\n    const conn = connections.get(requestId);\n    if (!conn) return;\n    if (direction === 'sent') conn.framesSent++;\n    else conn.framesReceived++;\n    if (conn.samples.length < 10 && response?.payloadData) {\n      const payload = String(response.payloadData).slice(0, 500);\n      conn.samples.push({ direction, payload, ts: new Date().toISOString() });\n      if (conn.protocol === 'unknown') conn.protocol = inferProtocol(payload);\n    }\n  };\n\n  const onSent = (evt: any) => onFrame(evt, 'sent');\n  const onReceived = (evt: any) => onFrame(evt, 'received');\n\n  if (cdpSession?.on) {\n    cdpSession.on('Network.webSocketCreated', onCreated);\n    cdpSession.on('Network.webSocketFrameSent', onSent);\n    cdpSession.on('Network.webSocketFrameReceived', onReceived);\n  }\n\n  return {\n    getConnections: () => [...connections.values()],\n    detach() {\n      if (cdpSession?.off) {\n        cdpSession.off('Network.webSocketCreated', onCreated);\n        cdpSession.off('Network.webSocketFrameSent', onSent);\n        cdpSession.off('Network.webSocketFrameReceived', onReceived);\n      }\n    },\n  };\n}\n"],
+  "mappings": "AAkBA,SAAS,cAAc,QAAwB;AAC7C,MAAI;AACF,UAAM,MAAM,KAAK,MAAM,MAAM;AAC7B,QAAI,KAAK,QAAS,QAAO;AACzB,QAAI,KAAK,SAAS,qBAAqB,KAAK,SAAS,YAAa,QAAO;AACzE,QAAI,OAAO,KAAK,QAAQ,YAAY,OAAO,WAAW,IAAI,EAAG,QAAO;AACpE,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,wBAAwB,YAGtC;AACA,QAAM,cAAc,oBAAI,IAA0B;AAElD,QAAM,YAAY,CAAC,QAAa;AAC9B,UAAM,EAAE,WAAW,KAAK,UAAU,IAAI,OAAO,CAAC;AAC9C,QAAI,CAAC,aAAa,CAAC,IAAK;AACxB,gBAAY,IAAI,WAAW;AAAA,MACzB;AAAA,MACA,YAAY;AAAA,MACZ,gBAAgB;AAAA,MAChB,SAAS,CAAC;AAAA,MACV,UAAU;AAAA,MACV,UAAU,IAAI,KAAK,YAAY,GAAI,EAAE,YAAY;AAAA,IACnD,CAAC;AAAA,EACH;AAEA,QAAM,UAAU,CAAC,KAAU,cAAmC;AAC5D,UAAM,EAAE,WAAW,SAAS,IAAI,OAAO,CAAC;AACxC,QAAI,CAAC,UAAW;AAChB,UAAM,OAAO,YAAY,IAAI,SAAS;AACtC,QAAI,CAAC,KAAM;AACX,QAAI,cAAc,OAAQ,MAAK;AAAA,QAC1B,MAAK;AACV,QAAI,KAAK,QAAQ,SAAS,MAAM,UAAU,aAAa;AACrD,YAAM,UAAU,OAAO,SAAS,WAAW,EAAE,MAAM,GAAG,GAAG;AACzD,WAAK,QAAQ,KAAK,EAAE,WAAW,SAAS,KAAI,oBAAI,KAAK,GAAE,YAAY,EAAE,CAAC;AACtE,UAAI,KAAK,aAAa,UAAW,MAAK,WAAW,cAAc,OAAO;AAAA,IACxE;AAAA,EACF;AAEA,QAAM,SAAS,CAAC,QAAa,QAAQ,KAAK,MAAM;AAChD,QAAM,aAAa,CAAC,QAAa,QAAQ,KAAK,UAAU;AAExD,MAAI,YAAY,IAAI;AAClB,eAAW,GAAG,4BAA4B,SAAS;AACnD,eAAW,GAAG,8BAA8B,MAAM;AAClD,eAAW,GAAG,kCAAkC,UAAU;AAAA,EAC5D;AAEA,SAAO;AAAA,IACL,gBAAgB,MAAM,CAAC,GAAG,YAAY,OAAO,CAAC;AAAA,IAC9C,SAAS;AACP,UAAI,YAAY,KAAK;AACnB,mBAAW,IAAI,4BAA4B,SAAS;AACpD,mBAAW,IAAI,8BAA8B,MAAM;AACnD,mBAAW,IAAI,kCAAkC,UAAU;AAAA,MAC7D;AAAA,IACF;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/worker.js

@@ -0,0 +1,33 @@
+import { startJobWorkerLoop, processDataRetentionPurge } from "./job-queue.js";
+import { logger } from "./logger.js";
+async function main() {
+  logger.info("[worker] starting standalone crawler worker process");
+  startJobWorkerLoop();
+  const runPurge = () => {
+    processDataRetentionPurge().catch((err) => {
+      logger.error({ err: String(err) }, "[worker] data-retention purge failed");
+    });
+  };
+  runPurge();
+  setInterval(runPurge, 24 * 60 * 60 * 1e3);
+  if (process.env.JOB_QUEUE_BACKEND === "sqs") {
+    const { startSqsWorker } = await import("./sqs-queue.js");
+    startSqsWorker().catch((err) => {
+      logger.error({ err: String(err) }, "[worker] SQS worker crashed");
+      process.exit(1);
+    });
+  }
+}
+process.on("SIGTERM", () => {
+  logger.info("[worker] SIGTERM received, exiting");
+  process.exit(0);
+});
+process.on("SIGINT", () => {
+  logger.info("[worker] SIGINT received, exiting");
+  process.exit(0);
+});
+main().catch((err) => {
+  logger.error({ err: String(err) }, "[worker] fatal startup error");
+  process.exit(1);
+});
+//# sourceMappingURL=worker.js.map

package/dist/worker.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/worker.ts"],
+  "sourcesContent": ["/**\n * Standalone worker process entrypoint for a dedicated crawler/worker container,\n * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).\n *\n * index.ts is a side-effect-free library barrel imported by @qip/api \u2014 it must NOT\n * start any background loop itself, or importing a single helper from @qip/crawler\n * would silently spin up a second worker inside the API process too. This file is\n * the actual entrypoint that keeps a dedicated worker container's process alive and\n * consuming jobs: `node dist/worker.js`.\n */\nimport { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';\nimport { logger } from './logger.js';\n\nasync function main() {\n  logger.info('[worker] starting standalone crawler worker process');\n\n  // Durable DB-backed queue \u2014 claims every non-CRAWL job type, and CRAWL too when\n  // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once.\n  startJobWorkerLoop();\n\n  // Daily data-retention purge: runs immediately on startup then every 24 h.\n  // Only runs in the dedicated worker container (this file), not in the API\n  // process, so there is exactly one purge sweep regardless of API replica count.\n  const runPurge = () => {\n    processDataRetentionPurge().catch((err) => {\n      logger.error({ err: String(err) }, '[worker] data-retention purge failed');\n    });\n  };\n  runPurge();\n  setInterval(runPurge, 24 * 60 * 60 * 1000);\n\n  // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets.\n  if (process.env.JOB_QUEUE_BACKEND === 'sqs') {\n    const { startSqsWorker } = await import('./sqs-queue.js');\n    startSqsWorker().catch((err) => {\n      logger.error({ err: String(err) }, '[worker] SQS worker crashed');\n      process.exit(1);\n    });\n  }\n}\n\nprocess.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });\nprocess.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });\n\nmain().catch((err) => {\n  logger.error({ err: String(err) }, '[worker] fatal startup error');\n  process.exit(1);\n});\n"],
+  "mappings": "AAUA,SAAS,oBAAoB,iCAAiC;AAC9D,SAAS,cAAc;AAEvB,eAAe,OAAO;AACpB,SAAO,KAAK,qDAAqD;AAIjE,qBAAmB;AAKnB,QAAM,WAAW,MAAM;AACrB,8BAA0B,EAAE,MAAM,CAAC,QAAQ;AACzC,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,sCAAsC;AAAA,IAC3E,CAAC;AAAA,EACH;AACA,WAAS;AACT,cAAY,UAAU,KAAK,KAAK,KAAK,GAAI;AAGzC,MAAI,QAAQ,IAAI,sBAAsB,OAAO;AAC3C,UAAM,EAAE,eAAe,IAAI,MAAM,OAAO,gBAAgB;AACxD,mBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,6BAA6B;AAChE,cAAQ,KAAK,CAAC;AAAA,IAChB,CAAC;AAAA,EACH;AACF;AAEA,QAAQ,GAAG,WAAW,MAAM;AAAE,SAAO,KAAK,oCAAoC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AACnG,QAAQ,GAAG,UAAU,MAAM;AAAE,SAAO,KAAK,mCAAmC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AAEjG,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,SAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,8BAA8B;AACjE,UAAQ,KAAK,CAAC;AAChB,CAAC;",
+  "names": []
+}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@detiq/crawler",
+  "version": "0.1.3",
+  "private": false,
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "types/index.d.ts",
+  "scripts": {
+    "build": "node -e \"const {execSync}=require('child_process'),{readdirSync,statSync}=require('fs'),path=require('path');function walk(d){let f=[];for(const e of readdirSync(d)){const p=path.join(d,e);statSync(p).isDirectory()?f.push(...walk(p)):p.endsWith('.ts')&&!p.endsWith('.test.ts')&&!p.endsWith('.d.ts')&&f.push(p);}return f;}const files=walk('src').join(' ');execSync('esbuild '+files+' --format=esm --platform=node --target=node20 --outdir=dist --sourcemap',{stdio:'inherit'});\" && tsc -d --emitDeclarationOnly --outDir types",
+    "typecheck": "tsc --noEmit",
+    "test": "tsx --test src/**/*.test.ts"
+  },
+  "dependencies": {
+    "@aws-sdk/client-sqs": "^3.1075.0",
+    "@browserbasehq/stagehand": "^3.6.0",
+    "@mozilla/readability": "^0.6.0",
+    "@detiq/agents": "*",
+    "@detiq/app-brain": "*",
+    "@detiq/core": "*",
+    "@detiq/database": "*",
+    "axe-core": "^4.12.1",
+    "camoufox": "^0.1.19",
+    "fast-xml-parser": "^5.9.3",
+    "imapflow": "^1.4.3",
+    "lighthouse": "^13.4.0",
+    "linkedom": "^0.18.12",
+    "mailparser": "^3.9.12",
+    "pino": "^10.3.1",
+    "pino-pretty": "^13.1.3",
+    "playwright": "^1.55.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "rebrowser-patches": "^1.0.19",
+    "rebrowser-playwright-core": "^1.52.0",
+    "sharp": "^0.33.5",
+    "turndown": "^7.2.4",
+    "webdriverio": "^9.0.0",
+    "zod": "^3.25.0"
+  },
+  "devDependencies": {
+    "@types/axe-core": "^2.0.2",
+    "@types/mailparser": "^3.4.6",
+    "@types/turndown": "^5.0.5"
+  }
+}

package/src/accessibility-auditor.ts

@@ -0,0 +1,102 @@
+/**
+ * Accessibility auditor — lightweight a11y checks using Playwright's aria snapshot
+ * and manual DOM queries. No axe-core dependency needed.
+ *
+ * Checks:
+ * 1. Images without alt text
+ * 2. Form inputs without associated labels
+ * 3. Buttons/links with no accessible name
+ * 4. Missing lang attribute on html element
+ * 5. Missing page title
+ * 6. Empty headings
+ * 7. Color contrast (limited — detects text with inline style color only)
+ * 8. Keyboard-unfocusable interactive elements (tabIndex=-1 on buttons)
+ */
+
+import type { Page } from 'playwright';
+
+export interface A11yViolation {
+  rule: string;
+  severity: 'critical' | 'serious' | 'moderate' | 'minor';
+  count: number;
+  examples: string[];
+}
+
+export interface A11yAuditResult {
+  url: string;
+  violations: A11yViolation[];
+  passCount: number;
+  score: number; // 0-100
+  auditedAt: string;
+}
+
+async function queryAll(page: Page, selector: string): Promise<any[]> {
+  try {
+    return await page.$$eval(selector, els => els.map((el: any) => ({
+      outerHTML: el.outerHTML?.slice(0, 200) ?? '',
+      textContent: (el.textContent ?? '').trim().slice(0, 100),
+      tabIndex: el.tabIndex,
+      getAttribute: (a: string) => el.getAttribute(a),
+    })));
+  } catch {
+    return [];
+  }
+}
+
+export async function auditAccessibility(page: Page, url: string): Promise<A11yAuditResult> {
+  const violations: A11yViolation[] = [];
+  let passCount = 0;
+  const auditedAt = new Date().toISOString();
+
+  // 1. Images without alt
+  const imgs = await page.$$eval('img:not([alt])', els => els.map((e: any) => e.outerHTML?.slice(0, 120) ?? ''));
+  if (imgs.length > 0) violations.push({ rule: 'image-alt', severity: 'critical', count: imgs.length, examples: imgs.slice(0, 3) });
+  else passCount++;
+
+  // 2. Inputs without labels
+  const unlabeledInputs = await page.$$eval(
+    'input:not([type="hidden"]):not([type="submit"]):not([type="button"]):not([aria-label]):not([aria-labelledby]):not([id])',
+    els => els.map((e: any) => e.outerHTML?.slice(0, 120) ?? ''),
+  );
+  if (unlabeledInputs.length > 0) violations.push({ rule: 'label', severity: 'serious', count: unlabeledInputs.length, examples: unlabeledInputs.slice(0, 3) });
+  else passCount++;
+
+  // 3. Buttons with no text
+  const emptyButtons = await page.$$eval(
+    'button:not([aria-label]):not([aria-labelledby])',
+    els => els.filter((e: any) => !(e.textContent ?? '').trim()).map((e: any) => e.outerHTML?.slice(0, 120) ?? ''),
+  );
+  if (emptyButtons.length > 0) violations.push({ rule: 'button-name', severity: 'critical', count: emptyButtons.length, examples: emptyButtons.slice(0, 3) });
+  else passCount++;
+
+  // 4. Missing html lang
+  const htmlLang = await page.$eval('html', (el: any) => el.getAttribute('lang') ?? '').catch(() => '');
+  if (!htmlLang) violations.push({ rule: 'html-has-lang', severity: 'serious', count: 1, examples: ['<html> missing lang attribute'] });
+  else passCount++;
+
+  // 5. Missing page title
+  const title = await page.title().catch(() => '');
+  if (!title) violations.push({ rule: 'document-title', severity: 'serious', count: 1, examples: ['Page has no <title>'] });
+  else passCount++;
+
+  // 6. Empty headings
+  const emptyHeadings = await page.$$eval(
+    'h1,h2,h3,h4,h5,h6',
+    els => els.filter((e: any) => !(e.textContent ?? '').trim()).map((e: any) => e.tagName.toLowerCase()),
+  );
+  if (emptyHeadings.length > 0) violations.push({ rule: 'empty-heading', severity: 'moderate', count: emptyHeadings.length, examples: emptyHeadings.slice(0, 3) });
+  else passCount++;
+
+  // 7. Links with no accessible name
+  const emptyLinks = await page.$$eval(
+    'a[href]:not([aria-label]):not([aria-labelledby])',
+    els => els.filter((e: any) => !(e.textContent ?? '').trim() && !e.querySelector('img[alt]')).map((e: any) => e.outerHTML?.slice(0, 120) ?? ''),
+  );
+  if (emptyLinks.length > 0) violations.push({ rule: 'link-name', severity: 'serious', count: emptyLinks.length, examples: emptyLinks.slice(0, 3) });
+  else passCount++;
+
+  const totalChecks = passCount + violations.length;
+  const score = totalChecks > 0 ? Math.round((passCount / totalChecks) * 100) : 100;
+
+  return { url, violations, passCount, score, auditedAt };
+}