@taodigital/crawler 0.0.0-stage → 1.2.0

package/src/url-pattern-completer.ts

@@ -0,0 +1,161 @@
+/**
+ * P1: URL Pattern LLM Completion
+ *
+ * After BFS crawl completes, runs a single LLM call to predict URLs that exist
+ * but were never linked from within the app. Validates each candidate with
+ * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).
+ *
+ * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on
+ * candidate count. Does NOT launch a browser.
+ */
+
+import { logger as rootLogger } from './logger.js';
+
+const logger = rootLogger.child({ module: 'url-pattern-completer' });
+
+export interface UrlCandidate {
+  url: string;
+  confidence: number;
+  reason: string;
+  status: 'hit' | 'miss' | 'error';
+}
+
+interface LlmCfg {
+  provider: string;
+  model: string;
+  apiKey?: string;
+  baseUrl?: string;
+}
+
+// Only these two providers are supported by this module (they use standard SDK calls).
+// NVIDIA, groq, ollama, openrouter, etc. are skipped — fall through to env fallback.
+const SUPPORTED = new Set(['openai', 'anthropic']);
+
+/**
+ * Resolve the effective LLM to use. Priority: openai → anthropic → skip others.
+ * Falls back to env vars when the platform-configured provider is unsupported.
+ */
+function resolveTextLLM(llmCfg: LlmCfg | null | undefined): { provider: 'openai' | 'anthropic'; apiKey: string; model: string; baseUrl?: string } | null {
+  // Use platform config if it's a supported provider with a key
+  if (llmCfg?.provider === 'openai' && llmCfg.apiKey) {
+    return { provider: 'openai', apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl };
+  }
+  if (llmCfg?.provider === 'anthropic' && llmCfg.apiKey) {
+    return { provider: 'anthropic', apiKey: llmCfg.apiKey, model: llmCfg.model };
+  }
+  // Platform provider is unsupported (nvidia, groq, etc.) or no key — fall back to env vars.
+  // OpenAI takes priority over Anthropic.
+  if (process.env.OPENAI_API_KEY) {
+    return { provider: 'openai', apiKey: process.env.OPENAI_API_KEY, model: 'gpt-4o-mini' };
+  }
+  if (process.env.ANTHROPIC_API_KEY) {
+    return { provider: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY, model: 'claude-haiku-4-5-20251001' };
+  }
+  return null;
+}
+
+/**
+ * Returns validated URL candidates (HEAD=200) predicted by LLM from
+ * the pattern of already-discovered URLs.
+ */
+export async function runUrlPatternCompletion(
+  discoveredUrls: string[],
+  appUrl: string,
+  llmCfg: LlmCfg | null | undefined,
+): Promise<UrlCandidate[]> {
+  const effective = resolveTextLLM(llmCfg);
+  if (!effective || discoveredUrls.length < 2) return [];
+
+  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();
+  if (!origin) return [];
+
+  // Only send internal URLs (same origin, no assets)
+  const ASSET_EXT = /\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;
+  const internalUrls = discoveredUrls
+    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })
+    .slice(0, 60); // cap context size
+
+  if (internalUrls.length < 2) return [];
+
+  const systemPrompt = 'You are a URL pattern analyst. Always respond with valid JSON only.';
+  const userPrompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.
+
+Discovered URLs:
+${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\n')}
+
+Based on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found — typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.
+
+Rules:
+- Only suggest URLs on the same domain
+- Use relative paths (starting with /)
+- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them
+- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)
+- Confidence 0.0–1.0: how certain you are the page exists
+
+Respond with ONLY valid JSON, no explanation:
+{"candidates":[{"path":"/settings/billing","confidence":0.85,"reason":"settings subpage pattern from /settings/profile"},{"path":"/admin/users","confidence":0.6,"reason":"admin prefix implied by /admin/dashboard"}]}`;
+
+  try {
+    let text = '';
+
+    if (effective.provider === 'openai') {
+      const { OpenAI } = await import('openai');
+      const client = new OpenAI({ apiKey: effective.apiKey, ...(effective.baseUrl ? { baseURL: effective.baseUrl } : {}) });
+      const response = await client.chat.completions.create({
+        model: effective.model ?? 'gpt-4o-mini',
+        max_tokens: 1024,
+        temperature: 0,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: userPrompt },
+        ],
+      });
+      text = response.choices[0]?.message?.content ?? '';
+    } else {
+      const Anthropic = (await import('@anthropic-ai/sdk')).default;
+      const client = new Anthropic({ apiKey: effective.apiKey });
+      const response = await client.messages.create({
+        model: effective.model ?? 'claude-haiku-4-5-20251001',
+        max_tokens: 1024,
+        temperature: 0,
+        // @ts-ignore: cache_control supported in beta
+        system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],
+        messages: [{ role: 'user', content: userPrompt }],
+      });
+      text = (response.content[0] as any)?.text ?? '';
+    }
+
+    const jsonMatch = text.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return [];
+    const parsed = JSON.parse(jsonMatch[0]);
+    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =
+      parsed.candidates ?? [];
+
+    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];
+
+    // HEAD-validate each candidate
+    const results: UrlCandidate[] = [];
+    for (const c of rawCandidates.slice(0, 20)) {
+      if (!c.path || typeof c.path !== 'string') continue;
+      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;
+      try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 6_000);
+        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);
+        clearTimeout(timeout);
+        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';
+        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });
+        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');
+      } catch {
+        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });
+      }
+    }
+
+    const hits = results.filter(r => r.status === 'hit');
+    logger.info({ total: results.length, hits: hits.length, provider: effective.provider }, '[url-completer] pattern completion done');
+    return results;
+  } catch (err) {
+    logger.warn({ err: String(err) }, '[url-completer] LLM call failed — skipping pattern completion');
+    return [];
+  }
+}

package/src/vendor.d.ts

@@ -0,0 +1,20 @@
+declare module 'pixelmatch' {
+  function pixelmatch(
+    img1: Buffer | Uint8Array,
+    img2: Buffer | Uint8Array,
+    output: Buffer | Uint8Array | null,
+    width: number,
+    height: number,
+    options?: {
+      threshold?: number;
+      includeAA?: boolean;
+      alpha?: number;
+      aaColor?: [number, number, number];
+      diffColor?: [number, number, number];
+      diffColorAlt?: [number, number, number];
+      diffMask?: boolean;
+    }
+  ): number;
+  export = pixelmatch;
+  export default pixelmatch;
+}

package/src/visual-diff-verdict.test.ts

@@ -0,0 +1,80 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { pHashVerdict, mergeAiVisualVerdict } from './visual-diff-verdict.js';
+
+test('pHashVerdict: IDENTICAL maps to NONE/APPROVE', () => {
+  const v = pHashVerdict(0, 'IDENTICAL', []);
+  assert.equal(v.changeType, 'NONE');
+  assert.equal(v.recommendation, 'APPROVE');
+});
+
+test('pHashVerdict: MODERATE change level maps to SIGNIFICANT changeType', () => {
+  const v = pHashVerdict(12, 'MODERATE', []);
+  assert.equal(v.changeType, 'SIGNIFICANT');
+});
+
+test('pHashVerdict: SIGNIFICANT change level maps to CRITICAL changeType (naming overlap between phash and VisualDiff scales)', () => {
+  const v = pHashVerdict(35, 'SIGNIFICANT', []);
+  assert.equal(v.changeType, 'CRITICAL');
+});
+
+test('pHashVerdict: recommendation escalates with distance regardless of changeLevel', () => {
+  assert.equal(pHashVerdict(0, 'MINOR', []).recommendation, 'APPROVE');
+  assert.equal(pHashVerdict(10, 'MINOR', []).recommendation, 'REVIEW');
+  assert.equal(pHashVerdict(25, 'MINOR', []).recommendation, 'FLAG');
+});
+
+test('mergeAiVisualVerdict: AI downgrading pHash CRITICAL to MINOR (anti-aliasing noise) produces no bug report draft', () => {
+  const pHash = pHashVerdict(35, 'SIGNIFICANT', [{ location: 'header', severity: 'Cosmetic', description: 'pixel shift' }]);
+  const merged = mergeAiVisualVerdict(
+    pHash,
+    { changeType: 'MINOR', recommendation: 'APPROVE', changes: [], diffScore: 3 },
+    'Login Screen',
+  );
+  assert.equal(merged.changeType, 'MINOR');
+  assert.equal(merged.recommendation, 'APPROVE');
+  assert.equal(merged.bugReportDraft, undefined, 'a downgraded verdict must not carry a stale bug report draft');
+});
+
+test('mergeAiVisualVerdict: AI confirming CRITICAL produces a bug report draft with the AI diff score, not the pHash one', () => {
+  const pHash = pHashVerdict(35, 'SIGNIFICANT', []);
+  const merged = mergeAiVisualVerdict(
+    pHash,
+    { changeType: 'CRITICAL', recommendation: 'FLAG', changes: [{ location: 'nav', severity: 'Functional', description: 'Logo missing' }], diffScore: 62.4 },
+    'Dashboard',
+  );
+  assert.equal(merged.changeType, 'CRITICAL');
+  assert.ok(merged.bugReportDraft);
+  assert.equal(merged.bugReportDraft!.severity, 'High');
+  assert.match(merged.bugReportDraft!.actual, /62\.4%/);
+  assert.equal(merged.changes.length, 1);
+  assert.equal(merged.changes[0].description, 'Logo missing');
+});
+
+test('mergeAiVisualVerdict: SIGNIFICANT (not just CRITICAL) also gets a bug report draft, with Medium severity', () => {
+  const pHash = pHashVerdict(20, 'MODERATE', []);
+  const merged = mergeAiVisualVerdict(
+    pHash,
+    { changeType: 'SIGNIFICANT', recommendation: 'REVIEW', changes: [], diffScore: 18 },
+    'Checkout',
+  );
+  assert.ok(merged.bugReportDraft);
+  assert.equal(merged.bugReportDraft!.severity, 'Medium');
+});
+
+test('mergeAiVisualVerdict: empty AI changes array keeps the pHash-derived changes instead of wiping them', () => {
+  const pHash = pHashVerdict(35, 'SIGNIFICANT', [{ location: 'footer', severity: 'Cosmetic', description: 'layout shift' }]);
+  const merged = mergeAiVisualVerdict(
+    pHash,
+    { changeType: 'CRITICAL', recommendation: 'FLAG', changes: [], diffScore: 40 },
+    'Settings',
+  );
+  assert.equal(merged.changes.length, 1);
+  assert.equal(merged.changes[0].description, 'layout shift');
+});
+
+test('mergeAiVisualVerdict: malformed AI result (no changeType) falls back to the pHash verdict untouched', () => {
+  const pHash = pHashVerdict(35, 'SIGNIFICANT', []);
+  const merged = mergeAiVisualVerdict(pHash, { changeType: '', recommendation: '', changes: [], diffScore: 0 }, 'Home');
+  assert.deepEqual(merged, pHash);
+});

package/src/visual-diff-verdict.ts

@@ -0,0 +1,59 @@
+/**
+ * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested
+ * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and
+ * transitively @taodigital/database, which instantiates a PrismaClient at module load and
+ * so requires a live DATABASE_URL just to import the module).
+ */
+
+export interface VisualDiffVerdict {
+  changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';
+  recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';
+  changes: Array<{ location: string; severity: string; description: string }>;
+  bugReportDraft?: { title: string; severity: string; steps: string[]; expected: string; actual: string };
+  aiSummary?: string;
+}
+
+/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */
+export function pHashVerdict(
+  pHashDistance: number,
+  changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT',
+  changes: Array<{ location: string; severity: string; description: string }>,
+): VisualDiffVerdict {
+  const changeType: VisualDiffVerdict['changeType'] = changeLevel === 'IDENTICAL' ? 'NONE'
+    : changeLevel === 'MINOR' ? 'MINOR'
+      : changeLevel === 'MODERATE' ? 'SIGNIFICANT'
+        : 'CRITICAL';
+  const approveThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_APPROVE_THRESHOLD ?? '') || 5;
+  const reviewThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_REVIEW_THRESHOLD ?? '') || 20;
+  const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'
+    : pHashDistance < approveThreshold ? 'APPROVE'
+      : pHashDistance < reviewThreshold ? 'REVIEW'
+        : 'FLAG';
+  return { changeType, recommendation, changes };
+}
+
+/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,
+ *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review
+ *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards
+ *  as NONE/MINOR — only what survives AI review is worth a draft). */
+export function mergeAiVisualVerdict(
+  pHashResult: VisualDiffVerdict,
+  aiResult: { changeType: string; changes: any[]; recommendation: string; diffScore: number; summary?: string },
+  screenName: string,
+): VisualDiffVerdict {
+  if (!aiResult.changeType) return pHashResult;
+  const changeType = aiResult.changeType as VisualDiffVerdict['changeType'];
+  const recommendation = aiResult.recommendation as VisualDiffVerdict['recommendation'];
+  const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;
+  const aiSummary = aiResult.summary ?? undefined;
+  const bugReportDraft = (changeType === 'CRITICAL' || changeType === 'SIGNIFICANT')
+    ? {
+        title: `Visual regression on "${screenName}"`,
+        severity: changeType === 'CRITICAL' ? 'High' : 'Medium',
+        steps: ['Navigate to the screen', 'Compare current view to baseline screenshot'],
+        expected: 'Screen appearance should match the stored baseline.',
+        actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`,
+      }
+    : undefined;
+  return { changeType, recommendation, changes, bugReportDraft, aiSummary };
+}

package/src/webhook-notifier.ts

@@ -0,0 +1,109 @@
+/**
+ * Webhook notifier — fire HTTP POST callbacks on crawl lifecycle events.
+ *
+ * Supports: crawl.started, crawl.progress, crawl.completed, crawl.failed
+ * Payload format compatible with Slack, Discord, and generic REST webhooks.
+ * Retries 3x on failure with exponential backoff.
+ */
+
+export type CrawlEventType = 'crawl.started' | 'crawl.progress' | 'crawl.completed' | 'crawl.failed';
+
+export interface WebhookPayload {
+  event: CrawlEventType;
+  jobId: string;
+  projectId: string;
+  tenantId: string;
+  timestamp: string;
+  data: Record<string, unknown>;
+}
+
+export interface WebhookNotifierOpts {
+  url: string;
+  secret?: string;       // HMAC-SHA256 signing secret — adds X-ZeTa-Signature header
+  headers?: Record<string, string>;
+  timeoutMs?: number;    // default 10s
+  maxRetries?: number;   // default 3
+}
+
+async function hmacSign(secret: string, payload: string): Promise<string> {
+  const key = await crypto.subtle.importKey(
+    'raw',
+    new TextEncoder().encode(secret),
+    { name: 'HMAC', hash: 'SHA-256' },
+    false,
+    ['sign'],
+  );
+  const sig = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(payload));
+  return Array.from(new Uint8Array(sig)).map(b => b.toString(16).padStart(2, '0')).join('');
+}
+
+export class WebhookNotifier {
+  private readonly opts: Required<Omit<WebhookNotifierOpts, 'secret'>> & { secret?: string };
+
+  constructor(opts: WebhookNotifierOpts) {
+    this.opts = {
+      url: opts.url,
+      secret: opts.secret,
+      headers: opts.headers ?? {},
+      timeoutMs: opts.timeoutMs ?? 10_000,
+      maxRetries: opts.maxRetries ?? 3,
+    };
+  }
+
+  async notify(
+    event: CrawlEventType,
+    jobId: string,
+    projectId: string,
+    tenantId: string,
+    data: Record<string, unknown> = {},
+  ): Promise<void> {
+    const payload: WebhookPayload = {
+      event,
+      jobId,
+      projectId,
+      tenantId,
+      timestamp: new Date().toISOString(),
+      data,
+    };
+    const body = JSON.stringify(payload);
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+      'User-Agent': 'ZeTa-Crawler/1.0',
+      ...this.opts.headers,
+    };
+    if (this.opts.secret) {
+      headers['X-ZeTa-Signature'] = `sha256=${await hmacSign(this.opts.secret, body)}`;
+    }
+
+    let lastErr: any;
+    for (let attempt = 0; attempt <= this.opts.maxRetries; attempt++) {
+      try {
+        const res = await fetch(this.opts.url, {
+          method: 'POST',
+          headers,
+          body,
+          signal: AbortSignal.timeout(this.opts.timeoutMs),
+        });
+        if (res.ok) return;
+        lastErr = new Error(`Webhook HTTP ${res.status}`);
+      } catch (e) {
+        lastErr = e;
+      }
+      if (attempt < this.opts.maxRetries) {
+        await new Promise(r => setTimeout(r, 1_000 * Math.pow(2, attempt)));
+      }
+    }
+    console.error(`[webhook] Failed to deliver ${event} after ${this.opts.maxRetries + 1} attempts:`, lastErr?.message);
+  }
+
+  /** Slack-compatible message format */
+  static slackPayload(event: CrawlEventType, data: Record<string, unknown>): Record<string, unknown> {
+    const emoji = event === 'crawl.completed' ? '✅' : event === 'crawl.failed' ? '❌' : '🔄';
+    return {
+      text: `${emoji} *ZeTa Crawl Event: ${event}*`,
+      blocks: [
+        { type: 'section', text: { type: 'mrkdwn', text: `${emoji} *${event}*\n${Object.entries(data).map(([k, v]) => `• *${k}*: ${v}`).join('\n')}` } },
+      ],
+    };
+  }
+}

package/src/websocket-detector.ts

@@ -0,0 +1,83 @@
+/**
+ * WebSocket detection — intercepts WS connections opened during page crawl.
+ * Records connection URLs, message schemas, and message counts.
+ * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
+ * that would be invisible to HTTP-only crawlers.
+ */
+
+export interface WsConnection {
+  url: string;
+  framesSent: number;
+  framesReceived: number;
+  /** Sampled message bodies (first 10 per connection, truncated to 500 chars) */
+  samples: Array<{ direction: 'sent' | 'received'; payload: string; ts: string }>;
+  /** Inferred protocol: 'json-rpc' | 'socket.io' | 'graphql-ws' | 'raw' */
+  protocol: string;
+  openedAt: string;
+}
+
+function inferProtocol(sample: string): string {
+  try {
+    const obj = JSON.parse(sample);
+    if (obj?.jsonrpc) return 'json-rpc';
+    if (obj?.type === 'connection_init' || obj?.type === 'subscribe') return 'graphql-ws';
+    if (typeof obj?.sid === 'string' || sample.startsWith('0{')) return 'socket.io';
+    return 'json';
+  } catch {
+    return 'raw';
+  }
+}
+
+export function attachWebSocketDetector(cdpSession: any): {
+  getConnections(): WsConnection[];
+  detach(): void;
+} {
+  const connections = new Map<string, WsConnection>();
+
+  const onCreated = (evt: any) => {
+    const { requestId, url, timestamp } = evt ?? {};
+    if (!requestId || !url) return;
+    connections.set(requestId, {
+      url,
+      framesSent: 0,
+      framesReceived: 0,
+      samples: [],
+      protocol: 'unknown',
+      openedAt: new Date(timestamp * 1000).toISOString(),
+    });
+  };
+
+  const onFrame = (evt: any, direction: 'sent' | 'received') => {
+    const { requestId, response } = evt ?? {};
+    if (!requestId) return;
+    const conn = connections.get(requestId);
+    if (!conn) return;
+    if (direction === 'sent') conn.framesSent++;
+    else conn.framesReceived++;
+    if (conn.samples.length < 10 && response?.payloadData) {
+      const payload = String(response.payloadData).slice(0, 500);
+      conn.samples.push({ direction, payload, ts: new Date().toISOString() });
+      if (conn.protocol === 'unknown') conn.protocol = inferProtocol(payload);
+    }
+  };
+
+  const onSent = (evt: any) => onFrame(evt, 'sent');
+  const onReceived = (evt: any) => onFrame(evt, 'received');
+
+  if (cdpSession?.on) {
+    cdpSession.on('Network.webSocketCreated', onCreated);
+    cdpSession.on('Network.webSocketFrameSent', onSent);
+    cdpSession.on('Network.webSocketFrameReceived', onReceived);
+  }
+
+  return {
+    getConnections: () => [...connections.values()],
+    detach() {
+      if (cdpSession?.off) {
+        cdpSession.off('Network.webSocketCreated', onCreated);
+        cdpSession.off('Network.webSocketFrameSent', onSent);
+        cdpSession.off('Network.webSocketFrameReceived', onReceived);
+      }
+    },
+  };
+}

package/src/worker.ts

@@ -0,0 +1,50 @@
+/**
+ * Standalone worker process entrypoint for a dedicated crawler/worker container,
+ * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).
+ *
+ * index.ts is a side-effect-free library barrel imported by @taodigital/api — it must NOT
+ * start any background loop itself, or importing a single helper from @taodigital/crawler
+ * would silently spin up a second worker inside the API process too. This file is
+ * the actual entrypoint that keeps a dedicated worker container's process alive and
+ * consuming jobs: `node dist/worker.js`.
+ */
+import { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';
+import { logger } from './logger.js';
+
+async function main() {
+  logger.info('[worker] starting standalone crawler worker process');
+
+  if (process.env.JOB_QUEUE_BACKEND === 'sqs') {
+    // SQS mode: messages arrive via long-poll; claimSpecificJob atomically moves
+    // QUEUED→RUNNING before dispatch, so multiple worker replicas are safe.
+    // DB poll loop also runs as a safety net for jobs whose SQS message was lost.
+    const { startSqsWorker } = await import('./sqs-queue.js');
+    startSqsWorker().catch((err) => {
+      logger.error({ err: String(err) }, '[worker] SQS worker crashed');
+      process.exit(1);
+    });
+  }
+
+  // DB-backed poll loop — primary in non-SQS mode, safety net in SQS mode.
+  // Claims all job types. Safe to run in many worker replicas (FOR UPDATE SKIP LOCKED).
+  startJobWorkerLoop();
+
+  // Daily data-retention purge: runs immediately on startup then every 24 h.
+  // Only runs in the dedicated worker container (this file), not in the API
+  // process, so there is exactly one purge sweep regardless of API replica count.
+  const runPurge = () => {
+    processDataRetentionPurge().catch((err) => {
+      logger.error({ err: String(err) }, '[worker] data-retention purge failed');
+    });
+  };
+  runPurge();
+  setInterval(runPurge, 24 * 60 * 60 * 1000);
+}
+
+process.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });
+process.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });
+
+main().catch((err) => {
+  logger.error({ err: String(err) }, '[worker] fatal startup error');
+  process.exit(1);
+});

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "isolatedModules": true,
+    "composite": false,
+    "noEmitOnError": false
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["src/**/*.test.ts"]
+}

package/types/accessibility-auditor.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Accessibility auditor — lightweight a11y checks using Playwright's aria snapshot
+ * and manual DOM queries. No axe-core dependency needed.
+ *
+ * Checks:
+ * 1. Images without alt text
+ * 2. Form inputs without associated labels
+ * 3. Buttons/links with no accessible name
+ * 4. Missing lang attribute on html element
+ * 5. Missing page title
+ * 6. Empty headings
+ * 7. Color contrast (limited — detects text with inline style color only)
+ * 8. Keyboard-unfocusable interactive elements (tabIndex=-1 on buttons)
+ */
+import type { Page } from 'playwright';
+export interface A11yViolation {
+    rule: string;
+    severity: 'critical' | 'serious' | 'moderate' | 'minor';
+    count: number;
+    examples: string[];
+}
+export interface A11yAuditResult {
+    url: string;
+    violations: A11yViolation[];
+    passCount: number;
+    score: number;
+    auditedAt: string;
+}
+export declare function auditAccessibility(page: Page, url: string): Promise<A11yAuditResult>;

package/types/accessibility-tree-crawler.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Accessibility tree crawler — extract landmark, heading, and ARIA metadata from crawled pages.
+ *
+ * Captures: landmarks with interactive counts, heading hierarchy with violation detection,
+ * interactive elements missing accessible names, live regions, and skip-link presence.
+ *
+ * This data feeds ZeTa's accessibility test generator to surface WCAG 2.1 Level A/AA
+ * failures (missing names, heading skips, absent skip links, unlabeled live regions).
+ */
+export interface LandmarkInfo {
+    role: string;
+    label: string;
+    elementId: string;
+    interactiveCount: number;
+}
+export interface HeadingInfo {
+    level: number;
+    text: string;
+    elementId: string;
+}
+export interface MissingNameEl {
+    tagName: string;
+    role: string;
+    elementId: string;
+    htmlSnippet: string;
+}
+export interface LiveRegionInfo {
+    role: string;
+    ariaLive: string;
+    ariaAtomic: string;
+    currentText: string;
+}
+export interface AccessibilityTreeResult {
+    landmarkCount: number;
+    landmarks: LandmarkInfo[];
+    headings: HeadingInfo[];
+    headingHierarchyViolations: string[];
+    missingNameElements: MissingNameEl[];
+    missingNameCount: number;
+    liveRegions: LiveRegionInfo[];
+    totalInteractiveCount: number;
+    hasSkipLink: boolean;
+}
+export declare function crawlAccessibilityTree(page: any): Promise<AccessibilityTreeResult>;