@taodigital/crawler 0.0.0-stage → 1.2.0

package/dist/worker.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/worker.ts"],
+  "sourcesContent": ["/**\n * Standalone worker process entrypoint for a dedicated crawler/worker container,\n * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).\n *\n * index.ts is a side-effect-free library barrel imported by @detiq/api \u2014 it must NOT\n * start any background loop itself, or importing a single helper from @detiq/crawler\n * would silently spin up a second worker inside the API process too. This file is\n * the actual entrypoint that keeps a dedicated worker container's process alive and\n * consuming jobs: `node dist/worker.js`.\n */\nimport { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';\nimport { logger } from './logger.js';\n\nasync function main() {\n  logger.info('[worker] starting standalone crawler worker process');\n\n  if (process.env.JOB_QUEUE_BACKEND === 'sqs') {\n    // SQS mode: messages arrive via long-poll; claimSpecificJob atomically moves\n    // QUEUED\u2192RUNNING before dispatch, so multiple worker replicas are safe.\n    // DB poll loop also runs as a safety net for jobs whose SQS message was lost.\n    const { startSqsWorker } = await import('./sqs-queue.js');\n    startSqsWorker().catch((err) => {\n      logger.error({ err: String(err) }, '[worker] SQS worker crashed');\n      process.exit(1);\n    });\n  }\n\n  // DB-backed poll loop \u2014 primary in non-SQS mode, safety net in SQS mode.\n  // Claims all job types. Safe to run in many worker replicas (FOR UPDATE SKIP LOCKED).\n  startJobWorkerLoop();\n\n  // Daily data-retention purge: runs immediately on startup then every 24 h.\n  // Only runs in the dedicated worker container (this file), not in the API\n  // process, so there is exactly one purge sweep regardless of API replica count.\n  const runPurge = () => {\n    processDataRetentionPurge().catch((err) => {\n      logger.error({ err: String(err) }, '[worker] data-retention purge failed');\n    });\n  };\n  runPurge();\n  setInterval(runPurge, 24 * 60 * 60 * 1000);\n}\n\nprocess.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });\nprocess.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });\n\nmain().catch((err) => {\n  logger.error({ err: String(err) }, '[worker] fatal startup error');\n  process.exit(1);\n});\n"],
+  "mappings": "AAUA,SAAS,oBAAoB,iCAAiC;AAC9D,SAAS,cAAc;AAEvB,eAAe,OAAO;AACpB,SAAO,KAAK,qDAAqD;AAEjE,MAAI,QAAQ,IAAI,sBAAsB,OAAO;AAI3C,UAAM,EAAE,eAAe,IAAI,MAAM,OAAO,gBAAgB;AACxD,mBAAe,EAAE,MAAM,CAAC,QAAQ;AAC9B,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,6BAA6B;AAChE,cAAQ,KAAK,CAAC;AAAA,IAChB,CAAC;AAAA,EACH;AAIA,qBAAmB;AAKnB,QAAM,WAAW,MAAM;AACrB,8BAA0B,EAAE,MAAM,CAAC,QAAQ;AACzC,aAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,sCAAsC;AAAA,IAC3E,CAAC;AAAA,EACH;AACA,WAAS;AACT,cAAY,UAAU,KAAK,KAAK,KAAK,GAAI;AAC3C;AAEA,QAAQ,GAAG,WAAW,MAAM;AAAE,SAAO,KAAK,oCAAoC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AACnG,QAAQ,GAAG,UAAU,MAAM;AAAE,SAAO,KAAK,mCAAmC;AAAG,UAAQ,KAAK,CAAC;AAAG,CAAC;AAEjG,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,SAAO,MAAM,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,8BAA8B;AACjE,UAAQ,KAAK,CAAC;AAChB,CAAC;",
+  "names": []
+}

package/package.json

@@ -1,6 +1,51 @@
 {
   "name": "@taodigital/crawler",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "types/index.d.ts",
+  "scripts": {
+    "build": "node -e \"const {execSync}=require('child_process'),{readdirSync,statSync}=require('fs'),path=require('path');function walk(d){let f=[];for(const e of readdirSync(d)){const p=path.join(d,e);statSync(p).isDirectory()?f.push(...walk(p)):p.endsWith('.ts')&&!p.endsWith('.test.ts')&&!p.endsWith('.d.ts')&&f.push(p);}return f;}const files=walk('src').join(' ');execSync('esbuild '+files+' --format=esm --platform=node --target=node20 --outdir=dist --sourcemap',{stdio:'inherit'});\" && tsc -d --emitDeclarationOnly --outDir types",
+    "typecheck": "tsc --noEmit",
+    "test": "tsx --test --test-force-exit src/**/*.test.ts"
+  },
+  "dependencies": {
+    "@aws-sdk/client-sqs": "^3.1075.0",
+    "@browserbasehq/stagehand": "^3.6.0",
+    "@taodigital/agents": "*",
+    "@taodigital/app-brain": "*",
+    "@taodigital/core": "*",
+    "@taodigital/database": "*",
+    "@mozilla/readability": "^0.6.0",
+    "axe-core": "^4.12.1",
+    "camoufox": "^0.1.19",
+    "fast-xml-parser": "^5.10.1",
+    "minimatch": "^10.0.0",
+    "imapflow": "^1.4.3",
+    "lighthouse": "^13.4.0",
+    "linkedom": "^0.18.12",
+    "mailparser": "^3.9.12",
+    "pino": "^10.3.1",
+    "pino-pretty": "^13.1.3",
+    "pixelmatch": "^6.0.0",
+    "playwright": "^1.55.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "rebrowser-patches": "^1.0.19",
+    "rebrowser-playwright-core": "^1.52.0",
+    "sharp": "^0.35.0",
+    "turndown": "^7.2.4",
+    "webdriverio": "^9.0.0",
+    "zod": "^3.25.0"
+  },
+  "devDependencies": {
+    "@types/axe-core": "^2.0.2",
+    "@types/mailparser": "^3.4.6",
+    "@types/turndown": "^5.0.5"
+  }
+}

package/src/accessibility-auditor.ts

@@ -0,0 +1,102 @@
+/**
+ * Accessibility auditor — lightweight a11y checks using Playwright's aria snapshot
+ * and manual DOM queries. No axe-core dependency needed.
+ *
+ * Checks:
+ * 1. Images without alt text
+ * 2. Form inputs without associated labels
+ * 3. Buttons/links with no accessible name
+ * 4. Missing lang attribute on html element
+ * 5. Missing page title
+ * 6. Empty headings
+ * 7. Color contrast (limited — detects text with inline style color only)
+ * 8. Keyboard-unfocusable interactive elements (tabIndex=-1 on buttons)
+ */
+
+import type { Page } from 'playwright';
+
+export interface A11yViolation {
+  rule: string;
+  severity: 'critical' | 'serious' | 'moderate' | 'minor';
+  count: number;
+  examples: string[];
+}
+
+export interface A11yAuditResult {
+  url: string;
+  violations: A11yViolation[];
+  passCount: number;
+  score: number; // 0-100
+  auditedAt: string;
+}
+
+async function queryAll(page: Page, selector: string): Promise<any[]> {
+  try {
+    return await page.$$eval(selector, els => els.map((el: any) => ({
+      outerHTML: el.outerHTML?.slice(0, 200) ?? '',
+      textContent: (el.textContent ?? '').trim().slice(0, 100),
+      tabIndex: el.tabIndex,
+      getAttribute: (a: string) => el.getAttribute(a),
+    })));
+  } catch {
+    return [];
+  }
+}
+
+export async function auditAccessibility(page: Page, url: string): Promise<A11yAuditResult> {
+  const violations: A11yViolation[] = [];
+  let passCount = 0;
+  const auditedAt = new Date().toISOString();
+
+  // 1. Images without alt
+  const imgs = await page.$$eval('img:not([alt])', els => els.map((e: any) => e.outerHTML?.slice(0, 120) ?? ''));
+  if (imgs.length > 0) violations.push({ rule: 'image-alt', severity: 'critical', count: imgs.length, examples: imgs.slice(0, 3) });
+  else passCount++;
+
+  // 2. Inputs without labels
+  const unlabeledInputs = await page.$$eval(
+    'input:not([type="hidden"]):not([type="submit"]):not([type="button"]):not([aria-label]):not([aria-labelledby]):not([id])',
+    els => els.map((e: any) => e.outerHTML?.slice(0, 120) ?? ''),
+  );
+  if (unlabeledInputs.length > 0) violations.push({ rule: 'label', severity: 'serious', count: unlabeledInputs.length, examples: unlabeledInputs.slice(0, 3) });
+  else passCount++;
+
+  // 3. Buttons with no text
+  const emptyButtons = await page.$$eval(
+    'button:not([aria-label]):not([aria-labelledby])',
+    els => els.filter((e: any) => !(e.textContent ?? '').trim()).map((e: any) => e.outerHTML?.slice(0, 120) ?? ''),
+  );
+  if (emptyButtons.length > 0) violations.push({ rule: 'button-name', severity: 'critical', count: emptyButtons.length, examples: emptyButtons.slice(0, 3) });
+  else passCount++;
+
+  // 4. Missing html lang
+  const htmlLang = await page.$eval('html', (el: any) => el.getAttribute('lang') ?? '').catch(() => '');
+  if (!htmlLang) violations.push({ rule: 'html-has-lang', severity: 'serious', count: 1, examples: ['<html> missing lang attribute'] });
+  else passCount++;
+
+  // 5. Missing page title
+  const title = await page.title().catch(() => '');
+  if (!title) violations.push({ rule: 'document-title', severity: 'serious', count: 1, examples: ['Page has no <title>'] });
+  else passCount++;
+
+  // 6. Empty headings
+  const emptyHeadings = await page.$$eval(
+    'h1,h2,h3,h4,h5,h6',
+    els => els.filter((e: any) => !(e.textContent ?? '').trim()).map((e: any) => e.tagName.toLowerCase()),
+  );
+  if (emptyHeadings.length > 0) violations.push({ rule: 'empty-heading', severity: 'moderate', count: emptyHeadings.length, examples: emptyHeadings.slice(0, 3) });
+  else passCount++;
+
+  // 7. Links with no accessible name
+  const emptyLinks = await page.$$eval(
+    'a[href]:not([aria-label]):not([aria-labelledby])',
+    els => els.filter((e: any) => !(e.textContent ?? '').trim() && !e.querySelector('img[alt]')).map((e: any) => e.outerHTML?.slice(0, 120) ?? ''),
+  );
+  if (emptyLinks.length > 0) violations.push({ rule: 'link-name', severity: 'serious', count: emptyLinks.length, examples: emptyLinks.slice(0, 3) });
+  else passCount++;
+
+  const totalChecks = passCount + violations.length;
+  const score = totalChecks > 0 ? Math.round((passCount / totalChecks) * 100) : 100;
+
+  return { url, violations, passCount, score, auditedAt };
+}

package/src/accessibility-tree-crawler.ts

@@ -0,0 +1,246 @@
+/**
+ * Accessibility tree crawler — extract landmark, heading, and ARIA metadata from crawled pages.
+ *
+ * Captures: landmarks with interactive counts, heading hierarchy with violation detection,
+ * interactive elements missing accessible names, live regions, and skip-link presence.
+ *
+ * This data feeds ZeTa's accessibility test generator to surface WCAG 2.1 Level A/AA
+ * failures (missing names, heading skips, absent skip links, unlabeled live regions).
+ */
+
+import type { Page } from 'playwright';
+
+export interface LandmarkInfo {
+  role: string;
+  label: string;
+  elementId: string;
+  interactiveCount: number;
+}
+
+export interface HeadingInfo {
+  level: number;
+  text: string;
+  elementId: string;
+}
+
+export interface MissingNameEl {
+  tagName: string;
+  role: string;
+  elementId: string;
+  htmlSnippet: string;
+}
+
+export interface LiveRegionInfo {
+  role: string;
+  ariaLive: string;
+  ariaAtomic: string;
+  currentText: string;
+}
+
+export interface AccessibilityTreeResult {
+  landmarkCount: number;
+  landmarks: LandmarkInfo[];
+  headings: HeadingInfo[];
+  headingHierarchyViolations: string[];
+  missingNameElements: MissingNameEl[];
+  missingNameCount: number;
+  liveRegions: LiveRegionInfo[];
+  totalInteractiveCount: number;
+  hasSkipLink: boolean;
+}
+
+async function extractLandmarks(page: any): Promise<LandmarkInfo[]> {
+  try {
+    const raw = await page.evaluate(() => {
+      const LANDMARK_SELECTORS = [
+        { role: 'banner', selector: 'header,[role="banner"]' },
+        { role: 'navigation', selector: 'nav,[role="navigation"]' },
+        { role: 'main', selector: 'main,[role="main"]' },
+        { role: 'complementary', selector: 'aside,[role="complementary"]' },
+        { role: 'contentinfo', selector: 'footer,[role="contentinfo"]' },
+        { role: 'search', selector: '[role="search"]' },
+        { role: 'form', selector: 'form[aria-label],[role="form"]' },
+        { role: 'region', selector: '[role="region"][aria-label]' },
+      ];
+      return LANDMARK_SELECTORS.flatMap(({ role, selector }) =>
+        [...document.querySelectorAll(selector)].slice(0, 10).map(el => ({
+          role,
+          label: el.getAttribute('aria-label') || el.getAttribute('aria-labelledby') || '',
+          elementId: el.id || '',
+          interactiveCount: el.querySelectorAll(
+            'button,a[href],input,select,textarea,[role="button"],[tabindex="0"]'
+          ).length,
+        }))
+      );
+    });
+    return raw.slice(0, 50);
+  } catch {
+    return [];
+  }
+}
+
+async function extractHeadings(page: any): Promise<HeadingInfo[]> {
+  try {
+    const raw = await page.evaluate(() => {
+      const headingEls = [...document.querySelectorAll('h1,h2,h3,h4,h5,h6,[role="heading"]')];
+      return headingEls
+        .map(el => ({
+          level:
+            parseInt(el.tagName.slice(1)) ||
+            parseInt(el.getAttribute('aria-level') || '0') ||
+            0,
+          text: el.textContent?.trim().slice(0, 100) || '',
+          elementId: el.id || '',
+        }))
+        .filter(h => h.level >= 1 && h.level <= 6);
+    });
+    return raw.slice(0, 100);
+  } catch {
+    return [];
+  }
+}
+
+function detectHeadingViolations(headings: HeadingInfo[]): string[] {
+  const violations: string[] = [];
+  if (headings.length === 0) return violations;
+
+  const h1s = headings.filter(h => h.level === 1);
+  if (h1s.length === 0) {
+    violations.push('Missing h1 landmark');
+  } else if (h1s.length > 1) {
+    violations.push('Multiple h1 elements found');
+  }
+
+  let prevLevel = 0;
+  for (const heading of headings) {
+    if (prevLevel > 0 && heading.level > prevLevel + 1) {
+      violations.push(
+        `Heading level skip: h${prevLevel} to h${heading.level}`
+      );
+    }
+    prevLevel = heading.level;
+  }
+
+  return violations;
+}
+
+async function findMissingNames(page: any): Promise<MissingNameEl[]> {
+  try {
+    return await page.evaluate(() => {
+      const INTERACTIVE_SELECTOR =
+        'button,a[href],input:not([type="hidden"]),select,textarea,' +
+        '[role="button"],[role="link"],[role="checkbox"],[role="radio"],' +
+        '[role="switch"],[role="menuitem"],[role="tab"]';
+      return [...document.querySelectorAll(INTERACTIVE_SELECTOR)]
+        .filter(el => {
+          const labelledById = el.getAttribute('aria-labelledby') || '';
+          const name =
+            (el.getAttribute('aria-label') || '').trim() ||
+            (labelledById
+              ? document.getElementById(labelledById)?.textContent?.trim() ?? ''
+              : '') ||
+            (el.textContent || '').trim() ||
+            ((el as HTMLInputElement).placeholder || '').trim() ||
+            ((el as HTMLElement).title || '').trim() ||
+            ((el as HTMLImageElement).alt || '').trim();
+          return !name || name.length === 0;
+        })
+        .slice(0, 30)
+        .map(el => ({
+          tagName: el.tagName.toLowerCase(),
+          role: el.getAttribute('role') || '',
+          elementId: el.id || '',
+          htmlSnippet: el.outerHTML.slice(0, 120),
+        }));
+    });
+  } catch {
+    return [];
+  }
+}
+
+async function extractLiveRegions(page: any): Promise<LiveRegionInfo[]> {
+  try {
+    return await page.evaluate(() => {
+      const els = [
+        ...document.querySelectorAll(
+          '[aria-live],[role="alert"],[role="status"],[role="log"]'
+        ),
+      ];
+      return els.slice(0, 20).map(el => ({
+        role: el.getAttribute('role') || '',
+        ariaLive: el.getAttribute('aria-live') || '',
+        ariaAtomic: el.getAttribute('aria-atomic') || '',
+        currentText: (el.textContent || '').trim().slice(0, 200),
+      }));
+    });
+  } catch {
+    return [];
+  }
+}
+
+async function checkSkipLink(page: any): Promise<boolean> {
+  try {
+    return await page.evaluate(() => {
+      const firstFocusable = document.querySelector('a[href^="#"],a:first-of-type');
+      return !!(
+        firstFocusable &&
+        /skip/i.test(
+          (firstFocusable.textContent || '') + (firstFocusable.getAttribute('href') || '')
+        )
+      );
+    });
+  } catch {
+    return false;
+  }
+}
+
+export async function crawlAccessibilityTree(page: any): Promise<AccessibilityTreeResult> {
+  const [
+    landmarksResult,
+    headingsResult,
+    missingNamesResult,
+    liveRegionsResult,
+    skipLinkResult,
+    interactiveCountResult,
+  ] = await Promise.allSettled([
+    extractLandmarks(page),
+    extractHeadings(page),
+    findMissingNames(page),
+    extractLiveRegions(page),
+    checkSkipLink(page),
+    page.evaluate(() => {
+      const INTERACTIVE_SELECTOR =
+        'button,a[href],input:not([type="hidden"]),select,textarea,' +
+        '[role="button"],[role="link"],[role="checkbox"],[role="radio"],' +
+        '[role="switch"],[role="menuitem"],[role="tab"]';
+      return document.querySelectorAll(INTERACTIVE_SELECTOR).length;
+    }).catch(() => 0),
+  ]);
+
+  const landmarks =
+    landmarksResult.status === 'fulfilled' ? landmarksResult.value : [];
+  const headings =
+    headingsResult.status === 'fulfilled' ? headingsResult.value : [];
+  const missingNameElements =
+    missingNamesResult.status === 'fulfilled' ? missingNamesResult.value : [];
+  const liveRegions =
+    liveRegionsResult.status === 'fulfilled' ? liveRegionsResult.value : [];
+  const hasSkipLink =
+    skipLinkResult.status === 'fulfilled' ? skipLinkResult.value : false;
+  const totalInteractiveCount =
+    interactiveCountResult.status === 'fulfilled' ? interactiveCountResult.value : 0;
+
+  const headingHierarchyViolations = detectHeadingViolations(headings);
+
+  return {
+    landmarkCount: landmarks.length,
+    landmarks,
+    headings,
+    headingHierarchyViolations,
+    missingNameElements,
+    missingNameCount: missingNameElements.length,
+    liveRegions,
+    totalInteractiveCount,
+    hasSkipLink,
+  };
+}