@taodigital/crawler 0.0.0-stage → 1.2.0

package/src/performance-metrics-collector.ts

@@ -0,0 +1,243 @@
+/**
+ * Performance metrics collector — captures Core Web Vitals and navigation timing
+ * for pages visited during crawl runs.
+ * Collects LCP, CLS, INP, TTFB, long tasks, and slow resources via PerformanceObserver
+ * and the Navigation/Resource Timing APIs.
+ */
+
+import type { Page } from 'playwright';
+
+export type CwvScore = 'good' | 'needs-improvement' | 'poor';
+
+export interface LcpMetric {
+  value: number;
+  score: CwvScore;
+  elementTag: string | null;
+  url: string | null;
+}
+
+export interface ClsMetric {
+  value: number;
+  score: CwvScore;
+  shiftCount: number;
+}
+
+export interface InpMetric {
+  p75: number;
+  score: CwvScore;
+  sampleCount: number;
+}
+
+export interface NavTimingMetric {
+  ttfb: number;
+  ttfbScore: CwvScore;
+  domContentLoaded: number;
+  domComplete: number;
+  loadComplete: number;
+  transferSizeBytes: number;
+  protocol: string;
+}
+
+export interface SlowResource {
+  path: string;
+  durationMs: number;
+  transferSizeBytes: number;
+  initiatorType: string;
+}
+
+export interface PerformanceMetricsResult {
+  lcp: LcpMetric | null;
+  cls: ClsMetric | null;
+  inp: InpMetric | null;
+  navTiming: NavTimingMetric | null;
+  slowResources: SlowResource[];
+  longTaskCount: number;
+  captureSuccess: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Observer injection — call before page navigation via addInitScript
+// ---------------------------------------------------------------------------
+
+export async function injectPerformanceObservers(page: Page): Promise<void> {
+  await page.addInitScript(() => {
+    (window as any).__zetaPerf = { lcp: null, cls: 0, clsCount: 0, inp: [], longTaskCount: 0 };
+
+    try {
+      new PerformanceObserver(list => {
+        const entries = list.getEntries();
+        const last = entries[entries.length - 1] as any;
+        if (last) {
+          (window as any).__zetaPerf.lcp = {
+            value: last.startTime,
+            elementTag: last.element?.tagName || null,
+            url: last.url || null,
+          };
+        }
+      }).observe({ entryTypes: ['largest-contentful-paint'] });
+    } catch {}
+
+    try {
+      new PerformanceObserver(list => {
+        for (const entry of list.getEntries() as any[]) {
+          if (!entry.hadRecentInput) {
+            (window as any).__zetaPerf.cls += entry.value;
+            (window as any).__zetaPerf.clsCount++;
+          }
+        }
+      }).observe({ entryTypes: ['layout-shift'] });
+    } catch {}
+
+    try {
+      new PerformanceObserver(list => {
+        for (const entry of list.getEntries() as any[]) {
+          (window as any).__zetaPerf.inp.push(entry.duration);
+        }
+      }).observe({ entryTypes: ['event'], durationThreshold: 40 } as any);
+    } catch {}
+
+    try {
+      new PerformanceObserver(list => {
+        (window as any).__zetaPerf.longTaskCount += list.getEntries().length;
+      }).observe({ entryTypes: ['longtask'] });
+    } catch {}
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Scoring helpers
+// ---------------------------------------------------------------------------
+
+function scoreLcp(value: number): CwvScore {
+  if (value <= 2500) return 'good';
+  if (value <= 4000) return 'needs-improvement';
+  return 'poor';
+}
+
+function scoreCls(value: number): CwvScore {
+  if (value <= 0.1) return 'good';
+  if (value <= 0.25) return 'needs-improvement';
+  return 'poor';
+}
+
+function scoreInp(p75: number): CwvScore {
+  if (p75 <= 200) return 'good';
+  if (p75 <= 500) return 'needs-improvement';
+  return 'poor';
+}
+
+function scoreTtfb(value: number): CwvScore {
+  if (value <= 800) return 'good';
+  if (value <= 1800) return 'needs-improvement';
+  return 'poor';
+}
+
+// ---------------------------------------------------------------------------
+// Internal collector
+// ---------------------------------------------------------------------------
+
+async function collectMetrics(page: Page): Promise<PerformanceMetricsResult> {
+  const rawMetrics = await page.evaluate(() => {
+    const p = (window as any).__zetaPerf || {};
+    const nav = performance.getEntriesByType('navigation')[0] as PerformanceNavigationTiming | undefined;
+    const resources = performance.getEntriesByType('resource')
+      .filter(r => r.duration > 500)
+      .sort((a, b) => b.duration - a.duration)
+      .slice(0, 10)
+      .map(r => ({
+        path: (() => { try { return new URL(r.name).pathname; } catch { return r.name.slice(0, 80); } })(),
+        durationMs: Math.round(r.duration),
+        transferSizeBytes: (r as any).transferSize || 0,
+        initiatorType: (r as any).initiatorType || '',
+      }));
+    const inpValues = p.inp || [];
+    const p75 = inpValues.length > 0
+      ? inpValues.sort((a: number, b: number) => a - b)[Math.floor(inpValues.length * 0.75)]
+      : 0;
+    return {
+      lcp: p.lcp || null,
+      cls: p.cls ?? null,
+      clsCount: p.clsCount ?? 0,
+      inpP75: p75,
+      inpCount: inpValues.length,
+      longTaskCount: p.longTaskCount || 0,
+      nav: nav ? {
+        ttfb: Math.round(nav.responseStart - nav.requestStart),
+        domContentLoaded: Math.round(nav.domContentLoadedEventEnd - nav.startTime),
+        domComplete: Math.round(nav.domComplete - nav.startTime),
+        loadComplete: Math.round(nav.loadEventEnd - nav.startTime),
+        transferSizeBytes: (nav as any).transferSize || 0,
+        protocol: (nav as any).nextHopProtocol || '',
+      } : null,
+      resources,
+    };
+  });
+
+  const lcp: LcpMetric | null = rawMetrics.lcp
+    ? {
+        value: rawMetrics.lcp.value,
+        score: scoreLcp(rawMetrics.lcp.value),
+        elementTag: rawMetrics.lcp.elementTag,
+        url: rawMetrics.lcp.url,
+      }
+    : null;
+
+  const cls: ClsMetric | null = rawMetrics.cls !== null
+    ? {
+        value: rawMetrics.cls,
+        score: scoreCls(rawMetrics.cls),
+        shiftCount: rawMetrics.clsCount,
+      }
+    : null;
+
+  const inp: InpMetric | null = rawMetrics.inpCount > 0
+    ? {
+        p75: rawMetrics.inpP75,
+        score: scoreInp(rawMetrics.inpP75),
+        sampleCount: rawMetrics.inpCount,
+      }
+    : null;
+
+  const navTiming: NavTimingMetric | null = rawMetrics.nav
+    ? {
+        ttfb: rawMetrics.nav.ttfb,
+        ttfbScore: scoreTtfb(rawMetrics.nav.ttfb),
+        domContentLoaded: rawMetrics.nav.domContentLoaded,
+        domComplete: rawMetrics.nav.domComplete,
+        loadComplete: rawMetrics.nav.loadComplete,
+        transferSizeBytes: rawMetrics.nav.transferSizeBytes,
+        protocol: rawMetrics.nav.protocol,
+      }
+    : null;
+
+  return {
+    lcp,
+    cls,
+    inp,
+    navTiming,
+    slowResources: rawMetrics.resources,
+    longTaskCount: rawMetrics.longTaskCount,
+    captureSuccess: true,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+const emptyResult: Omit<PerformanceMetricsResult, 'captureSuccess'> = {
+  lcp: null,
+  cls: null,
+  inp: null,
+  navTiming: null,
+  slowResources: [],
+  longTaskCount: 0,
+};
+
+export async function collectPagePerformance(page: Page): Promise<PerformanceMetricsResult> {
+  try {
+    return await collectMetrics(page);
+  } catch {
+    return { ...emptyResult, captureSuccess: false };
+  }
+}

package/src/performance-metrics.ts

@@ -0,0 +1,134 @@
+/**
+ * Performance metrics collection via CDP + Playwright page.evaluate().
+ *
+ * Collects Core Web Vitals and resource timing after page navigation.
+ * Stored alongside screen data for test generation context and regression detection.
+ *
+ * Metrics:
+ * - LCP (Largest Contentful Paint) — via PerformanceObserver in page
+ * - CLS (Cumulative Layout Shift) — via PerformanceObserver in page
+ * - TTFB (Time to First Byte) — from navigation timing
+ * - DOMContentLoaded — from navigation timing
+ * - Resource count and size by type
+ */
+
+import type { Page } from 'playwright';
+
+export interface CoreWebVitals {
+  lcp?: number;   // ms
+  cls?: number;   // unitless score
+  ttfb?: number;  // ms
+  fcp?: number;   // First Contentful Paint ms
+  domContentLoaded?: number; // ms
+  loadEventEnd?: number;     // ms
+}
+
+export interface ResourceSummary {
+  scripts: { count: number; totalBytes: number };
+  styles: { count: number; totalBytes: number };
+  images: { count: number; totalBytes: number };
+  fonts: { count: number; totalBytes: number };
+  other: { count: number; totalBytes: number };
+}
+
+export interface PagePerfMetrics {
+  url: string;
+  vitals: CoreWebVitals;
+  resources: ResourceSummary;
+  totalTransferBytes: number;
+  requestCount: number;
+  measuredAt: string;
+}
+
+export async function collectPerfMetrics(page: any, url: string): Promise<PagePerfMetrics> {
+  const measuredAt = new Date().toISOString();
+
+  // Inject PerformanceObserver for LCP + CLS, wait up to 3s
+  let vitals: CoreWebVitals = {};
+  try {
+    vitals = await page.evaluate(() => {
+      return new Promise<any>((resolve) => {
+        const result: any = {};
+
+        // Navigation timing
+        const nav = performance.getEntriesByType('navigation')[0] as any;
+        if (nav) {
+          result.ttfb = Math.round(nav.responseStart - nav.requestStart);
+          result.domContentLoaded = Math.round(nav.domContentLoadedEventEnd);
+          result.loadEventEnd = Math.round(nav.loadEventEnd);
+        }
+
+        let lcpDone = false;
+        let clsDone = false;
+        const tryResolve = () => { if (lcpDone && clsDone) resolve(result); };
+
+        // LCP
+        try {
+          new PerformanceObserver((list) => {
+            const entries = list.getEntries();
+            if (entries.length > 0) result.lcp = Math.round((entries[entries.length - 1] as any).startTime);
+            lcpDone = true;
+            tryResolve();
+          }).observe({ type: 'largest-contentful-paint', buffered: true });
+        } catch { lcpDone = true; }
+
+        // CLS
+        try {
+          let cls = 0;
+          new PerformanceObserver((list) => {
+            for (const e of list.getEntries() as any[]) {
+              if (!e.hadRecentInput) cls += e.value;
+            }
+            result.cls = Math.round(cls * 1000) / 1000;
+            clsDone = true;
+            tryResolve();
+          }).observe({ type: 'layout-shift', buffered: true });
+        } catch { clsDone = true; }
+
+        // FCP
+        try {
+          const fcp = performance.getEntriesByName('first-contentful-paint')[0] as any;
+          if (fcp) result.fcp = Math.round(fcp.startTime);
+        } catch {}
+
+        // Fallback resolve after 3s
+        setTimeout(() => resolve(result), 3_000);
+      });
+    }).catch(() => ({}));
+  } catch { /* page closed or navigate already */ }
+
+  // Resource timing
+  let resources: ResourceSummary = {
+    scripts: { count: 0, totalBytes: 0 },
+    styles: { count: 0, totalBytes: 0 },
+    images: { count: 0, totalBytes: 0 },
+    fonts: { count: 0, totalBytes: 0 },
+    other: { count: 0, totalBytes: 0 },
+  };
+  let totalTransferBytes = 0;
+  let requestCount = 0;
+
+  try {
+    const entries = await page.evaluate(() =>
+      performance.getEntriesByType('resource').map((e: any) => ({
+        name: e.name,
+        initiatorType: e.initiatorType,
+        transferSize: e.transferSize ?? 0,
+      }))
+    ).catch(() => [] as any[]);
+
+    for (const e of entries) {
+      requestCount++;
+      totalTransferBytes += e.transferSize;
+      const cat = e.initiatorType === 'script' ? 'scripts'
+        : e.initiatorType === 'css' || e.name?.endsWith('.css') ? 'styles'
+        : e.initiatorType === 'img' ? 'images'
+        : e.name?.match(/\.(woff|woff2|ttf|otf|eot)/) ? 'fonts'
+        : 'other';
+      resources[cat as keyof ResourceSummary].count++;
+      resources[cat as keyof ResourceSummary].totalBytes += e.transferSize;
+    }
+  } catch {}
+
+  return { url, vitals, resources, totalTransferBytes, requestCount, measuredAt };
+}

package/src/phash.ts

@@ -0,0 +1,184 @@
+import sharp from 'sharp';
+import type { OverlayOptions } from 'sharp';
+
+/**
+ * Composites neutral grey (128,128,128) over each ignoredRegion (fractional
+ * 0-1 coordinates) so masked content — timestamps, live counters, ads —
+ * contributes nothing to any downstream pixel comparison. Shared by every
+ * comparison path (whole-image hash, per-cell grid hash, AI vision) so a
+ * region masked once is actually ignored everywhere, not just in the
+ * cheapest check.
+ */
+export async function applyMask(
+  imagePath: string,
+  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }>,
+): Promise<Buffer> {
+  const meta = await sharp(imagePath).metadata();
+  const W = meta.width ?? 1200;
+  const H = meta.height ?? 800;
+
+  const overlays: OverlayOptions[] = await Promise.all(
+    ignoredRegions.map(async (r) => {
+      const rW = Math.max(1, Math.round(r.width * W));
+      const rH = Math.max(1, Math.round(r.height * H));
+      const buf = await sharp({
+        create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } },
+      }).png().toBuffer();
+      return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) } as OverlayOptions;
+    }),
+  );
+
+  return sharp(imagePath).composite(overlays).png().toBuffer();
+}
+
+/**
+ * Compute an 8x8 average hash (aHash) of an image file, with optional mask overlays.
+ * ignoredRegions are fractional (0-1 range), converted to pixels here.
+ * Masked regions are filled with neutral grey (128,128,128) before hashing.
+ */
+async function hashWithMasks(
+  imagePath: string,
+  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],
+): Promise<bigint> {
+  const pipeline = ignoredRegions.length > 0
+    ? sharp(await applyMask(imagePath, ignoredRegions))
+    : sharp(imagePath);
+
+  const buf = await pipeline.resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer();
+  const pixels = Array.from(buf);
+  const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
+  return pixels.reduce((h, p, i) => (p >= mean ? h | (1n << BigInt(63 - i)) : h), 0n);
+}
+
+function hammingDistance(a: bigint, b: bigint): number {
+  let xor = a ^ b;
+  let count = 0;
+  while (xor !== 0n) { count += Number(xor & 1n); xor >>= 1n; }
+  return count;
+}
+
+/**
+ * Generates a pixelmatch diff image highlighting changed pixels in red.
+ * Applies the same ignoredRegions mask as screenshotDiffPercent so masked
+ * zones don't show as "changed" in the visual output.
+ * Returns a PNG Buffer suitable for upload to S3.
+ */
+export async function generatePixelmatchDiff(
+  pathA: string,
+  pathB: string,
+  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],
+): Promise<Buffer> {
+  const metaA = await sharp(pathA).metadata();
+  const W = metaA.width ?? 1200;
+  const H = metaA.height ?? 800;
+
+  const toRawRGBA = async (path: string): Promise<Buffer> => {
+    const base = ignoredRegions.length > 0 ? sharp(await applyMask(path, ignoredRegions)) : sharp(path);
+    const { data } = await base.resize(W, H, { fit: 'fill' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+    return data;
+  };
+
+  const [rawA, rawB] = await Promise.all([toRawRGBA(pathA), toRawRGBA(pathB)]);
+  const { default: pixelmatch } = await import('pixelmatch');
+  const diffData = Buffer.alloc(W * H * 4);
+  pixelmatch(rawA, rawB, diffData, W, H, {
+    threshold: 0.1,
+    alpha: 0.3,
+    diffColor: [255, 0, 0],
+    aaColor: [255, 255, 0],
+  });
+  return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
+}
+
+export async function screenshotDiffPercent(
+  pathA: string,
+  pathB: string,
+  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],
+): Promise<{ pHashDistance: number; changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT' }> {
+  const [hashA, hashB] = await Promise.all([
+    hashWithMasks(pathA, ignoredRegions),
+    hashWithMasks(pathB, ignoredRegions),
+  ]);
+  let xor = hashA ^ hashB;
+  let dist = 0;
+  while (xor !== 0n) { dist += Number(xor & 1n); xor >>= 1n; }
+  const pct = parseFloat(((dist / 64) * 100).toFixed(1));
+  const changeLevel = pct === 0 ? 'IDENTICAL' : pct < 10 ? 'MINOR' : pct < 30 ? 'MODERATE' : 'SIGNIFICANT';
+  return { pHashDistance: pct, changeLevel };
+}
+
+/**
+ * Grid-based layout analysis: divides screenshots into a 3×3 grid and computes
+ * pHash per cell. Returns which regions changed and where, for the VisualDiff.changes field.
+ * Uses sharp's `extract` to crop each cell before hashing.
+ */
+export async function gridLayoutDiff(
+  pathA: string,
+  pathB: string,
+  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],
+): Promise<Array<{ location: string; severity: string; description: string }>> {
+  const GRID_LABELS = [
+    'top-left',    'top-center',    'top-right',
+    'mid-left',    'mid-center',    'mid-right',
+    'bottom-left', 'bottom-center', 'bottom-right',
+  ];
+
+  try {
+    // Mask before cropping into cells — a masked region (e.g. a timestamp)
+    // then contributes nothing to any cell's hash, instead of only being
+    // ignored by the separate whole-image hash in screenshotDiffPercent().
+    const [sourceA, sourceB] = ignoredRegions.length > 0
+      ? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)])
+      : [pathA, pathB];
+
+    const [metaA, metaB] = await Promise.all([
+      sharp(sourceA).metadata(),
+      sharp(sourceB).metadata(),
+    ]);
+    const W = Math.min(metaA.width ?? 1200, metaB.width ?? 1200);
+    const H = Math.min(metaA.height ?? 800, metaB.height ?? 800);
+    const cW = Math.floor(W / 3);
+    const cH = Math.floor(H / 3);
+
+    const changes: Array<{ location: string; severity: string; description: string }> = [];
+
+    for (let row = 0; row < 3; row++) {
+      for (let col = 0; col < 3; col++) {
+        const region = { left: col * cW, top: row * cH, width: cW, height: cH };
+        const label = GRID_LABELS[row * 3 + col];
+
+        const [hashA, hashB] = await Promise.all([
+          sharp(sourceA).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()
+            .then((buf) => {
+              const pixels = Array.from(buf);
+              const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
+              return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);
+            }),
+          sharp(sourceB).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()
+            .then((buf) => {
+              const pixels = Array.from(buf);
+              const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
+              return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);
+            }),
+        ]);
+
+        let xor = hashA ^ hashB;
+        let dist = 0;
+        while (xor !== 0n) { dist += Number(xor & 1n); xor >>= 1n; }
+        const pct = (dist / 64) * 100;
+
+        if (pct >= 5) {
+          const severity = pct >= 20 ? 'HIGH' : pct >= 10 ? 'MEDIUM' : 'LOW';
+          changes.push({
+            location: label,
+            severity,
+            description: `${Math.round(pct)}% visual change in ${label.replace('-', ' ')} region`,
+          });
+        }
+      }
+    }
+    return changes;
+  } catch {
+    return [];
+  }
+}

package/src/pii-scrubber.ts

@@ -0,0 +1,99 @@
+/**
+ * PII scrubber — detects and redacts Personally Identifiable Information
+ * from crawled markdown, HTML, and extracted text before storage.
+ *
+ * GDPR Article 4(1): "any information relating to an identified or identifiable natural person"
+ */
+
+export interface PiiMatch {
+  type: 'email' | 'phone' | 'ssn' | 'credit-card' | 'ip-address' | 'passport' | 'iban' | 'dob';
+  value: string;
+  start: number;
+  end: number;
+}
+
+export interface ScrubResult {
+  text: string;
+  matches: PiiMatch[];
+  scrubbed: boolean;
+}
+
+// Patterns — ordered from most-specific to least-specific
+const PATTERNS: Array<{ type: PiiMatch['type']; re: RegExp }> = [
+  // Email — RFC 5321 simplified
+  { type: 'email', re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g },
+  // US SSN — 9 digits with dashes or spaces
+  { type: 'ssn', re: /\b(?!000|666|9\d\d)\d{3}[-\s](?!00)\d{2}[-\s](?!0000)\d{4}\b/g },
+  // Credit card — Visa/MC/Amex/Discover with optional spaces/dashes
+  { type: 'credit-card', re: /\b(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{3,4}\b/g },
+  // IBAN — international bank account (15-34 chars)
+  { type: 'iban', re: /\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b/g },
+  // Phone — international or US formats
+  { type: 'phone', re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b|\+\d{1,3}[-.\s]\d{2,4}[-.\s]\d{3,4}[-.\s]\d{3,4}/g },
+  // Date of birth — common formats
+  { type: 'dob', re: /\b(?:0?[1-9]|1[0-2])\/(?:0?[1-9]|[12]\d|3[01])\/(?:19|20)\d{2}\b/g },
+  // IPv4 private/internal addresses — only flag private-looking ones in content
+  { type: 'ip-address', re: /\b(?:192\.168\.\d{1,3}\.\d{1,3}|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b/g },
+];
+
+const REDACT_MAP: Record<PiiMatch['type'], string> = {
+  email: '[EMAIL]',
+  phone: '[PHONE]',
+  ssn: '[SSN]',
+  'credit-card': '[CARD]',
+  'ip-address': '[IP]',
+  passport: '[PASSPORT]',
+  iban: '[IBAN]',
+  dob: '[DOB]',
+};
+
+export function detectPii(text: string): PiiMatch[] {
+  const matches: PiiMatch[] = [];
+  const seen = new Set<string>();
+
+  for (const { type, re } of PATTERNS) {
+    const r = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g');
+    let m: RegExpExecArray | null;
+    while ((m = r.exec(text)) !== null) {
+      const key = `${type}:${m.index}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length });
+      }
+    }
+  }
+
+  return matches.sort((a, b) => a.start - b.start);
+}
+
+export function scrubPii(text: string): ScrubResult {
+  const matches = detectPii(text);
+  if (matches.length === 0) return { text, matches: [], scrubbed: false };
+
+  // Replace from end to start so indices stay valid
+  let scrubbed = text;
+  for (let i = matches.length - 1; i >= 0; i--) {
+    const { type, start, end } = matches[i];
+    scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end);
+  }
+
+  return { text: scrubbed, matches, scrubbed: true };
+}
+
+export function hasPii(text: string): boolean {
+  for (const { re } of PATTERNS) {
+    const r = new RegExp(re.source, re.flags);
+    if (r.test(text)) return true;
+  }
+  return false;
+}
+
+export function scrubObject<T extends Record<string, unknown>>(obj: T, fields: (keyof T)[]): T {
+  const result = { ...obj };
+  for (const field of fields) {
+    if (typeof result[field] === 'string') {
+      (result as any)[field] = scrubPii(result[field] as string).text;
+    }
+  }
+  return result;
+}