@detiq/crawler 0.1.3

package/types/job-queue.d.ts

@@ -0,0 +1,171 @@
+/**
+ * ZeTa-AI Background Job Queue
+ *
+ * Postgres-backed durable queue. Each job becomes a PlatformJob record.
+ * enqueueJob() writes it QUEUED and, for same-process responsiveness, kicks
+ * an immediate best-effort claim attempt — but durability comes from
+ * startJobWorkerLoop()'s poll: it atomically claims QUEUED jobs (Postgres
+ * `FOR UPDATE SKIP LOCKED`, safe across any number of concurrent worker
+ * processes/containers) and requeues jobs whose heartbeat went stale,
+ * so a crashed/restarted worker never silently loses a job.
+ *
+ * CRAWL-type jobs additionally support AWS SQS as an alternate transport
+ * (JOB_QUEUE_BACKEND=sqs, see sqs-queue.ts) for horizontally-scaled crawler
+ * fleets; every other job type always goes through the DB-backed queue.
+ */
+export declare function generateTOTP(secret: string): string;
+export interface PlatformJobConfig {
+    tenantId: string;
+    projectId: string;
+    appUrl?: string;
+    credentials?: {
+        username: string;
+        password: string;
+        oneTimeCode?: string;
+    };
+    /** Decrypted Playwright storageState JSON — if present, session is restored without AI login. */
+    storageState?: string;
+    screenshotDir?: string;
+    maxScreens?: number;
+    /** Seed URLs added to the initial crawl queue — link discovery still runs from each page. */
+    startUrls?: string[];
+    /** When provided, crawl ONLY these exact URLs (no link-following). Used for selected-screen recrawl. */
+    selectiveUrls?: string[];
+    /** Device profiles for multi-viewport screenshot pass after main DESKTOP crawl. */
+    deviceProfiles?: string[];
+    /** HTTP Basic Auth credentials — injected as Authorization header on same-origin requests. */
+    httpBasicUsername?: string;
+    httpBasicPassword?: string;
+    /** Bearer/API token — injected as a request header and into localStorage. Skips AI login. */
+    authToken?: string;
+    /** Header name for authToken (default: 'Authorization'). */
+    authTokenHeader?: string;
+    /** Prefix for authToken value (default: 'Bearer '). */
+    authTokenPrefix?: string;
+    /** CAPTCHA solving API key (2captcha or CapSolver). */
+    captchaSolverApiKey?: string;
+    /** CAPTCHA solving service provider (default: '2captcha'). */
+    captchaSolverProvider?: '2captcha' | 'capsolver';
+    /**
+     * Path to a Chrome user data directory for profile inheritance (local deployments only).
+     * When set, the crawler exports the existing session from this profile and injects it.
+     */
+    chromeProfilePath?: string;
+    /** MailSlurp API key for email OTP auto-retrieval. */
+    mailslurpApiKey?: string;
+    /** MailSlurp inbox ID to poll for OTP emails. */
+    mailslurpInboxId?: string;
+    /** SSO provider ('okta' | 'azure_ad' | 'generic_oidc'). */
+    ssoProvider?: string;
+    /** SSO domain or tenant ID. */
+    ssoDomain?: string;
+    /** SSO OAuth2 client ID. */
+    ssoClientId?: string;
+    /** SSO OAuth2 client secret (decrypted). */
+    ssoClientSecret?: string;
+    /** SSO scopes (space-separated). Default: 'openid profile'. */
+    ssoScope?: string;
+    /** HTTP/HTTPS/SOCKS5 proxy URL — routes browser through this proxy for bot-protected sites. */
+    proxyUrl?: string;
+}
+type ClaimedJob = {
+    id: string;
+    tenantId: string;
+    projectId: string;
+    type: string;
+    config: any;
+};
+/** Atomically claims one specific QUEUED job for this worker. Returns null if it was already claimed. */
+export declare function claimSpecificJob(jobId: string): Promise<ClaimedJob | null>;
+/** Atomically claims the oldest QUEUED job across all tenants. SKIP LOCKED makes this
+ *  race-safe against any number of other pollers doing the same thing concurrently. */
+export declare function claimNextQueuedJob(): Promise<ClaimedJob | null>;
+/** Requeues RUNNING jobs whose heartbeat has gone stale — the worker that claimed them
+ *  died (crash, restart, OOM) without ever marking them DONE/FAILED. Idempotent and
+ *  safe to run from every worker process; only affects rows past the timeout. */
+export declare function requeueStaleRunningJobs(): Promise<void>;
+/**
+ * Starts the durable job worker poll loop. This is what makes job processing
+ * survive a restart — enqueueJob()'s immediate same-process claim attempt is
+ * only a latency optimization; if the process dies before that fires (or
+ * mid-job), this loop's next tick claims/requeues it. Call once at process
+ * startup (API server and/or a dedicated crawler worker process — safe to
+ * run in both, and safe to run in many container replicas at once).
+ */
+export declare function startJobWorkerLoop(opts?: {
+    pollIntervalMs?: number;
+    concurrency?: number;
+}): void;
+/** Stops the poll loop. Mainly for tests. */
+export declare function stopJobWorkerLoop(): void;
+/** Create a job record that tracks a synchronous operation (telemetry only, no background processor). */
+export declare function createTrackedJob(type: string, config: {
+    tenantId: string;
+    projectId: string;
+    [k: string]: any;
+}): Promise<string>;
+/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors. */
+export declare function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void>;
+/** Enqueue a generic platform job — returns the job ID immediately. */
+export declare function enqueueJob(type: string, config: any): Promise<string>;
+/** Get the current state of a platform job. */
+export declare function getJob(tenantId: string, jobId: string): Promise<{
+    progress: number;
+    error: string | null;
+    type: string;
+    status: string;
+    tenantId: string;
+    projectId: string;
+    name: string | null;
+    result: import("@prisma/client/runtime/library").JsonValue | null;
+    id: string;
+    config: import("@prisma/client/runtime/library").JsonValue;
+    queuedAt: Date;
+    startedAt: Date | null;
+    finishedAt: Date | null;
+    workerId: string | null;
+    heartbeatAt: Date | null;
+    totalPromptTokens: number | null;
+    totalCompletionTokens: number | null;
+    totalCacheReadTokens: number | null;
+    totalEstimatedCostUsd: number | null;
+    llmCallCount: number;
+} | null>;
+/** List recent jobs for a project. */
+export declare function listJobs(tenantId: string, projectId: string): Promise<{
+    progress: number;
+    error: string | null;
+    type: string;
+    status: string;
+    tenantId: string;
+    projectId: string;
+    name: string | null;
+    result: import("@prisma/client/runtime/library").JsonValue | null;
+    id: string;
+    config: import("@prisma/client/runtime/library").JsonValue;
+    queuedAt: Date;
+    startedAt: Date | null;
+    finishedAt: Date | null;
+    workerId: string | null;
+    heartbeatAt: Date | null;
+    totalPromptTokens: number | null;
+    totalCompletionTokens: number | null;
+    totalCacheReadTokens: number | null;
+    totalEstimatedCostUsd: number | null;
+    llmCallCount: number;
+}[]>;
+/** Cancel a RUNNING or QUEUED job. Aborts the in-process crawl and marks DB CANCELLED. */
+export declare function cancelJob(tenantId: string, jobId: string): Promise<void>;
+/** Re-enqueue a job using the same config as a previous job. */
+export declare function rerunJob(tenantId: string, jobId: string): Promise<string>;
+export declare function resumeJob(tenantId: string, jobId: string): Promise<string>;
+/** Update progress percentage and current step label on a running job (non-fatal). */
+export declare function updateJobProgress(tenantId: string, jobId: string, progress: number, step: string): Promise<void>;
+export interface ProcessJobOpts {
+    onProgress?: (pct: number) => Promise<void>;
+}
+export declare function processJob(jobId: string, config: any, opts?: ProcessJobOpts): Promise<void>;
+/** After a job finishes, sum LLMCallLog entries for that project during the job window. */
+export declare function aggregateJobTokens(tenantId: string, jobId: string, projectId: string, startedAt: Date): Promise<void>;
+export declare function processDataRetentionPurge(): Promise<void>;
+export {};

package/types/js-coverage.d.ts

@@ -0,0 +1,29 @@
+/**
+ * JS/CSS coverage collector via Playwright's built-in coverage API.
+ *
+ * For each crawled page, records what percentage of loaded JS/CSS was actually
+ * executed/applied. High unused code % → performance opportunity (code splitting, lazy loading).
+ *
+ * Uses Playwright's page.coverage API (wraps CDP Coverage domain).
+ */
+import type { Page } from 'playwright';
+export interface CoverageEntry {
+    url: string;
+    type: 'js' | 'css';
+    totalBytes: number;
+    usedBytes: number;
+    unusedPercent: number;
+}
+export interface CoverageReport {
+    pageUrl: string;
+    entries: CoverageEntry[];
+    totalJsBytes: number;
+    usedJsBytes: number;
+    totalCssBytes: number;
+    usedCssBytes: number;
+    jsUnusedPercent: number;
+    cssUnusedPercent: number;
+    collectedAt: string;
+}
+export declare function startCoverage(page: Page): Promise<void>;
+export declare function stopAndCollectCoverage(page: Page, pageUrl: string): Promise<CoverageReport | null>;

package/types/js-router-detector.d.ts

@@ -0,0 +1,17 @@
+/**
+ * P2: JS Router Detection
+ *
+ * Scans the loaded page for client-side route tables declared by React Router v6,
+ * Vue Router, Angular Router, or custom window globals. Supplements the existing
+ * history.pushState interception (which only captures runtime navigation) with a
+ * static analysis of routes declared in the JS bundle.
+ *
+ * Called once per page after goto(). Runs entirely in-browser via page.evaluate().
+ * Adds discovered routes to the BFS frontier so the crawler visits them.
+ */
+/**
+ * Extract absolute page URLs from the current page's JS router config.
+ * Returns deduplicated absolute URLs (same origin). Dynamic segments (:id, *)
+ * are skipped — the crawler can't resolve them without knowing valid IDs.
+ */
+export declare function detectJsRoutes(page: any, origin: string): Promise<string[]>;

package/types/language-detector.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Language detector — identify page language from HTML attributes and content analysis.
+ *
+ * Sources (in priority order):
+ * 1. <html lang="..."> attribute
+ * 2. <meta http-equiv="Content-Language"> header
+ * 3. hreflang alternate links
+ * 4. Content-based n-gram fingerprint (detects top 20 languages)
+ *
+ * Used to: route pages to the correct markdown processor, group screens by locale,
+ * and detect missing hreflang implementations.
+ */
+import type { Page } from 'playwright';
+export interface LanguageDetectionResult {
+    url: string;
+    declaredLang: string;
+    detectedLang: string;
+    confidence: 'high' | 'medium' | 'low';
+    alternates: Record<string, string>;
+    isMultilingual: boolean;
+}
+export declare function detectLanguage(page: Page, url: string): Promise<LanguageDetectionResult>;

package/types/link-classifier.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Link classifier — assigns priority and category to discovered URLs.
+ *
+ * Categories:
+ * - 'nav'       : site navigation (homepage, top-level pages) — highest priority
+ * - 'auth'      : login/register/password flows — high priority
+ * - 'app'       : authenticated app screens — high priority
+ * - 'content'   : blog/docs/articles — medium priority
+ * - 'api'       : REST/GraphQL API endpoints — detected, not crawled as screens
+ * - 'resource'  : static assets (JS/CSS/img/fonts) — skip
+ * - 'external'  : different domain — skip unless cross-domain crawl enabled
+ * - 'low-value' : pagination, sort, filter query strings — deprioritize
+ */
+export type LinkCategory = 'nav' | 'auth' | 'app' | 'content' | 'api' | 'resource' | 'external' | 'low-value';
+export interface ClassifiedLink {
+    url: string;
+    category: LinkCategory;
+    priority: number;
+    reason: string;
+}
+export declare function classifyLink(url: string, baseUrl: string): ClassifiedLink;
+export declare function classifyLinks(urls: string[], baseUrl: string): ClassifiedLink[];
+export declare function filterCrawlable(classified: ClassifiedLink[]): ClassifiedLink[];

package/types/live-view.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Start a live view session: launches a headless browser, navigates to appUrl,
+ * begins capturing frames. Returns the session ID.
+ */
+export declare function startLiveView(appUrl: string, storageState?: string): Promise<string>;
+/**
+ * Get the latest frame for a session (base64 JPEG).
+ */
+export declare function getFrame(id: string): string | null;
+/**
+ * Navigate the live view browser to a URL.
+ */
+export declare function liveViewNavigate(id: string, url: string): Promise<void>;
+/**
+ * Capture storageState from the live view session, close the browser, return JSON string.
+ */
+export declare function completeLiveView(id: string): Promise<string>;
+/**
+ * Send a mouse or keyboard input event to the live view browser.
+ */
+export declare function sendInput(id: string, event: {
+    type: 'click' | 'type' | 'key';
+    x?: number;
+    y?: number;
+    button?: 'left' | 'right' | 'middle';
+    text?: string;
+    key?: string;
+}): Promise<void>;
+/**
+ * Close a live view session without capturing.
+ */
+export declare function closeLiveView(id: string): Promise<void>;
+/**
+ * List all active session IDs (for health checks / cleanup).
+ */
+export declare function listLiveViews(): string[];

package/types/logger.d.ts

@@ -0,0 +1,2 @@
+import pino from 'pino';
+export declare const logger: pino.Logger<never, boolean>;

package/types/markdown-extract.d.ts

@@ -0,0 +1,8 @@
+export interface ExtractedMarkdown {
+    markdown: string;
+    method: 'readability' | 'full-page';
+    title: string | null;
+    wordCount: number;
+}
+/** Best-effort HTML→Markdown for one crawled page. Never throws. */
+export declare function extractMarkdown(html: string, url: string): ExtractedMarkdown | null;

package/types/native-crawler.d.ts

@@ -0,0 +1,69 @@
+/**
+ * ZeTa Native App Crawler — Appium-based crawler for Android APK and iOS IPA.
+ *
+ * Architecture:
+ *   Appium Server (running at APPIUM_SERVER_URL) → WebDriverIO session
+ *   → UiAutomator2 (Android) / XCUITest (iOS)
+ *   → accessibility tree XML → ZeTa Element records
+ *
+ * TODO: requires Appium server running at APPIUM_SERVER_URL env var
+ * TODO: requires `webdriverio` package — add "webdriverio": "^9.x" to package.json
+ *
+ * Device farm support:
+ *   - local_emulator: Appium server + local emulator/simulator
+ *   - aws_device_farm: presigned S3 URL uploaded to Device Farm, session via WDIO
+ *   - browserstack: BrowserStack Automate Appium endpoint
+ */
+export interface NativeCrawlerOptions {
+    tenantId: string;
+    projectId: string;
+    jobId: string;
+    platform: 'android' | 'ios' | 'windows' | 'macos';
+    appS3Path: string;
+    packageName?: string;
+    deviceModel?: string;
+    osVersion?: string;
+    appActivity?: string;
+    appWaitActivity?: string;
+    deviceFarm: 'local_emulator' | 'aws_device_farm' | 'browserstack';
+    appiumServerUrl?: string;
+    browserstackUsername?: string;
+    browserstackAccessKey?: string;
+    awsAccessKeyId?: string;
+    awsSecretAccessKey?: string;
+    awsRegion?: string;
+    awsDeviceFarmProjectArn?: string;
+    credentials?: {
+        username?: string;
+        password?: string;
+        usernameFieldLabel?: string;
+        passwordFieldLabel?: string;
+        loginButtonLabel?: string;
+    };
+    visionLlmConfig?: {
+        provider: string;
+        model: string;
+        apiKey?: string;
+        baseUrl?: string;
+    } | null;
+    gestureConfig?: {
+        enableSwipe?: boolean;
+        enableLongPress?: boolean;
+    };
+    mitmProxyPort?: number;
+}
+/**
+ * Run a native app crawl session via Appium.
+ *
+ * Flow:
+ *   1. Download APK/IPA from S3 to local temp dir
+ *   2. Build Appium WebDriverIO capabilities for the target platform
+ *   3. Create WebDriverIO remote session (Appium server at APPIUM_SERVER_URL)
+ *   4. For each discovered screen:
+ *      a. Get page source XML → parse elements
+ *      b. Take screenshot → save to screenshotDir → persist via AppBrain
+ *      c. Save screen + elements to DB
+ *      d. Tap nav elements to discover next screens
+ *   5. Update PlatformJob as DONE (or FAILED on error)
+ */
+export declare function runNativeCrawl(opts: NativeCrawlerOptions): Promise<void>;

package/types/network-observer.d.ts

@@ -0,0 +1,49 @@
+import type { BrowserContext } from 'playwright';
+export interface ApiEndpointObservationInput {
+    method: string;
+    urlPattern: string;
+    statusCode?: number;
+    requestKind?: string;
+    responseShape?: unknown;
+    durationMs?: number;
+    errorBody?: string;
+}
+export declare function attachNetworkObserver(context: BrowserContext, sink: ApiEndpointObservationInput[], onPageUrlDiscovered?: (relPath: string) => void): void;
+export interface HarEntry {
+    startedAt: string;
+    method: string;
+    url: string;
+    status: number;
+    statusText: string;
+    requestHeaders: Record<string, string>;
+    responseHeaders: Record<string, string>;
+    contentType: string;
+    bodySize: number;
+    durationMs: number;
+    timings: {
+        send: number;
+        wait: number;
+        receive: number;
+    };
+}
+export interface HarLog {
+    version: '1.2';
+    creator: {
+        name: 'ZeTa-Crawler';
+        version: '1.0';
+    };
+    pages: Array<{
+        id: string;
+        title: string;
+        startedAt: string;
+    }>;
+    entries: HarEntry[];
+}
+/**
+ * Full HAR capture via CDP — intercepts ALL network traffic, not just API endpoints.
+ * Returns a controller with getHar() and detach().
+ */
+export declare function attachHarCapture(cdpSession: any, pageTitle?: string): {
+    getHar(): HarLog;
+    detach(): void;
+};

package/types/network-throttle.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Network throttle simulation via CDP.
+ *
+ * Simulates different network conditions during crawl to test how the app
+ * behaves under slow connections. Useful for detecting loading states, error
+ * handling, and performance degradation on poor networks.
+ *
+ * Conditions mirror Chrome DevTools presets:
+ * - slow3g: ~500kbps download, high latency
+ * - fast3g: ~1.6Mbps download, medium latency
+ * - 4g: ~9Mbps download, low latency
+ * - offline: no connectivity
+ * - none: no throttle (default)
+ */
+export type NetworkConditionPreset = 'none' | 'slow3g' | 'fast3g' | '4g' | 'offline';
+export declare function applyNetworkThrottle(cdpSession: any, preset: NetworkConditionPreset): Promise<void>;
+export declare function clearNetworkThrottle(cdpSession: any): Promise<void>;
+export interface ThrottledCrawlOpts {
+    /** Network preset to apply during this URL's crawl */
+    preset: NetworkConditionPreset;
+    /** Restore to 'none' after navigation (default: true) */
+    restoreAfter?: boolean;
+}
+/** Apply throttle, run fn, restore. Returns fn's result. */
+export declare function withNetworkThrottle<T>(cdpSession: any, preset: NetworkConditionPreset, fn: () => Promise<T>, restoreAfter?: boolean): Promise<T>;

package/types/openapi-generator.d.ts

@@ -0,0 +1,49 @@
+/**
+ * OpenAPI 3.0 spec generator from observed network traffic.
+ *
+ * Takes the ApiEndpointObservationInput[] collected by the network observer
+ * during crawl and generates a minimal but valid OpenAPI 3.0 spec.
+ *
+ * The spec captures: endpoints, methods, observed response shapes, status codes.
+ * It does NOT capture request bodies (we'd need to intercept requests too) — that's
+ * marked as a TODO. But it's enough for generating smoke-test suites.
+ */
+import type { ApiEndpointObservationInput } from './network-observer.js';
+export interface OpenApiSpec {
+    openapi: '3.0.3';
+    info: {
+        title: string;
+        version: string;
+        description?: string;
+    };
+    servers: Array<{
+        url: string;
+    }>;
+    paths: Record<string, Record<string, OpenApiOperation>>;
+}
+interface OpenApiOperation {
+    summary: string;
+    parameters?: OpenApiParam[];
+    responses: Record<string, {
+        description: string;
+        content?: Record<string, {
+            schema: unknown;
+        }>;
+    }>;
+    tags?: string[];
+}
+interface OpenApiParam {
+    name: string;
+    in: 'path' | 'query';
+    required: boolean;
+    schema: {
+        type: string;
+    };
+}
+export declare function generateOpenApiSpec(observations: ApiEndpointObservationInput[], opts?: {
+    title?: string;
+    baseUrl?: string;
+    version?: string;
+}): OpenApiSpec;
+export declare function specToYaml(spec: OpenApiSpec): string;
+export {};

package/types/pdf-extractor.d.ts

@@ -0,0 +1,28 @@
+/**
+ * PDF extraction — discovers and extracts text from PDF URLs found during crawl.
+ *
+ * When the HTTP engine or browser engine discovers a link to a .pdf file,
+ * this module fetches the PDF, extracts its text content, and returns it
+ * as a screen-compatible markdown string for test generation.
+ *
+ * Uses pdf-parse (already installed in API package, accessible via dynamic import).
+ */
+export interface PdfScreen {
+    url: string;
+    title: string;
+    markdown: string;
+    pageCount: number;
+    info: Record<string, string>;
+    sizeBytes: number;
+}
+export declare function isPdfUrl(url: string, contentType?: string): boolean;
+export declare function extractPdfFromUrl(url: string, opts?: {
+    userAgent?: string;
+    timeoutMs?: number;
+    maxSizeBytes?: number;
+}): Promise<PdfScreen | null>;
+/**
+ * Filter a list of discovered URLs to find PDF links.
+ * Used by http-engine and browser crawler to decide which links to pass to extractPdfFromUrl.
+ */
+export declare function filterPdfUrls(urls: string[]): string[];

package/types/performance-metrics.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Performance metrics collection via CDP + Playwright page.evaluate().
+ *
+ * Collects Core Web Vitals and resource timing after page navigation.
+ * Stored alongside screen data for test generation context and regression detection.
+ *
+ * Metrics:
+ * - LCP (Largest Contentful Paint) — via PerformanceObserver in page
+ * - CLS (Cumulative Layout Shift) — via PerformanceObserver in page
+ * - TTFB (Time to First Byte) — from navigation timing
+ * - DOMContentLoaded — from navigation timing
+ * - Resource count and size by type
+ */
+import type { Page } from 'playwright';
+export interface CoreWebVitals {
+    lcp?: number;
+    cls?: number;
+    ttfb?: number;
+    fcp?: number;
+    domContentLoaded?: number;
+    loadEventEnd?: number;
+}
+export interface ResourceSummary {
+    scripts: {
+        count: number;
+        totalBytes: number;
+    };
+    styles: {
+        count: number;
+        totalBytes: number;
+    };
+    images: {
+        count: number;
+        totalBytes: number;
+    };
+    fonts: {
+        count: number;
+        totalBytes: number;
+    };
+    other: {
+        count: number;
+        totalBytes: number;
+    };
+}
+export interface PagePerfMetrics {
+    url: string;
+    vitals: CoreWebVitals;
+    resources: ResourceSummary;
+    totalTransferBytes: number;
+    requestCount: number;
+    measuredAt: string;
+}
+export declare function collectPerfMetrics(page: Page, url: string): Promise<PagePerfMetrics>;

package/types/phash.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Composites neutral grey (128,128,128) over each ignoredRegion (fractional
+ * 0-1 coordinates) so masked content — timestamps, live counters, ads —
+ * contributes nothing to any downstream pixel comparison. Shared by every
+ * comparison path (whole-image hash, per-cell grid hash, AI vision) so a
+ * region masked once is actually ignored everywhere, not just in the
+ * cheapest check.
+ */
+export declare function applyMask(imagePath: string, ignoredRegions: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<Buffer>;
+export declare function screenshotDiffPercent(pathA: string, pathB: string, ignoredRegions?: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<{
+    pHashDistance: number;
+    changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT';
+}>;
+/**
+ * Grid-based layout analysis: divides screenshots into a 3×3 grid and computes
+ * pHash per cell. Returns which regions changed and where, for the VisualDiff.changes field.
+ * Uses sharp's `extract` to crop each cell before hashing.
+ */
+export declare function gridLayoutDiff(pathA: string, pathB: string, ignoredRegions?: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<Array<{
+    location: string;
+    severity: string;
+    description: string;
+}>>;

package/types/pii-scrubber.d.ts

@@ -0,0 +1,21 @@
+/**
+ * PII scrubber — detects and redacts Personally Identifiable Information
+ * from crawled markdown, HTML, and extracted text before storage.
+ *
+ * GDPR Article 4(1): "any information relating to an identified or identifiable natural person"
+ */
+export interface PiiMatch {
+    type: 'email' | 'phone' | 'ssn' | 'credit-card' | 'ip-address' | 'passport' | 'iban' | 'dob';
+    value: string;
+    start: number;
+    end: number;
+}
+export interface ScrubResult {
+    text: string;
+    matches: PiiMatch[];
+    scrubbed: boolean;
+}
+export declare function detectPii(text: string): PiiMatch[];
+export declare function scrubPii(text: string): ScrubResult;
+export declare function hasPii(text: string): boolean;
+export declare function scrubObject<T extends Record<string, unknown>>(obj: T, fields: (keyof T)[]): T;