@taodigital/crawler 0.0.0-stage → 1.2.0

package/types/network-action-correlator.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Network-action correlator — captures XHR/fetch requests that occur during
+ * a named UI action (click, form submit, navigation) and maps them to API
+ * endpoint contracts.
+ *
+ * Usage:
+ *   const correlator = createCorrelator();
+ *   const detach = attachNetworkInterceptor(page, correlator);
+ *   correlator.startWindow('Submit login form');
+ *   await page.click('#login-btn');
+ *   const correlation = correlator.endWindow();
+ *   detach();
+ *   const result = correlator.getResult();
+ */
+import type { Page } from 'playwright';
+export interface NetworkRequestRecord {
+    url: string;
+    urlPattern: string;
+    method: string;
+    bodySchema: Record<string, string> | null;
+    status: number | null;
+    responseSchema: Record<string, string> | null;
+    isGraphQL: boolean;
+    graphQLOperation?: string;
+    resourceType: string;
+    correlatedActionLabel?: string;
+}
+export interface ActionCorrelation {
+    actionLabel: string;
+    requests: NetworkRequestRecord[];
+}
+export interface NetworkCorrelationResult {
+    correlations: ActionCorrelation[];
+    uniqueEndpoints: string[];
+    graphqlOperations: string[];
+    totalRequestsCaptured: number;
+    coverage: number;
+}
+interface NetworkCorrelator {
+    startWindow(actionLabel: string): void;
+    recordRequest(rec: NetworkRequestRecord): void;
+    endWindow(): ActionCorrelation;
+    getResult(): NetworkCorrelationResult;
+}
+export declare function createCorrelator(): NetworkCorrelator;
+export declare function attachNetworkInterceptor(page: Page, correlator: ReturnType<typeof createCorrelator>): () => void;
+export {};

package/types/network-observer.d.ts

@@ -0,0 +1,49 @@
+import type { BrowserContext } from 'playwright';
+export interface ApiEndpointObservationInput {
+    method: string;
+    urlPattern: string;
+    statusCode?: number;
+    requestKind?: string;
+    responseShape?: unknown;
+    durationMs?: number;
+    errorBody?: string;
+}
+export declare function attachNetworkObserver(context: BrowserContext, sink: ApiEndpointObservationInput[], onPageUrlDiscovered?: (relPath: string) => void): void;
+export interface HarEntry {
+    startedAt: string;
+    method: string;
+    url: string;
+    status: number;
+    statusText: string;
+    requestHeaders: Record<string, string>;
+    responseHeaders: Record<string, string>;
+    contentType: string;
+    bodySize: number;
+    durationMs: number;
+    timings: {
+        send: number;
+        wait: number;
+        receive: number;
+    };
+}
+export interface HarLog {
+    version: '1.2';
+    creator: {
+        name: 'ZeTa-Crawler';
+        version: '1.0';
+    };
+    pages: Array<{
+        id: string;
+        title: string;
+        startedAt: string;
+    }>;
+    entries: HarEntry[];
+}
+/**
+ * Full HAR capture via CDP — intercepts ALL network traffic, not just API endpoints.
+ * Returns a controller with getHar() and detach().
+ */
+export declare function attachHarCapture(cdpSession: any, pageTitle?: string): {
+    getHar(): HarLog;
+    detach(): void;
+};

package/types/network-throttle.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Network throttle simulation via CDP.
+ *
+ * Simulates different network conditions during crawl to test how the app
+ * behaves under slow connections. Useful for detecting loading states, error
+ * handling, and performance degradation on poor networks.
+ *
+ * Conditions mirror Chrome DevTools presets:
+ * - slow3g: ~500kbps download, high latency
+ * - fast3g: ~1.6Mbps download, medium latency
+ * - 4g: ~9Mbps download, low latency
+ * - offline: no connectivity
+ * - none: no throttle (default)
+ */
+export type NetworkConditionPreset = 'none' | 'slow3g' | 'fast3g' | '4g' | 'offline';
+export declare function applyNetworkThrottle(cdpSession: any, preset: NetworkConditionPreset): Promise<void>;
+export declare function clearNetworkThrottle(cdpSession: any): Promise<void>;
+export interface ThrottledCrawlOpts {
+    /** Network preset to apply during this URL's crawl */
+    preset: NetworkConditionPreset;
+    /** Restore to 'none' after navigation (default: true) */
+    restoreAfter?: boolean;
+}
+/** Apply throttle, run fn, restore. Returns fn's result. */
+export declare function withNetworkThrottle<T>(cdpSession: any, preset: NetworkConditionPreset, fn: () => Promise<T>, restoreAfter?: boolean): Promise<T>;

package/types/openapi-generator.d.ts

@@ -0,0 +1,49 @@
+/**
+ * OpenAPI 3.0 spec generator from observed network traffic.
+ *
+ * Takes the ApiEndpointObservationInput[] collected by the network observer
+ * during crawl and generates a minimal but valid OpenAPI 3.0 spec.
+ *
+ * The spec captures: endpoints, methods, observed response shapes, status codes.
+ * It does NOT capture request bodies (we'd need to intercept requests too) — that's
+ * marked as a TODO. But it's enough for generating smoke-test suites.
+ */
+import type { ApiEndpointObservationInput } from './network-observer.js';
+export interface OpenApiSpec {
+    openapi: '3.0.3';
+    info: {
+        title: string;
+        version: string;
+        description?: string;
+    };
+    servers: Array<{
+        url: string;
+    }>;
+    paths: Record<string, Record<string, OpenApiOperation>>;
+}
+interface OpenApiOperation {
+    summary: string;
+    parameters?: OpenApiParam[];
+    responses: Record<string, {
+        description: string;
+        content?: Record<string, {
+            schema: unknown;
+        }>;
+    }>;
+    tags?: string[];
+}
+interface OpenApiParam {
+    name: string;
+    in: 'path' | 'query';
+    required: boolean;
+    schema: {
+        type: string;
+    };
+}
+export declare function generateOpenApiSpec(observations: ApiEndpointObservationInput[], opts?: {
+    title?: string;
+    baseUrl?: string;
+    version?: string;
+}): OpenApiSpec;
+export declare function specToYaml(spec: OpenApiSpec): string;
+export {};

package/types/pdf-extractor.d.ts

@@ -0,0 +1,28 @@
+/**
+ * PDF extraction — discovers and extracts text from PDF URLs found during crawl.
+ *
+ * When the HTTP engine or browser engine discovers a link to a .pdf file,
+ * this module fetches the PDF, extracts its text content, and returns it
+ * as a screen-compatible markdown string for test generation.
+ *
+ * Uses pdf-parse (already installed in API package, accessible via dynamic import).
+ */
+export interface PdfScreen {
+    url: string;
+    title: string;
+    markdown: string;
+    pageCount: number;
+    info: Record<string, string>;
+    sizeBytes: number;
+}
+export declare function isPdfUrl(url: string, contentType?: string): boolean;
+export declare function extractPdfFromUrl(url: string, opts?: {
+    userAgent?: string;
+    timeoutMs?: number;
+    maxSizeBytes?: number;
+}): Promise<PdfScreen | null>;
+/**
+ * Filter a list of discovered URLs to find PDF links.
+ * Used by http-engine and browser crawler to decide which links to pass to extractPdfFromUrl.
+ */
+export declare function filterPdfUrls(urls: string[]): string[];

package/types/performance-metrics-collector.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Performance metrics collector — captures Core Web Vitals and navigation timing
+ * for pages visited during crawl runs.
+ * Collects LCP, CLS, INP, TTFB, long tasks, and slow resources via PerformanceObserver
+ * and the Navigation/Resource Timing APIs.
+ */
+import type { Page } from 'playwright';
+export type CwvScore = 'good' | 'needs-improvement' | 'poor';
+export interface LcpMetric {
+    value: number;
+    score: CwvScore;
+    elementTag: string | null;
+    url: string | null;
+}
+export interface ClsMetric {
+    value: number;
+    score: CwvScore;
+    shiftCount: number;
+}
+export interface InpMetric {
+    p75: number;
+    score: CwvScore;
+    sampleCount: number;
+}
+export interface NavTimingMetric {
+    ttfb: number;
+    ttfbScore: CwvScore;
+    domContentLoaded: number;
+    domComplete: number;
+    loadComplete: number;
+    transferSizeBytes: number;
+    protocol: string;
+}
+export interface SlowResource {
+    path: string;
+    durationMs: number;
+    transferSizeBytes: number;
+    initiatorType: string;
+}
+export interface PerformanceMetricsResult {
+    lcp: LcpMetric | null;
+    cls: ClsMetric | null;
+    inp: InpMetric | null;
+    navTiming: NavTimingMetric | null;
+    slowResources: SlowResource[];
+    longTaskCount: number;
+    captureSuccess: boolean;
+}
+export declare function injectPerformanceObservers(page: Page): Promise<void>;
+export declare function collectPagePerformance(page: Page): Promise<PerformanceMetricsResult>;

package/types/performance-metrics.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Performance metrics collection via CDP + Playwright page.evaluate().
+ *
+ * Collects Core Web Vitals and resource timing after page navigation.
+ * Stored alongside screen data for test generation context and regression detection.
+ *
+ * Metrics:
+ * - LCP (Largest Contentful Paint) — via PerformanceObserver in page
+ * - CLS (Cumulative Layout Shift) — via PerformanceObserver in page
+ * - TTFB (Time to First Byte) — from navigation timing
+ * - DOMContentLoaded — from navigation timing
+ * - Resource count and size by type
+ */
+export interface CoreWebVitals {
+    lcp?: number;
+    cls?: number;
+    ttfb?: number;
+    fcp?: number;
+    domContentLoaded?: number;
+    loadEventEnd?: number;
+}
+export interface ResourceSummary {
+    scripts: {
+        count: number;
+        totalBytes: number;
+    };
+    styles: {
+        count: number;
+        totalBytes: number;
+    };
+    images: {
+        count: number;
+        totalBytes: number;
+    };
+    fonts: {
+        count: number;
+        totalBytes: number;
+    };
+    other: {
+        count: number;
+        totalBytes: number;
+    };
+}
+export interface PagePerfMetrics {
+    url: string;
+    vitals: CoreWebVitals;
+    resources: ResourceSummary;
+    totalTransferBytes: number;
+    requestCount: number;
+    measuredAt: string;
+}
+export declare function collectPerfMetrics(page: any, url: string): Promise<PagePerfMetrics>;

package/types/phash.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Composites neutral grey (128,128,128) over each ignoredRegion (fractional
+ * 0-1 coordinates) so masked content — timestamps, live counters, ads —
+ * contributes nothing to any downstream pixel comparison. Shared by every
+ * comparison path (whole-image hash, per-cell grid hash, AI vision) so a
+ * region masked once is actually ignored everywhere, not just in the
+ * cheapest check.
+ */
+export declare function applyMask(imagePath: string, ignoredRegions: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<Buffer>;
+/**
+ * Generates a pixelmatch diff image highlighting changed pixels in red.
+ * Applies the same ignoredRegions mask as screenshotDiffPercent so masked
+ * zones don't show as "changed" in the visual output.
+ * Returns a PNG Buffer suitable for upload to S3.
+ */
+export declare function generatePixelmatchDiff(pathA: string, pathB: string, ignoredRegions?: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<Buffer>;
+export declare function screenshotDiffPercent(pathA: string, pathB: string, ignoredRegions?: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<{
+    pHashDistance: number;
+    changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT';
+}>;
+/**
+ * Grid-based layout analysis: divides screenshots into a 3×3 grid and computes
+ * pHash per cell. Returns which regions changed and where, for the VisualDiff.changes field.
+ * Uses sharp's `extract` to crop each cell before hashing.
+ */
+export declare function gridLayoutDiff(pathA: string, pathB: string, ignoredRegions?: Array<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}>): Promise<Array<{
+    location: string;
+    severity: string;
+    description: string;
+}>>;

package/types/pii-scrubber.d.ts

@@ -0,0 +1,21 @@
+/**
+ * PII scrubber — detects and redacts Personally Identifiable Information
+ * from crawled markdown, HTML, and extracted text before storage.
+ *
+ * GDPR Article 4(1): "any information relating to an identified or identifiable natural person"
+ */
+export interface PiiMatch {
+    type: 'email' | 'phone' | 'ssn' | 'credit-card' | 'ip-address' | 'passport' | 'iban' | 'dob';
+    value: string;
+    start: number;
+    end: number;
+}
+export interface ScrubResult {
+    text: string;
+    matches: PiiMatch[];
+    scrubbed: boolean;
+}
+export declare function detectPii(text: string): PiiMatch[];
+export declare function scrubPii(text: string): ScrubResult;
+export declare function hasPii(text: string): boolean;
+export declare function scrubObject<T extends Record<string, unknown>>(obj: T, fields: (keyof T)[]): T;

package/types/playwright-recorder.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Playwright test recorder — generate Playwright test code from crawl data.
+ *
+ * Given a list of crawled screens, generates a minimal Playwright test file that:
+ * 1. Navigates to each screen URL
+ * 2. Verifies the page title
+ * 3. Checks key elements visible on the page
+ * 4. Verifies no console errors (optional)
+ *
+ * This is a "smoke test" generator — not a full interaction recorder.
+ * Output: valid TypeScript Playwright test file ready to run with npx playwright test.
+ */
+export interface RecorderScreen {
+    id: string;
+    name: string;
+    url: string;
+    title?: string;
+    heading?: string;
+    markdownWordCount?: number;
+    smartLocator?: {
+        tag: string;
+        text?: string;
+        role?: string;
+        placeholder?: string;
+        nearText?: string;
+    };
+}
+export interface RecorderOpts {
+    baseUrl?: string;
+    includeConsoleCheck?: boolean;
+    testTimeout?: number;
+    projectName?: string;
+    useSmartLocators?: boolean;
+}
+export declare function generatePlaywrightTest(screens: RecorderScreen[], opts?: RecorderOpts): string;
+export declare function generatePlaywrightConfig(opts?: {
+    baseUrl?: string;
+    reportDir?: string;
+}): string;

package/types/policies/crawl-safety.d.ts

@@ -0,0 +1,9 @@
+import type { DomainProfile } from '../profiles/domain-profile.js';
+export interface CrawlSafetyDecision {
+    allowed: boolean;
+    reason?: string;
+    severity: 'info' | 'warn' | 'block';
+}
+export declare function assessActionSafety(label: string, profile: DomainProfile): CrawlSafetyDecision;
+export declare function redactSensitiveValue(value: string): string;
+export declare function redactJson(value: unknown): unknown;

package/types/profiles/domain-profile.d.ts

@@ -0,0 +1,19 @@
+export type Industry = 'ecommerce' | 'banking' | 'fintech' | 'healthcare' | 'pharma' | 'agriculture' | 'legacy-enterprise' | 'saas' | 'unknown';
+export interface DomainProfile {
+    industry: Industry;
+    confidence: number;
+    importantScreenPatterns: string[];
+    workflowGoals: string[];
+    piiRules: string[];
+    riskyActions: string[];
+    safeProbeRules: string[];
+    requiredArtifacts: string[];
+    evidence: string[];
+}
+export declare function classifyDomainProfile(input: {
+    appUrl: string;
+    projectName?: string | null;
+    appType?: string | null;
+    seedText?: string;
+}): DomainProfile;
+export declare function getProfile(industry: Industry): DomainProfile;

package/types/proxy.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Free proxy auto-selection for bot-protection bypass.
+ *
+ * Source: proxifly/free-proxy-list (MIT) via jsDelivr CDN.
+ * These are datacenter proxies — they bypass simple IP blocks but NOT
+ * full Cloudflare Bot Manager / Akamai. For Cloudflare-protected sites,
+ * users need a residential proxy (Bright Data, Oxylabs, etc.).
+ */
+/**
+ * Pick a working free proxy. Tests in parallel batches of 10 with a 4s timeout.
+ * Returns null if no working proxy found (not an error — crawl can proceed without proxy).
+ */
+export declare function pickFreeProxy(): Promise<string | null>;
+export type ProxyTier = 'free' | 'datacenter' | 'residential' | 'premium';
+export interface ProxyTierConfig {
+    /** List of proxy URLs for this tier. E.g. ['http://user:pass@dc1.proxy:8080'] */
+    urls: string[];
+    tier: ProxyTier;
+}
+/**
+ * Tiered proxy manager with per-domain error histograms.
+ *
+ * Tracks success/failure per domain and automatically escalates to higher proxy tiers
+ * on consecutive failures. Applies time-decay to probe returning to lower tiers.
+ *
+ * Usage:
+ *   const mgr = new TieredProxyManager([{ tier: 'datacenter', urls: [...] }]);
+ *   const proxy = await mgr.selectProxy('api.salesforce.com');
+ *   // ... make request ...
+ *   mgr.recordResult('api.salesforce.com', success);
+ */
+export declare class TieredProxyManager {
+    private tiers;
+    private domainHistory;
+    private readonly ESCALATE_AFTER;
+    private readonly DECAY_AFTER_MS;
+    constructor(configs: ProxyTierConfig[]);
+    private getHistory;
+    private lowestAvailableTier;
+    /** Pick a proxy URL for this domain, based on its tier history. */
+    selectProxy(domain: string): string | null;
+    /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
+    recordResult(domain: string, success: boolean): void;
+    stats(): Array<{
+        domain: string;
+        tier: ProxyTier;
+        successRate: number;
+        totalRequests: number;
+    }>;
+}

package/types/rate-limiter.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Domain-aware rate limiter — ensures the crawler respects per-domain rate limits.
+ *
+ * Uses a token bucket algorithm per domain. When the bucket is empty, requests
+ * wait until a token is available. Tokens refill at a configurable rate.
+ *
+ * Default: 2 requests/second per domain, burst of 5.
+ * Can be configured per-domain via the rateMap (domain → requests/second).
+ */
+export interface RateLimiterOpts {
+    defaultRps?: number;
+    burstSize?: number;
+    rateMap?: Record<string, number>;
+}
+export declare class DomainRateLimiter {
+    private buckets;
+    private readonly defaultRps;
+    private readonly burstSize;
+    private readonly rateMap;
+    constructor(opts?: RateLimiterOpts);
+    private getBucket;
+    private refill;
+    /** Wait until a token is available for the given URL's domain. */
+    throttle(url: string): Promise<void>;
+    /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
+    setRate(domain: string, rps: number): void;
+    stats(): Record<string, {
+        tokens: number;
+        rps: number;
+    }>;
+}

package/types/redirect-tracker.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Redirect chain tracker — records the full redirect chain for crawled URLs.
+ *
+ * Why: redirect chains (A→B→C) slow down pages; circular redirects crash crawlers;
+ * redirect loops indicate config errors. ZeTa records chains for SEO/perf audit.
+ *
+ * Integration: hook into Playwright's page.on('response') to see each redirect hop.
+ */
+export interface RedirectHop {
+    from: string;
+    to: string;
+    statusCode: number;
+    durationMs: number;
+}
+export interface RedirectChain {
+    originalUrl: string;
+    finalUrl: string;
+    hops: RedirectHop[];
+    isCircular: boolean;
+    hopCount: number;
+    totalDurationMs: number;
+}
+import type { Page } from 'playwright';
+export declare function trackRedirects(page: Page, url: string): Promise<RedirectChain>;
+export declare class RedirectRegistry {
+    private chains;
+    register(chain: RedirectChain): void;
+    get(url: string): RedirectChain | undefined;
+    getAll(): RedirectChain[];
+    getProblematic(): RedirectChain[];
+    stats(): {
+        total: number;
+        withRedirects: number;
+        circular: number;
+        deepChains: number;
+    };
+}

package/types/redis-frontier.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Redis-backed distributed crawl frontier.
+ *
+ * Replaces the in-memory CrawlFrontier for horizontally-scaled crawler deployments.
+ * Multiple crawler workers on different containers can share the same URL queue for
+ * a given job — enabling 10x crawl throughput on large enterprise apps.
+ *
+ * Redis key namespace:  zeta:frontier:{jobId}
+ *   LIST  zeta:frontier:{jobId}:queue   — BFS queue (LPUSH/RPOP)
+ *   SET   zeta:frontier:{jobId}:visited — deduplication set
+ *   EXPIRE                               — 24h TTL on all keys
+ *
+ * Graceful fallback: if REDIS_URL is not set, RedisCrawlFrontier silently falls
+ * back to in-memory arrays so local development works without Redis.
+ */
+export declare class RedisCrawlFrontier {
+    private readonly jobId;
+    private redis;
+    private queueKey;
+    private visitedKey;
+    private inMemoryQueue;
+    private inMemoryVisited;
+    private useRedis;
+    constructor(jobId: string);
+    init(): Promise<void>;
+    /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
+    push(url: string): Promise<boolean>;
+    /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
+    shift(): Promise<string | null>;
+    /** Check if a URL has already been visited/queued. */
+    hasVisited(url: string): Promise<boolean>;
+    /** Current queue length. */
+    length(): Promise<number>;
+    /** Total visited count. */
+    visitedCount(): Promise<number>;
+    /** Seed initial URLs (used at job start). */
+    seed(urls: string[]): Promise<void>;
+    /** Clean up all Redis keys for this frontier. Call at job completion. */
+    cleanup(): Promise<void>;
+    /** Stats for monitoring. */
+    stats(): Promise<{
+        backend: 'redis' | 'memory';
+        queued: number;
+        visited: number;
+    }>;
+}
+/** Check if Redis is configured for distributed frontier. */
+export declare function isRedisFrontierEnabled(): boolean;

package/types/redis-queue.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Redis Streams distributed job queue.
+ *
+ * Provides horizontal crawler scaling via Redis Streams consumer groups.
+ * Multiple crawler nodes can pull from the same stream — each job is
+ * delivered to exactly one consumer (at-least-once via ACK/PENDING).
+ *
+ * Used when REDIS_URL is set AND JOB_QUEUE_BACKEND=redis.
+ * Falls back to the Postgres-backed queue when Redis is unavailable.
+ *
+ * Stream key:   zeta:crawl:stream
+ * Group name:   zeta:crawl:workers
+ * Consumer id:  zeta:worker:<hostname>:<pid>
+ */
+export interface RedisQueueJob {
+    id: string;
+    jobId: string;
+    tenantId: string;
+    projectId: string;
+    payload: Record<string, unknown>;
+    enqueuedAt: string;
+}
+export interface RedisQueueClient {
+    enqueue(jobId: string, tenantId: string, projectId: string, payload?: Record<string, unknown>): Promise<string>;
+    dequeue(timeoutMs?: number): Promise<RedisQueueJob | null>;
+    ack(streamEntryId: string): Promise<void>;
+    nack(streamEntryId: string): Promise<void>;
+    reclaimStale(maxIdleMs?: number): Promise<RedisQueueJob[]>;
+    stats(): Promise<{
+        pending: number;
+        delivered: number;
+        groupLag: number;
+    }>;
+    close(): Promise<void>;
+}
+/**
+ * Create a Redis Streams job queue client.
+ * Throws if REDIS_URL is not set.
+ */
+export declare function createRedisQueue(): Promise<RedisQueueClient>;
+/**
+ * Check if Redis queue backend is configured.
+ */
+export declare function isRedisQueueEnabled(): boolean;
+/**
+ * Convenience: create queue only if enabled, else return null.
+ */
+export declare function maybeCreateRedisQueue(): Promise<RedisQueueClient | null>;