@taodigital/crawler 0.0.0-stage → 1.2.0

package/types/custom-js-injector.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Custom JS injector — run user-provided JavaScript in the context of each crawled page.
+ *
+ * Use cases:
+ * - Extract custom data points not captured by standard extractors
+ * - Set localStorage/sessionStorage values before crawl
+ * - Override fetch() to intercept API calls
+ * - Simulate user events (keydown, focus, etc.)
+ * - Bypass specific anti-bot checks
+ *
+ * Security: scripts run in the page context (sandboxed by Chromium's renderer).
+ * They cannot access the Node.js process or file system.
+ * ZeTa validates scripts are strings before injection.
+ */
+import type { Page } from 'playwright';
+export interface JsInjectionOpts {
+    /** Script to run on every page BEFORE navigation (via page.addInitScript) */
+    initScript?: string;
+    /** Script to run on every page AFTER navigation completes (via page.evaluate) */
+    postLoadScript?: string;
+    /** Key-value pairs to set in localStorage before page load */
+    localStorage?: Record<string, string>;
+    /** Key-value pairs to set in sessionStorage before page load */
+    sessionStorage?: Record<string, string>;
+    /** Cookie key-value pairs to add before page load */
+    cookies?: Array<{
+        name: string;
+        value: string;
+        domain?: string;
+        path?: string;
+    }>;
+}
+export interface JsInjectionResult {
+    initScriptApplied: boolean;
+    postLoadResult: unknown;
+    storageSet: boolean;
+    cookiesSet: number;
+    error?: string;
+}
+export declare function injectCustomJs(page: Page, opts: JsInjectionOpts): Promise<JsInjectionResult>;
+/** Safe string validator — ensures script is a string, not an object/function reference */
+export declare function validateScript(script: unknown): string;

package/types/dark-mode-capture.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Dark mode capture — screenshot pages in both light and dark color schemes.
+ *
+ * Uses Playwright's page.emulateMedia({ colorScheme: 'dark' }) to force dark mode,
+ * then captures a screenshot. Compares with light mode to determine if dark mode
+ * is supported (significant visual difference).
+ *
+ * Also detects prefers-color-scheme media query support in CSS.
+ */
+import type { Page } from 'playwright';
+export interface DarkModeCapture {
+    url: string;
+    lightScreenshot?: Buffer;
+    darkScreenshot?: Buffer;
+    hasDarkMode: boolean;
+    hasCssDarkMode: boolean;
+    backgroundColorLight?: string;
+    backgroundColorDark?: string;
+    capturedAt: string;
+}
+export declare function captureDarkMode(page: Page, url: string, opts?: {
+    takeScreenshots?: boolean;
+}): Promise<DarkModeCapture>;

package/types/dead-link-detector.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Dead link detector — tracks broken links discovered during crawl.
+ *
+ * A "dead link" is a URL that:
+ * - Returns 404, 410, or other 4xx/5xx responses
+ * - Times out after maxRetries
+ * - Redirects to an error page (detected by title heuristic)
+ *
+ * Results: list of broken links with source page, status code, and anchor text.
+ * Used for: SEO audit export, test report enrichment.
+ */
+export interface DeadLink {
+    url: string;
+    statusCode: number;
+    sourceUrl?: string;
+    anchorText?: string;
+    checkedAt: string;
+    reason: 'http-error' | 'timeout' | 'dns-error';
+}
+export declare class DeadLinkTracker {
+    private deadLinks;
+    private checked;
+    record(url: string, statusCode: number, source?: {
+        sourceUrl?: string;
+        anchorText?: string;
+    }): void;
+    recordTimeout(url: string, sourceUrl?: string): void;
+    getDeadLinks(): DeadLink[];
+    isKnownBroken(url: string): boolean;
+    stats(): {
+        total: number;
+        by404: number;
+        by5xx: number;
+        byTimeout: number;
+    };
+    toCsv(): string;
+}

package/types/dropdown-enumerator.d.ts

@@ -0,0 +1,20 @@
+import type { Page } from 'playwright';
+export interface DropdownOption {
+    value: string;
+    label: string;
+    disabled?: boolean;
+}
+export interface DropdownDef {
+    kind: 'native-select' | 'combobox' | 'listbox' | 'multiselect';
+    label: string;
+    locatorHint: string;
+    options: DropdownOption[];
+    defaultValue?: string;
+    cascadesFrom?: string;
+}
+export interface DropdownEnumerationResult {
+    dropdowns: DropdownDef[];
+    conditionalRevealMap: Record<string, string[]>;
+    totalOptionsFound: number;
+}
+export declare function enumerateDropdowns(page: Page): Promise<DropdownEnumerationResult>;

package/types/element-hash.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Compute a stable fingerprint of a screen's element set.
+ * Sorted so insertion-order changes (React re-renders) don't produce false positives.
+ * Returns first 16 hex chars of SHA-256.
+ */
+export declare function computeElementHash(elements: Array<{
+    role: string;
+    meaning: string;
+}>): string;

package/types/email-otp.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Email OTP retrieval — supports 3 providers:
+ *   mailslurp  — managed SaaS (existing, no self-hosting)
+ *   imap       — any real IMAP mailbox (Gmail, Outlook, custom domain catch-all)
+ *   mailpit    — self-hosted SMTP trap (dev/staging only)
+ *
+ * Use mailslurp or imap for production apps.
+ * Use mailpit only when you can point the app's SMTP at the trap server.
+ */
+export type EmailOTPProvider = 'mailslurp' | 'imap' | 'mailpit';
+export interface EmailContent {
+    /** 4-8 digit OTP code, if found. */
+    otp?: string;
+    /** Magic link URL to navigate to, if found. */
+    magicLink?: string;
+}
+export interface EmailOTPConfig {
+    provider: EmailOTPProvider;
+    apiKey?: string;
+    inboxId?: string;
+    imapHost?: string;
+    imapPort?: number;
+    imapUser?: string;
+    imapPass?: string;
+    imapTls?: boolean;
+    mailpitBaseUrl?: string;
+}
+/**
+ * Wait for an email and return its OTP code or magic link URL.
+ * Prefer this over waitForEmailOTP when the app may send passwordless magic links.
+ */
+export declare function waitForEmailContent(config: EmailOTPConfig, since: string | Date): Promise<EmailContent | null>;
+/**
+ * Wait for an OTP email and extract the code.
+ * @deprecated Use waitForEmailContent which also handles magic links.
+ */
+export declare function waitForEmailOTP(config: EmailOTPConfig, since: string | Date): Promise<string | null>;
+/**
+ * Get the email address for a MailSlurp inbox (used to display in UI).
+ * Returns null for IMAP/Mailpit providers — use imapUser directly.
+ */
+export declare function getInboxEmail(configOrApiKey: EmailOTPConfig | string, inboxId?: string): Promise<string | null>;

package/types/engine-router.d.ts

@@ -0,0 +1,47 @@
+import type { DomainProfile } from './profiles/domain-profile.js';
+export type CrawlEngineName = 'http' | 'browser' | 'browser-stealth' | 'stagehand' | 'vision' | 'recorded-session' | 'native';
+export interface CrawlRouteDecisionInput {
+    appUrl: string;
+    appType?: string | null;
+    hasCredentials: boolean;
+    hasStorageState: boolean;
+    hasAuthToken: boolean;
+    hasSkyvern: boolean;
+    hasProxy: boolean;
+    profile: DomainProfile;
+    previousFailureReason?: string;
+}
+export interface CrawlRouteDecision {
+    engines: CrawlEngineName[];
+    reasons: string[];
+    confidence: number;
+}
+/**
+ * Per-domain rendering predictor — learns whether a domain requires a full browser
+ * or can be served by the fast HTTP engine. Implements adaptive crawlee-python pattern.
+ *
+ * Strategy:
+ * 1. On first encounter of a domain, classify the HTTP response body.
+ * 2. Store the decision in memory (Map). All future URLs on same domain skip re-testing.
+ * 3. "Requires browser" signals: React/Vue/Angular root with no content, tiny body (<500 chars meaningful text).
+ */
+export declare class RenderingTypePredictor {
+    private cache;
+    /** Return cached decision if available, undefined if unknown. */
+    getCached(url: string): 'http' | 'browser' | undefined;
+    /**
+     * Analyze an HTTP response body to predict if JS rendering is required.
+     * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
+     */
+    predict(url: string, htmlBody: string, statusCode: number): 'http' | 'browser';
+    /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
+    set(url: string, engine: 'http' | 'browser'): void;
+    stats(): {
+        cached: number;
+        browserDomains: string[];
+        httpDomains: string[];
+    };
+}
+/** Singleton predictor — shared across crawl jobs in the same process. */
+export declare const renderingPredictor: RenderingTypePredictor;
+export declare function chooseCrawlEngines(input: CrawlRouteDecisionInput): CrawlRouteDecision;

package/types/export/bulk-export.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Bulk export — export crawled screens to CSV / JSON / Markdown digest.
+ *
+ * Outputs:
+ * - CSV: url, name, wordCount, screenshotPath, markdownPath, createdAt — for spreadsheet analysis
+ * - JSON: full screen records array — for API integration
+ * - Markdown digest: concatenated markdown of all screens — for LLM context windows
+ */
+export interface ExportScreen {
+    id: string;
+    name: string;
+    url: string;
+    projectId: string;
+    wordCount?: number;
+    screenshotPath?: string | null;
+    markdownPath?: string | null;
+    markdown?: string | null;
+    createdAt?: Date;
+    updatedAt?: Date;
+}
+export declare function exportToCsv(screens: ExportScreen[], outputPath: string): Promise<void>;
+export declare function exportToJson(screens: ExportScreen[], outputPath: string): Promise<void>;
+export declare function exportToMarkdownDigest(screens: ExportScreen[], outputPath: string, opts?: {
+    readMarkdownFromPath?: boolean;
+    maxScreens?: number;
+}): Promise<void>;

package/types/export/vector-export.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Vector DB export — convert crawled screens to embedding-ready chunks.
+ *
+ * Produces a JSONL file where each line is one chunk ready for upsert into
+ * Pinecone, Qdrant, Weaviate, or any vector store. The embedding itself is
+ * NOT generated here (requires an embedding model at the user's end) — we
+ * produce the text chunks + metadata, and the caller embeds them.
+ *
+ * Format per line (JSONL):
+ * {
+ *   "id": "<screenId>-<chunkIndex>",
+ *   "text": "<chunk text>",
+ *   "metadata": {
+ *     "screenId": "...",
+ *     "projectId": "...",
+ *     "tenantId": "...",
+ *     "url": "...",
+ *     "screenName": "...",
+ *     "chunkIndex": 0,
+ *     "totalChunks": 3,
+ *     "crawledAt": "2026-...",
+ *     "wordCount": 142
+ *   }
+ * }
+ */
+export interface VectorChunk {
+    id: string;
+    text: string;
+    metadata: {
+        screenId: string;
+        projectId: string;
+        tenantId: string;
+        url: string;
+        screenName: string;
+        chunkIndex: number;
+        totalChunks: number;
+        crawledAt: string;
+        wordCount: number;
+    };
+}
+export interface ScreenRecord {
+    id: string;
+    name: string;
+    url: string;
+    projectId: string;
+    tenantId: string;
+    markdown?: string | null;
+    markdownPath?: string | null;
+    updatedAt?: Date;
+}
+export declare function exportScreensToVectorJsonl(screens: ScreenRecord[], outputPath: string, opts?: {
+    maxChunkWords?: number;
+    readMarkdownFromPath?: boolean;
+}): Promise<{
+    totalChunks: number;
+    skipped: number;
+}>;
+/**
+ * Pinecone-compatible upsert batch format.
+ * Call this after generating embeddings to produce the final upsert payload.
+ */
+export declare function toPineconeUpsert(chunks: VectorChunk[], embeddings: number[][], namespace?: string): Array<{
+    id: string;
+    values: number[];
+    metadata: VectorChunk['metadata'];
+}>;
+/**
+ * Qdrant-compatible point format.
+ */
+export declare function toQdrantPoints(chunks: VectorChunk[], embeddings: number[][]): Array<{
+    id: string;
+    vector: number[];
+    payload: VectorChunk['metadata'] & {
+        text: string;
+    };
+}>;

package/types/extractors/advanced.d.ts

@@ -0,0 +1,64 @@
+export interface ExtractionResult {
+    matches: string[];
+    count: number;
+    method: string;
+}
+export type TableRow = Record<string, string>;
+export interface PageMetadata {
+    title: string | null;
+    description: string | null;
+    keywords: string | null;
+    ogTitle: string | null;
+    ogDescription: string | null;
+    ogImage: string | null;
+    ogUrl: string | null;
+    ogType: string | null;
+    ogSiteName: string | null;
+    ogLocale: string | null;
+    twitterCard: string | null;
+    twitterTitle: string | null;
+    twitterDescription: string | null;
+    twitterImage: string | null;
+    twitterSite: string | null;
+    canonical: string | null;
+    robots: string | null;
+    hreflang: Array<{
+        lang: string;
+        href: string;
+    }>;
+    themeColor: string | null;
+    favicon: string | null;
+    jsonLd: object[];
+    schemaTypes: string[];
+    lang: string | null;
+}
+export declare function extractByCssSelector(html: string, selector: string, attr?: string): ExtractionResult;
+export declare function extractByXPath(html: string, xpath: string): ExtractionResult;
+export declare function extractByRegex(text: string, pattern: string, flags?: string): ExtractionResult;
+export declare function extractTables(html: string): TableRow[][];
+export declare function extractMetadata(html: string, url: string): PageMetadata;
+export interface ProductData {
+    name: string | null;
+    description: string | null;
+    brand: string | null;
+    sku: string | null;
+    price: number | null;
+    priceCurrency: string | null;
+    availability: string | null;
+    rating: number | null;
+    reviewCount: number | null;
+    image: string | null;
+    url: string | null;
+}
+export declare function extractStructuredProduct(jsonLd: object[]): ProductData | null;
+export declare function extractImages(html: string, baseUrl: string): Array<{
+    src: string;
+    alt: string;
+    width?: number;
+    height?: number;
+}>;
+export declare function extractMediaFiles(html: string, baseUrl: string): Array<{
+    url: string;
+    type: 'image' | 'video' | 'audio' | 'document';
+    tag: string;
+}>;

package/types/extractors/artifact-envelope.d.ts

@@ -0,0 +1,63 @@
+import type { RawElement } from '../screen-reader.js';
+export interface CrawlEvidenceFact {
+    type: string;
+    payload: unknown;
+}
+export interface CrawlArtifactEnvelope {
+    tenantId: string;
+    projectId: string;
+    jobId?: string;
+    url: string;
+    canonicalUrl: string;
+    screenId?: string;
+    status: 'ok' | 'blocked' | 'auth_expired' | 'error' | 'skipped';
+    outputs: {
+        screenshot?: string;
+        html?: string;
+        markdown?: string;
+        fitMarkdown?: string;
+        accessibilityTree?: string;
+        structuredJson?: unknown;
+        networkTrace?: string;
+        consoleLog?: string;
+    };
+    fingerprints: {
+        textHash?: string;
+        domHash?: string;
+        accessibilityHash?: string;
+        urlCanonicalHash: string;
+    };
+    evidence: CrawlEvidenceFact[];
+}
+export declare function sha256(input: string): string;
+export declare function buildFitMarkdown(markdown: string, keywords?: string[]): string;
+export declare function summarizeStructuredElements(elements: RawElement[]): {
+    elementCount: number;
+    formFieldCount: number;
+    actionCount: number;
+    fields: {
+        label: string;
+        role: string;
+        expectedData: string | undefined;
+    }[];
+    actions: {
+        label: string;
+        role: string;
+    }[];
+};
+export declare function buildCrawlArtifactEnvelope(input: {
+    tenantId: string;
+    projectId: string;
+    jobId?: string;
+    url: string;
+    canonicalUrl: string;
+    screenId?: string;
+    screenshotPath?: string;
+    domPath?: string;
+    domContent?: string;
+    markdown?: string;
+    fitMarkdown?: string;
+    accessibilityElements?: RawElement[];
+    structuredJson?: unknown;
+    evidence?: CrawlEvidenceFact[];
+}): CrawlArtifactEnvelope;

package/types/extractors/branding-extractor.d.ts

@@ -0,0 +1,46 @@
+import type { Page } from 'playwright';
+export interface BrandingColors {
+    primary?: string;
+    accent?: string;
+    background?: string;
+    textPrimary?: string;
+    link?: string;
+}
+export interface BrandingFont {
+    family: string;
+    role: 'heading' | 'body';
+}
+export interface BrandingImages {
+    logo?: string;
+    favicon?: string;
+    ogImage?: string;
+}
+export interface BrandingPersonality {
+    tone?: string;
+    energy?: 'high' | 'medium' | 'low';
+    targetAudience?: string;
+}
+export interface BrandingResult {
+    colorScheme: 'light' | 'dark';
+    colors: BrandingColors;
+    fonts: BrandingFont[];
+    images: BrandingImages;
+    personality?: BrandingPersonality;
+    designSystem?: {
+        framework?: string;
+        componentLibrary?: string;
+    };
+    metadata?: {
+        title?: string;
+        description?: string;
+        themeColor?: string;
+        language?: string;
+    };
+    confidence: {
+        colors: number;
+        fonts: number;
+        images: number;
+        overall: number;
+    };
+}
+export declare function extractBranding(page: Page): Promise<BrandingResult>;

package/types/extractors/summary-extractor.d.ts

@@ -0,0 +1,10 @@
+export interface SummaryResult {
+    summary: string;
+    wordCount: number;
+    method: 'llm' | 'truncated';
+    model?: string;
+}
+export declare function extractSummary(markdown: string, options?: {
+    model?: string;
+    maxTokens?: number;
+}): Promise<SummaryResult>;

package/types/failure-analysis-agent.d.ts

@@ -0,0 +1,16 @@
+export interface FailureContext {
+    tenantId?: string;
+    error: string;
+    stepDescription?: string;
+    consoleLogs?: Array<{
+        type: string;
+        text: string;
+    }>;
+    networkErrors?: Array<{
+        url: string;
+        status: number;
+        method: string;
+    }>;
+    screenshotPath?: string;
+}
+export declare function analyzeFailure(ctx: FailureContext): Promise<string | null>;

package/types/feature-flag-detector.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Feature flag + A/B test detector.
+ *
+ * Detects popular feature flag / experiment SDKs loaded on the page:
+ * LaunchDarkly, Optimizely, Split.io, Unleash, GrowthBook, Statsig, Flagsmith, Amplitude Experiment.
+ *
+ * Also detects common A/B test patterns: data-testid attributes, gtm.js experiment layers,
+ * Optimizely variation IDs, VWO, etc.
+ *
+ * Why: ZeTa needs to know when a page shows different content to different users
+ * so test generation can account for variants and not generate flaky tests.
+ */
+import type { Page } from 'playwright';
+export interface FeatureFlagInfo {
+    sdk: string;
+    confidence: 'high' | 'medium' | 'low';
+    details?: string;
+}
+export interface AbTestInfo {
+    tool: string;
+    experiments?: string[];
+}
+export interface FeatureFlagDetectionResult {
+    url: string;
+    flagSdks: FeatureFlagInfo[];
+    abTools: AbTestInfo[];
+    hasAnyFlags: boolean;
+    detectedAt: string;
+}
+export declare function detectFeatureFlags(page: Page, url: string): Promise<FeatureFlagDetectionResult>;

package/types/filter-pagination-crawler.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Filter state crawling and pagination capture.
+ *
+ * Discovers filter controls (select, checkbox-group, radio-group, search),
+ * applies each meaningful option and records result counts and first-row
+ * values, then detects and walks pagination (numbered, load-more, infinite
+ * scroll). Also captures sort column behaviour.
+ *
+ * All per-filter / per-page errors are caught and skipped so one bad control
+ * never aborts the full crawl.
+ */
+import type { Page } from 'playwright';
+export interface FilterOption {
+    value: string;
+    label: string;
+}
+export interface FilterDef {
+    kind: 'select' | 'checkbox-group' | 'radio-group' | 'search';
+    label: string;
+    locatorHint: string;
+    options: FilterOption[];
+}
+export interface FilterState {
+    filterLabel: string;
+    filterValue: string;
+    resultCount: number | null;
+    firstRowValues: string[];
+}
+export interface PaginationCapture {
+    kind: 'numbered' | 'load-more' | 'infinite-scroll' | 'none';
+    totalPages?: number;
+    totalItems?: number;
+    pageSizes?: number[];
+    pages: Array<{
+        pageNum: number;
+        rowValues: string[];
+    }>;
+}
+export interface FilterPaginationResult {
+    filters: FilterDef[];
+    filterStates: FilterState[];
+    pagination: PaginationCapture;
+    sortColumns: Array<{
+        header: string;
+        ariaSortAfterClick: string | null;
+    }>;
+    searchCapture?: {
+        noResultsText: string | null;
+        sampleSuggestions: string[];
+    };
+}
+export declare function crawlFiltersAndPagination(page: Page): Promise<FilterPaginationResult>;

package/types/form-detector.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Form detector — extract form metadata from crawled pages.
+ *
+ * For each form found on a page, captures: action URL, method, all input fields
+ * with their types/names/required/validation constraints, and submit button text.
+ *
+ * This data feeds ZeTa's test generator to create form validation test cases
+ * automatically (happy path + boundary/error cases per field type).
+ */
+import type { Page } from 'playwright';
+export interface FormField {
+    name: string;
+    type: string;
+    label?: string;
+    required: boolean;
+    placeholder?: string;
+    minLength?: number;
+    maxLength?: number;
+    min?: string;
+    max?: string;
+    pattern?: string;
+    options?: string[];
+    autocomplete?: string;
+}
+export interface DetectedForm {
+    index: number;
+    action?: string;
+    method: string;
+    fields: FormField[];
+    submitText?: string;
+    hasFileUpload: boolean;
+    purpose: 'login' | 'search' | 'registration' | 'contact' | 'checkout' | 'settings' | 'unknown';
+}
+export declare function detectForms(page: Page): Promise<DetectedForm[]>;