@taodigital/crawler 0.0.0-stage → 1.2.1

package/types/job-queue.d.ts

@@ -0,0 +1,308 @@
+/**
+ * ZeTa-AI Background Job Queue
+ *
+ * Postgres-backed durable queue. Each job becomes a PlatformJob record.
+ * enqueueJob() writes it QUEUED and, for same-process responsiveness, kicks
+ * an immediate best-effort claim attempt — but durability comes from
+ * startJobWorkerLoop()'s poll: it atomically claims QUEUED jobs (Postgres
+ * `FOR UPDATE SKIP LOCKED`, safe across any number of concurrent worker
+ * processes/containers) and requeues jobs whose heartbeat went stale,
+ * so a crashed/restarted worker never silently loses a job.
+ *
+ * CRAWL-type jobs additionally support AWS SQS as an alternate transport
+ * (JOB_QUEUE_BACKEND=sqs, see sqs-queue.ts) for horizontally-scaled crawler
+ * fleets; every other job type always goes through the DB-backed queue.
+ */
+export declare enum FailureType {
+    TRANSIENT = "TRANSIENT",// network timeout, rate limit, 429, 503
+    INFRASTRUCTURE = "INFRASTRUCTURE",// DB connection, OOM, worker crash
+    LOGICAL = "LOGICAL",// bad input, business rule violation, 400/422
+    FATAL = "FATAL"
+}
+export declare function classifyFailure(error: string | Error | unknown): FailureType;
+/**
+ * Centralized failure recorder for all PlatformJobs.
+ * - Classifies error into FailureType (TRANSIENT, INFRASTRUCTURE, LOGICAL, FATAL)
+ * - Computes exponential backoff per failure classification
+ * - If FATAL or retries exhausted: marks status='FAILED', dlq=true, dlqAt=NOW(), dlqReason=error
+ * - If retryable: marks status='RETRY_WAIT', nextRetryAt=NOW() + backoff, retryCount=retryCount+1
+ * - Persists checkpoint and checkpointAt if checkpointData provided
+ * - Updates failureType on the row and in result.failureType
+ */
+export declare function recordJobFailure(tenantId: string | null | undefined, jobId: string, err: unknown, checkpointData?: any, options?: {
+    isAborted?: boolean;
+}): Promise<void>;
+export declare function generateTOTP(secret: string): string;
+export interface PlatformJobConfig {
+    tenantId: string;
+    projectId: string;
+    appUrl?: string;
+    credentials?: {
+        username: string;
+        password: string;
+        oneTimeCode?: string;
+    };
+    /** Decrypted Playwright storageState JSON — if present, session is restored without AI login. */
+    storageState?: string;
+    screenshotDir?: string;
+    maxScreens?: number;
+    /** Seed URLs added to the initial crawl queue — link discovery still runs from each page. */
+    startUrls?: string[];
+    /** When provided, crawl ONLY these exact URLs (no link-following). Used for selected-screen recrawl. */
+    selectiveUrls?: string[];
+    /** Device profiles for multi-viewport screenshot pass after main DESKTOP crawl. */
+    deviceProfiles?: string[];
+    /** HTTP Basic Auth credentials — injected as Authorization header on same-origin requests. */
+    httpBasicUsername?: string;
+    httpBasicPassword?: string;
+    /** Bearer/API token — injected as a request header and into localStorage. Skips AI login. */
+    authToken?: string;
+    /** Header name for authToken (default: 'Authorization'). */
+    authTokenHeader?: string;
+    /** Prefix for authToken value (default: 'Bearer '). */
+    authTokenPrefix?: string;
+    /** CAPTCHA solving API key (2captcha or CapSolver). */
+    captchaSolverApiKey?: string;
+    /** CAPTCHA solving service provider (default: '2captcha'). */
+    captchaSolverProvider?: '2captcha' | 'capsolver';
+    /**
+     * Path to a Chrome user data directory for profile inheritance (local deployments only).
+     * When set, the crawler exports the existing session from this profile and injects it.
+     */
+    chromeProfilePath?: string;
+    /** MailSlurp API key for email OTP auto-retrieval. */
+    mailslurpApiKey?: string;
+    /** MailSlurp inbox ID to poll for OTP emails. */
+    mailslurpInboxId?: string;
+    /** SSO provider ('okta' | 'azure_ad' | 'generic_oidc'). */
+    ssoProvider?: string;
+    /** SSO domain or tenant ID. */
+    ssoDomain?: string;
+    /** SSO OAuth2 client ID. */
+    ssoClientId?: string;
+    /** SSO OAuth2 client secret (decrypted). */
+    ssoClientSecret?: string;
+    /** SSO scopes (space-separated). Default: 'openid profile'. */
+    ssoScope?: string;
+    /** HTTP/HTTPS/SOCKS5 proxy URL — routes browser through this proxy for bot-protected sites. */
+    proxyUrl?: string;
+    /** HTML extraction mode. 'normal'=raw, 'reader'=Readability article, 'prune'=strip nav/ads. Default: 'normal'. */
+    htmlMode?: 'normal' | 'reader' | 'prune';
+    /** Per-request screenshot dimensions and quality. Defaults: 1440×900, quality 80. */
+    screenshotConfig?: {
+        fullPage?: boolean;
+        width?: number;
+        height?: number;
+        quality?: number;
+    };
+    /** Enable/disable Playwright stealth plugin. Default: true (existing behaviour). */
+    stealth?: boolean;
+    /** Mock mode — intercept all navigation and return synthetic content. Good for CI. */
+    mock?: boolean | {
+        minKb?: number;
+        maxKb?: number;
+        minSleep?: number;
+        maxSleep?: number;
+    };
+    /** MIME type allowlist — crawler skips URLs returning disallowed content types. */
+    allowedTypes?: string[];
+    /** Glob-style include patterns for URL paths (e.g. '/blog/*'). Only matching paths crawled. */
+    includePatterns?: string[];
+    /** Glob-style exclude patterns for URL paths (e.g. '/admin/*'). Matching paths skipped. */
+    excludePatterns?: string[];
+    /** Allow crawler to follow links to external domains. Default: false. */
+    allowExternal?: boolean;
+    /** Requested output formats for a single-pass multi-format extraction. */
+    requestedFormats?: Array<'markdown' | 'html' | 'links' | 'images' | 'summary' | 'json' | 'branding' | 'screenshot'>;
+    /** LLM chunker configuration for AI extraction passes. */
+    chunkerConfig?: {
+        size?: number | 'dynamic';
+        overlap?: number;
+    };
+    /** ISO-3166-1 alpha-2 country code for geo-targeted proxy routing (e.g. 'us', 'gb'). */
+    proxyCountry?: string;
+    /** When true, after the primary crawl auto-enqueue one CRAWL job per credentialProfile. */
+    crawlAllRoles?: boolean;
+    /** Decrypted credential profiles used when crawlAllRoles is true. */
+    credentialProfiles?: Array<{
+        id: string;
+        name: string;
+        username: string;
+        password: string;
+    }>;
+    /** Set on role-crawl sub-jobs — profile name tagged on discovered screens via roleAccess column. */
+    roleProfileName?: string;
+}
+export declare function getActiveExecutionCount(): number;
+type ClaimedJob = {
+    id: string;
+    tenantId: string;
+    projectId: string;
+    type: string;
+    config: any;
+};
+/** Atomically claims one specific QUEUED job for this worker. Returns null if it was already claimed. */
+export declare function claimSpecificJob(jobId: string): Promise<ClaimedJob | null>;
+/** Atomically claims the oldest QUEUED job across all tenants. SKIP LOCKED makes this
+ *  race-safe against any number of other pollers doing the same thing concurrently.
+ *  DISCOVER jobs are subject to MAX_CONCURRENT_DISCOVER — skipped when that many are
+ *  already RUNNING to prevent Playwright/DB pool exhaustion from simultaneous crawls. */
+export declare function claimNextQueuedJob(): Promise<ClaimedJob | null>;
+/** Requeues RUNNING jobs whose heartbeat has gone stale — the worker that claimed them
+ *  died (crash, restart, OOM) without ever marking them DONE/FAILED. Idempotent and
+ *  safe to run from every worker process; only affects rows past the timeout.
+ *  API-in-process job types (GENERATE_SCRIPT etc.) are failed directly — requeueing them
+ *  would cause the worker to claim and fail them with a confusing "claimed by worker" error. */
+export declare function requeueStaleRunningJobs(): Promise<void>;
+/**
+ * Requeues any RUNNING jobs claimed by a different worker (i.e., a previous
+ * process instance). Call once at startup before the poll loop begins — this
+ * provides immediate recovery instead of waiting the full STALE_JOB_TIMEOUT_MINUTES.
+ * Safe to call from multiple replicas simultaneously (each uses its own WORKER_ID).
+ */
+export declare function requeueOrphanedJobs(): Promise<void>;
+/** Dispatches a claimed job to its type's processor. Last-resort safety net — every
+ *  process*Job function already catches and records its own failures. */
+export declare function dispatchClaimedJob(job: ClaimedJob): Promise<void>;
+/** Re-promotes FAILED jobs to RETRY_WAIT if they are eligible for retry based on per-type limits.
+ *  RETRY_WAIT makes the waiting period visible in the UI. A separate sweep then promotes
+ *  RETRY_WAIT → QUEUED once nextRetryAt <= NOW(). */
+export declare function retryEligibleFailedJobs(): Promise<void>;
+/** Promotes FAILED jobs to DLQ if they exceed their per-type retry limits. */
+export declare function promoteToDeadLetterQueue(): Promise<void>;
+/**
+ * Starts the durable job worker poll loop. This is what makes job processing
+ * survive a restart — enqueueJob()'s immediate same-process claim attempt is
+ * only a latency optimization; if the process dies before that fires (or
+ * mid-job), this loop's next tick claims/requeues it. Call once at process
+ * startup (API server and/or a dedicated crawler worker process — safe to
+ * run in both, and safe to run in many container replicas at once).
+ */
+export declare function startJobWorkerLoop(opts?: {
+    pollIntervalMs?: number;
+    concurrency?: number;
+}): void;
+/** Stops the poll loop. Mainly for tests. */
+export declare function stopJobWorkerLoop(): void;
+/** Create a job record that tracks a synchronous operation (telemetry only, no background processor). */
+export declare function createTrackedJob(type: string, config: {
+    tenantId: string;
+    projectId: string;
+    [k: string]: any;
+}): Promise<string>;
+/** Mark a tracked job DONE or FAILED. Safe to call in finally — swallows errors.
+ * GAP-14/15: Uses failure-type classification for per-type backoff on FAILED. */
+export declare function finishTrackedJob(tenantId: string, jobId: string, error?: string): Promise<void>;
+/** Enqueue a generic platform job — returns the job ID immediately. */
+export declare function enqueueJob(type: string, config: any): Promise<string>;
+/** Get the current state of a platform job. */
+export declare function getJob(tenantId: string, jobId: string): Promise<{
+    progress: number;
+    id: string;
+    name: string | null;
+    type: string;
+    error: string | null;
+    result: import("@prisma/client/runtime/library").JsonValue | null;
+    status: string;
+    tenantId: string;
+    projectId: string;
+    inline: boolean;
+    config: import("@prisma/client/runtime/library").JsonValue;
+    retryCount: number;
+    failureType: string | null;
+    queuedAt: Date;
+    startedAt: Date | null;
+    finishedAt: Date | null;
+    workerId: string | null;
+    heartbeatAt: Date | null;
+    totalPromptTokens: number | null;
+    totalCompletionTokens: number | null;
+    totalCacheReadTokens: number | null;
+    totalEstimatedCostUsd: number | null;
+    llmCallCount: number;
+    nextRetryAt: Date | null;
+    dlq: boolean;
+    dlqReason: string | null;
+    leaseExpiresAt: Date | null;
+    fencingToken: string | null;
+    checkpoint: import("@prisma/client/runtime/library").JsonValue | null;
+    checkpointAt: Date | null;
+    parentJobId: string | null;
+    rootJobId: string | null;
+    currentStep: string | null;
+    requeuedAt: Date | null;
+    dlqAt: Date | null;
+    dlqRedriveCount: number;
+    submittedBy: string | null;
+    priority: number;
+} | null>;
+/** List recent jobs for a project. */
+export declare function listJobs(tenantId: string, projectId: string): Promise<{
+    progress: number;
+    id: string;
+    name: string | null;
+    type: string;
+    error: string | null;
+    result: import("@prisma/client/runtime/library").JsonValue | null;
+    status: string;
+    tenantId: string;
+    projectId: string;
+    inline: boolean;
+    config: import("@prisma/client/runtime/library").JsonValue;
+    retryCount: number;
+    failureType: string | null;
+    queuedAt: Date;
+    startedAt: Date | null;
+    finishedAt: Date | null;
+    workerId: string | null;
+    heartbeatAt: Date | null;
+    totalPromptTokens: number | null;
+    totalCompletionTokens: number | null;
+    totalCacheReadTokens: number | null;
+    totalEstimatedCostUsd: number | null;
+    llmCallCount: number;
+    nextRetryAt: Date | null;
+    dlq: boolean;
+    dlqReason: string | null;
+    leaseExpiresAt: Date | null;
+    fencingToken: string | null;
+    checkpoint: import("@prisma/client/runtime/library").JsonValue | null;
+    checkpointAt: Date | null;
+    parentJobId: string | null;
+    rootJobId: string | null;
+    currentStep: string | null;
+    requeuedAt: Date | null;
+    dlqAt: Date | null;
+    dlqRedriveCount: number;
+    submittedBy: string | null;
+    priority: number;
+}[]>;
+/** Cancel a RUNNING or QUEUED job.
+ *  Sets status to CANCEL_REQUESTED (GAP-08) so the worker's abort handler can
+ *  detect the request and set CANCELLED + finishedAt after the underlying work stops.
+ *  The in-process AbortController is also triggered for same-process jobs. */
+export declare function cancelJob(tenantId: string, jobId: string): Promise<void>;
+/** Re-enqueue a job using the same config as a previous job. */
+export declare function rerunJob(tenantId: string, jobId: string): Promise<string>;
+export declare function resumeJob(tenantId: string, jobId: string): Promise<string>;
+/** Update progress percentage and current step label on a running job (non-fatal). */
+export declare function updateJobProgress(tenantId: string, jobId: string, progress: number, step: string): Promise<void>;
+export interface ProcessJobOpts {
+    onProgress?: (pct: number) => Promise<void>;
+}
+export declare function processJob(jobId: string, config: any, opts?: ProcessJobOpts): Promise<void>;
+/** After a job finishes, sum LLMCallLog entries for that project during the job window. */
+export declare function aggregateJobTokens(tenantId: string, jobId: string, projectId: string, startedAt: Date): Promise<void>;
+export declare function processDataRetentionPurge(): Promise<void>;
+export declare function generateK6Script(params: {
+    endpoints: Array<{
+        method: string;
+        path: string;
+        summary?: string;
+    }>;
+    baseUrl: string;
+    vus?: number;
+    duration?: string;
+    thinkTimeSeconds?: number;
+}): string;
+export {};

package/types/js-coverage.d.ts

@@ -0,0 +1,29 @@
+/**
+ * JS/CSS coverage collector via Playwright's built-in coverage API.
+ *
+ * For each crawled page, records what percentage of loaded JS/CSS was actually
+ * executed/applied. High unused code % → performance opportunity (code splitting, lazy loading).
+ *
+ * Uses Playwright's page.coverage API (wraps CDP Coverage domain).
+ */
+import type { Page } from 'playwright';
+export interface CoverageEntry {
+    url: string;
+    type: 'js' | 'css';
+    totalBytes: number;
+    usedBytes: number;
+    unusedPercent: number;
+}
+export interface CoverageReport {
+    pageUrl: string;
+    entries: CoverageEntry[];
+    totalJsBytes: number;
+    usedJsBytes: number;
+    totalCssBytes: number;
+    usedCssBytes: number;
+    jsUnusedPercent: number;
+    cssUnusedPercent: number;
+    collectedAt: string;
+}
+export declare function startCoverage(page: Page): Promise<void>;
+export declare function stopAndCollectCoverage(page: Page, pageUrl: string): Promise<CoverageReport | null>;

package/types/js-router-detector.d.ts

@@ -0,0 +1,17 @@
+/**
+ * P2: JS Router Detection
+ *
+ * Scans the loaded page for client-side route tables declared by React Router v6,
+ * Vue Router, Angular Router, or custom window globals. Supplements the existing
+ * history.pushState interception (which only captures runtime navigation) with a
+ * static analysis of routes declared in the JS bundle.
+ *
+ * Called once per page after goto(). Runs entirely in-browser via page.evaluate().
+ * Adds discovered routes to the BFS frontier so the crawler visits them.
+ */
+/**
+ * Extract absolute page URLs from the current page's JS router config.
+ * Returns deduplicated absolute URLs (same origin). Dynamic segments (:id, *)
+ * are skipped — the crawler can't resolve them without knowing valid IDs.
+ */
+export declare function detectJsRoutes(page: any, origin: string): Promise<string[]>;

package/types/language-detector.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Language detector — identify page language from HTML attributes and content analysis.
+ *
+ * Sources (in priority order):
+ * 1. <html lang="..."> attribute
+ * 2. <meta http-equiv="Content-Language"> header
+ * 3. hreflang alternate links
+ * 4. Content-based n-gram fingerprint (detects top 20 languages)
+ *
+ * Used to: route pages to the correct markdown processor, group screens by locale,
+ * and detect missing hreflang implementations.
+ */
+import type { Page } from 'playwright';
+export interface LanguageDetectionResult {
+    url: string;
+    declaredLang: string;
+    detectedLang: string;
+    confidence: 'high' | 'medium' | 'low';
+    alternates: Record<string, string>;
+    isMultilingual: boolean;
+}
+export declare function detectLanguage(page: Page, url: string): Promise<LanguageDetectionResult>;

package/types/link-classifier.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Link classifier — assigns priority and category to discovered URLs.
+ *
+ * Categories:
+ * - 'nav'       : site navigation (homepage, top-level pages) — highest priority
+ * - 'auth'      : login/register/password flows — high priority
+ * - 'app'       : authenticated app screens — high priority
+ * - 'content'   : blog/docs/articles — medium priority
+ * - 'api'       : REST/GraphQL API endpoints — detected, not crawled as screens
+ * - 'resource'  : static assets (JS/CSS/img/fonts) — skip
+ * - 'external'  : different domain — skip unless cross-domain crawl enabled
+ * - 'low-value' : pagination, sort, filter query strings — deprioritize
+ */
+export type LinkCategory = 'nav' | 'auth' | 'app' | 'content' | 'api' | 'resource' | 'external' | 'low-value';
+export interface ClassifiedLink {
+    url: string;
+    category: LinkCategory;
+    priority: number;
+    reason: string;
+}
+export declare function classifyLink(url: string, baseUrl: string): ClassifiedLink;
+export declare function classifyLinks(urls: string[], baseUrl: string): ClassifiedLink[];
+export declare function filterCrawlable(classified: ClassifiedLink[]): ClassifiedLink[];

package/types/live-view.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Start a live view session: launches a headless browser, navigates to appUrl,
+ * begins capturing frames. Returns the session ID.
+ */
+export declare function startLiveView(appUrl: string, storageState?: string): Promise<string>;
+/**
+ * Get the latest frame for a session (base64 JPEG).
+ */
+export declare function getFrame(id: string): string | null;
+/**
+ * Navigate the live view browser to a URL.
+ */
+export declare function liveViewNavigate(id: string, url: string): Promise<void>;
+/**
+ * Capture storageState from the live view session, close the browser, return JSON string.
+ */
+export declare function completeLiveView(id: string): Promise<string>;
+/**
+ * Send a mouse or keyboard input event to the live view browser.
+ */
+export declare function sendInput(id: string, event: {
+    type: 'click' | 'type' | 'key';
+    x?: number;
+    y?: number;
+    button?: 'left' | 'right' | 'middle';
+    text?: string;
+    key?: string;
+}): Promise<void>;
+/**
+ * Close a live view session without capturing.
+ */
+export declare function closeLiveView(id: string): Promise<void>;
+/**
+ * List all active session IDs (for health checks / cleanup).
+ */
+export declare function listLiveViews(): string[];

package/types/locator-extractor.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+export interface PlaywrightLocator {
+    key: string;
+    expression: string;
+    elementType: string;
+    confidence: 'high' | 'medium' | 'low';
+    source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+    description: string;
+}
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export declare function elementToPlaywrightLocator(element: {
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+    ariaState?: Record<string, unknown> | null;
+}): PlaywrightLocator | null;
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ * Filters out crawl-analysis meta-annotations and produces an auth-wall warning when applicable.
+ */
+export declare function elementsToPlaywrightLocators(elements: Array<{
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+    isInteractive?: boolean | null;
+    ariaState?: Record<string, unknown> | null;
+}>): PlaywrightLocator[];
+/**
+ * Detect whether a set of locators was extracted from an auth-wall page.
+ * Returns a warning message if auth-wall detected, null otherwise.
+ * Heuristic: >50% of locators are low-confidence text matches AND at least one matches auth-wall pattern.
+ */
+export declare function detectAuthWallLocators(locators: PlaywrightLocator[]): string | null;
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export declare function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string;

package/types/logger.d.ts

@@ -0,0 +1,2 @@
+import pino from 'pino';
+export declare const logger: pino.Logger<never, boolean>;

package/types/markdown-extract.d.ts

@@ -0,0 +1,8 @@
+export interface ExtractedMarkdown {
+    markdown: string;
+    method: 'readability' | 'full-page';
+    title: string | null;
+    wordCount: number;
+}
+/** Best-effort HTML→Markdown for one crawled page. Never throws. */
+export declare function extractMarkdown(html: string, url: string): ExtractedMarkdown | null;

package/types/multi-context-handler.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Multi-context browser handling — detects and records new tabs, iframes,
+ * popups, and file downloads triggered during a page crawl.
+ * This lets ZeTa discover cross-origin flows, OAuth redirects, embedded
+ * third-party content, and downloadable resources that single-page HTTP
+ * crawlers would miss entirely.
+ */
+import type { Page, BrowserContext } from 'playwright';
+export interface NewTabRecord {
+    url: string;
+    linkText: string;
+    isInternal: boolean;
+    httpStatus?: number;
+}
+export interface IframeRecord {
+    src: string;
+    crossOrigin: boolean;
+    sandboxed: boolean;
+    sandboxValue: string | null;
+    accessible: boolean;
+    contentSample: string;
+    hasForm: boolean;
+}
+export interface PopupRecord {
+    url: string;
+    triggerLabel: string;
+    isOAuthLike: boolean;
+}
+export interface DownloadRecord {
+    filename: string;
+    downloadUrl: string;
+    triggerLabel: string;
+}
+export interface MultiContextResult {
+    newTabs: NewTabRecord[];
+    iframes: IframeRecord[];
+    popups: PopupRecord[];
+    downloads: DownloadRecord[];
+}
+/**
+ * Attaches a listener to the BrowserContext that records every new tab
+ * opened while the listener is active. Returns an unsubscribe function.
+ *
+ * Call this before navigating / clicking so that tabs opened during the
+ * crawl are captured, then call the returned function to stop listening.
+ */
+export declare function attachNewTabListener(context: BrowserContext, seedUrl: string, result: MultiContextResult): () => void;
+/**
+ * Probes the current page for multi-context browser events: iframes,
+ * popup windows, and file downloads.
+ *
+ * New-tab tracking is intentionally excluded here — it must be set up at
+ * the crawler level (before navigation) using `attachNewTabListener`.
+ */
+export declare function probeMultiContext(page: Page, context: BrowserContext, seedUrl: string): Promise<MultiContextResult>;

package/types/native-crawler.d.ts

@@ -0,0 +1,69 @@
+/**
+ * ZeTa Native App Crawler — Appium-based crawler for Android APK and iOS IPA.
+ *
+ * Architecture:
+ *   Appium Server (running at APPIUM_SERVER_URL) → WebDriverIO session
+ *   → UiAutomator2 (Android) / XCUITest (iOS)
+ *   → accessibility tree XML → ZeTa Element records
+ *
+ * TODO: requires Appium server running at APPIUM_SERVER_URL env var
+ * TODO: requires `webdriverio` package — add "webdriverio": "^9.x" to package.json
+ *
+ * Device farm support:
+ *   - local_emulator: Appium server + local emulator/simulator
+ *   - aws_device_farm: presigned S3 URL uploaded to Device Farm, session via WDIO
+ *   - browserstack: BrowserStack Automate Appium endpoint
+ */
+export interface NativeCrawlerOptions {
+    tenantId: string;
+    projectId: string;
+    jobId: string;
+    platform: 'android' | 'ios' | 'windows' | 'macos';
+    appS3Path: string;
+    packageName?: string;
+    deviceModel?: string;
+    osVersion?: string;
+    appActivity?: string;
+    appWaitActivity?: string;
+    deviceFarm: 'local_emulator' | 'aws_device_farm' | 'browserstack';
+    appiumServerUrl?: string;
+    browserstackUsername?: string;
+    browserstackAccessKey?: string;
+    awsAccessKeyId?: string;
+    awsSecretAccessKey?: string;
+    awsRegion?: string;
+    awsDeviceFarmProjectArn?: string;
+    credentials?: {
+        username?: string;
+        password?: string;
+        usernameFieldLabel?: string;
+        passwordFieldLabel?: string;
+        loginButtonLabel?: string;
+    };
+    visionLlmConfig?: {
+        provider: string;
+        model: string;
+        apiKey?: string;
+        baseUrl?: string;
+    } | null;
+    gestureConfig?: {
+        enableSwipe?: boolean;
+        enableLongPress?: boolean;
+    };
+    mitmProxyPort?: number;
+}
+/**
+ * Run a native app crawl session via Appium.
+ *
+ * Flow:
+ *   1. Download APK/IPA from S3 to local temp dir
+ *   2. Build Appium WebDriverIO capabilities for the target platform
+ *   3. Create WebDriverIO remote session (Appium server at APPIUM_SERVER_URL)
+ *   4. For each discovered screen:
+ *      a. Get page source XML → parse elements
+ *      b. Take screenshot → save to screenshotDir → persist via AppBrain
+ *      c. Save screen + elements to DB
+ *      d. Tap nav elements to discover next screens
+ *   5. Update PlatformJob as DONE (or FAILED on error)
+ */
+export declare function runNativeCrawl(opts: NativeCrawlerOptions): Promise<void>;

package/types/nested-window-crawler.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Nested window / browsing-context crawler — discovers and records frames,
+ * embedded apps, and popup windows that are already open in the browser
+ * context at the time of a crawl step.
+ *
+ * Complements multi-context-handler.ts: that module handles proactive popup
+ * interception via context.on('page'); this module snapshots what is already
+ * present in the frame tree and context page list.
+ */
+import type { BrowserContext } from 'playwright';
+export type BrowsingContextType = 'page' | 'frame' | 'popup' | 'embedded-app';
+export interface BrowsingContextNode {
+    id: string;
+    type: BrowsingContextType;
+    url: string;
+    parentId: string | null;
+    depth: number;
+    title: string;
+    interactiveCount: number;
+    accessible: boolean;
+    isEmbeddedApp: boolean;
+}
+export interface NestedCrawlResult {
+    contextTree: BrowsingContextNode[];
+    totalContextsFound: number;
+    maxDepthReached: number;
+    inaccessibleCount: number;
+    frameUrls: string[];
+    embeddedAppUrls: string[];
+}
+/**
+ * Discovers all nested browsing contexts reachable from the current page:
+ * sub-frames, embedded applications, and already-open popup pages.
+ *
+ * All failures are non-fatal — errors are caught and an empty result is
+ * returned so the caller's crawl step can continue uninterrupted.
+ */
+export declare function crawlNestedContexts(page: any, context: BrowserContext | null, seedUrl: string, opts?: {
+    maxFrameDepth?: number;
+    maxPopupDepth?: number;
+}): Promise<NestedCrawlResult>;