@taodigital/crawler 0.0.0-stage → 1.2.0

package/dist/crawler.d.ts

@@ -0,0 +1,233 @@
+/**
+ * ZeTa-AI Crawler — Stagehand hybrid architecture
+ *
+ * Three-layer stack:
+ *   Layer 1 — Playwright (deterministic): navigation, screenshots, DOM fallbacks
+ *   Layer 2 — Stagehand act/extract (AI): login, element understanding, smart link discovery
+ *   Layer 3 — Skyvern (vision): fallback for CAPTCHAs, legacy apps, visual-only flows
+ *
+ * enableCaching: true → repeat crawls of the same site cost ~zero AI tokens after first run.
+ */
+import type { Credentials } from './auth.js';
+import { type SkyvernConfig } from './skyvern.js';
+import { type TraversalStrategy } from './frontier.js';
+import { type ApiEndpointObservationInput } from './network-observer.js';
+import type { ProxyTierConfig } from './proxy.js';
+interface PerformanceMetric {
+    url: string;
+    screenName: string;
+    lcp?: number;
+    fcp?: number;
+    ttfb?: number;
+    cls?: number;
+}
+export interface DiscoveredLinkMeta {
+    url: string;
+    text?: string;
+    landmark?: string;
+    source: 'DOM' | 'AI' | 'INTERACTIVE';
+}
+export declare function getNonHeadlessUserAgent(executablePath?: string): Promise<string>;
+/**
+ * Lightweight pre-crawl link discovery: loads the app URL in a headless (proxy-aware) browser
+ * and returns same-site `<a href>` links. Complements sitemap discovery for SPA/app sites that
+ * have no sitemap.xml. Best-effort — returns [] on any failure (bot wall, timeout, etc.).
+ * chromium.launch({ proxy }) handles proxy 407 auth itself (same client), so no CDP handler needed.
+ */
+export declare function discoverLinksViaBrowser(appUrl: string, opts?: {
+    proxyUrl?: string;
+    maxLinks?: number;
+}): Promise<string[]>;
+export declare function crawlProject(opts: {
+    tenantId: string;
+    projectId: string;
+    appUrl: string;
+    /** The viewport used for this crawl pass */
+    viewport?: 'DESKTOP' | 'MOBILE_PORTRAIT' | 'TABLET';
+    credentials?: Credentials;
+    /** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */
+    storageState?: string;
+    screenshotDir?: string;
+    /** Cloud storage config for immediate screenshot upload. If set, screenshots are uploaded to cloud immediately after capture and cloudUrl is persisted. */
+    storageConfig?: {
+        provider: string;
+        bucket: string;
+        region?: string;
+        accountId?: string;
+        endpointUrl?: string;
+        accessKeyId?: string;
+        secretAccessKey?: string;
+        prefix?: string;
+        publicUrl?: string;
+    } | null;
+    /** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */
+    discoveryMode?: boolean;
+    maxScreens?: number;
+    /** Seed URLs added to the initial crawl queue — link discovery still runs from each page. */
+    startUrls?: string[];
+    /** When provided, crawl ONLY these exact URLs (no link-following, no full screen clear). Used for selected-screen recrawl. */
+    selectiveUrls?: string[];
+    /**
+     * URL path patterns that bypass the CUID/UUID generated-ID filter during BFS link discovery.
+     * Use {id} as a placeholder for any generated path segment (CUID, UUID, numeric ID, etc.).
+     * Example: ['/projects/{id}/tests', '/workspaces/{id}/settings']
+     * Without this, all URLs containing generated IDs are silently skipped by the BFS.
+     */
+    seedPathPatterns?: string[];
+    /** When true, run the interactive probe after static crawl to capture real validation errors, toasts, and modals. */
+    interactive?: boolean;
+    /** Optional abort signal — abort() cancels the crawl between page navigations. */
+    abortSignal?: AbortSignal;
+    /** Platform job ID — stored on screens so we can diff which were found in each crawl. */
+    jobId?: string;
+    /** Device profiles to capture screenshots for after the main DESKTOP crawl.
+     *  Values: 'MOBILE_PORTRAIT' | 'TABLET'
+     *  The main crawl always runs as DESKTOP. Extra profiles trigger a screenshot-only pass.
+     */
+    deviceProfiles?: string[];
+    /** HTTP Basic Auth credentials — injected as Authorization header on same-origin requests. */
+    httpBasicUsername?: string;
+    httpBasicPassword?: string;
+    /** Bearer/API token — injected as a request header and into localStorage. Skips AI login. */
+    authToken?: string;
+    /** Header name for authToken (default: 'Authorization'). */
+    authTokenHeader?: string;
+    /** Prefix for authToken value (default: 'Bearer '). */
+    authTokenPrefix?: string;
+    /** CAPTCHA solving API key (2captcha or CapSolver). */
+    captchaSolverApiKey?: string;
+    /** CAPTCHA solving service provider (default: '2captcha'). */
+    captchaSolverProvider?: '2captcha' | 'capsolver';
+    /**
+     * Path to a Chrome user data directory for profile inheritance.
+     * When set, the crawler exports the existing session from this profile and injects it,
+     * so the agent is already logged in. Works for local/self-hosted deployments only.
+     * Ignored on cloud/Docker (headless env cannot open another Chrome instance safely).
+     */
+    chromeProfilePath?: string;
+    /** MailSlurp API key for email OTP auto-retrieval (GAP 5). */
+    mailslurpApiKey?: string;
+    /** MailSlurp inbox ID to poll for OTP emails (GAP 5). */
+    mailslurpInboxId?: string;
+    /** SSO provider for ROPC token flow — 'okta' | 'azure_ad' | 'generic_oidc' (GAP 18). */
+    ssoProvider?: string;
+    /** SSO domain or Azure tenant ID (GAP 18). */
+    ssoDomain?: string;
+    /** SSO OAuth2 client ID (GAP 18). */
+    ssoClientId?: string;
+    /** SSO OAuth2 client secret (GAP 18). */
+    ssoClientSecret?: string;
+    /** SSO scopes (space-separated, default: 'openid profile') (GAP 18). */
+    ssoScope?: string;
+    /** HTTP/HTTPS/SOCKS5 proxy URL — e.g. 'http://user:pass@proxy.example.com:8080' or 'socks5://proxy:1080'.
+     *  Useful for sites that block datacenter IPs (Akamai, Cloudflare Bot Manager). */
+    proxyUrl?: string;
+    /** Tiered proxy configuration for automatic per-domain escalation (free → datacenter → residential → premium). */
+    proxyTiers?: ProxyTierConfig[];
+    /** Use Camoufox (Firefox) as a cookie bootstrapper before the main Chromium crawl.
+     *  Camoufox visits the URL first to solve Cloudflare JS challenges and extract session
+     *  cookies, which are then injected into the Chromium crawl as storageState.
+     *  Enable via Project Settings → Crawl Config → useFirefox: true. */
+    useFirefox?: boolean;
+    /** Natural language login instructions for Stagehand AI login. E.g. "Click Sign In in the nav, fill email, click Continue". */
+    loginInstructions?: string;
+    /** Skyvern vision fallback config from workspace settings (overrides SKYVERN_API_URL/KEY env vars). */
+    skyvernConfig?: SkyvernConfig;
+    /**
+     * Max screens to crawl per segment (default 50).
+     * After this limit the crawl pauses and remaining URLs are saved so the
+     * next job can continue. Total crawl is unlimited — just runs in multiple jobs.
+     */
+    segmentSize?: number;
+    /** URLs still queued from a previous segment — injected by the job-queue auto-continuation. */
+    resumeUrls?: string[];
+    /** Segment number (1-based) — used for progress display only. */
+    segmentIndex?: number;
+    /** Seed the crawl queue from the project's sitemap.xml, in addition to link-following. */
+    useSitemap?: boolean;
+    /** Explicit sitemap URL override — defaults to `${origin}/sitemap.xml` when useSitemap is set. */
+    sitemapUrl?: string;
+    /** Crawl queue traversal order. Default BFS — unchanged behavior from before this option existed. */
+    traversalStrategy?: TraversalStrategy;
+    /** BEST_FIRST only: URLs containing any of these (case-insensitive) are prioritized. */
+    bestFirstKeywords?: string[];
+    /** When true, skip URLs disallowed by the site's robots.txt. Default false. */
+    respectRobotsTxt?: boolean;
+    /** When true, dynamically slow crawl speed when server response latency is high (Scrapy-style AutoThrottle). Default false. */
+    autoThrottle?: boolean;
+    /** Regex pattern — only crawl URLs whose path+query match this pattern. */
+    urlFilter?: string;
+    /** Glob patterns — only enqueue URLs whose pathname matches at least one (supports * and **). */
+    includePatterns?: string[];
+    /** Glob patterns — skip URLs whose pathname matches any of these (supports * and **). */
+    excludePatterns?: string[];
+    /** Already-captured URLs from a previous failed job — pre-seeded into visited map so they're never re-crawled. */
+    skipUrls?: string[];
+    /** DOM hashes collected in a prior segment — prevents re-screenshotting structurally identical pages across segments. */
+    knownDomHashes?: string[];
+    /** When false, disables the puppeteer-extra stealth plugin (playwright-extra) for this crawl.
+     *  Default true — stealth is applied unless explicitly set to false. */
+    stealth?: boolean;
+    /** Desktop viewport width in pixels (default: 1440). */
+    viewportWidth?: number;
+    /** Desktop viewport height in pixels (default: 900). */
+    viewportHeight?: number;
+    /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
+    onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
+    /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
+    onUrlsDiscovered?: (urls: string[]) => Promise<void>;
+}, onProgress?: (pct: number, step: string) => Promise<void>): Promise<{
+    discoveredUrls: string[];
+    screensFound: number;
+    screensErrored: number;
+    screensSkippedToNextSegment: number;
+    isPartial: boolean;
+    discovered: never[];
+    apiEndpoints: never[];
+    performanceMetrics: never[];
+    pendingUrls: never[];
+    segmentIndex: number;
+    hasMoreSegments: boolean;
+    domHashes: string[];
+    visitedUrls?: undefined;
+    wasTimeout?: undefined;
+    crawlWarnings?: undefined;
+} | {
+    screensFound: number;
+    screensErrored: number;
+    screensSkippedToNextSegment: number;
+    isPartial: boolean;
+    discovered: {
+        url: string;
+        screenId?: string;
+        elements?: number;
+        elementFacts?: Array<{
+            meaning: string;
+            role: string;
+            expectedData?: string;
+        }>;
+        screenshotPath?: string;
+        version?: number;
+        error?: string;
+    }[];
+    apiEndpoints: ApiEndpointObservationInput[];
+    performanceMetrics: PerformanceMetric[];
+    pendingUrls: string[];
+    visitedUrls: string[];
+    wasTimeout: boolean;
+    segmentIndex: number;
+    hasMoreSegments: boolean;
+    domHashes: string[];
+    crawlWarnings: string[] | undefined;
+    discoveredUrls?: undefined;
+}>;
+export declare function domFallbackElements(page: any, rawPage?: any): Promise<Array<{
+    meaning: string;
+    role: string;
+    expectedData?: string;
+    notes?: string;
+    boundingRect?: Record<string, number>;
+    ariaState?: Record<string, string>;
+    parentLandmark?: string;
+}>>;
+export {};