@detiq/crawler 0.1.3

package/dist/cookie-consent.js

@@ -0,0 +1,78 @@
+const CONSENT_SELECTORS = [
+  // Cookiebot
+  "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
+  "#CybotCookiebotDialogBodyButtonAccept",
+  // OneTrust
+  "#onetrust-accept-btn-handler",
+  ".ot-sdk-accept-btn",
+  // TrustArc
+  '.truste_popframe [title*="Accept"]',
+  "#truste-consent-button",
+  // Usercentrics
+  '[data-testid="uc-accept-all-button"]',
+  // Generic high-confidence
+  '[id*="accept"][id*="cookie"]',
+  '[class*="accept"][class*="cookie"]',
+  '[class*="cookie"][class*="accept"]',
+  // GDPR Cookie Compliance
+  ".gdpr-cookie-notice-accept",
+  // Quantcast
+  ".qc-cmp2-summary-buttons button:first-child"
+];
+const ACCEPT_TEXT_PATTERNS = [
+  /^accept all$/i,
+  /^accept all cookies$/i,
+  /^allow all$/i,
+  /^i accept$/i,
+  /^agree$/i,
+  /^ok$/i,
+  /^got it$/i,
+  /^allow cookies$/i
+];
+async function acceptCookieConsent(page, opts = {}) {
+  const { waitMs = 1500 } = opts;
+  await page.waitForTimeout(waitMs).catch(() => {
+  });
+  for (const selector of CONSENT_SELECTORS) {
+    try {
+      const el = await page.$(selector);
+      if (el && await el.isVisible()) {
+        await el.click({ timeout: 2e3 });
+        await page.waitForTimeout(500);
+        return { found: true, clicked: true, method: `selector: ${selector}` };
+      }
+    } catch {
+    }
+  }
+  try {
+    const buttons = await page.$$('button, [role="button"], a[class*="cookie"]');
+    for (const btn of buttons) {
+      const text = (await btn.textContent() ?? "").trim();
+      if (ACCEPT_TEXT_PATTERNS.some((re) => re.test(text))) {
+        if (await btn.isVisible()) {
+          await btn.click({ timeout: 2e3 });
+          await page.waitForTimeout(500);
+          return { found: true, clicked: true, method: `text: "${text}"` };
+        }
+      }
+    }
+  } catch {
+  }
+  return { found: false, clicked: false };
+}
+async function isConsentBannerVisible(page) {
+  try {
+    for (const sel of CONSENT_SELECTORS.slice(0, 5)) {
+      const el = await page.$(sel);
+      if (el && await el.isVisible()) return true;
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}
+export {
+  acceptCookieConsent,
+  isConsentBannerVisible
+};
+//# sourceMappingURL=cookie-consent.js.map

package/dist/cookie-consent.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/cookie-consent.ts"],
+  "sourcesContent": ["/**\n * Cookie consent auto-acceptor.\n *\n * Detects and clicks \"Accept\" on common cookie consent banners before the\n * crawler proceeds. Without this, many EU sites show a consent wall that\n * blocks content or shows a different layout, making crawl results unreliable.\n *\n * Supports: Cookiebot, OneTrust, TrustArc, Usercentrics, GDPR Cookie Compliance,\n * generic \"Accept All\" / \"Accept Cookies\" buttons.\n *\n * Strategy: try known selectors first, then fall back to text-based button search.\n * Non-blocking \u2014 if no banner is found or click fails, crawl proceeds normally.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface ConsentResult {\n  found: boolean;\n  clicked: boolean;\n  method?: string; // which selector/strategy matched\n  error?: string;\n}\n\n// Known consent platform selectors\nconst CONSENT_SELECTORS = [\n  // Cookiebot\n  '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',\n  '#CybotCookiebotDialogBodyButtonAccept',\n  // OneTrust\n  '#onetrust-accept-btn-handler',\n  '.ot-sdk-accept-btn',\n  // TrustArc\n  '.truste_popframe [title*=\"Accept\"]',\n  '#truste-consent-button',\n  // Usercentrics\n  '[data-testid=\"uc-accept-all-button\"]',\n  // Generic high-confidence\n  '[id*=\"accept\"][id*=\"cookie\"]',\n  '[class*=\"accept\"][class*=\"cookie\"]',\n  '[class*=\"cookie\"][class*=\"accept\"]',\n  // GDPR Cookie Compliance\n  '.gdpr-cookie-notice-accept',\n  // Quantcast\n  '.qc-cmp2-summary-buttons button:first-child',\n];\n\n// Fallback: button text matching\nconst ACCEPT_TEXT_PATTERNS = [\n  /^accept all$/i,\n  /^accept all cookies$/i,\n  /^allow all$/i,\n  /^i accept$/i,\n  /^agree$/i,\n  /^ok$/i,\n  /^got it$/i,\n  /^allow cookies$/i,\n];\n\nexport async function acceptCookieConsent(page: Page, opts: { waitMs?: number } = {}): Promise<ConsentResult> {\n  const { waitMs = 1_500 } = opts;\n\n  // Wait briefly for consent banners to appear\n  await page.waitForTimeout(waitMs).catch(() => {});\n\n  // Try known selectors first\n  for (const selector of CONSENT_SELECTORS) {\n    try {\n      const el = await page.$(selector);\n      if (el && await el.isVisible()) {\n        await el.click({ timeout: 2_000 });\n        await page.waitForTimeout(500);\n        return { found: true, clicked: true, method: `selector: ${selector}` };\n      }\n    } catch { /* try next */ }\n  }\n\n  // Fallback: find buttons with accept-like text\n  try {\n    const buttons = await page.$$('button, [role=\"button\"], a[class*=\"cookie\"]');\n    for (const btn of buttons) {\n      const text = ((await btn.textContent()) ?? '').trim();\n      if (ACCEPT_TEXT_PATTERNS.some(re => re.test(text))) {\n        if (await btn.isVisible()) {\n          await btn.click({ timeout: 2_000 });\n          await page.waitForTimeout(500);\n          return { found: true, clicked: true, method: `text: \"${text}\"` };\n        }\n      }\n    }\n  } catch { /* non-fatal */ }\n\n  return { found: false, clicked: false };\n}\n\n/** Check if a consent banner is still visible after attempted dismissal */\nexport async function isConsentBannerVisible(page: Page): Promise<boolean> {\n  try {\n    for (const sel of CONSENT_SELECTORS.slice(0, 5)) {\n      const el = await page.$(sel);\n      if (el && await el.isVisible()) return true;\n    }\n    return false;\n  } catch {\n    return false;\n  }\n}\n"],
+  "mappings": "AAwBA,MAAM,oBAAoB;AAAA;AAAA,EAExB;AAAA,EACA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AACF;AAGA,MAAM,uBAAuB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,eAAsB,oBAAoB,MAAY,OAA4B,CAAC,GAA2B;AAC5G,QAAM,EAAE,SAAS,KAAM,IAAI;AAG3B,QAAM,KAAK,eAAe,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAGhD,aAAW,YAAY,mBAAmB;AACxC,QAAI;AACF,YAAM,KAAK,MAAM,KAAK,EAAE,QAAQ;AAChC,UAAI,MAAM,MAAM,GAAG,UAAU,GAAG;AAC9B,cAAM,GAAG,MAAM,EAAE,SAAS,IAAM,CAAC;AACjC,cAAM,KAAK,eAAe,GAAG;AAC7B,eAAO,EAAE,OAAO,MAAM,SAAS,MAAM,QAAQ,aAAa,QAAQ,GAAG;AAAA,MACvE;AAAA,IACF,QAAQ;AAAA,IAAiB;AAAA,EAC3B;AAGA,MAAI;AACF,UAAM,UAAU,MAAM,KAAK,GAAG,6CAA6C;AAC3E,eAAW,OAAO,SAAS;AACzB,YAAM,QAAS,MAAM,IAAI,YAAY,KAAM,IAAI,KAAK;AACpD,UAAI,qBAAqB,KAAK,QAAM,GAAG,KAAK,IAAI,CAAC,GAAG;AAClD,YAAI,MAAM,IAAI,UAAU,GAAG;AACzB,gBAAM,IAAI,MAAM,EAAE,SAAS,IAAM,CAAC;AAClC,gBAAM,KAAK,eAAe,GAAG;AAC7B,iBAAO,EAAE,OAAO,MAAM,SAAS,MAAM,QAAQ,UAAU,IAAI,IAAI;AAAA,QACjE;AAAA,MACF;AAAA,IACF;AAAA,EACF,QAAQ;AAAA,EAAkB;AAE1B,SAAO,EAAE,OAAO,OAAO,SAAS,MAAM;AACxC;AAGA,eAAsB,uBAAuB,MAA8B;AACzE,MAAI;AACF,eAAW,OAAO,kBAAkB,MAAM,GAAG,CAAC,GAAG;AAC/C,YAAM,KAAK,MAAM,KAAK,EAAE,GAAG;AAC3B,UAAI,MAAM,MAAM,GAAG,UAAU,EAAG,QAAO;AAAA,IACzC;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/cookie-jar.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Cookie jar — persist and restore Playwright storageState cookies between
+ * crawl segments and restarts. Enables continuous auth across session boundaries
+ * (e.g. segment 1 logs in, segment 2–N inherit the session without re-login).
+ *
+ * Storage: JSON file per project in the crawl cache dir.
+ * Format: Playwright's storageState shape (cookies + origins).
+ */
+export interface CookieJarEntry {
+    projectId: string;
+    tenantId: string;
+    storageState: string;
+    savedAt: string;
+    domain: string;
+}
+export declare function saveCookieJar(cacheDir: string, projectId: string, tenantId: string, storageState: string, domain: string): Promise<void>;
+export declare function loadCookieJar(cacheDir: string, projectId: string, maxAgeMs?: number): Promise<string | null>;
+export declare function clearCookieJar(cacheDir: string, projectId: string): Promise<void>;
+/** Merge two storageState JSON strings — second takes priority for same-domain cookies. */
+export declare function mergeStorageStates(base: string, overlay: string): string;

package/dist/cookie-jar.js

@@ -0,0 +1,61 @@
+import * as fs from "fs/promises";
+import * as path from "path";
+function jarPath(cacheDir, projectId) {
+  return path.join(cacheDir, `cookie-jar-${projectId}.json`);
+}
+async function saveCookieJar(cacheDir, projectId, tenantId, storageState, domain) {
+  await fs.mkdir(cacheDir, { recursive: true });
+  const entry = {
+    projectId,
+    tenantId,
+    storageState,
+    savedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    domain
+  };
+  await fs.writeFile(jarPath(cacheDir, projectId), JSON.stringify(entry, null, 2), "utf8");
+}
+async function loadCookieJar(cacheDir, projectId, maxAgeMs = 8 * 60 * 60 * 1e3) {
+  try {
+    const raw = await fs.readFile(jarPath(cacheDir, projectId), "utf8");
+    const entry = JSON.parse(raw);
+    const age = Date.now() - new Date(entry.savedAt).getTime();
+    if (age > maxAgeMs) return null;
+    return entry.storageState;
+  } catch {
+    return null;
+  }
+}
+async function clearCookieJar(cacheDir, projectId) {
+  try {
+    await fs.unlink(jarPath(cacheDir, projectId));
+  } catch {
+  }
+}
+function mergeStorageStates(base, overlay) {
+  try {
+    const a = JSON.parse(base);
+    const b = JSON.parse(overlay);
+    const cookies = [...a.cookies ?? []];
+    for (const ck of b.cookies ?? []) {
+      const idx = cookies.findIndex(
+        (c) => c.name === ck.name && c.domain === ck.domain && c.path === ck.path
+      );
+      if (idx >= 0) cookies[idx] = ck;
+      else cookies.push(ck);
+    }
+    const originsMap = /* @__PURE__ */ new Map();
+    for (const o of [...a.origins ?? [], ...b.origins ?? []]) {
+      originsMap.set(o.origin, o);
+    }
+    return JSON.stringify({ cookies, origins: [...originsMap.values()] });
+  } catch {
+    return overlay || base;
+  }
+}
+export {
+  clearCookieJar,
+  loadCookieJar,
+  mergeStorageStates,
+  saveCookieJar
+};
+//# sourceMappingURL=cookie-jar.js.map

package/dist/cookie-jar.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/cookie-jar.ts"],
+  "sourcesContent": ["/**\n * Cookie jar \u2014 persist and restore Playwright storageState cookies between\n * crawl segments and restarts. Enables continuous auth across session boundaries\n * (e.g. segment 1 logs in, segment 2\u2013N inherit the session without re-login).\n *\n * Storage: JSON file per project in the crawl cache dir.\n * Format: Playwright's storageState shape (cookies + origins).\n */\n\nimport * as fs from 'fs/promises';\nimport * as path from 'path';\n\nexport interface CookieJarEntry {\n  projectId: string;\n  tenantId: string;\n  storageState: string; // JSON-stringified Playwright StorageState\n  savedAt: string;      // ISO timestamp\n  domain: string;       // hostname for quick lookup\n}\n\nfunction jarPath(cacheDir: string, projectId: string): string {\n  return path.join(cacheDir, `cookie-jar-${projectId}.json`);\n}\n\nexport async function saveCookieJar(\n  cacheDir: string,\n  projectId: string,\n  tenantId: string,\n  storageState: string,\n  domain: string,\n): Promise<void> {\n  await fs.mkdir(cacheDir, { recursive: true });\n  const entry: CookieJarEntry = {\n    projectId,\n    tenantId,\n    storageState,\n    savedAt: new Date().toISOString(),\n    domain,\n  };\n  await fs.writeFile(jarPath(cacheDir, projectId), JSON.stringify(entry, null, 2), 'utf8');\n}\n\nexport async function loadCookieJar(\n  cacheDir: string,\n  projectId: string,\n  maxAgeMs = 8 * 60 * 60 * 1000, // 8 hours\n): Promise<string | null> {\n  try {\n    const raw = await fs.readFile(jarPath(cacheDir, projectId), 'utf8');\n    const entry: CookieJarEntry = JSON.parse(raw);\n    const age = Date.now() - new Date(entry.savedAt).getTime();\n    if (age > maxAgeMs) return null; // expired\n    return entry.storageState;\n  } catch {\n    return null;\n  }\n}\n\nexport async function clearCookieJar(cacheDir: string, projectId: string): Promise<void> {\n  try {\n    await fs.unlink(jarPath(cacheDir, projectId));\n  } catch { /* already gone */ }\n}\n\n/** Merge two storageState JSON strings \u2014 second takes priority for same-domain cookies. */\nexport function mergeStorageStates(base: string, overlay: string): string {\n  try {\n    const a = JSON.parse(base);\n    const b = JSON.parse(overlay);\n\n    // Cookies: overlay replaces same-name+domain+path cookies from base\n    const cookies = [...(a.cookies ?? [])];\n    for (const ck of (b.cookies ?? [])) {\n      const idx = cookies.findIndex(\n        (c: any) => c.name === ck.name && c.domain === ck.domain && c.path === ck.path,\n      );\n      if (idx >= 0) cookies[idx] = ck;\n      else cookies.push(ck);\n    }\n\n    // Origins: merge origins, overlay wins on same-origin\n    const originsMap = new Map<string, any>();\n    for (const o of [...(a.origins ?? []), ...(b.origins ?? [])]) {\n      originsMap.set(o.origin, o);\n    }\n\n    return JSON.stringify({ cookies, origins: [...originsMap.values()] });\n  } catch {\n    return overlay || base;\n  }\n}\n"],
+  "mappings": "AASA,YAAY,QAAQ;AACpB,YAAY,UAAU;AAUtB,SAAS,QAAQ,UAAkB,WAA2B;AAC5D,SAAO,KAAK,KAAK,UAAU,cAAc,SAAS,OAAO;AAC3D;AAEA,eAAsB,cACpB,UACA,WACA,UACA,cACA,QACe;AACf,QAAM,GAAG,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;AAC5C,QAAM,QAAwB;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAS,oBAAI,KAAK,GAAE,YAAY;AAAA,IAChC;AAAA,EACF;AACA,QAAM,GAAG,UAAU,QAAQ,UAAU,SAAS,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,MAAM;AACzF;AAEA,eAAsB,cACpB,UACA,WACA,WAAW,IAAI,KAAK,KAAK,KACD;AACxB,MAAI;AACF,UAAM,MAAM,MAAM,GAAG,SAAS,QAAQ,UAAU,SAAS,GAAG,MAAM;AAClE,UAAM,QAAwB,KAAK,MAAM,GAAG;AAC5C,UAAM,MAAM,KAAK,IAAI,IAAI,IAAI,KAAK,MAAM,OAAO,EAAE,QAAQ;AACzD,QAAI,MAAM,SAAU,QAAO;AAC3B,WAAO,MAAM;AAAA,EACf,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,eAAe,UAAkB,WAAkC;AACvF,MAAI;AACF,UAAM,GAAG,OAAO,QAAQ,UAAU,SAAS,CAAC;AAAA,EAC9C,QAAQ;AAAA,EAAqB;AAC/B;AAGO,SAAS,mBAAmB,MAAc,SAAyB;AACxE,MAAI;AACF,UAAM,IAAI,KAAK,MAAM,IAAI;AACzB,UAAM,IAAI,KAAK,MAAM,OAAO;AAG5B,UAAM,UAAU,CAAC,GAAI,EAAE,WAAW,CAAC,CAAE;AACrC,eAAW,MAAO,EAAE,WAAW,CAAC,GAAI;AAClC,YAAM,MAAM,QAAQ;AAAA,QAClB,CAAC,MAAW,EAAE,SAAS,GAAG,QAAQ,EAAE,WAAW,GAAG,UAAU,EAAE,SAAS,GAAG;AAAA,MAC5E;AACA,UAAI,OAAO,EAAG,SAAQ,GAAG,IAAI;AAAA,UACxB,SAAQ,KAAK,EAAE;AAAA,IACtB;AAGA,UAAM,aAAa,oBAAI,IAAiB;AACxC,eAAW,KAAK,CAAC,GAAI,EAAE,WAAW,CAAC,GAAI,GAAI,EAAE,WAAW,CAAC,CAAE,GAAG;AAC5D,iBAAW,IAAI,EAAE,QAAQ,CAAC;AAAA,IAC5B;AAEA,WAAO,KAAK,UAAU,EAAE,SAAS,SAAS,CAAC,GAAG,WAAW,OAAO,CAAC,EAAE,CAAC;AAAA,EACtE,QAAQ;AACN,WAAO,WAAW;AAAA,EACpB;AACF;",
+  "names": []
+}

package/dist/crawler.d.ts

@@ -0,0 +1,179 @@
+/**
+ * ZeTa-AI Crawler — Stagehand hybrid architecture
+ *
+ * Three-layer stack:
+ *   Layer 1 — Playwright (deterministic): navigation, screenshots, DOM fallbacks
+ *   Layer 2 — Stagehand act/extract (AI): login, element understanding, smart link discovery
+ *   Layer 3 — Skyvern (vision): fallback for CAPTCHAs, legacy apps, visual-only flows
+ *
+ * enableCaching: true → repeat crawls of the same site cost ~zero AI tokens after first run.
+ */
+import type { Credentials } from './auth.js';
+import { type SkyvernConfig } from './skyvern.js';
+import { type TraversalStrategy } from './frontier.js';
+import { type ApiEndpointObservationInput } from './network-observer.js';
+import type { ProxyTierConfig } from './proxy.js';
+interface PerformanceMetric {
+    url: string;
+    screenName: string;
+    lcp?: number;
+    fcp?: number;
+    ttfb?: number;
+    cls?: number;
+}
+/**
+ * Lightweight pre-crawl link discovery: loads the app URL in a headless (proxy-aware) browser
+ * and returns same-site `<a href>` links. Complements sitemap discovery for SPA/app sites that
+ * have no sitemap.xml. Best-effort — returns [] on any failure (bot wall, timeout, etc.).
+ * chromium.launch({ proxy }) handles proxy 407 auth itself (same client), so no CDP handler needed.
+ */
+export declare function discoverLinksViaBrowser(appUrl: string, opts?: {
+    proxyUrl?: string;
+    maxLinks?: number;
+}): Promise<string[]>;
+export declare function crawlProject(opts: {
+    tenantId: string;
+    projectId: string;
+    appUrl: string;
+    credentials?: Credentials;
+    /** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */
+    storageState?: string;
+    screenshotDir?: string;
+    /** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */
+    discoveryMode?: boolean;
+    maxScreens?: number;
+    /** Seed URLs added to the initial crawl queue — link discovery still runs from each page. */
+    startUrls?: string[];
+    /** When provided, crawl ONLY these exact URLs (no link-following, no full screen clear). Used for selected-screen recrawl. */
+    selectiveUrls?: string[];
+    /** When true, run the interactive probe after static crawl to capture real validation errors, toasts, and modals. */
+    interactive?: boolean;
+    /** Optional abort signal — abort() cancels the crawl between page navigations. */
+    abortSignal?: AbortSignal;
+    /** Platform job ID — stored on screens so we can diff which were found in each crawl. */
+    jobId?: string;
+    /** Device profiles to capture screenshots for after the main DESKTOP crawl.
+     *  Values: 'MOBILE_PORTRAIT' | 'TABLET'
+     *  The main crawl always runs as DESKTOP. Extra profiles trigger a screenshot-only pass.
+     */
+    deviceProfiles?: string[];
+    /** HTTP Basic Auth credentials — injected as Authorization header on same-origin requests. */
+    httpBasicUsername?: string;
+    httpBasicPassword?: string;
+    /** Bearer/API token — injected as a request header and into localStorage. Skips AI login. */
+    authToken?: string;
+    /** Header name for authToken (default: 'Authorization'). */
+    authTokenHeader?: string;
+    /** Prefix for authToken value (default: 'Bearer '). */
+    authTokenPrefix?: string;
+    /** CAPTCHA solving API key (2captcha or CapSolver). */
+    captchaSolverApiKey?: string;
+    /** CAPTCHA solving service provider (default: '2captcha'). */
+    captchaSolverProvider?: '2captcha' | 'capsolver';
+    /**
+     * Path to a Chrome user data directory for profile inheritance.
+     * When set, the crawler exports the existing session from this profile and injects it,
+     * so the agent is already logged in. Works for local/self-hosted deployments only.
+     * Ignored on cloud/Docker (headless env cannot open another Chrome instance safely).
+     */
+    chromeProfilePath?: string;
+    /** MailSlurp API key for email OTP auto-retrieval (GAP 5). */
+    mailslurpApiKey?: string;
+    /** MailSlurp inbox ID to poll for OTP emails (GAP 5). */
+    mailslurpInboxId?: string;
+    /** SSO provider for ROPC token flow — 'okta' | 'azure_ad' | 'generic_oidc' (GAP 18). */
+    ssoProvider?: string;
+    /** SSO domain or Azure tenant ID (GAP 18). */
+    ssoDomain?: string;
+    /** SSO OAuth2 client ID (GAP 18). */
+    ssoClientId?: string;
+    /** SSO OAuth2 client secret (GAP 18). */
+    ssoClientSecret?: string;
+    /** SSO scopes (space-separated, default: 'openid profile') (GAP 18). */
+    ssoScope?: string;
+    /** HTTP/HTTPS/SOCKS5 proxy URL — e.g. 'http://user:pass@proxy.example.com:8080' or 'socks5://proxy:1080'.
+     *  Useful for sites that block datacenter IPs (Akamai, Cloudflare Bot Manager). */
+    proxyUrl?: string;
+    /** Tiered proxy configuration for automatic per-domain escalation (free → datacenter → residential → premium). */
+    proxyTiers?: ProxyTierConfig[];
+    /** Use Camoufox (Firefox) as a cookie bootstrapper before the main Chromium crawl.
+     *  Camoufox visits the URL first to solve Cloudflare JS challenges and extract session
+     *  cookies, which are then injected into the Chromium crawl as storageState.
+     *  Enable via Project Settings → Crawl Config → useFirefox: true. */
+    useFirefox?: boolean;
+    /** Natural language login instructions for Stagehand AI login. E.g. "Click Sign In in the nav, fill email, click Continue". */
+    loginInstructions?: string;
+    /** Skyvern vision fallback config from workspace settings (overrides SKYVERN_API_URL/KEY env vars). */
+    skyvernConfig?: SkyvernConfig;
+    /**
+     * Max screens to crawl per segment (default 50).
+     * After this limit the crawl pauses and remaining URLs are saved so the
+     * next job can continue. Total crawl is unlimited — just runs in multiple jobs.
+     */
+    segmentSize?: number;
+    /** URLs still queued from a previous segment — injected by the job-queue auto-continuation. */
+    resumeUrls?: string[];
+    /** Segment number (1-based) — used for progress display only. */
+    segmentIndex?: number;
+    /** Seed the crawl queue from the project's sitemap.xml, in addition to link-following. */
+    useSitemap?: boolean;
+    /** Explicit sitemap URL override — defaults to `${origin}/sitemap.xml` when useSitemap is set. */
+    sitemapUrl?: string;
+    /** Crawl queue traversal order. Default BFS — unchanged behavior from before this option existed. */
+    traversalStrategy?: TraversalStrategy;
+    /** BEST_FIRST only: URLs containing any of these (case-insensitive) are prioritized. */
+    bestFirstKeywords?: string[];
+    /** When true, skip URLs disallowed by the site's robots.txt. Default false. */
+    respectRobotsTxt?: boolean;
+    /** When true, dynamically slow crawl speed when server response latency is high (Scrapy-style AutoThrottle). Default false. */
+    autoThrottle?: boolean;
+    /** Regex pattern — only crawl URLs whose path+query match this pattern. */
+    urlFilter?: string;
+    /** Already-captured URLs from a previous failed job — pre-seeded into visited map so they're never re-crawled. */
+    skipUrls?: string[];
+    /** DOM hashes collected in a prior segment — prevents re-screenshotting structurally identical pages across segments. */
+    knownDomHashes?: string[];
+    /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
+    onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
+    /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
+    onUrlsDiscovered?: (urls: string[]) => Promise<void>;
+}, onProgress?: (pct: number, step: string) => Promise<void>): Promise<{
+    discoveredUrls: string[];
+    screensFound: number;
+    screensErrored: number;
+    screensSkippedToNextSegment: number;
+    isPartial: boolean;
+    discovered: never[];
+    apiEndpoints: never[];
+    performanceMetrics: never[];
+    pendingUrls: never[];
+    segmentIndex: number;
+    hasMoreSegments: boolean;
+    domHashes: string[];
+} | {
+    screensFound: number;
+    screensErrored: number;
+    screensSkippedToNextSegment: number;
+    isPartial: boolean;
+    discovered: {
+        url: string;
+        screenId?: string;
+        elements?: number;
+        elementFacts?: Array<{
+            meaning: string;
+            role: string;
+            expectedData?: string;
+        }>;
+        screenshotPath?: string;
+        version?: number;
+        error?: string;
+    }[];
+    apiEndpoints: ApiEndpointObservationInput[];
+    performanceMetrics: PerformanceMetric[];
+    pendingUrls: string[];
+    segmentIndex: number;
+    hasMoreSegments: boolean;
+    domHashes: string[];
+    discoveredUrls?: undefined;
+}>;
+export {};