@detiq/crawler 0.1.3

package/types/playwright-recorder.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Playwright test recorder — generate Playwright test code from crawl data.
+ *
+ * Given a list of crawled screens, generates a minimal Playwright test file that:
+ * 1. Navigates to each screen URL
+ * 2. Verifies the page title
+ * 3. Checks key elements visible on the page
+ * 4. Verifies no console errors (optional)
+ *
+ * This is a "smoke test" generator — not a full interaction recorder.
+ * Output: valid TypeScript Playwright test file ready to run with npx playwright test.
+ */
+export interface RecorderScreen {
+    id: string;
+    name: string;
+    url: string;
+    title?: string;
+    heading?: string;
+    markdownWordCount?: number;
+    smartLocator?: {
+        tag: string;
+        text?: string;
+        role?: string;
+        placeholder?: string;
+        nearText?: string;
+    };
+}
+export interface RecorderOpts {
+    baseUrl?: string;
+    includeConsoleCheck?: boolean;
+    testTimeout?: number;
+    projectName?: string;
+    useSmartLocators?: boolean;
+}
+export declare function generatePlaywrightTest(screens: RecorderScreen[], opts?: RecorderOpts): string;
+export declare function generatePlaywrightConfig(opts?: {
+    baseUrl?: string;
+    reportDir?: string;
+}): string;

package/types/policies/crawl-safety.d.ts

@@ -0,0 +1,9 @@
+import type { DomainProfile } from '../profiles/domain-profile.js';
+export interface CrawlSafetyDecision {
+    allowed: boolean;
+    reason?: string;
+    severity: 'info' | 'warn' | 'block';
+}
+export declare function assessActionSafety(label: string, profile: DomainProfile): CrawlSafetyDecision;
+export declare function redactSensitiveValue(value: string): string;
+export declare function redactJson(value: unknown): unknown;

package/types/profiles/domain-profile.d.ts

@@ -0,0 +1,19 @@
+export type Industry = 'ecommerce' | 'banking' | 'fintech' | 'healthcare' | 'pharma' | 'agriculture' | 'legacy-enterprise' | 'saas' | 'unknown';
+export interface DomainProfile {
+    industry: Industry;
+    confidence: number;
+    importantScreenPatterns: string[];
+    workflowGoals: string[];
+    piiRules: string[];
+    riskyActions: string[];
+    safeProbeRules: string[];
+    requiredArtifacts: string[];
+    evidence: string[];
+}
+export declare function classifyDomainProfile(input: {
+    appUrl: string;
+    projectName?: string | null;
+    appType?: string | null;
+    seedText?: string;
+}): DomainProfile;
+export declare function getProfile(industry: Industry): DomainProfile;

package/types/proxy.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Free proxy auto-selection for bot-protection bypass.
+ *
+ * Source: proxifly/free-proxy-list (MIT) via jsDelivr CDN.
+ * These are datacenter proxies — they bypass simple IP blocks but NOT
+ * full Cloudflare Bot Manager / Akamai. For Cloudflare-protected sites,
+ * users need a residential proxy (Bright Data, Oxylabs, etc.).
+ */
+/**
+ * Pick a working free proxy. Tests in parallel batches of 10 with a 4s timeout.
+ * Returns null if no working proxy found (not an error — crawl can proceed without proxy).
+ */
+export declare function pickFreeProxy(): Promise<string | null>;
+export type ProxyTier = 'free' | 'datacenter' | 'residential' | 'premium';
+export interface ProxyTierConfig {
+    /** List of proxy URLs for this tier. E.g. ['http://user:pass@dc1.proxy:8080'] */
+    urls: string[];
+    tier: ProxyTier;
+}
+/**
+ * Tiered proxy manager with per-domain error histograms.
+ *
+ * Tracks success/failure per domain and automatically escalates to higher proxy tiers
+ * on consecutive failures. Applies time-decay to probe returning to lower tiers.
+ *
+ * Usage:
+ *   const mgr = new TieredProxyManager([{ tier: 'datacenter', urls: [...] }]);
+ *   const proxy = await mgr.selectProxy('api.salesforce.com');
+ *   // ... make request ...
+ *   mgr.recordResult('api.salesforce.com', success);
+ */
+export declare class TieredProxyManager {
+    private tiers;
+    private domainHistory;
+    private readonly ESCALATE_AFTER;
+    private readonly DECAY_AFTER_MS;
+    constructor(configs: ProxyTierConfig[]);
+    private getHistory;
+    private lowestAvailableTier;
+    /** Pick a proxy URL for this domain, based on its tier history. */
+    selectProxy(domain: string): string | null;
+    /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
+    recordResult(domain: string, success: boolean): void;
+    stats(): Array<{
+        domain: string;
+        tier: ProxyTier;
+        successRate: number;
+        totalRequests: number;
+    }>;
+}

package/types/rate-limiter.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Domain-aware rate limiter — ensures the crawler respects per-domain rate limits.
+ *
+ * Uses a token bucket algorithm per domain. When the bucket is empty, requests
+ * wait until a token is available. Tokens refill at a configurable rate.
+ *
+ * Default: 2 requests/second per domain, burst of 5.
+ * Can be configured per-domain via the rateMap (domain → requests/second).
+ */
+export interface RateLimiterOpts {
+    defaultRps?: number;
+    burstSize?: number;
+    rateMap?: Record<string, number>;
+}
+export declare class DomainRateLimiter {
+    private buckets;
+    private readonly defaultRps;
+    private readonly burstSize;
+    private readonly rateMap;
+    constructor(opts?: RateLimiterOpts);
+    private getBucket;
+    private refill;
+    /** Wait until a token is available for the given URL's domain. */
+    throttle(url: string): Promise<void>;
+    /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
+    setRate(domain: string, rps: number): void;
+    stats(): Record<string, {
+        tokens: number;
+        rps: number;
+    }>;
+}

package/types/redirect-tracker.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Redirect chain tracker — records the full redirect chain for crawled URLs.
+ *
+ * Why: redirect chains (A→B→C) slow down pages; circular redirects crash crawlers;
+ * redirect loops indicate config errors. ZeTa records chains for SEO/perf audit.
+ *
+ * Integration: hook into Playwright's page.on('response') to see each redirect hop.
+ */
+export interface RedirectHop {
+    from: string;
+    to: string;
+    statusCode: number;
+    durationMs: number;
+}
+export interface RedirectChain {
+    originalUrl: string;
+    finalUrl: string;
+    hops: RedirectHop[];
+    isCircular: boolean;
+    hopCount: number;
+    totalDurationMs: number;
+}
+import type { Page } from 'playwright';
+export declare function trackRedirects(page: Page, url: string): Promise<RedirectChain>;
+export declare class RedirectRegistry {
+    private chains;
+    register(chain: RedirectChain): void;
+    get(url: string): RedirectChain | undefined;
+    getAll(): RedirectChain[];
+    getProblematic(): RedirectChain[];
+    stats(): {
+        total: number;
+        withRedirects: number;
+        circular: number;
+        deepChains: number;
+    };
+}

package/types/redis-frontier.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Redis-backed distributed crawl frontier.
+ *
+ * Replaces the in-memory CrawlFrontier for horizontally-scaled crawler deployments.
+ * Multiple crawler workers on different containers can share the same URL queue for
+ * a given job — enabling 10x crawl throughput on large enterprise apps.
+ *
+ * Redis key namespace:  zeta:frontier:{jobId}
+ *   LIST  zeta:frontier:{jobId}:queue   — BFS queue (LPUSH/RPOP)
+ *   SET   zeta:frontier:{jobId}:visited — deduplication set
+ *   EXPIRE                               — 24h TTL on all keys
+ *
+ * Graceful fallback: if REDIS_URL is not set, RedisCrawlFrontier silently falls
+ * back to in-memory arrays so local development works without Redis.
+ */
+export declare class RedisCrawlFrontier {
+    private readonly jobId;
+    private redis;
+    private queueKey;
+    private visitedKey;
+    private inMemoryQueue;
+    private inMemoryVisited;
+    private useRedis;
+    constructor(jobId: string);
+    init(): Promise<void>;
+    /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
+    push(url: string): Promise<boolean>;
+    /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
+    shift(): Promise<string | null>;
+    /** Check if a URL has already been visited/queued. */
+    hasVisited(url: string): Promise<boolean>;
+    /** Current queue length. */
+    length(): Promise<number>;
+    /** Total visited count. */
+    visitedCount(): Promise<number>;
+    /** Seed initial URLs (used at job start). */
+    seed(urls: string[]): Promise<void>;
+    /** Clean up all Redis keys for this frontier. Call at job completion. */
+    cleanup(): Promise<void>;
+    /** Stats for monitoring. */
+    stats(): Promise<{
+        backend: 'redis' | 'memory';
+        queued: number;
+        visited: number;
+    }>;
+}
+/** Check if Redis is configured for distributed frontier. */
+export declare function isRedisFrontierEnabled(): boolean;

package/types/redis-queue.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Redis Streams distributed job queue.
+ *
+ * Provides horizontal crawler scaling via Redis Streams consumer groups.
+ * Multiple crawler nodes can pull from the same stream — each job is
+ * delivered to exactly one consumer (at-least-once via ACK/PENDING).
+ *
+ * Used when REDIS_URL is set AND JOB_QUEUE_BACKEND=redis.
+ * Falls back to the Postgres-backed queue when Redis is unavailable.
+ *
+ * Stream key:   zeta:crawl:stream
+ * Group name:   zeta:crawl:workers
+ * Consumer id:  zeta:worker:<hostname>:<pid>
+ */
+export interface RedisQueueJob {
+    id: string;
+    jobId: string;
+    tenantId: string;
+    projectId: string;
+    payload: Record<string, unknown>;
+    enqueuedAt: string;
+}
+export interface RedisQueueClient {
+    enqueue(jobId: string, tenantId: string, projectId: string, payload?: Record<string, unknown>): Promise<string>;
+    dequeue(timeoutMs?: number): Promise<RedisQueueJob | null>;
+    ack(streamEntryId: string): Promise<void>;
+    nack(streamEntryId: string): Promise<void>;
+    reclaimStale(maxIdleMs?: number): Promise<RedisQueueJob[]>;
+    stats(): Promise<{
+        pending: number;
+        delivered: number;
+        groupLag: number;
+    }>;
+    close(): Promise<void>;
+}
+/**
+ * Create a Redis Streams job queue client.
+ * Throws if REDIS_URL is not set.
+ */
+export declare function createRedisQueue(): Promise<RedisQueueClient>;
+/**
+ * Check if Redis queue backend is configured.
+ */
+export declare function isRedisQueueEnabled(): boolean;
+/**
+ * Convenience: create queue only if enabled, else return null.
+ */
+export declare function maybeCreateRedisQueue(): Promise<RedisQueueClient | null>;

package/types/response-cache.d.ts

@@ -0,0 +1,38 @@
+export interface ResponseCacheEntry {
+    url: string;
+    etag?: string;
+    lastModified?: string;
+    cacheControl?: string;
+    body: string;
+    contentType: string;
+    statusCode: number;
+    cachedAt: number;
+    hitCount: number;
+}
+export declare class ResponseCache {
+    private readonly dir;
+    private totalHits;
+    private totalRequests;
+    constructor(cacheDir: string);
+    private key;
+    private path;
+    get(url: string): ResponseCacheEntry | null;
+    private read;
+    isFresh(url: string): boolean;
+    staleIfError(url: string): boolean;
+    set(url: string, entry: Omit<ResponseCacheEntry, 'cachedAt' | 'hitCount'>): void;
+    getValidationHeaders(url: string): {
+        'If-None-Match'?: string;
+        'If-Modified-Since'?: string;
+    };
+    markHit(url: string): void;
+    invalidate(url: string): void;
+    clear(): void;
+    stats(): {
+        entries: number;
+        totalSizeBytes: number;
+        hitRate: number;
+        totalHits: number;
+    };
+    prune(maxAgeMs: number): number;
+}

package/types/responsive-detector.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Responsive design detector — captures page at multiple breakpoints
+ * and detects layout differences between desktop and mobile viewports.
+ *
+ * Breakpoints: mobile(375), tablet(768), desktop(1280), wide(1920)
+ * Detects: viewport-specific content visibility, menu collapse state,
+ * font size changes, hidden/shown elements.
+ */
+import type { Page } from 'playwright';
+export interface ViewportCapture {
+    width: number;
+    height: number;
+    label: 'mobile' | 'tablet' | 'desktop' | 'wide';
+    screenshotBuffer?: Buffer;
+    visibleTextLength: number;
+    hasHamburgerMenu: boolean;
+    hiddenElements: number;
+    scrollHeight: number;
+}
+export interface ResponsiveReport {
+    url: string;
+    captures: ViewportCapture[];
+    isResponsive: boolean;
+    hasMobileMenu: boolean;
+    issues: string[];
+}
+export declare function detectResponsive(page: Page, url: string, opts?: {
+    captureScreenshots?: boolean;
+}): Promise<ResponsiveReport>;

package/types/retry-policy.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Retry policy — configurable exponential backoff with jitter.
+ * Covers: 5xx server errors, network timeouts, ECONNREFUSED, ECONNRESET.
+ * NOT retried: 4xx client errors (except 429 — handled separately), redirects.
+ */
+export interface RetryPolicyOpts {
+    maxRetries?: number;
+    baseDelayMs?: number;
+    maxDelayMs?: number;
+    jitterMs?: number;
+    retryOn?: number[];
+}
+export interface RetryResult<T> {
+    value: T;
+    attempts: number;
+    totalDelayMs: number;
+}
+export declare function withRetry<T>(fn: (attempt: number) => Promise<T>, opts?: RetryPolicyOpts): Promise<RetryResult<T>>;
+/** Wraps fetch() with retry policy */
+export declare function fetchWithRetry(url: string, init?: RequestInit, retryOpts?: RetryPolicyOpts): Promise<Response>;

package/types/robots-txt.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Minimal robots.txt fetcher + parser. No external deps. Caches per origin.
+ * Implements the subset of robots.txt spec used in practice:
+ *   User-agent, Disallow, Allow — longest match wins (RFC 9309).
+ */
+export declare class RobotsCache {
+    private readonly cache;
+    isAllowed(url: string, userAgent?: string): Promise<boolean>;
+}

package/types/screen-reader.d.ts

@@ -0,0 +1,16 @@
+import type { Page } from 'playwright';
+/**
+ * Reads a page as a structured outline (cheap + reliable), the way
+ * Playwright MCP / the accessibility tree works — NOT by sending pixels to a
+ * vision model. Vision is a fallback for image-only pages (added later).
+ *
+ * Returns the interactive elements found, which the Screen Reader Agent then
+ * enriches with MEANING and saves to the Zeta Core.
+ */
+export interface RawElement {
+    role: string;
+    label: string;
+    selectorHint: string;
+    expectedData?: string;
+}
+export declare function readScreen(page: Page): Promise<RawElement[]>;

package/types/screenshot-masker.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Screenshot masker — blur or black-box PII regions in captured screenshots.
+ *
+ * Strategy: inject a CSS overlay into the page before screenshot capture that
+ * covers detected PII-likely elements (input[type=email], input[type=password],
+ * elements with PII-containing text) with a blur or solid black mask.
+ *
+ * This is a CSS-based approach (no image processing library needed):
+ * 1. Identify likely-PII DOM elements via selectors
+ * 2. Add an overlaid <div> with blur/black filter at each element's bounding box
+ * 3. Take screenshot
+ * 4. Remove overlays
+ *
+ * Falls back gracefully if elements are not found or overlay injection fails.
+ */
+import type { Page } from 'playwright';
+export interface MaskingOpts {
+    /** Additional CSS selectors to mask (in addition to defaults) */
+    extraSelectors?: string[];
+    /** Masking style: 'blur' or 'blackout' */
+    style?: 'blur' | 'blackout';
+    /** Blur strength in px (default 8) */
+    blurPx?: number;
+}
+export declare function maskScreenshot(page: Page, opts?: MaskingOpts): Promise<{
+    maskedCount: number;
+    screenshot: Buffer;
+}>;

package/types/seo-checker.d.ts

@@ -0,0 +1,33 @@
+/**
+ * SEO indexability checker — determines if a page is crawlable and indexable.
+ *
+ * Checks:
+ * 1. X-Robots-Tag response header (noindex, nofollow, noarchive)
+ * 2. <meta name="robots"> tag (noindex, nofollow)
+ * 3. Canonical URL (self-canonical vs cross-canonical)
+ * 4. rel="nofollow" on outbound links
+ * 5. hreflang tags (multilingual)
+ * 6. Open Graph completeness (og:title, og:description, og:image)
+ * 7. Structured data presence (JSON-LD)
+ * 8. Meta description
+ * 9. Title tag length (< 60 chars recommended)
+ * 10. URL structure (query params, fragments, excessive depth)
+ */
+import type { Page } from 'playwright';
+export interface SeoIssue {
+    rule: string;
+    severity: 'critical' | 'warning' | 'info';
+    message: string;
+    value?: string;
+}
+export interface SeoCheckResult {
+    url: string;
+    isIndexable: boolean;
+    canonicalUrl?: string;
+    isSelfCanonical?: boolean;
+    hreflang: Record<string, string>;
+    issues: SeoIssue[];
+    score: number;
+    checkedAt: string;
+}
+export declare function checkSeo(page: Page, url: string, responseHeaders?: Record<string, string>): Promise<SeoCheckResult>;

package/types/session-capture.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Opens a headed (visible) browser at appUrl and returns a captureId.
+ * The user logs in manually, then calls completeSessionCapture() to harvest storageState.
+ * Sessions expire after 10 minutes if not completed.
+ *
+ * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.
+ * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.
+ */
+export declare function startSessionCapture(appUrl: string): Promise<string>;
+/** Captures storageState from the open browser, closes it, and returns the JSON string. */
+export declare function completeSessionCapture(captureId: string): Promise<string>;
+/** Closes the browser without capturing — called on Cancel. */
+export declare function cancelSessionCapture(captureId: string): Promise<void>;
+/**
+ * Headless login + session capture — works on Docker/cloud (no display needed).
+ * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,
+ * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).
+ *
+ * Returns { storageState, cookies, localStorage } on success.
+ * Throws a human-readable error on failure.
+ */
+export declare function validateAndCaptureSession(appUrl: string, creds: {
+    username: string;
+    password: string;
+    totpSecret?: string;
+    httpBasicUsername?: string;
+    httpBasicPassword?: string;
+}, llmConfig?: {
+    modelName: string;
+    apiKey: string;
+}, captchaOpts?: {
+    apiKey?: string;
+    provider?: '2captcha' | 'capsolver';
+}): Promise<{
+    storageState: string;
+    cookies: number;
+    localStorage: number;
+}>;

package/types/session-export.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Converts a Playwright storageState (the format ZeTa already captures via
+ * session-capture.ts/live-view.ts and stores per-project) into snippets for
+ * other automation frameworks. Playwright's storageState is the de-facto
+ * standard for browser session export — see docs/UNIVERSAL-SESSION-CAPTURE-
+ * STRATEGY.md — so every other framework's import mechanism is a small,
+ * mechanical conversion away, not a new capture flow.
+ *
+ * Pure, dependency-free functions — no I/O, no network, fully unit-testable.
+ */
+export interface StorageStateCookie {
+    name: string;
+    value: string;
+    domain: string;
+    path: string;
+    expires?: number;
+    httpOnly?: boolean;
+    secure?: boolean;
+    sameSite?: 'Strict' | 'Lax' | 'None';
+}
+export interface StorageStateOrigin {
+    origin: string;
+    localStorage: Array<{
+        name: string;
+        value: string;
+    }>;
+}
+export interface StorageState {
+    cookies: StorageStateCookie[];
+    origins: StorageStateOrigin[];
+}
+export type SessionExportFormat = 'cypress' | 'selenium-python' | 'webdriverio';
+/** cy.setCookie() + localStorage.setItem() inside a reusable custom command,
+ *  following Cypress's documented cy.session() pattern (see references in
+ *  docs/UNIVERSAL-SESSION-CAPTURE-STRATEGY.md). */
+export declare function toCypressSnippet(state: StorageState): string;
+/** driver.add_cookie() (Selenium requires the browser already be on the
+ *  cookie's domain before adding it) + execute_script() for localStorage,
+ *  per Selenium's documented Cookie API. */
+export declare function toSeleniumPythonDict(state: StorageState): string;
+/** browser.setCookies() accepts an array shape very close to Playwright's own
+ *  cookie objects (per WebdriverIO's documented API) — closest 1:1 mapping
+ *  of the three target formats. */
+export declare function toWebdriverIOSnippet(state: StorageState): string;
+export declare function exportSessionAs(format: SessionExportFormat, state: StorageState): {
+    content: string;
+    filename: string;
+    mimeType: string;
+};

package/types/sitemap-writer.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Sitemap writer — generate XML sitemap from crawled screens.
+ *
+ * Compliant with sitemaps.org protocol (sitemap index + individual sitemap files).
+ * Splits into 50k-URL files per the spec when needed.
+ * Used for: SEO audit export, future crawl seed, and crawler/robots.txt comparison.
+ */
+export interface SitemapUrl {
+    loc: string;
+    lastmod?: string;
+    changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+    priority?: number;
+}
+export declare function writeSitemap(urls: SitemapUrl[], outputPath: string): Promise<void>;
+/** Convert crawled screen records to SitemapUrl objects */
+export declare function screensToSitemapUrls(screens: Array<{
+    url: string;
+    updatedAt?: Date;
+    depth?: number;
+}>): SitemapUrl[];
+export declare function buildSitemapString(urls: SitemapUrl[]): string;

package/types/sitemap.d.ts

@@ -0,0 +1,51 @@
+import { ProxyAgent } from 'undici';
+/**
+ * Same-site check that tolerates www/non-www and http/https differences — a sitemap on
+ * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).
+ */
+export declare function sameSite(candidateUrl: string, siteOrigin: string): boolean;
+/**
+ * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap
+ * document (either a <urlset> or a <sitemapindex>), filtered to same-origin
+ * and deduped. Pure function — no I/O — for direct unit testing without a
+ * real HTTP fetch.
+ */
+export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string): {
+    pageUrls: string[];
+    subSitemapUrls: string[];
+};
+/**
+ * Fetches and parses a sitemap (or sitemap index, recursed up to a small
+ * depth) into a flat, deduped list of same-origin URLs.
+ */
+export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any): Promise<string[]>;
+/**
+ * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
+ * explicit override) and returns the URLs it lists. Never throws.
+ */
+export declare function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]>;
+/**
+ * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because
+ * DataImpulse-style usernames contain commas (country codes) that break new URL().
+ * Returns undefined for no/auto proxy — the caller then fetches directly.
+ */
+export declare function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined;
+/** Extract same-origin `Sitemap:` directives from a robots.txt body. */
+export declare function parseRobotsSitemaps(robotsTxt: string, origin: string): string[];
+export interface UrlCandidate {
+    url: string;
+    source: 'sitemap' | 'robots' | 'link';
+}
+/**
+ * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap
+ * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped
+ * candidate list the user can review and seed before launching the Playwright crawl.
+ * Never throws — bot-protected sites (Akamai/Cloudflare) simply yield an empty list.
+ */
+export declare function discoverProjectUrls(appUrl: string, opts?: {
+    sitemapUrl?: string;
+    proxyUrl?: string;
+}): Promise<{
+    candidates: UrlCandidate[];
+    sitemapFound: boolean;
+}>;