@detiq/crawler 0.1.3

package/types/bull-worker.d.ts

@@ -0,0 +1,6 @@
+import { Worker } from 'bullmq';
+export declare function startBullWorker(redisUrl?: string): Worker<{
+    jobId: string;
+    tenantId: string;
+    config: any;
+}, any, string>;

package/types/captcha-solver.d.ts

@@ -0,0 +1,16 @@
+/**
+ * CAPTCHA auto-solving via 2captcha or CapSolver APIs.
+ * Supports reCAPTCHA v2, hCaptcha, and Cloudflare Turnstile.
+ */
+export type CaptchaSolverProvider = '2captcha' | 'capsolver';
+interface SolveResult {
+    token: string;
+    provider: CaptchaSolverProvider;
+}
+/**
+ * Detect CAPTCHA type and sitekey on the page, then solve it.
+ * Returns null if no CAPTCHA found or solver not configured.
+ * Throws on solve failure.
+ */
+export declare function solveCaptcha(page: any, pageUrl: string, apiKey: string, provider?: CaptchaSolverProvider): Promise<SolveResult | null>;
+export {};

package/types/color-contrast.d.ts

@@ -0,0 +1,30 @@
+/**
+ * WCAG color contrast checker.
+ *
+ * Evaluates text contrast ratios against backgrounds for all visible text elements.
+ * Reports elements that fail WCAG 2.1 AA (4.5:1 for normal text, 3:1 for large text)
+ * or AAA (7:1 for normal, 4.5:1 for large text) requirements.
+ *
+ * Uses computedStyle to get actual rendered colors — accounts for CSS variables
+ * and inheritance.
+ */
+import type { Page } from 'playwright';
+export interface ContrastViolation {
+    selector: string;
+    text: string;
+    foreground: string;
+    background: string;
+    ratio: number;
+    required: number;
+    level: 'AA' | 'AAA';
+    isLargeText: boolean;
+    severity: 'fail' | 'warn';
+}
+export interface ContrastAuditResult {
+    url: string;
+    violations: ContrastViolation[];
+    passCount: number;
+    score: number;
+    auditedAt: string;
+}
+export declare function auditColorContrast(page: Page, url: string): Promise<ContrastAuditResult>;

package/types/content-chunker.d.ts

@@ -0,0 +1,18 @@
+export type ChunkingStrategy = 'fixed' | 'overlap' | 'sentence' | 'paragraph' | 'markdown-heading' | 'semantic' | 'recursive';
+export interface ChunkOpts {
+    strategy: ChunkingStrategy;
+    chunkSize?: number;
+    overlap?: number;
+    separators?: string[];
+}
+export interface Chunk {
+    index: number;
+    text: string;
+    tokenEstimate: number;
+    startChar: number;
+    endChar: number;
+    heading?: string;
+}
+export declare function estimateTokens(text: string): number;
+export declare function chunkContent(text: string, opts: ChunkOpts): Chunk[];
+export declare function mergeSmallChunks(chunks: Chunk[], minTokens: number): Chunk[];

package/types/content-dedup.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Content deduplication — detect near-duplicate pages during crawl.
+ *
+ * Uses MinHash-inspired approach: instead of full MinHash (needs many hash functions),
+ * we use 8 structural fingerprints extracted from the markdown to detect pages that
+ * share the same layout/template even if the data differs.
+ *
+ * Typical duplicates: /products?page=1 vs /products?page=2 (same template, different data).
+ * These waste crawl budget and generate redundant test cases.
+ */
+export interface DedupResult {
+    isDuplicate: boolean;
+    similarUrl?: string;
+    similarityScore: number;
+}
+export declare class ContentDeduplicator {
+    private seen;
+    private readonly threshold;
+    constructor(threshold?: number);
+    check(url: string, markdown: string): DedupResult;
+    size(): number;
+    reset(): void;
+}

package/types/cookie-consent.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Cookie consent auto-acceptor.
+ *
+ * Detects and clicks "Accept" on common cookie consent banners before the
+ * crawler proceeds. Without this, many EU sites show a consent wall that
+ * blocks content or shows a different layout, making crawl results unreliable.
+ *
+ * Supports: Cookiebot, OneTrust, TrustArc, Usercentrics, GDPR Cookie Compliance,
+ * generic "Accept All" / "Accept Cookies" buttons.
+ *
+ * Strategy: try known selectors first, then fall back to text-based button search.
+ * Non-blocking — if no banner is found or click fails, crawl proceeds normally.
+ */
+import type { Page } from 'playwright';
+export interface ConsentResult {
+    found: boolean;
+    clicked: boolean;
+    method?: string;
+    error?: string;
+}
+export declare function acceptCookieConsent(page: Page, opts?: {
+    waitMs?: number;
+}): Promise<ConsentResult>;
+/** Check if a consent banner is still visible after attempted dismissal */
+export declare function isConsentBannerVisible(page: Page): Promise<boolean>;

package/types/cookie-jar.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Cookie jar — persist and restore Playwright storageState cookies between
+ * crawl segments and restarts. Enables continuous auth across session boundaries
+ * (e.g. segment 1 logs in, segment 2–N inherit the session without re-login).
+ *
+ * Storage: JSON file per project in the crawl cache dir.
+ * Format: Playwright's storageState shape (cookies + origins).
+ */
+export interface CookieJarEntry {
+    projectId: string;
+    tenantId: string;
+    storageState: string;
+    savedAt: string;
+    domain: string;
+}
+export declare function saveCookieJar(cacheDir: string, projectId: string, tenantId: string, storageState: string, domain: string): Promise<void>;
+export declare function loadCookieJar(cacheDir: string, projectId: string, maxAgeMs?: number): Promise<string | null>;
+export declare function clearCookieJar(cacheDir: string, projectId: string): Promise<void>;
+/** Merge two storageState JSON strings — second takes priority for same-domain cookies. */
+export declare function mergeStorageStates(base: string, overlay: string): string;

package/types/crawler.d.ts

@@ -0,0 +1,179 @@
+/**
+ * ZeTa-AI Crawler — Stagehand hybrid architecture
+ *
+ * Three-layer stack:
+ *   Layer 1 — Playwright (deterministic): navigation, screenshots, DOM fallbacks
+ *   Layer 2 — Stagehand act/extract (AI): login, element understanding, smart link discovery
+ *   Layer 3 — Skyvern (vision): fallback for CAPTCHAs, legacy apps, visual-only flows
+ *
+ * enableCaching: true → repeat crawls of the same site cost ~zero AI tokens after first run.
+ */
+import type { Credentials } from './auth.js';
+import { type SkyvernConfig } from './skyvern.js';
+import { type TraversalStrategy } from './frontier.js';
+import { type ApiEndpointObservationInput } from './network-observer.js';
+import type { ProxyTierConfig } from './proxy.js';
+interface PerformanceMetric {
+    url: string;
+    screenName: string;
+    lcp?: number;
+    fcp?: number;
+    ttfb?: number;
+    cls?: number;
+}
+/**
+ * Lightweight pre-crawl link discovery: loads the app URL in a headless (proxy-aware) browser
+ * and returns same-site `<a href>` links. Complements sitemap discovery for SPA/app sites that
+ * have no sitemap.xml. Best-effort — returns [] on any failure (bot wall, timeout, etc.).
+ * chromium.launch({ proxy }) handles proxy 407 auth itself (same client), so no CDP handler needed.
+ */
+export declare function discoverLinksViaBrowser(appUrl: string, opts?: {
+    proxyUrl?: string;
+    maxLinks?: number;
+}): Promise<string[]>;
+export declare function crawlProject(opts: {
+    tenantId: string;
+    projectId: string;
+    appUrl: string;
+    credentials?: Credentials;
+    /** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */
+    storageState?: string;
+    screenshotDir?: string;
+    /** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */
+    discoveryMode?: boolean;
+    maxScreens?: number;
+    /** Seed URLs added to the initial crawl queue — link discovery still runs from each page. */
+    startUrls?: string[];
+    /** When provided, crawl ONLY these exact URLs (no link-following, no full screen clear). Used for selected-screen recrawl. */
+    selectiveUrls?: string[];
+    /** When true, run the interactive probe after static crawl to capture real validation errors, toasts, and modals. */
+    interactive?: boolean;
+    /** Optional abort signal — abort() cancels the crawl between page navigations. */
+    abortSignal?: AbortSignal;
+    /** Platform job ID — stored on screens so we can diff which were found in each crawl. */
+    jobId?: string;
+    /** Device profiles to capture screenshots for after the main DESKTOP crawl.
+     *  Values: 'MOBILE_PORTRAIT' | 'TABLET'
+     *  The main crawl always runs as DESKTOP. Extra profiles trigger a screenshot-only pass.
+     */
+    deviceProfiles?: string[];
+    /** HTTP Basic Auth credentials — injected as Authorization header on same-origin requests. */
+    httpBasicUsername?: string;
+    httpBasicPassword?: string;
+    /** Bearer/API token — injected as a request header and into localStorage. Skips AI login. */
+    authToken?: string;
+    /** Header name for authToken (default: 'Authorization'). */
+    authTokenHeader?: string;
+    /** Prefix for authToken value (default: 'Bearer '). */
+    authTokenPrefix?: string;
+    /** CAPTCHA solving API key (2captcha or CapSolver). */
+    captchaSolverApiKey?: string;
+    /** CAPTCHA solving service provider (default: '2captcha'). */
+    captchaSolverProvider?: '2captcha' | 'capsolver';
+    /**
+     * Path to a Chrome user data directory for profile inheritance.
+     * When set, the crawler exports the existing session from this profile and injects it,
+     * so the agent is already logged in. Works for local/self-hosted deployments only.
+     * Ignored on cloud/Docker (headless env cannot open another Chrome instance safely).
+     */
+    chromeProfilePath?: string;
+    /** MailSlurp API key for email OTP auto-retrieval (GAP 5). */
+    mailslurpApiKey?: string;
+    /** MailSlurp inbox ID to poll for OTP emails (GAP 5). */
+    mailslurpInboxId?: string;
+    /** SSO provider for ROPC token flow — 'okta' | 'azure_ad' | 'generic_oidc' (GAP 18). */
+    ssoProvider?: string;
+    /** SSO domain or Azure tenant ID (GAP 18). */
+    ssoDomain?: string;
+    /** SSO OAuth2 client ID (GAP 18). */
+    ssoClientId?: string;
+    /** SSO OAuth2 client secret (GAP 18). */
+    ssoClientSecret?: string;
+    /** SSO scopes (space-separated, default: 'openid profile') (GAP 18). */
+    ssoScope?: string;
+    /** HTTP/HTTPS/SOCKS5 proxy URL — e.g. 'http://user:pass@proxy.example.com:8080' or 'socks5://proxy:1080'.
+     *  Useful for sites that block datacenter IPs (Akamai, Cloudflare Bot Manager). */
+    proxyUrl?: string;
+    /** Tiered proxy configuration for automatic per-domain escalation (free → datacenter → residential → premium). */
+    proxyTiers?: ProxyTierConfig[];
+    /** Use Camoufox (Firefox) as a cookie bootstrapper before the main Chromium crawl.
+     *  Camoufox visits the URL first to solve Cloudflare JS challenges and extract session
+     *  cookies, which are then injected into the Chromium crawl as storageState.
+     *  Enable via Project Settings → Crawl Config → useFirefox: true. */
+    useFirefox?: boolean;
+    /** Natural language login instructions for Stagehand AI login. E.g. "Click Sign In in the nav, fill email, click Continue". */
+    loginInstructions?: string;
+    /** Skyvern vision fallback config from workspace settings (overrides SKYVERN_API_URL/KEY env vars). */
+    skyvernConfig?: SkyvernConfig;
+    /**
+     * Max screens to crawl per segment (default 50).
+     * After this limit the crawl pauses and remaining URLs are saved so the
+     * next job can continue. Total crawl is unlimited — just runs in multiple jobs.
+     */
+    segmentSize?: number;
+    /** URLs still queued from a previous segment — injected by the job-queue auto-continuation. */
+    resumeUrls?: string[];
+    /** Segment number (1-based) — used for progress display only. */
+    segmentIndex?: number;
+    /** Seed the crawl queue from the project's sitemap.xml, in addition to link-following. */
+    useSitemap?: boolean;
+    /** Explicit sitemap URL override — defaults to `${origin}/sitemap.xml` when useSitemap is set. */
+    sitemapUrl?: string;
+    /** Crawl queue traversal order. Default BFS — unchanged behavior from before this option existed. */
+    traversalStrategy?: TraversalStrategy;
+    /** BEST_FIRST only: URLs containing any of these (case-insensitive) are prioritized. */
+    bestFirstKeywords?: string[];
+    /** When true, skip URLs disallowed by the site's robots.txt. Default false. */
+    respectRobotsTxt?: boolean;
+    /** When true, dynamically slow crawl speed when server response latency is high (Scrapy-style AutoThrottle). Default false. */
+    autoThrottle?: boolean;
+    /** Regex pattern — only crawl URLs whose path+query match this pattern. */
+    urlFilter?: string;
+    /** Already-captured URLs from a previous failed job — pre-seeded into visited map so they're never re-crawled. */
+    skipUrls?: string[];
+    /** DOM hashes collected in a prior segment — prevents re-screenshotting structurally identical pages across segments. */
+    knownDomHashes?: string[];
+    /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
+    onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
+    /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
+    onUrlsDiscovered?: (urls: string[]) => Promise<void>;
+}, onProgress?: (pct: number, step: string) => Promise<void>): Promise<{
+    discoveredUrls: string[];
+    screensFound: number;
+    screensErrored: number;
+    screensSkippedToNextSegment: number;
+    isPartial: boolean;
+    discovered: never[];
+    apiEndpoints: never[];
+    performanceMetrics: never[];
+    pendingUrls: never[];
+    segmentIndex: number;
+    hasMoreSegments: boolean;
+    domHashes: string[];
+} | {
+    screensFound: number;
+    screensErrored: number;
+    screensSkippedToNextSegment: number;
+    isPartial: boolean;
+    discovered: {
+        url: string;
+        screenId?: string;
+        elements?: number;
+        elementFacts?: Array<{
+            meaning: string;
+            role: string;
+            expectedData?: string;
+        }>;
+        screenshotPath?: string;
+        version?: number;
+        error?: string;
+    }[];
+    apiEndpoints: ApiEndpointObservationInput[];
+    performanceMetrics: PerformanceMetric[];
+    pendingUrls: string[];
+    segmentIndex: number;
+    hasMoreSegments: boolean;
+    domHashes: string[];
+    discoveredUrls?: undefined;
+}>;
+export {};

package/types/custom-js-injector.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Custom JS injector — run user-provided JavaScript in the context of each crawled page.
+ *
+ * Use cases:
+ * - Extract custom data points not captured by standard extractors
+ * - Set localStorage/sessionStorage values before crawl
+ * - Override fetch() to intercept API calls
+ * - Simulate user events (keydown, focus, etc.)
+ * - Bypass specific anti-bot checks
+ *
+ * Security: scripts run in the page context (sandboxed by Chromium's renderer).
+ * They cannot access the Node.js process or file system.
+ * ZeTa validates scripts are strings before injection.
+ */
+import type { Page } from 'playwright';
+export interface JsInjectionOpts {
+    /** Script to run on every page BEFORE navigation (via page.addInitScript) */
+    initScript?: string;
+    /** Script to run on every page AFTER navigation completes (via page.evaluate) */
+    postLoadScript?: string;
+    /** Key-value pairs to set in localStorage before page load */
+    localStorage?: Record<string, string>;
+    /** Key-value pairs to set in sessionStorage before page load */
+    sessionStorage?: Record<string, string>;
+    /** Cookie key-value pairs to add before page load */
+    cookies?: Array<{
+        name: string;
+        value: string;
+        domain?: string;
+        path?: string;
+    }>;
+}
+export interface JsInjectionResult {
+    initScriptApplied: boolean;
+    postLoadResult: unknown;
+    storageSet: boolean;
+    cookiesSet: number;
+    error?: string;
+}
+export declare function injectCustomJs(page: Page, opts: JsInjectionOpts): Promise<JsInjectionResult>;
+/** Safe string validator — ensures script is a string, not an object/function reference */
+export declare function validateScript(script: unknown): string;

package/types/dark-mode-capture.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Dark mode capture — screenshot pages in both light and dark color schemes.
+ *
+ * Uses Playwright's page.emulateMedia({ colorScheme: 'dark' }) to force dark mode,
+ * then captures a screenshot. Compares with light mode to determine if dark mode
+ * is supported (significant visual difference).
+ *
+ * Also detects prefers-color-scheme media query support in CSS.
+ */
+import type { Page } from 'playwright';
+export interface DarkModeCapture {
+    url: string;
+    lightScreenshot?: Buffer;
+    darkScreenshot?: Buffer;
+    hasDarkMode: boolean;
+    hasCssDarkMode: boolean;
+    backgroundColorLight?: string;
+    backgroundColorDark?: string;
+    capturedAt: string;
+}
+export declare function captureDarkMode(page: Page, url: string, opts?: {
+    takeScreenshots?: boolean;
+}): Promise<DarkModeCapture>;

package/types/dead-link-detector.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Dead link detector — tracks broken links discovered during crawl.
+ *
+ * A "dead link" is a URL that:
+ * - Returns 404, 410, or other 4xx/5xx responses
+ * - Times out after maxRetries
+ * - Redirects to an error page (detected by title heuristic)
+ *
+ * Results: list of broken links with source page, status code, and anchor text.
+ * Used for: SEO audit export, test report enrichment.
+ */
+export interface DeadLink {
+    url: string;
+    statusCode: number;
+    sourceUrl?: string;
+    anchorText?: string;
+    checkedAt: string;
+    reason: 'http-error' | 'timeout' | 'dns-error';
+}
+export declare class DeadLinkTracker {
+    private deadLinks;
+    private checked;
+    record(url: string, statusCode: number, source?: {
+        sourceUrl?: string;
+        anchorText?: string;
+    }): void;
+    recordTimeout(url: string, sourceUrl?: string): void;
+    getDeadLinks(): DeadLink[];
+    isKnownBroken(url: string): boolean;
+    stats(): {
+        total: number;
+        by404: number;
+        by5xx: number;
+        byTimeout: number;
+    };
+    toCsv(): string;
+}

package/types/element-hash.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Compute a stable fingerprint of a screen's element set.
+ * Sorted so insertion-order changes (React re-renders) don't produce false positives.
+ * Returns first 16 hex chars of SHA-256.
+ */
+export declare function computeElementHash(elements: Array<{
+    role: string;
+    meaning: string;
+}>): string;

package/types/email-otp.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Email OTP retrieval — supports 3 providers:
+ *   mailslurp  — managed SaaS (existing, no self-hosting)
+ *   imap       — any real IMAP mailbox (Gmail, Outlook, custom domain catch-all)
+ *   mailpit    — self-hosted SMTP trap (dev/staging only)
+ *
+ * Use mailslurp or imap for production apps.
+ * Use mailpit only when you can point the app's SMTP at the trap server.
+ */
+export type EmailOTPProvider = 'mailslurp' | 'imap' | 'mailpit';
+export interface EmailContent {
+    /** 4-8 digit OTP code, if found. */
+    otp?: string;
+    /** Magic link URL to navigate to, if found. */
+    magicLink?: string;
+}
+export interface EmailOTPConfig {
+    provider: EmailOTPProvider;
+    apiKey?: string;
+    inboxId?: string;
+    imapHost?: string;
+    imapPort?: number;
+    imapUser?: string;
+    imapPass?: string;
+    imapTls?: boolean;
+    mailpitBaseUrl?: string;
+}
+/**
+ * Wait for an email and return its OTP code or magic link URL.
+ * Prefer this over waitForEmailOTP when the app may send passwordless magic links.
+ */
+export declare function waitForEmailContent(config: EmailOTPConfig, since: string | Date): Promise<EmailContent | null>;
+/**
+ * Wait for an OTP email and extract the code.
+ * @deprecated Use waitForEmailContent which also handles magic links.
+ */
+export declare function waitForEmailOTP(config: EmailOTPConfig, since: string | Date): Promise<string | null>;
+/**
+ * Get the email address for a MailSlurp inbox (used to display in UI).
+ * Returns null for IMAP/Mailpit providers — use imapUser directly.
+ */
+export declare function getInboxEmail(configOrApiKey: EmailOTPConfig | string, inboxId?: string): Promise<string | null>;

package/types/engine-router.d.ts

@@ -0,0 +1,47 @@
+import type { DomainProfile } from './profiles/domain-profile.js';
+export type CrawlEngineName = 'http' | 'browser' | 'browser-stealth' | 'stagehand' | 'vision' | 'recorded-session' | 'native';
+export interface CrawlRouteDecisionInput {
+    appUrl: string;
+    appType?: string | null;
+    hasCredentials: boolean;
+    hasStorageState: boolean;
+    hasAuthToken: boolean;
+    hasSkyvern: boolean;
+    hasProxy: boolean;
+    profile: DomainProfile;
+    previousFailureReason?: string;
+}
+export interface CrawlRouteDecision {
+    engines: CrawlEngineName[];
+    reasons: string[];
+    confidence: number;
+}
+/**
+ * Per-domain rendering predictor — learns whether a domain requires a full browser
+ * or can be served by the fast HTTP engine. Implements adaptive crawlee-python pattern.
+ *
+ * Strategy:
+ * 1. On first encounter of a domain, classify the HTTP response body.
+ * 2. Store the decision in memory (Map). All future URLs on same domain skip re-testing.
+ * 3. "Requires browser" signals: React/Vue/Angular root with no content, tiny body (<500 chars meaningful text).
+ */
+export declare class RenderingTypePredictor {
+    private cache;
+    /** Return cached decision if available, undefined if unknown. */
+    getCached(url: string): 'http' | 'browser' | undefined;
+    /**
+     * Analyze an HTTP response body to predict if JS rendering is required.
+     * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
+     */
+    predict(url: string, htmlBody: string, statusCode: number): 'http' | 'browser';
+    /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
+    set(url: string, engine: 'http' | 'browser'): void;
+    stats(): {
+        cached: number;
+        browserDomains: string[];
+        httpDomains: string[];
+    };
+}
+/** Singleton predictor — shared across crawl jobs in the same process. */
+export declare const renderingPredictor: RenderingTypePredictor;
+export declare function chooseCrawlEngines(input: CrawlRouteDecisionInput): CrawlRouteDecision;

package/types/export/bulk-export.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Bulk export — export crawled screens to CSV / JSON / Markdown digest.
+ *
+ * Outputs:
+ * - CSV: url, name, wordCount, screenshotPath, markdownPath, createdAt — for spreadsheet analysis
+ * - JSON: full screen records array — for API integration
+ * - Markdown digest: concatenated markdown of all screens — for LLM context windows
+ */
+export interface ExportScreen {
+    id: string;
+    name: string;
+    url: string;
+    projectId: string;
+    wordCount?: number;
+    screenshotPath?: string | null;
+    markdownPath?: string | null;
+    markdown?: string | null;
+    createdAt?: Date;
+    updatedAt?: Date;
+}
+export declare function exportToCsv(screens: ExportScreen[], outputPath: string): Promise<void>;
+export declare function exportToJson(screens: ExportScreen[], outputPath: string): Promise<void>;
+export declare function exportToMarkdownDigest(screens: ExportScreen[], outputPath: string, opts?: {
+    readMarkdownFromPath?: boolean;
+    maxScreens?: number;
+}): Promise<void>;

package/types/export/vector-export.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Vector DB export — convert crawled screens to embedding-ready chunks.
+ *
+ * Produces a JSONL file where each line is one chunk ready for upsert into
+ * Pinecone, Qdrant, Weaviate, or any vector store. The embedding itself is
+ * NOT generated here (requires an embedding model at the user's end) — we
+ * produce the text chunks + metadata, and the caller embeds them.
+ *
+ * Format per line (JSONL):
+ * {
+ *   "id": "<screenId>-<chunkIndex>",
+ *   "text": "<chunk text>",
+ *   "metadata": {
+ *     "screenId": "...",
+ *     "projectId": "...",
+ *     "tenantId": "...",
+ *     "url": "...",
+ *     "screenName": "...",
+ *     "chunkIndex": 0,
+ *     "totalChunks": 3,
+ *     "crawledAt": "2026-...",
+ *     "wordCount": 142
+ *   }
+ * }
+ */
+export interface VectorChunk {
+    id: string;
+    text: string;
+    metadata: {
+        screenId: string;
+        projectId: string;
+        tenantId: string;
+        url: string;
+        screenName: string;
+        chunkIndex: number;
+        totalChunks: number;
+        crawledAt: string;
+        wordCount: number;
+    };
+}
+export interface ScreenRecord {
+    id: string;
+    name: string;
+    url: string;
+    projectId: string;
+    tenantId: string;
+    markdown?: string | null;
+    markdownPath?: string | null;
+    updatedAt?: Date;
+}
+export declare function exportScreensToVectorJsonl(screens: ScreenRecord[], outputPath: string, opts?: {
+    maxChunkWords?: number;
+    readMarkdownFromPath?: boolean;
+}): Promise<{
+    totalChunks: number;
+    skipped: number;
+}>;
+/**
+ * Pinecone-compatible upsert batch format.
+ * Call this after generating embeddings to produce the final upsert payload.
+ */
+export declare function toPineconeUpsert(chunks: VectorChunk[], embeddings: number[][], namespace?: string): Array<{
+    id: string;
+    values: number[];
+    metadata: VectorChunk['metadata'];
+}>;
+/**
+ * Qdrant-compatible point format.
+ */
+export declare function toQdrantPoints(chunks: VectorChunk[], embeddings: number[][]): Array<{
+    id: string;
+    vector: number[];
+    payload: VectorChunk['metadata'] & {
+        text: string;
+    };
+}>;