@xberg-io/crawlberg 0.0.1 → 1.0.0-rc.1

package/index.d.ts

@@ -0,0 +1,1079 @@
+/* auto-generated by NAPI-RS */
+/* eslint-disable */
+/**
+ * This type implements JavaScript's async iterable protocol.
+ * It can be used with `for await...of` loops.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols#the_async_iterator_and_async_iterable_protocols
+ */
+export declare class BatchCrawlStreamIterator {
+
+}
+
+/**
+ * Opaque handle to a configured crawl engine.
+ *
+ * Constructed via `create_engine` with an optional `CrawlConfig`.
+ * Default implementations for all pluggable components are used internally.
+ */
+export declare class CrawlEngineHandle {
+  /**
+   * Stream a single-URL crawl, yielding `CrawlEvent`s as pages are processed.
+   *
+   * Returns an async stream that emits one event per crawled page, plus a
+   * terminal `Complete` event. On per-URL failure during the crawl, emits an
+   * `Error` event followed by `Complete`. The stream item type is wrapped in
+   * a `Result` to surface transport-level errors; today every emit is `Ok`.
+   *
+   * Language bindings expose this as the native streaming shape for each
+   * target. WASM does not expose native streaming wrappers.
+   */
+  crawlStream(req: CrawlStreamRequest): Promise<CrawlStreamIterator>
+  /**
+   * Stream a multi-URL crawl, yielding `CrawlEvent`s across all seeds.
+   *
+   * Returns an async stream that emits one event per crawled page across all
+   * seeds, plus terminal `Complete` and `Error` events as appropriate. The
+   * stream item type is wrapped in a `Result` to surface transport-level
+   * errors; today every emit is `Ok`.
+   *
+   * Language bindings expose this as the native streaming shape for each
+   * target. WASM does not expose native streaming wrappers.
+   */
+  batchCrawlStream(req: BatchCrawlStreamRequest): Promise<BatchCrawlStreamIterator>
+}
+export type JsCrawlEngineHandle = CrawlEngineHandle
+
+/**
+ * This type implements JavaScript's async iterable protocol.
+ * It can be used with `for await...of` loops.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols#the_async_iterator_and_async_iterable_protocols
+ */
+export declare class CrawlStreamIterator {
+
+}
+
+/** Result from a single page action execution. */
+export interface ActionResult {
+  /** Zero-based index of the action in the sequence. */
+  actionIndex?: number
+  /** The type of action that was executed. */
+  actionType?: string
+  /** Whether the action completed successfully. */
+  success?: boolean
+  /** Action-specific return data (screenshot bytes, JS return value, scraped HTML). */
+  data?: any
+  /** Error message if the action failed. */
+  error?: string
+}
+
+/** Article metadata extracted from `article:*` Open Graph tags. */
+export interface ArticleMetadata {
+  /** The article publication time. */
+  publishedTime?: string
+  /** The article modification time. */
+  modifiedTime?: string
+  /** The article author. */
+  author?: string
+  /** The article section. */
+  section?: string
+  /** The article tags. */
+  tags?: Array<string>
+}
+
+/** The category of a downloaded asset. */
+export declare const enum AssetCategory {
+  /** A document file (PDF, DOC, etc.). */
+  Document = 'document',
+  /** An image file. */
+  Image = 'image',
+  /** An audio file. */
+  Audio = 'audio',
+  /** A video file. */
+  Video = 'video',
+  /** A font file. */
+  Font = 'font',
+  /** A CSS stylesheet. */
+  Stylesheet = 'stylesheet',
+  /** A JavaScript file. */
+  Script = 'script',
+  /** An archive file (ZIP, TAR, etc.). */
+  Archive = 'archive',
+  /** A data file (JSON, XML, CSV, etc.). */
+  Data = 'data',
+  /** An unrecognized asset type. */
+  Other = 'other'
+}
+
+/** Authentication configuration. */
+export interface AuthConfig {
+  type: string
+  username?: string
+  password?: string
+  token?: string
+  name?: string
+  value?: string
+}
+
+/** Crawl multiple seed URLs concurrently, each following links to configured depth. */
+export declare function batchCrawl(engine: CrawlEngineHandle, urls: Array<string>): Promise<BatchCrawlResults>
+
+/** Result from a single URL in a batch crawl operation. */
+export interface BatchCrawlResult {
+  /** The seed URL that was crawled. */
+  url?: string
+  /** The crawl result, if successful. */
+  result?: CrawlResult
+  /** The error message, if the crawl failed. */
+  error?: string
+}
+
+/**
+ * Aggregate result of a batch crawl, exposing per-URL results plus precomputed counts.
+ *
+ * The counts are derived once at construction so every binding language can read them
+ * as plain integer fields without re-iterating the `results` vector.
+ */
+export interface BatchCrawlResults {
+  /** Per-URL crawl results, in the order seed URLs were submitted. */
+  results?: Array<BatchCrawlResult>
+  /** Total number of seed URLs in the batch (equal to `results.len()`). */
+  totalCount?: number
+  /** Number of seed URLs whose crawl succeeded (`error` is `None`). */
+  completedCount?: number
+  /** Number of seed URLs whose crawl failed (`error` is `Some`). */
+  failedCount?: number
+}
+
+export declare function batchCrawlStream(engine: CrawlEngineHandle, urls: Array<string>): Promise<BatchCrawlStreamIterator>
+
+/**
+ * Request to begin a multi-URL streaming crawl.
+ *
+ * Wraps a set of seed URLs for delivery through the streaming-adapter binding
+ * surface. Required as a struct because alef's streaming adapter requires a
+ * named request type — primitives are not supported.
+ */
+export interface BatchCrawlStreamRequest {
+  /**
+   * The seed URLs to crawl. Each URL is followed independently up to the
+   * engine's configured depth.
+   */
+  urls?: Array<string>
+}
+
+/** Scrape multiple URLs concurrently. */
+export declare function batchScrape(engine: CrawlEngineHandle, urls: Array<string>): Promise<BatchScrapeResults>
+
+/** Result from a single URL in a batch scrape operation. */
+export interface BatchScrapeResult {
+  /** The URL that was scraped. */
+  url?: string
+  /** The scrape result, if successful. */
+  result?: ScrapeResult
+  /** The error message, if the scrape failed. */
+  error?: string
+}
+
+/**
+ * Aggregate result of a batch scrape, exposing per-URL results plus precomputed counts.
+ *
+ * The counts are derived once at construction so every binding language can read them
+ * as plain integer fields without re-iterating the `results` vector.
+ */
+export interface BatchScrapeResults {
+  /** Per-URL scrape results, in the order URLs were submitted. */
+  results?: Array<BatchScrapeResult>
+  /** Total number of URLs in the batch (equal to `results.len()`). */
+  totalCount?: number
+  /** Number of URLs whose scrape succeeded (`error` is `None`). */
+  completedCount?: number
+  /** Number of URLs whose scrape failed (`error` is `Some`). */
+  failedCount?: number
+}
+
+/** Browser backend used for JavaScript rendering. */
+export declare const enum BrowserBackend {
+  /** Existing Chromium/CDP backend powered by chromiumoxide. */
+  Chromiumoxide = 'chromiumoxide',
+  /** Crawlberg-owned native browser backend derived from Obscura. */
+  Native = 'native'
+}
+
+/** Browser fallback configuration. */
+export interface BrowserConfig {
+  /** When to use the headless browser fallback. */
+  mode?: JsBrowserMode
+  /** Browser backend used to render JavaScript-heavy pages. */
+  backend?: JsBrowserBackend
+  /** CDP WebSocket endpoint for connecting to an external browser instance. */
+  endpoint?: string
+  /** Timeout for browser page load and rendering (in milliseconds when serialized). */
+  timeout?: number
+  /** Wait strategy after browser navigation. */
+  wait?: JsBrowserWait
+  /** CSS selector to wait for when `wait` is `Selector`. */
+  waitSelector?: string
+  /** Extra time to wait after the wait condition is met. */
+  extraWait?: number
+  /**
+   * Proxy for browser fetches. Overrides `CrawlConfig.proxy` when set.
+   * Native backend supports http/https only (no SOCKS5).
+   */
+  proxy?: ProxyConfig
+  /**
+   * URL patterns to block before the network request fires. Supports `*`
+   * wildcards. Useful for skipping ads/analytics/large images. Honored by
+   * `BrowserBackend.Native`; chromiumoxide ignores this field today.
+   */
+  blockUrlPatterns?: Array<string>
+  /**
+   * JavaScript snippet evaluated after navigation completes.
+   *
+   * Scraping captures the native backend result in `ScrapeResult.browser.eval_result`.
+   * Interactions run this script before page actions on both browser backends but do
+   * not include the script result in `InteractionResult`.
+   */
+  evalScript?: string
+  /**
+   * User-agent used when fetching robots.txt. Defaults to `BrowserConfig.user_agent`
+   * (or crawlberg's default) if unset. Native only.
+   */
+  robotsUserAgent?: string
+  /**
+   * Capture the full network event stream into the result. Default false
+   * (only the document event is captured). Native only.
+   */
+  captureNetworkEvents?: boolean
+  /**
+   * Enable session affinity: reuse chromiumoxide Pages for same-domain
+   * requests so cookies + fingerprint + solved challenges persist.
+   * Default: true. When false, each request gets a fresh Page.
+   */
+  sessionAffinity?: boolean
+}
+
+export declare function browserConfigDefault(): BrowserConfig
+
+/**
+ * Browser-specific extras populated when the native browser backend was used.
+ *
+ * Available on `ScrapeResult.browser` when `BrowserBackend.Native` handled the request.
+ */
+export interface BrowserExtras {
+  /** Return value of `BrowserConfig.eval_script`, if provided. */
+  evalResult?: any
+  /**
+   * Network events captured during page navigation (only populated when
+   * `BrowserConfig.capture_network_events` is true).
+   */
+  networkEvents?: Array<JsResponseMeta>
+  /**
+   * All non-expired cookies present in the browser's cookie jar after
+   * navigation completes (includes both prior cookies and server Set-Cookie).
+   */
+  cookies?: Array<JsCookieInfo>
+}
+
+/** When to use the headless browser fallback. */
+export declare const enum BrowserMode {
+  /** Automatically detect when JS rendering is needed and fall back to browser. */
+  Auto = 'auto',
+  /** Always use the browser for every request. */
+  Always = 'always',
+  /** Never use the browser fallback. */
+  Never = 'never',
+  /**
+   * Always use the browser with all stealth surfaces enabled.
+   *
+   * Behaves like `Always` for escalation purposes
+   * (every request is routed through the browser tier), but additionally
+   * enables:
+   *
+   * - browser JavaScript stealth patches
+   * - native-backend TLS fingerprint spoofing
+   * - stealth-aware default user-agent when no explicit UA is set
+   * - 1920×1080 viewport override
+   *
+   * Use this instead of setting the now-removed `BrowserConfig.stealth`
+   * boolean field.
+   */
+  Stealth = 'stealth'
+}
+
+/** Wait strategy for browser page rendering. */
+export declare const enum BrowserWait {
+  /** Wait until network activity is idle. */
+  NetworkIdle = 'network_idle',
+  /** Wait for a specific CSS selector to appear in the DOM. */
+  Selector = 'selector',
+  /** Wait for a fixed duration after navigation. */
+  Fixed = 'fixed'
+}
+
+/**
+ * A single numbered reference in a citation list — produced by the citation
+ * extractor when content uses inline `[N]`-style markers.
+ */
+export interface CitationReference {
+  /** 1-based reference number as it appears in the source text. */
+  index?: number
+  /** Resolved absolute URL for this reference. */
+  url?: string
+  /** Human-readable anchor text or title for the reference. */
+  text?: string
+}
+
+/** Result of citation conversion. */
+export interface CitationResult {
+  /** Markdown with links replaced by numbered citations. */
+  content?: string
+  /** Numbered reference list: (index, url, text). */
+  references?: Array<JsCitationReference>
+}
+
+/**
+ * Content extraction and conversion configuration.
+ *
+ * Controls how HTML is converted to the output format. Uses
+ * html-to-markdown-rs as the conversion engine for all formats
+ * (markdown, plain text, djot).
+ */
+export interface ContentConfig {
+  /** Output format: `"markdown"` (default), `"plain"`, `"djot"`. */
+  outputFormat?: string
+  /**
+   * Preprocessing aggressiveness: `"minimal"`, `"standard"` (default), `"aggressive"`.
+   *
+   * - Minimal: only scripts/styles removed.
+   * - Standard: also removes nav, nav-hinted headers/footers/asides, forms.
+   * - Aggressive: removes all footers/asides unconditionally.
+   */
+  preprocessingPreset?: string
+  /** Remove navigation elements (nav, breadcrumbs, menus). Default: `true`. */
+  removeNavigation?: boolean
+  /** Remove form elements. Default: `true`. */
+  removeForms?: boolean
+  /**
+   * HTML tag names to strip (render children only, remove the tag wrapper).
+   * Default: `["noscript"]`.
+   */
+  stripTags?: Array<string>
+  /** HTML tag names to preserve as raw HTML in output. */
+  preserveTags?: Array<string>
+  /**
+   * CSS selectors for elements to exclude entirely (element + all content).
+   *
+   * Unlike `strip_tags` (which removes the wrapper but keeps children),
+   * excluded elements and all descendants are dropped. Supports CSS selectors:
+   * `.class`, `#id`, `[attribute]`, compound selectors.
+   *
+   * Example: `[".cookie-banner", "#ad-container", "[role='complementary']"]`
+   */
+  excludeSelectors?: Array<string>
+  /** Skip image elements in output. Default: `false`. */
+  skipImages?: boolean
+  /** Max DOM traversal depth. Prevents stack overflow on deeply nested HTML. */
+  maxDepth?: number
+  /** Enable line wrapping. Default: `false`. */
+  wrap?: boolean
+  /** Wrap width when `wrap` is enabled. Default: `80`. */
+  wrapWidth?: number
+  /** Include document structure tree in output. Default: `true`. */
+  includeDocumentStructure?: boolean
+}
+
+export declare function contentConfigDefault(): ContentConfig
+
+/** Information about an HTTP cookie received from a response. */
+export interface CookieInfo {
+  /** The cookie name. */
+  name?: string
+  /** The cookie value. */
+  value?: string
+  /** The cookie domain, if specified. */
+  domain?: string
+  /** The cookie path, if specified. */
+  path?: string
+}
+
+/** Crawl a website starting from `url`, following links up to the configured depth. */
+export declare function crawl(engine: CrawlEngineHandle, url: string): Promise<CrawlResult>
+
+/** Configuration for crawl, scrape, and map operations. */
+export interface CrawlConfig {
+  /** Maximum crawl depth (number of link hops from the start URL). */
+  maxDepth?: number
+  /** Maximum number of pages to crawl. */
+  maxPages?: number
+  /** Maximum number of concurrent requests. */
+  maxConcurrent?: number
+  /** Whether to respect robots.txt directives. */
+  respectRobotsTxt?: boolean
+  /**
+   * When true, HTTP-level error responses (404 NotFound, 403 Forbidden, WAF blocks)
+   * are surfaced as `ScrapeResult` records with the matching `status_code` rather
+   * than raised as `CrawlError`. Default `false` preserves the historical
+   * throw-on-error contract for direct fetches. Independently of this flag,
+   * 404s reached at the end of a redirect chain are *always* surfaced softly —
+   * the user opted into redirect-following, so receiving a 404 there is part of
+   * the normal flow rather than an unexpected error.
+   */
+  softHttpErrors?: boolean
+  /** Custom user-agent string. */
+  userAgent?: string
+  /** Whether to restrict crawling to the same domain. */
+  stayOnDomain?: boolean
+  /** Whether to allow subdomains when `stay_on_domain` is true. */
+  allowSubdomains?: boolean
+  /** Regex patterns for paths to include during crawling. */
+  includePaths?: Array<string>
+  /** Regex patterns for paths to exclude during crawling. */
+  excludePaths?: Array<string>
+  /** Custom HTTP headers to send with each request. */
+  customHeaders?: Record<string, string>
+  /** Timeout for individual HTTP requests (in milliseconds when serialized). */
+  requestTimeout?: number
+  /**
+   * Per-domain rate limit in milliseconds. When set, enforces a minimum delay
+   * between requests to the same domain. Defaults to 200ms when `None`.
+   */
+  rateLimitMs?: number
+  /** Maximum number of redirects to follow. */
+  maxRedirects?: number
+  /** Number of retry attempts for failed requests. */
+  retryCount?: number
+  /** HTTP status codes that should trigger a retry. */
+  retryCodes?: Array<number>
+  /** Whether to enable cookie handling. */
+  cookiesEnabled?: boolean
+  /** Authentication configuration. */
+  auth?: JsAuthConfig
+  /** Maximum response body size in bytes. */
+  maxBodySize?: number
+  /** CSS selectors for tags to remove from HTML before processing. */
+  removeTags?: Array<string>
+  /** Content extraction and conversion configuration. */
+  content?: ContentConfig
+  /** Maximum number of URLs to return from a map operation. */
+  mapLimit?: number
+  /** Search filter for map results (case-insensitive substring match on URLs). */
+  mapSearch?: string
+  /** Whether to download assets (CSS, JS, images, etc.) from the page. */
+  downloadAssets?: boolean
+  /** Filter for asset categories to download. */
+  assetTypes?: Array<JsAssetCategory>
+  /** Maximum size in bytes for individual asset downloads. */
+  maxAssetSize?: number
+  /** Browser configuration. */
+  browser?: BrowserConfig
+  /** Proxy configuration for HTTP requests. */
+  proxy?: ProxyConfig
+  /** List of user-agent strings for rotation. If non-empty, overrides `user_agent`. */
+  userAgents?: Array<string>
+  /** Whether to capture a screenshot when using the browser. */
+  captureScreenshot?: boolean
+  /**
+   * Re-enqueue discovered `LinkType.Document` URLs into the crawl frontier so
+   * the crawl follows links *from* document pages (PDFs, etc.) as it would
+   * from HTML pages. Default: `false` (documents terminate at materialisation).
+   */
+  followDocumentUrls?: boolean
+  /**
+   * Maximum document-depth (from the seed URL through document links only)
+   * when `follow_document_urls` is true. `None` means inherit `max_depth`.
+   * Independent of `max_depth`: a document URL is enqueued only if BOTH the
+   * outer `max_depth` and (if set) `document_url_depth` permit it.
+   */
+  documentUrlDepth?: number
+  /** Whether to download non-HTML documents (PDF, DOCX, images, code, etc.) instead of skipping them. */
+  downloadDocuments?: boolean
+  /** Maximum size in bytes for document downloads. Defaults to 50 MB. */
+  documentMaxSize?: number
+  /** Allowlist of MIME types to download. If empty, uses built-in defaults. */
+  documentMimeTypes?: Array<string>
+  /** Path to write WARC output. If `None`, WARC output is disabled. */
+  warcOutput?: string
+  /** Named browser profile for persistent sessions (cookies, localStorage). */
+  browserProfile?: string
+  /** Whether to save changes back to the browser profile on exit. */
+  saveBrowserProfile?: boolean
+  /**
+   * SSRF policy for outbound network requests. Default: deny private networks,
+   * allow http/https only, max 5 redirects.
+   *
+   * Phase 1: `deny_private` and `max_redirects` are exposed to all language
+   * bindings. `allowlist` is skipped (see `SsrfPolicy` fields) and will be
+   * added in a follow-up when `HostMatcher`'s tagged-enum FFI form is decided.
+   */
+  ssrf?: JsSsrfPolicy
+}
+
+export declare function crawlConfigDefault(): CrawlConfig
+
+/**
+ * An event emitted during a streaming crawl operation.
+ *
+ * Not available on `wasm32` targets — streaming requires native concurrency
+ * primitives (tokio channels, `JoinSet`) that are not supported on wasm32.
+ *
+ * Delivered to bindings through each target's native streaming idiom.
+ */
+export interface CrawlEvent {
+  type: string
+  result?: CrawlPageResult
+  url?: string
+  error?: string
+  pagesCrawled?: number
+}
+
+/** The result of crawling a single page during a crawl operation. */
+export interface CrawlPageResult {
+  /** The original URL of the page. */
+  url?: string
+  /** The normalized URL of the page. */
+  normalizedUrl?: string
+  /** The HTTP status code of the response. */
+  statusCode?: number
+  /** The Content-Type header value. */
+  contentType?: string
+  /** The HTML body of the response. */
+  html?: string
+  /** The size of the response body in bytes. */
+  bodySize?: number
+  /** Extracted metadata from the page. */
+  metadata?: JsPageMetadata
+  /** Links found on the page. */
+  links?: Array<JsLinkInfo>
+  /** Images found on the page. */
+  images?: Array<JsImageInfo>
+  /** Feed links found on the page. */
+  feeds?: Array<JsFeedInfo>
+  /** JSON-LD entries found on the page. */
+  jsonLd?: Array<JsJsonLdEntry>
+  /** The depth of this page from the start URL. */
+  depth?: number
+  /** Whether this page is on the same domain as the start URL. */
+  stayedOnDomain?: boolean
+  /** Whether this page was skipped (binary or PDF content). */
+  wasSkipped?: boolean
+  /** Whether the content is a PDF. */
+  isPdf?: boolean
+  /** The detected character set encoding. */
+  detectedCharset?: string
+  /** Markdown conversion of the page content. */
+  markdown?: JsMarkdownResult
+  /** Structured data extracted by LLM. Populated when extraction is configured. */
+  extractedData?: any
+  /** Metadata about the LLM extraction pass (cost, tokens, model). */
+  extractionMeta?: ExtractionMeta
+  /** Downloaded non-HTML document (PDF, DOCX, image, code, etc.). */
+  downloadedDocument?: DownloadedDocument
+  /** Whether the browser fallback was used to fetch this page. */
+  browserUsed?: boolean
+}
+
+/** The result of a multi-page crawl operation. */
+export interface CrawlResult {
+  /** The list of crawled pages. */
+  pages?: Array<CrawlPageResult>
+  /** The final URL after following redirects. */
+  finalUrl?: string
+  /** The number of redirects followed. */
+  redirectCount?: number
+  /** Whether any page was skipped during crawling. */
+  wasSkipped?: boolean
+  /** An error message, if the crawl encountered an issue. */
+  error?: string
+  /** Cookies collected during the crawl. */
+  cookies?: Array<JsCookieInfo>
+  /** Whether all crawled pages stayed on the same domain as the start URL. */
+  stayedOnDomain?: boolean
+  /** Whether the browser fallback was used for any page in this crawl. */
+  browserUsed?: boolean
+}
+
+export declare function crawlStream(engine: CrawlEngineHandle, url: string): Promise<CrawlStreamIterator>
+
+/**
+ * Request to begin a single-URL streaming crawl.
+ *
+ * Wraps a single seed URL for delivery through the streaming-adapter binding
+ * surface. Required as a struct because alef's streaming adapter requires a
+ * named request type — primitives are not supported.
+ */
+export interface CrawlStreamRequest {
+  /** The seed URL to crawl. */
+  url?: string
+}
+
+/**
+ * Create a new crawl engine with the given configuration.
+ *
+ * If `config` is `None`, uses `CrawlConfig.default()`.
+ * Returns an error if the configuration is invalid.
+ */
+export declare function createEngine(config?: CrawlConfig | undefined | null): CrawlEngineHandle
+
+/** A downloaded asset from a page. */
+export interface DownloadedAsset {
+  /** The original URL of the asset. */
+  url?: string
+  /** The SHA-256 content hash of the asset. */
+  contentHash?: string
+  /** The MIME type from the Content-Type header. */
+  mimeType?: string
+  /** The size of the asset in bytes. */
+  size?: number
+  /** The category of the asset. */
+  assetCategory?: JsAssetCategory
+  /** The HTML tag that referenced this asset (e.g., "link", "script", "img"). */
+  htmlTag?: string
+}
+
+/**
+ * A downloaded non-HTML document (PDF, DOCX, image, code file, etc.).
+ *
+ * When the crawler encounters non-HTML content and `download_documents` is
+ * enabled, it downloads the raw bytes and populates this struct instead of
+ * skipping the resource.
+ */
+export interface DownloadedDocument {
+  /** The URL the document was fetched from. */
+  url?: string
+  /** The MIME type from the Content-Type header. */
+  mimeType?: string
+  /** Size of the document in bytes. */
+  size?: number
+  /** Filename extracted from Content-Disposition or URL path. */
+  filename?: string
+  /** SHA-256 hex digest of the content. */
+  contentHash?: string
+  /** Selected response headers. */
+  headers?: Record<string, string>
+}
+
+/** Metadata about an LLM extraction pass. */
+export interface ExtractionMeta {
+  /** Estimated cost of the LLM call in USD. */
+  cost?: number
+  /** Number of prompt (input) tokens consumed. */
+  promptTokens?: number
+  /** Number of completion (output) tokens generated. */
+  completionTokens?: number
+  /** The model identifier used for extraction. */
+  model?: string
+  /** Number of content chunks sent to the LLM. */
+  chunksProcessed?: number
+}
+
+/** Information about a favicon or icon link. */
+export interface FaviconInfo {
+  /** The icon URL. */
+  url?: string
+  /** The `rel` attribute (e.g., "icon", "apple-touch-icon"). */
+  rel?: string
+  /** The `sizes` attribute, if present. */
+  sizes?: string
+  /** The MIME type, if present. */
+  mimeType?: string
+}
+
+/** Information about a feed link found on a page. */
+export interface FeedInfo {
+  /** The feed URL. */
+  url?: string
+  /** The feed title, if present. */
+  title?: string
+  /** The type of feed. */
+  feedType?: JsFeedType
+}
+
+/** The type of a feed (RSS, Atom, or JSON Feed). */
+export declare const enum FeedType {
+  /** RSS feed. */
+  Rss = 'rss',
+  /** Atom feed. */
+  Atom = 'atom',
+  /** JSON Feed. */
+  JsonFeed = 'json_feed'
+}
+
+/**
+ * Convert markdown links to numbered citations.
+ *
+ * `[Example](https://example.com)` becomes `Example[1]`
+ * with `[1]: https://example.com` in the reference list.
+ * Images `![alt](url)` are preserved unchanged.
+ */
+export declare function generateCitations(markdown: string): CitationResult
+
+/** A heading element extracted from the page. */
+export interface HeadingInfo {
+  /** The heading level (1-6). */
+  level?: number
+  /** The heading text content. */
+  text?: string
+}
+
+/** An hreflang alternate link entry. */
+export interface HreflangEntry {
+  /** The language code (e.g., "en", "fr", "x-default"). */
+  lang?: string
+  /** The URL for this language variant. */
+  url?: string
+}
+
+/** Information about an image found on a page. */
+export interface ImageInfo {
+  /** The image URL. */
+  url?: string
+  /** The alt text, if present. */
+  alt?: string
+  /** The width attribute, if present and parseable. */
+  width?: number
+  /** The height attribute, if present and parseable. */
+  height?: number
+  /** The source of the image reference. */
+  source?: JsImageSource
+}
+
+/** The source of an image reference. */
+export declare const enum ImageSource {
+  /** An `<img>` tag. */
+  Img = 'img',
+  /** A `<source>` tag inside `<picture>`. */
+  PictureSource = 'picture_source',
+  /** An `og:image` meta tag. */
+  OgImage = 'og:image',
+  /** A `twitter:image` meta tag. */
+  TwitterImage = 'twitter:image'
+}
+
+/** Execute browser actions on a single page. */
+export declare function interact(engine: CrawlEngineHandle, url: string, actions: Array<PageAction>): Promise<InteractionResult>
+
+/** Result of executing a sequence of page interaction actions. */
+export interface InteractionResult {
+  /** Results from each executed action. */
+  actionResults?: Array<JsActionResult>
+  /** Final page HTML after all actions completed. */
+  finalHtml?: string
+  /** Final page URL (may have changed due to navigation). */
+  finalUrl?: string
+}
+
+/** A JSON-LD structured data entry found on a page. */
+export interface JsonLdEntry {
+  /** The `@type` value from the JSON-LD object. */
+  schemaType?: string
+  /** The `name` value, if present. */
+  name?: string
+  /** The raw JSON-LD string. */
+  raw?: string
+}
+
+/** Information about a link found on a page. */
+export interface LinkInfo {
+  /** The resolved URL of the link. */
+  url?: string
+  /** The visible text of the link. */
+  text?: string
+  /** The classification of the link. */
+  linkType?: JsLinkType
+  /** The `rel` attribute value, if present. */
+  rel?: string
+  /** Whether the link has `rel="nofollow"`. */
+  nofollow?: boolean
+}
+
+/** The classification of a link. */
+export declare const enum LinkType {
+  /** A link to the same domain. */
+  Internal = 'internal',
+  /** A link to a different domain. */
+  External = 'external',
+  /** A fragment-only link (e.g., `#section`). */
+  Anchor = 'anchor',
+  /** A link to a downloadable document (PDF, DOC, etc.). */
+  Document = 'document'
+}
+
+/** The result of a map operation, containing discovered URLs. */
+export interface MapResult {
+  /** The list of discovered URLs. */
+  urls?: Array<SitemapUrl>
+}
+
+/** Discover all pages on a website by following links and sitemaps. */
+export declare function mapUrls(engine: CrawlEngineHandle, url: string): Promise<MapResult>
+
+/** Rich markdown conversion result from HTML processing. */
+export interface MarkdownResult {
+  /** Converted markdown text. */
+  content?: string
+  /** Structured document tree with semantic nodes. */
+  documentStructure?: any
+  /** Extracted tables with structured cell data. */
+  tables?: Array<any>
+  /** Non-fatal processing warnings. */
+  warnings?: Array<string>
+  /**
+   * Whether citation conversion was applied and produced at least one reference.
+   *
+   * `true` when the markdown contained inline links that were converted to
+   * numbered citation references. The converted content (with `[N]` markers)
+   * is available in `content`; the full reference list is accessible via
+   * `generate_citations` if needed separately.
+   */
+  citations?: boolean
+  /** Content-filtered markdown optimized for LLM consumption. */
+  fitContent?: string
+}
+
+/**
+ * A single page interaction action.
+ *
+ * Actions are serialized with a `type` tag using camelCase naming,
+ * except `ExecuteJs` which is explicitly renamed to `"executeJs"`.
+ */
+export interface PageAction {
+  type: string
+  selector?: string
+  text?: string
+  key?: string
+  direction?: JsScrollDirection
+  amount?: number
+  milliseconds?: number
+  fullPage?: boolean
+  script?: string
+}
+
+/** Metadata extracted from an HTML page's `<meta>` tags and `<title>` element. */
+export interface PageMetadata {
+  /** The page title from the `<title>` element. */
+  title?: string
+  /** The meta description. */
+  description?: string
+  /** The canonical URL from `<link rel="canonical">`. */
+  canonicalUrl?: string
+  /** Keywords from `<meta name="keywords">`. */
+  keywords?: string
+  /** Author from `<meta name="author">`. */
+  author?: string
+  /** Viewport content from `<meta name="viewport">`. */
+  viewport?: string
+  /** Theme color from `<meta name="theme-color">`. */
+  themeColor?: string
+  /** Generator from `<meta name="generator">`. */
+  generator?: string
+  /** Robots content from `<meta name="robots">`. */
+  robots?: string
+  /** The `lang` attribute from the `<html>` element. */
+  htmlLang?: string
+  /** The `dir` attribute from the `<html>` element. */
+  htmlDir?: string
+  /** Open Graph title. */
+  ogTitle?: string
+  /** Open Graph type. */
+  ogType?: string
+  /** Open Graph image URL. */
+  ogImage?: string
+  /** Open Graph description. */
+  ogDescription?: string
+  /** Open Graph URL. */
+  ogUrl?: string
+  /** Open Graph site name. */
+  ogSiteName?: string
+  /** Open Graph locale. */
+  ogLocale?: string
+  /** Open Graph video URL. */
+  ogVideo?: string
+  /** Open Graph audio URL. */
+  ogAudio?: string
+  /** Open Graph locale alternates. */
+  ogLocaleAlternates?: Array<string>
+  /** Twitter card type. */
+  twitterCard?: string
+  /** Twitter title. */
+  twitterTitle?: string
+  /** Twitter description. */
+  twitterDescription?: string
+  /** Twitter image URL. */
+  twitterImage?: string
+  /** Twitter site handle. */
+  twitterSite?: string
+  /** Twitter creator handle. */
+  twitterCreator?: string
+  /** Dublin Core title. */
+  dcTitle?: string
+  /** Dublin Core creator. */
+  dcCreator?: string
+  /** Dublin Core subject. */
+  dcSubject?: string
+  /** Dublin Core description. */
+  dcDescription?: string
+  /** Dublin Core publisher. */
+  dcPublisher?: string
+  /** Dublin Core date. */
+  dcDate?: string
+  /** Dublin Core type. */
+  dcType?: string
+  /** Dublin Core format. */
+  dcFormat?: string
+  /** Dublin Core identifier. */
+  dcIdentifier?: string
+  /** Dublin Core language. */
+  dcLanguage?: string
+  /** Dublin Core rights. */
+  dcRights?: string
+  /** Article metadata from `article:*` Open Graph tags. */
+  article?: ArticleMetadata
+  /** Hreflang alternate links. */
+  hreflangs?: Array<HreflangEntry>
+  /** Favicon and icon links. */
+  favicons?: Array<FaviconInfo>
+  /** Heading elements (h1-h6). */
+  headings?: Array<HeadingInfo>
+  /** Computed word count of the page body text. */
+  wordCount?: number
+}
+
+/** Proxy configuration for HTTP requests. */
+export interface ProxyConfig {
+  /** Proxy URL (e.g. "http://proxy:8080", "socks5://proxy:1080"). */
+  url?: string
+  /** Optional username for proxy authentication. */
+  username?: string
+  /** Optional password for proxy authentication. */
+  password?: string
+}
+
+/** Response metadata extracted from HTTP headers. */
+export interface ResponseMeta {
+  /** The ETag header value. */
+  etag?: string
+  /** The Last-Modified header value. */
+  lastModified?: string
+  /** The Cache-Control header value. */
+  cacheControl?: string
+  /** The Server header value. */
+  server?: string
+  /** The X-Powered-By header value. */
+  xPoweredBy?: string
+  /** The Content-Language header value. */
+  contentLanguage?: string
+  /** The Content-Encoding header value. */
+  contentEncoding?: string
+}
+
+/** Scrape a single URL, returning extracted page data. */
+export declare function scrape(engine: CrawlEngineHandle, url: string): Promise<ScrapeResult>
+
+/** The result of a single-page scrape operation. */
+export interface ScrapeResult {
+  /** The HTTP status code of the response. */
+  statusCode?: number
+  /** The final URL after following all redirects. */
+  finalUrl?: string
+  /** The Content-Type header value. */
+  contentType?: string
+  /** The HTML body of the response. */
+  html?: string
+  /** The size of the response body in bytes. */
+  bodySize?: number
+  /** Extracted metadata from the page. */
+  metadata?: JsPageMetadata
+  /** Links found on the page. */
+  links?: Array<JsLinkInfo>
+  /** Images found on the page. */
+  images?: Array<JsImageInfo>
+  /** Feed links found on the page. */
+  feeds?: Array<JsFeedInfo>
+  /** JSON-LD entries found on the page. */
+  jsonLd?: Array<JsJsonLdEntry>
+  /** Whether the URL is allowed by robots.txt. */
+  isAllowed?: boolean
+  /** The crawl delay from robots.txt, in seconds. */
+  crawlDelay?: number
+  /** Whether a noindex directive was detected. */
+  noindexDetected?: boolean
+  /** Whether a nofollow directive was detected. */
+  nofollowDetected?: boolean
+  /** The X-Robots-Tag header value, if present. */
+  xRobotsTag?: string
+  /** Whether the content is a PDF. */
+  isPdf?: boolean
+  /** Whether the page was skipped (binary or PDF content). */
+  wasSkipped?: boolean
+  /** The detected character set encoding. */
+  detectedCharset?: string
+  /** Whether an authentication header was sent with the request. */
+  authHeaderSent?: boolean
+  /** Response metadata extracted from HTTP headers. */
+  responseMeta?: JsResponseMeta
+  /** Downloaded assets from the page. */
+  assets?: Array<JsDownloadedAsset>
+  /** Whether the page content suggests JavaScript rendering is needed. */
+  jsRenderHint?: boolean
+  /** Whether the browser fallback was used to fetch this page. */
+  browserUsed?: boolean
+  /** Markdown conversion of the page content. */
+  markdown?: JsMarkdownResult
+  /** Structured data extracted by LLM. Populated when extraction is configured. */
+  extractedData?: any
+  /** Metadata about the LLM extraction pass (cost, tokens, model). */
+  extractionMeta?: ExtractionMeta
+  /** Downloaded non-HTML document (PDF, DOCX, image, code, etc.). */
+  downloadedDocument?: DownloadedDocument
+  /**
+   * Browser-specific extras (eval result, network events, cookies). Only
+   * populated when `BrowserBackend.Native` was used for this request.
+   */
+  browser?: BrowserExtras
+}
+
+/** Direction for a scroll action. */
+export declare const enum ScrollDirection {
+  /** Scroll upward. */
+  Up = 'up',
+  /** Scroll downward. */
+  Down = 'down'
+}
+
+/** A URL entry from a sitemap. */
+export interface SitemapUrl {
+  /** The URL. */
+  url?: string
+  /** The last modification date, if present. */
+  lastmod?: string
+  /** The change frequency, if present. */
+  changefreq?: string
+  /** The priority, if present. */
+  priority?: string
+}
+
+/** SSRF policy configuration. */
+export interface SsrfPolicy {
+  /** If true, reject URLs that resolve to private/metadata IP ranges. */
+  denyPrivate?: boolean
+  /** Maximum number of HTTP redirects to follow during validation. */
+  maxRedirects?: number
+}
+
+export declare function ssrfPolicyDefault(): SsrfPolicy
+
+/**
+ * Create a policy from environment variables.
+ *
+ * On native platforms, reads `CRAWLBERG_ALLOW_PRIVATE_NETWORK` — if set to "1" or "true"
+ * (case-insensitive), sets `deny_private = false`. Otherwise, defaults to `deny_private = true`.
+ *
+ * On wasm32 targets (browser/Node.js), environment variables are not accessible to the
+ * compiled module. Defaults to `deny_private = false` because:
+ * - Outbound requests in a browser go through the fetch API, which enforces its own network policies.
+ * - Rust-side SSRF checking is unenforceable and redundant in a wasm32 context.
+ * - For testing and localhost access, the host's network sandbox is the enforcing boundary.
+ */
+export declare function ssrfPolicyFromEnv(): SsrfPolicy