crawlberg 1.2.1 → 1.4.0

data/sig/types.rbs

@@ -1,547 +1,554 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:0ce4d753fdb4854e44358639dcbaebee3449a4afa142dbc4f0a72aa72c214648
+# alef:hash:71edb0ad11af2e26ac8c388c9aac9e12715a6a126f85a44c69e785b59fe8ad1f
 # To regenerate: alef generate
-# To verify freshness: alef verify --exit-code
+# To verify freshness: alef verify
 
 module Crawlberg
 
-    VERSION: String
+  VERSION: String
 
-type json_value = Hash[String, untyped] | Array[untyped] | String | Integer | Float | bool | nil
+  type json_value = Hash[String, untyped] | Array[untyped] | String | Integer | Float | bool | nil
 
-    class ExtractionMeta
-        attr_accessor cost: Float?
-        attr_accessor prompt_tokens: Integer?
-        attr_accessor completion_tokens: Integer?
-        attr_accessor model: String?
-        attr_accessor chunks_processed: Integer?
+  class ActionResult
+    attr_reader action_index: Integer
+    attr_reader action_type: String
+    attr_reader success: bool
+    attr_reader data: String?
+    attr_reader error: String?
 
-    def initialize: (?cost: Float, ?prompt_tokens: Integer, ?completion_tokens: Integer, ?model: String, ?chunks_processed: Integer) -> void
-    end
-
-    class ProxyConfig
-        attr_accessor url: String?
-        attr_accessor username: String?
-        attr_accessor password: String?
-
-    def initialize: (?url: String, ?username: String, ?password: String) -> void
-    end
-
-    class ContentConfig
-        attr_accessor output_format: String?
-        attr_accessor preprocessing_preset: String?
-        attr_accessor remove_navigation: bool?
-        attr_accessor remove_forms: bool?
-    attr_accessor strip_tags: Array[String]?
-    attr_accessor preserve_tags: Array[String]?
-    attr_accessor exclude_selectors: Array[String]?
-        attr_accessor skip_images: bool?
-        attr_accessor max_depth: Integer?
-        attr_accessor wrap: bool?
-        attr_accessor wrap_width: Integer?
-        attr_accessor include_document_structure: bool?
-
-def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool) -> void
-    def self.default: () -> ContentConfig
-    end
-
-    class BrowserConfig
-        attr_accessor mode: BrowserMode?
-        attr_accessor backend: BrowserBackend?
-        attr_accessor endpoint: String?
-        attr_accessor timeout: Integer?
-        attr_accessor wait: BrowserWait?
-        attr_accessor wait_selector: String?
-        attr_accessor extra_wait: Integer?
-        attr_accessor proxy: ProxyConfig?
-    attr_accessor block_url_patterns: Array[String]?
-        attr_accessor eval_script: String?
-        attr_accessor robots_user_agent: String?
-        attr_accessor capture_network_events: bool?
-        attr_accessor session_affinity: bool?
-
-def initialize: (?mode: BrowserMode, ?backend: BrowserBackend, ?endpoint: String, ?timeout: Integer, ?wait: BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool) -> void
-    def self.default: () -> BrowserConfig
-    end
-
-    class CrawlConfig
-        attr_accessor max_depth: Integer?
-        attr_accessor max_pages: Integer?
-        attr_accessor max_links_per_page: Integer?
-        attr_accessor max_concurrent: Integer?
-        attr_accessor respect_robots_txt: bool?
-        attr_accessor soft_http_errors: bool?
-        attr_accessor user_agent: String?
-        attr_accessor stay_on_domain: bool?
-        attr_accessor allow_subdomains: bool?
-    attr_accessor include_paths: Array[String]?
-    attr_accessor exclude_paths: Array[String]?
-    attr_accessor custom_headers: Hash[String, String]?
-        attr_accessor request_timeout: Integer?
-        attr_accessor rate_limit_ms: Integer?
-        attr_accessor max_redirects: Integer?
-        attr_accessor retry_count: Integer?
-    attr_accessor retry_codes: Array[Integer]?
-        attr_accessor cookies_enabled: bool?
-        attr_accessor auth: AuthConfig?
-        attr_accessor max_body_size: Integer?
-    attr_accessor remove_tags: Array[String]?
-        attr_accessor content: ContentConfig?
-        attr_accessor map_limit: Integer?
-        attr_accessor map_search: String?
-        attr_accessor download_assets: bool?
-    attr_accessor asset_types: Array[AssetCategory]?
-        attr_accessor max_asset_size: Integer?
-        attr_accessor browser: BrowserConfig?
-        attr_accessor proxy: ProxyConfig?
-    attr_accessor user_agents: Array[String]?
-        attr_accessor capture_screenshot: bool?
-        attr_accessor follow_document_urls: bool?
-        attr_accessor document_url_depth: Integer?
-        attr_accessor download_documents: bool?
-        attr_accessor document_max_size: Integer?
-    attr_accessor document_mime_types: Array[String]?
-        attr_accessor document_output_dir: String?
-        attr_accessor document_content_encoding: DocumentContentEncoding?
-        attr_accessor warc_output: String?
-        attr_accessor browser_profile: String?
-        attr_accessor save_browser_profile: bool?
-        attr_accessor ssrf: SsrfPolicy?
-        attr_accessor ssrf_deny_private_explicit: bool?
-
-def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
-    def validate: () -> void
-    def self.default: () -> CrawlConfig
-    end
-
-    class BrowserExtras
-        attr_accessor eval_result: json_value?
-    attr_accessor network_events: Array[ResponseMeta]?
-    attr_accessor cookies: Array[CookieInfo]?
-
-def initialize: (?eval_result: json_value, ?network_events: Array[ResponseMeta], ?cookies: Array[CookieInfo]) -> void
-    end
-
-    class DownloadedDocument
-        attr_accessor url: String?
-        attr_accessor mime_type: String?
-        attr_accessor size: Integer?
-        attr_accessor filename: String?
-        attr_accessor content_hash: String?
-    attr_accessor headers: Hash[String, String]?
-        attr_accessor truncated: bool?
-        attr_accessor content_path: String?
-        attr_accessor content_base64: String?
-
-def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: String, ?content_hash: String, ?headers: Hash[String, String], ?truncated: bool, ?content_path: String, ?content_base64: String) -> void
-    end
-
-    class InteractionResult
-    attr_accessor action_results: Array[ActionResult]?
-        attr_accessor final_html: String?
-        attr_accessor final_url: String?
-        attr_accessor screenshot_base64: String?
-
-def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?final_url: String, ?screenshot_base64: String) -> void
-    end
-
-    class ActionResult
-        attr_accessor action_index: Integer?
-        attr_accessor action_type: String?
-        attr_accessor success: bool?
-        attr_accessor data: json_value?
-        attr_accessor error: String?
-
-    def initialize: (?action_index: Integer, ?action_type: String, ?success: bool, ?data: json_value, ?error: String) -> void
-    end
-
-    class ScrapeResult
-        attr_accessor status_code: Integer?
-        attr_accessor final_url: String?
-        attr_accessor content_type: String?
-        attr_accessor html: String?
-        attr_accessor body_size: Integer?
-        attr_accessor metadata: PageMetadata?
-    attr_accessor links: Array[LinkInfo]?
-    attr_accessor images: Array[ImageInfo]?
-    attr_accessor feeds: Array[FeedInfo]?
-    attr_accessor json_ld: Array[JsonLdEntry]?
-        attr_accessor is_allowed: bool?
-        attr_accessor crawl_delay: Integer?
-        attr_accessor noindex_detected: bool?
-        attr_accessor nofollow_detected: bool?
-        attr_accessor x_robots_tag: String?
-        attr_accessor is_pdf: bool?
-        attr_accessor was_skipped: bool?
-        attr_accessor detected_charset: String?
-        attr_accessor auth_header_sent: bool?
-        attr_accessor response_meta: ResponseMeta?
-    attr_accessor assets: Array[DownloadedAsset]?
-        attr_accessor js_render_hint: bool?
-        attr_accessor browser_used: bool?
-        attr_accessor markdown: MarkdownResult?
-        attr_accessor extracted_data: json_value?
-        attr_accessor extraction_meta: ExtractionMeta?
-        attr_accessor screenshot_base64: String?
-        attr_accessor downloaded_document: DownloadedDocument?
-        attr_accessor browser: BrowserExtras?
-
-def initialize: (?status_code: Integer, ?final_url: String, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?is_allowed: bool, ?crawl_delay: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?x_robots_tag: String, ?is_pdf: bool, ?was_skipped: bool, ?detected_charset: String, ?auth_header_sent: bool, ?response_meta: ResponseMeta, ?assets: Array[DownloadedAsset], ?js_render_hint: bool, ?browser_used: bool, ?markdown: MarkdownResult, ?extracted_data: json_value, ?extraction_meta: ExtractionMeta, ?screenshot_base64: String, ?downloaded_document: DownloadedDocument, ?browser: BrowserExtras) -> void
-    end
-
-    class CrawlPageResult
-        attr_accessor url: String?
-        attr_accessor normalized_url: String?
-        attr_accessor status_code: Integer?
-        attr_accessor content_type: String?
-        attr_accessor html: String?
-        attr_accessor body_size: Integer?
-        attr_accessor metadata: PageMetadata?
-    attr_accessor links: Array[LinkInfo]?
-    attr_accessor images: Array[ImageInfo]?
-    attr_accessor feeds: Array[FeedInfo]?
-    attr_accessor json_ld: Array[JsonLdEntry]?
-        attr_accessor depth: Integer?
-        attr_accessor stayed_on_domain: bool?
-        attr_accessor was_skipped: bool?
-        attr_accessor is_pdf: bool?
-        attr_accessor detected_charset: String?
-        attr_accessor markdown: MarkdownResult?
-        attr_accessor extracted_data: json_value?
-        attr_accessor extraction_meta: ExtractionMeta?
-        attr_accessor downloaded_document: DownloadedDocument?
-        attr_accessor browser_used: bool?
-
-def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?depth: Integer, ?stayed_on_domain: bool, ?was_skipped: bool, ?is_pdf: bool, ?detected_charset: String, ?markdown: MarkdownResult, ?extracted_data: json_value, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser_used: bool) -> void
-    end
-
-    class CrawlResult
-    attr_accessor pages: Array[CrawlPageResult]?
-        attr_accessor final_url: String?
-        attr_accessor redirect_count: Integer?
-        attr_accessor was_skipped: bool?
-        attr_accessor error: String?
-    attr_accessor cookies: Array[CookieInfo]?
-        attr_accessor stayed_on_domain: bool?
-        attr_accessor browser_used: bool?
-
-def initialize: (?pages: Array[CrawlPageResult], ?final_url: String, ?redirect_count: Integer, ?was_skipped: bool, ?error: String, ?cookies: Array[CookieInfo], ?stayed_on_domain: bool, ?browser_used: bool) -> void
-    def unique_normalized_urls: () -> Integer
-    end
-
-    class SitemapUrl
-        attr_accessor url: String?
-        attr_accessor lastmod: String?
-        attr_accessor changefreq: String?
-        attr_accessor priority: String?
-
-    def initialize: (?url: String, ?lastmod: String, ?changefreq: String, ?priority: String) -> void
-    end
-
-    class MapResult
-    attr_accessor urls: Array[SitemapUrl]?
-
-def initialize: (?urls: Array[SitemapUrl]) -> void
-    end
+    def initialize: (?action_index: Integer, ?action_type: String, ?success: bool, ?data: String, ?error: String) -> void
+  end
 
-    class MarkdownResult
-        attr_accessor content: String?
-        attr_accessor document_structure: json_value?
-    attr_accessor tables: Array[json_value]?
-    attr_accessor warnings: Array[String]?
-        attr_accessor citations: bool?
-        attr_accessor fit_content: String?
+  class ArticleMetadata
+    attr_reader published_time: String?
+    attr_reader modified_time: String?
+    attr_reader author: String?
+    attr_reader section: String?
+    attr_reader tags: Array[String]
 
-def initialize: (?content: String, ?document_structure: json_value, ?tables: Array[json_value], ?warnings: Array[String], ?citations: bool, ?fit_content: String) -> void
-    end
+    def initialize: (?published_time: String, ?modified_time: String, ?author: String, ?section: String, ?tags: Array[String]) -> void
+  end
 
-    class LinkInfo
-        attr_accessor url: String?
-        attr_accessor text: String?
-        attr_accessor link_type: LinkType?
-        attr_accessor rel: String?
-        attr_accessor nofollow: bool?
+  class BatchCrawlResult
+    attr_reader url: String
+    attr_reader result: CrawlResult?
+    attr_reader error: String?
 
-    def initialize: (?url: String, ?text: String, ?link_type: LinkType, ?rel: String, ?nofollow: bool) -> void
-    end
+    def initialize: (?url: String, ?result: CrawlResult, ?error: String) -> void
+  end
 
-    class ImageInfo
-        attr_accessor url: String?
-        attr_accessor alt: String?
-        attr_accessor width: Integer?
-        attr_accessor height: Integer?
-        attr_accessor source: ImageSource?
+  class BatchCrawlResults
+    attr_reader results: Array[BatchCrawlResult]
+    attr_reader total_count: Integer
+    attr_reader completed_count: Integer
+    attr_reader failed_count: Integer
 
-    def initialize: (?url: String, ?alt: String, ?width: Integer, ?height: Integer, ?source: ImageSource) -> void
-    end
+    def initialize: (?results: Array[BatchCrawlResult], ?total_count: Integer, ?completed_count: Integer, ?failed_count: Integer) -> void
+  end
 
-    class FeedInfo
-        attr_accessor url: String?
-        attr_accessor title: String?
-        attr_accessor feed_type: FeedType?
+  class BatchCrawlStreamRequest
+    attr_reader urls: Array[String]
 
-    def initialize: (?url: String, ?title: String, ?feed_type: FeedType) -> void
-    end
+    def initialize: (?urls: Array[String]) -> void
+  end
 
-    class JsonLdEntry
-        attr_accessor schema_type: String?
-        attr_accessor name: String?
-        attr_accessor raw: String?
+  class BatchScrapeResult
+    attr_reader url: String
+    attr_reader result: ScrapeResult?
+    attr_reader error: String?
 
-    def initialize: (?schema_type: String, ?name: String, ?raw: String) -> void
-    end
+    def initialize: (?url: String, ?result: ScrapeResult, ?error: String) -> void
+  end
+
+  class BatchScrapeResults
+    attr_reader results: Array[BatchScrapeResult]
+    attr_reader total_count: Integer
+    attr_reader completed_count: Integer
+    attr_reader failed_count: Integer
+
+    def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?completed_count: Integer, ?failed_count: Integer) -> void
+  end
+
+  class BrowserConfig
+    attr_reader mode: BrowserMode
+    attr_reader backend: BrowserBackend
+    attr_reader endpoint: String?
+    attr_reader timeout: Integer
+    attr_reader wait: BrowserWait
+    attr_reader wait_selector: String?
+    attr_reader extra_wait: Integer?
+    attr_reader proxy: ProxyConfig?
+    attr_reader block_url_patterns: Array[String]
+    attr_reader eval_script: String?
+    attr_reader robots_user_agent: String?
+    attr_reader capture_network_events: bool
+    attr_reader session_affinity: bool
+
+    def initialize: (?mode: BrowserMode, ?backend: BrowserBackend, ?endpoint: String, ?timeout: Integer, ?wait: BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool) -> void
+  end
+
+  class BrowserExtras
+    attr_reader eval_result: String?
+    attr_reader network_events: Array[ResponseMeta]
+    attr_reader cookies: Array[CookieInfo]
+
+    def initialize: (?eval_result: String, ?network_events: Array[ResponseMeta], ?cookies: Array[CookieInfo]) -> void
+  end
+
+  class CitationReference
+    attr_reader index: Integer
+    attr_reader url: String
+    attr_reader text: String
 
-    class CookieInfo
-        attr_accessor name: String?
-        attr_accessor value: String?
-        attr_accessor domain: String?
-        attr_accessor path: String?
+    def initialize: (?index: Integer, ?url: String, ?text: String) -> void
+  end
+
+  class CitationResult
+    attr_reader content: String
+    attr_reader references: Array[CitationReference]
+
+    def initialize: (?content: String, ?references: Array[CitationReference]) -> void
+  end
+
+  class ContentConfig
+    attr_reader output_format: String
+    attr_reader preprocessing_preset: String
+    attr_reader remove_navigation: bool
+    attr_reader remove_forms: bool
+    attr_reader strip_tags: Array[String]
+    attr_reader preserve_tags: Array[String]
+    attr_reader exclude_selectors: Array[String]
+    attr_reader skip_images: bool
+    attr_reader max_depth: Integer?
+    attr_reader wrap: bool
+    attr_reader wrap_width: Integer
+    attr_reader include_document_structure: bool
+
+    def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool) -> void
+  end
+
+  class CookieInfo
+    attr_reader name: String
+    attr_reader value: String
+    attr_reader domain: String?
+    attr_reader path: String?
 
     def initialize: (?name: String, ?value: String, ?domain: String, ?path: String) -> void
-    end
+  end
+
+  class CrawlConfig
+    attr_reader max_depth: Integer?
+    attr_reader max_pages: Integer?
+    attr_reader max_links_per_page: Integer?
+    attr_reader max_concurrent: Integer?
+    attr_reader crawl_strategy: CrawlStrategyKind
+    attr_reader content_filter: ContentFilterKind?
+    attr_reader bm25_query: String?
+    attr_reader bm25_threshold: Float?
+    attr_reader respect_robots_txt: bool
+    attr_reader soft_http_errors: bool
+    attr_reader user_agent: String?
+    attr_reader stay_on_domain: bool
+    attr_reader allow_subdomains: bool
+    attr_reader include_paths: Array[String]
+    attr_reader exclude_paths: Array[String]
+    attr_reader custom_headers: Hash[String, String]
+    attr_reader request_timeout: Integer
+    attr_reader rate_limit_ms: Integer?
+    attr_reader max_redirects: Integer
+    attr_reader retry_count: Integer
+    attr_reader retry_codes: Array[Integer]
+    attr_reader cookies_enabled: bool
+    attr_reader auth: AuthConfig?
+    attr_reader max_body_size: Integer?
+    attr_reader remove_tags: Array[String]
+    attr_reader content: ContentConfig
+    attr_reader map_limit: Integer?
+    attr_reader map_search: String?
+    attr_reader download_assets: bool
+    attr_reader asset_types: Array[AssetCategory]
+    attr_reader max_asset_size: Integer?
+    attr_reader browser: BrowserConfig
+    attr_reader proxy: ProxyConfig?
+    attr_reader user_agents: Array[String]
+    attr_reader capture_screenshot: bool
+    attr_reader follow_document_urls: bool
+    attr_reader document_url_depth: Integer?
+    attr_reader download_documents: bool
+    attr_reader document_max_size: Integer?
+    attr_reader document_mime_types: Array[String]
+    attr_reader document_output_dir: String?
+    attr_reader document_content_encoding: DocumentContentEncoding?
+    attr_reader warc_output: String?
+    attr_reader browser_profile: String?
+    attr_reader save_browser_profile: bool
+    attr_reader ssrf: SsrfPolicy
+    attr_reader ssrf_deny_private_explicit: bool?
+
+    def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: CrawlStrategyKind, ?content_filter: ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
+    def validate: () -> void
+  end
+
+  class CrawlEngineHandle
+    def crawl_stream: (CrawlStreamRequest req) -> Enumerator[CrawlEvent]
+    def batch_crawl_stream: (BatchCrawlStreamRequest req) -> Enumerator[CrawlEvent]
+  end
+
+  class CrawlPageResult
+    attr_reader url: String
+    attr_reader normalized_url: String
+    attr_reader status_code: Integer
+    attr_reader content_type: String
+    attr_reader html: String
+    attr_reader body_size: Integer
+    attr_reader metadata: PageMetadata
+    attr_reader links: Array[LinkInfo]
+    attr_reader images: Array[ImageInfo]
+    attr_reader feeds: Array[FeedInfo]
+    attr_reader json_ld: Array[JsonLdEntry]
+    attr_reader depth: Integer
+    attr_reader stayed_on_domain: bool
+    attr_reader was_skipped: bool
+    attr_reader is_pdf: bool
+    attr_reader detected_charset: String?
+    attr_reader markdown: MarkdownResult?
+    attr_reader extracted_data: String?
+    attr_reader extraction_meta: ExtractionMeta?
+    attr_reader downloaded_document: DownloadedDocument?
+    attr_reader browser_used: bool
+
+    def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?depth: Integer, ?stayed_on_domain: bool, ?was_skipped: bool, ?is_pdf: bool, ?detected_charset: String, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser_used: bool) -> void
+  end
+
+  class CrawlResult
+    attr_reader pages: Array[CrawlPageResult]
+    attr_reader final_url: String
+    attr_reader redirect_count: Integer
+    attr_reader was_skipped: bool
+    attr_reader error: String?
+    attr_reader cookies: Array[CookieInfo]
+    attr_reader stayed_on_domain: bool
+    attr_reader browser_used: bool
+
+    def initialize: (?pages: Array[CrawlPageResult], ?final_url: String, ?redirect_count: Integer, ?was_skipped: bool, ?error: String, ?cookies: Array[CookieInfo], ?stayed_on_domain: bool, ?browser_used: bool) -> void
+    def unique_normalized_urls: () -> Integer
+  end
 
-    class DownloadedAsset
-        attr_accessor url: String?
-        attr_accessor content_hash: String?
-        attr_accessor mime_type: String?
-        attr_accessor size: Integer?
-        attr_accessor asset_category: AssetCategory?
-        attr_accessor html_tag: String?
+  class CrawlStreamRequest
+    attr_reader url: String
 
-    def initialize: (?url: String, ?content_hash: String, ?mime_type: String, ?size: Integer, ?asset_category: AssetCategory, ?html_tag: String) -> void
-    end
-
-    class ArticleMetadata
-        attr_accessor published_time: String?
-        attr_accessor modified_time: String?
-        attr_accessor author: String?
-        attr_accessor section: String?
-    attr_accessor tags: Array[String]?
+    def initialize: (?url: String) -> void
+  end
 
-def initialize: (?published_time: String, ?modified_time: String, ?author: String, ?section: String, ?tags: Array[String]) -> void
-    end
+  class DownloadedAsset
+    attr_reader url: String
+    attr_reader content_hash: String
+    attr_reader mime_type: String?
+    attr_reader size: Integer
+    attr_reader asset_category: AssetCategory
+    attr_reader html_tag: String?
 
-    class HreflangEntry
-        attr_accessor lang: String?
-        attr_accessor url: String?
+    def initialize: (?url: String, ?content_hash: String, ?mime_type: String, ?size: Integer, ?asset_category: AssetCategory, ?html_tag: String) -> void
+  end
+
+  class DownloadedDocument
+    attr_reader url: String
+    attr_reader mime_type: String
+    attr_reader size: Integer
+    attr_reader filename: String?
+    attr_reader content_hash: String
+    attr_reader headers: Hash[String, String]
+    attr_reader truncated: bool
+    attr_reader content_path: String?
+    attr_reader content_base64: String?
+
+    def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: String, ?content_hash: String, ?headers: Hash[String, String], ?truncated: bool, ?content_path: String, ?content_base64: String) -> void
+  end
+
+  class ExtractionMeta
+    attr_reader cost: Float?
+    attr_reader prompt_tokens: Integer?
+    attr_reader completion_tokens: Integer?
+    attr_reader model: String?
+    attr_reader chunks_processed: Integer
 
-    def initialize: (?lang: String, ?url: String) -> void
-    end
+    def initialize: (?cost: Float, ?prompt_tokens: Integer, ?completion_tokens: Integer, ?model: String, ?chunks_processed: Integer) -> void
+  end
 
-    class FaviconInfo
-        attr_accessor url: String?
-        attr_accessor rel: String?
-        attr_accessor sizes: String?
-        attr_accessor mime_type: String?
+  class FaviconInfo
+    attr_reader url: String
+    attr_reader rel: String
+    attr_reader sizes: String?
+    attr_reader mime_type: String?
 
     def initialize: (?url: String, ?rel: String, ?sizes: String, ?mime_type: String) -> void
-    end
+  end
 
-    class HeadingInfo
-        attr_accessor level: Integer?
-        attr_accessor text: String?
+  class FeedInfo
+    attr_reader url: String
+    attr_reader title: String?
+    attr_reader feed_type: FeedType
 
-    def initialize: (?level: Integer, ?text: String) -> void
-    end
+    def initialize: (?url: String, ?title: String, ?feed_type: FeedType) -> void
+  end
 
-    class ResponseMeta
-        attr_accessor etag: String?
-        attr_accessor last_modified: String?
-        attr_accessor cache_control: String?
-        attr_accessor server: String?
-        attr_accessor x_powered_by: String?
-        attr_accessor content_language: String?
-        attr_accessor content_encoding: String?
+  class HeadingInfo
+    attr_reader level: Integer
+    attr_reader text: String
 
-    def initialize: (?etag: String, ?last_modified: String, ?cache_control: String, ?server: String, ?x_powered_by: String, ?content_language: String, ?content_encoding: String) -> void
-    end
-
-    class PageMetadata
-        attr_accessor title: String?
-        attr_accessor description: String?
-        attr_accessor canonical_url: String?
-        attr_accessor keywords: String?
-        attr_accessor author: String?
-        attr_accessor viewport: String?
-        attr_accessor theme_color: String?
-        attr_accessor generator: String?
-        attr_accessor robots: String?
-        attr_accessor html_lang: String?
-        attr_accessor html_dir: String?
-        attr_accessor og_title: String?
-        attr_accessor og_type: String?
-        attr_accessor og_image: String?
-        attr_accessor og_description: String?
-        attr_accessor og_url: String?
-        attr_accessor og_site_name: String?
-        attr_accessor og_locale: String?
-        attr_accessor og_video: String?
-        attr_accessor og_audio: String?
-    attr_accessor og_locale_alternates: Array[String]?
-        attr_accessor twitter_card: String?
-        attr_accessor twitter_title: String?
-        attr_accessor twitter_description: String?
-        attr_accessor twitter_image: String?
-        attr_accessor twitter_site: String?
-        attr_accessor twitter_creator: String?
-        attr_accessor dc_title: String?
-        attr_accessor dc_creator: String?
-        attr_accessor dc_subject: String?
-        attr_accessor dc_description: String?
-        attr_accessor dc_publisher: String?
-        attr_accessor dc_date: String?
-        attr_accessor dc_type: String?
-        attr_accessor dc_format: String?
-        attr_accessor dc_identifier: String?
-        attr_accessor dc_language: String?
-        attr_accessor dc_rights: String?
-        attr_accessor article: ArticleMetadata?
-    attr_accessor hreflangs: Array[HreflangEntry]?
-    attr_accessor favicons: Array[FaviconInfo]?
-    attr_accessor headings: Array[HeadingInfo]?
-        attr_accessor word_count: Integer?
-
-def initialize: (?title: String, ?description: String, ?canonical_url: String, ?keywords: String, ?author: String, ?viewport: String, ?theme_color: String, ?generator: String, ?robots: String, ?html_lang: String, ?html_dir: String, ?og_title: String, ?og_type: String, ?og_image: String, ?og_description: String, ?og_url: String, ?og_site_name: String, ?og_locale: String, ?og_video: String, ?og_audio: String, ?og_locale_alternates: Array[String], ?twitter_card: String, ?twitter_title: String, ?twitter_description: String, ?twitter_image: String, ?twitter_site: String, ?twitter_creator: String, ?dc_title: String, ?dc_creator: String, ?dc_subject: String, ?dc_description: String, ?dc_publisher: String, ?dc_date: String, ?dc_type: String, ?dc_format: String, ?dc_identifier: String, ?dc_language: String, ?dc_rights: String, ?article: ArticleMetadata, ?hreflangs: Array[HreflangEntry], ?favicons: Array[FaviconInfo], ?headings: Array[HeadingInfo], ?word_count: Integer) -> void
-    end
-
-    class CrawlStreamRequest
-        attr_accessor url: String?
+    def initialize: (?level: Integer, ?text: String) -> void
+  end
 
-    def initialize: (?url: String) -> void
-    end
+  class HreflangEntry
+    attr_reader lang: String
+    attr_reader url: String
 
-    class BatchCrawlStreamRequest
-    attr_accessor urls: Array[String]?
+    def initialize: (?lang: String, ?url: String) -> void
+  end
 
-def initialize: (?urls: Array[String]) -> void
-    end
+  class ImageInfo
+    attr_reader url: String
+    attr_reader alt: String?
+    attr_reader width: Integer?
+    attr_reader height: Integer?
+    attr_reader source: ImageSource
 
-    class CitationResult
-        attr_accessor content: String?
-    attr_accessor references: Array[CitationReference]?
+    def initialize: (?url: String, ?alt: String, ?width: Integer, ?height: Integer, ?source: ImageSource) -> void
+  end
 
-def initialize: (?content: String, ?references: Array[CitationReference]) -> void
-    end
+  class InteractionResult
+    attr_reader action_results: Array[ActionResult]
+    attr_reader final_html: String
+    attr_reader final_url: String
+    attr_reader screenshot_base64: String?
 
-    class CitationReference
-        attr_accessor index: Integer?
-        attr_accessor url: String?
-        attr_accessor text: String?
+    def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?final_url: String, ?screenshot_base64: String) -> void
+  end
 
-    def initialize: (?index: Integer, ?url: String, ?text: String) -> void
-    end
+  class JsonLdEntry
+    attr_reader schema_type: String
+    attr_reader name: String?
+    attr_reader raw: String
 
-    class CrawlEngineHandle
-def crawl_stream: (CrawlStreamRequest req) -> Enumerator[CrawlEvent]
-def batch_crawl_stream: (BatchCrawlStreamRequest req) -> Enumerator[CrawlEvent]
-    end
+    def initialize: (?schema_type: String, ?name: String, ?raw: String) -> void
+  end
 
-    class BatchScrapeResult
-        attr_accessor url: String?
-        attr_accessor result: ScrapeResult?
-        attr_accessor error: String?
+  class LinkInfo
+    attr_reader url: String
+    attr_reader text: String
+    attr_reader link_type: LinkType
+    attr_reader rel: String?
+    attr_reader nofollow: bool
 
-    def initialize: (?url: String, ?result: ScrapeResult, ?error: String) -> void
-    end
+    def initialize: (?url: String, ?text: String, ?link_type: LinkType, ?rel: String, ?nofollow: bool) -> void
+  end
+
+  class MapResult
+    attr_reader urls: Array[SitemapUrl]
+
+    def initialize: (?urls: Array[SitemapUrl]) -> void
+  end
+
+  class MarkdownResult
+    attr_reader content: String
+    attr_reader document_structure: String?
+    attr_reader tables: Array[String]
+    attr_reader warnings: Array[String]
+    attr_reader citations: bool
+    attr_reader fit_content: String?
+
+    def initialize: (?content: String, ?document_structure: String, ?tables: Array[String], ?warnings: Array[String], ?citations: bool, ?fit_content: String) -> void
+  end
+
+  class PageMetadata
+    attr_reader title: String?
+    attr_reader description: String?
+    attr_reader canonical_url: String?
+    attr_reader keywords: String?
+    attr_reader author: String?
+    attr_reader viewport: String?
+    attr_reader theme_color: String?
+    attr_reader generator: String?
+    attr_reader robots: String?
+    attr_reader html_lang: String?
+    attr_reader html_dir: String?
+    attr_reader og_title: String?
+    attr_reader og_type: String?
+    attr_reader og_image: String?
+    attr_reader og_description: String?
+    attr_reader og_url: String?
+    attr_reader og_site_name: String?
+    attr_reader og_locale: String?
+    attr_reader og_video: String?
+    attr_reader og_audio: String?
+    attr_reader og_locale_alternates: Array[String]?
+    attr_reader twitter_card: String?
+    attr_reader twitter_title: String?
+    attr_reader twitter_description: String?
+    attr_reader twitter_image: String?
+    attr_reader twitter_site: String?
+    attr_reader twitter_creator: String?
+    attr_reader dc_title: String?
+    attr_reader dc_creator: String?
+    attr_reader dc_subject: String?
+    attr_reader dc_description: String?
+    attr_reader dc_publisher: String?
+    attr_reader dc_date: String?
+    attr_reader dc_type: String?
+    attr_reader dc_format: String?
+    attr_reader dc_identifier: String?
+    attr_reader dc_language: String?
+    attr_reader dc_rights: String?
+    attr_reader article: ArticleMetadata?
+    attr_reader hreflangs: Array[HreflangEntry]?
+    attr_reader favicons: Array[FaviconInfo]?
+    attr_reader headings: Array[HeadingInfo]?
+    attr_reader word_count: Integer?
+
+    def initialize: (?title: String, ?description: String, ?canonical_url: String, ?keywords: String, ?author: String, ?viewport: String, ?theme_color: String, ?generator: String, ?robots: String, ?html_lang: String, ?html_dir: String, ?og_title: String, ?og_type: String, ?og_image: String, ?og_description: String, ?og_url: String, ?og_site_name: String, ?og_locale: String, ?og_video: String, ?og_audio: String, ?og_locale_alternates: Array[String], ?twitter_card: String, ?twitter_title: String, ?twitter_description: String, ?twitter_image: String, ?twitter_site: String, ?twitter_creator: String, ?dc_title: String, ?dc_creator: String, ?dc_subject: String, ?dc_description: String, ?dc_publisher: String, ?dc_date: String, ?dc_type: String, ?dc_format: String, ?dc_identifier: String, ?dc_language: String, ?dc_rights: String, ?article: ArticleMetadata, ?hreflangs: Array[HreflangEntry], ?favicons: Array[FaviconInfo], ?headings: Array[HeadingInfo], ?word_count: Integer) -> void
+  end
+
+  class ProxyConfig
+    attr_reader url: String
+    attr_reader username: String?
+    attr_reader password: String?
 
-    class BatchCrawlResult
-        attr_accessor url: String?
-        attr_accessor result: CrawlResult?
-        attr_accessor error: String?
+    def initialize: (?url: String, ?username: String, ?password: String) -> void
+  end
 
-    def initialize: (?url: String, ?result: CrawlResult, ?error: String) -> void
-    end
+  class ResponseMeta
+    attr_reader etag: String?
+    attr_reader last_modified: String?
+    attr_reader cache_control: String?
+    attr_reader server: String?
+    attr_reader x_powered_by: String?
+    attr_reader content_language: String?
+    attr_reader content_encoding: String?
 
-    class BatchScrapeResults
-    attr_accessor results: Array[BatchScrapeResult]?
-        attr_accessor total_count: Integer?
-        attr_accessor completed_count: Integer?
-        attr_accessor failed_count: Integer?
+    def initialize: (?etag: String, ?last_modified: String, ?cache_control: String, ?server: String, ?x_powered_by: String, ?content_language: String, ?content_encoding: String) -> void
+  end
+
+  class ScrapeResult
+    attr_reader status_code: Integer
+    attr_reader final_url: String
+    attr_reader content_type: String
+    attr_reader html: String
+    attr_reader body_size: Integer
+    attr_reader metadata: PageMetadata
+    attr_reader links: Array[LinkInfo]
+    attr_reader images: Array[ImageInfo]
+    attr_reader feeds: Array[FeedInfo]
+    attr_reader json_ld: Array[JsonLdEntry]
+    attr_reader is_allowed: bool
+    attr_reader crawl_delay: Integer?
+    attr_reader noindex_detected: bool
+    attr_reader nofollow_detected: bool
+    attr_reader x_robots_tag: String?
+    attr_reader is_pdf: bool
+    attr_reader was_skipped: bool
+    attr_reader detected_charset: String?
+    attr_reader auth_header_sent: bool
+    attr_reader response_meta: ResponseMeta?
+    attr_reader assets: Array[DownloadedAsset]
+    attr_reader js_render_hint: bool
+    attr_reader browser_used: bool
+    attr_reader markdown: MarkdownResult?
+    attr_reader extracted_data: String?
+    attr_reader extraction_meta: ExtractionMeta?
+    attr_reader screenshot_base64: String?
+    attr_reader downloaded_document: DownloadedDocument?
+    attr_reader browser: BrowserExtras?
+
+    def initialize: (?status_code: Integer, ?final_url: String, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?is_allowed: bool, ?crawl_delay: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?x_robots_tag: String, ?is_pdf: bool, ?was_skipped: bool, ?detected_charset: String, ?auth_header_sent: bool, ?response_meta: ResponseMeta, ?assets: Array[DownloadedAsset], ?js_render_hint: bool, ?browser_used: bool, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?screenshot_base64: String, ?downloaded_document: DownloadedDocument, ?browser: BrowserExtras) -> void
+  end
+
+  class SitemapUrl
+    attr_reader url: String
+    attr_reader lastmod: String?
+    attr_reader changefreq: String?
+    attr_reader priority: String?
 
-def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?completed_count: Integer, ?failed_count: Integer) -> void
-    end
+    def initialize: (?url: String, ?lastmod: String, ?changefreq: String, ?priority: String) -> void
+  end
 
-    class BatchCrawlResults
-    attr_accessor results: Array[BatchCrawlResult]?
-        attr_accessor total_count: Integer?
-        attr_accessor completed_count: Integer?
-        attr_accessor failed_count: Integer?
+  class SsrfPolicy
+    attr_reader deny_private: bool
+    attr_reader allowlist: Array[HostMatcher]
+    attr_reader max_redirects: Integer
 
-def initialize: (?results: Array[BatchCrawlResult], ?total_count: Integer, ?completed_count: Integer, ?failed_count: Integer) -> void
-    end
+    def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?max_redirects: Integer) -> void
+  end
 
-    class SsrfPolicy
-        attr_accessor deny_private: bool?
-    attr_accessor allowlist: Array[HostMatcher]?
-        attr_accessor max_redirects: Integer?
+  class AssetCategory
+    type value = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
+  end
 
-def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?max_redirects: Integer) -> void
-    def self.default: () -> SsrfPolicy
-    def self.from_env: () -> SsrfPolicy
-    end
+  class AuthConfig
+  end
 
-    class BrowserMode
-        type value = :auto | :always | :never | :stealth
-    end
+  class BrowserBackend
+    type value = :chromiumoxide | :native
+  end
 
-    class BrowserWait
-        type value = :network_idle | :selector | :fixed
-    end
+  class BrowserMode
+    type value = :auto | :always | :never | :stealth
+  end
 
-    class BrowserBackend
-        type value = :chromiumoxide | :native
-    end
+  class BrowserWait
+    type value = :network_idle | :selector | :fixed
+  end
 
-    class DocumentContentEncoding
-        type value = :base64
-    end
+  class ContentFilterKind
+    type value = :bm25
+  end
 
-    class AuthConfig
-    end
+  class CrawlEvent
+  end
 
-    class LinkType
-        type value = :internal | :external | :anchor | :document
-    end
+  class CrawlStrategyKind
+    type value = :bfs | :dfs | :best_first | :adaptive
+  end
 
-    class ImageSource
-        type value = :img | :picture_source | :og_image | :twitter_image
-    end
+  class DocumentContentEncoding
+    type value = :base64
+  end
 
-    class FeedType
-        type value = :rss | :atom | :json_feed
-    end
+  class FeedType
+    type value = :rss | :atom | :json_feed
+  end
 
-    class AssetCategory
-        type value = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
-    end
+  class HostMatcher
+  end
 
-    class CrawlEvent
-    end
+  class ImageSource
+    type value = :img | :picture_source | :"og:image" | :"twitter:image"
+  end
 
-    class PageAction
-    end
+  class LinkType
+    type value = :internal | :external | :anchor | :document
+  end
 
-    class ScrollDirection
-        type value = :up | :down
-    end
+  class PageAction
+  end
 
-    class HostMatcher
-    end
+  class ScrollDirection
+    type value = :up | :down
+  end
 
-def self.generate_citations: (String markdown) -> CitationResult
+  def self.batch_crawl: (CrawlEngineHandle engine, Array[String] urls) -> BatchCrawlResults
 
-def self.create_engine: (?CrawlConfig config) -> CrawlEngineHandle
+  def self.batch_scrape: (CrawlEngineHandle engine, Array[String] urls) -> BatchScrapeResults
 
-def self.scrape: (CrawlEngineHandle engine, String url) -> ScrapeResult
+  def self.crawl: (CrawlEngineHandle engine, String url) -> CrawlResult
 
-def self.crawl: (CrawlEngineHandle engine, String url) -> CrawlResult
+  def self.create_engine: (?CrawlConfig config) -> CrawlEngineHandle
 
-def self.map_urls: (CrawlEngineHandle engine, String url) -> MapResult
+  def self.generate_citations: (String markdown) -> CitationResult
 
-def self.interact: (CrawlEngineHandle engine, String url, Array[PageAction] actions) -> InteractionResult
+  def self.interact: (CrawlEngineHandle engine, String url, Array[PageAction] actions) -> InteractionResult
 
-def self.batch_scrape: (CrawlEngineHandle engine, Array[String] urls) -> BatchScrapeResults
+  def self.map_urls: (CrawlEngineHandle engine, String url) -> MapResult
 
-def self.batch_crawl: (CrawlEngineHandle engine, Array[String] urls) -> BatchCrawlResults
+  def self.scrape: (CrawlEngineHandle engine, String url) -> ScrapeResult
 
 end