crawlberg 1.11.0-x86_64-linux

data/sig/types.rbs

@@ -0,0 +1,563 @@
+# This file is auto-generated by alef — DO NOT EDIT.
+# alef:hash:ad470f62b746634fbd629f163e7bae4a88fb011bbde28d998e38ade4877adf3f
+# To regenerate: alef generate
+# To verify freshness: alef verify
+
+module Crawlberg
+
+    VERSION: String
+
+type json_value = Hash[String, untyped] | Array[untyped] | String | Integer | Float | bool | nil
+
+    class ActionResult
+        attr_reader action_index: Integer
+        attr_reader action_type: String
+        attr_reader success: bool
+        attr_reader data: String?
+        attr_reader error: String?
+
+    def initialize: (?action_index: Integer, ?action_type: String, ?success: bool, ?data: String, ?error: String) -> void
+    end
+
+    class ArticleMetadata
+        attr_reader published_time: String?
+        attr_reader modified_time: String?
+        attr_reader author: String?
+        attr_reader section: String?
+    attr_reader tags: Array[String]
+
+def initialize: (?published_time: String, ?modified_time: String, ?author: String, ?section: String, ?tags: Array[String]) -> void
+    end
+
+    class BatchCrawlResult
+        attr_reader url: String
+        attr_reader result: CrawlResult?
+        attr_reader error: String?
+
+    def initialize: (?url: String, ?result: CrawlResult, ?error: String) -> void
+    end
+
+    class BatchCrawlResults
+    attr_reader results: Array[BatchCrawlResult]
+        attr_reader total_count: Integer
+        attr_reader completed_count: Integer
+        attr_reader failed_count: Integer
+
+def initialize: (?results: Array[BatchCrawlResult], ?total_count: Integer, ?completed_count: Integer, ?failed_count: Integer) -> void
+    end
+
+    class BatchCrawlStreamRequest
+    attr_reader urls: Array[String]
+
+def initialize: (?urls: Array[String]) -> void
+    end
+
+    class BatchScrapeResult
+        attr_reader url: String
+        attr_reader result: ScrapeResult?
+        attr_reader error: String?
+
+    def initialize: (?url: String, ?result: ScrapeResult, ?error: String) -> void
+    end
+
+    class BatchScrapeResults
+    attr_reader results: Array[BatchScrapeResult]
+        attr_reader total_count: Integer
+        attr_reader completed_count: Integer
+        attr_reader failed_count: Integer
+
+def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?completed_count: Integer, ?failed_count: Integer) -> void
+    end
+
+    class BrowserConfig
+        attr_reader mode: enum_BrowserMode
+        attr_reader backend: enum_BrowserBackend
+        attr_reader endpoint: String?
+        attr_reader timeout: Integer
+        attr_reader overall_timeout: Integer
+        attr_reader shutdown_timeout: Integer
+        attr_reader wait: enum_BrowserWait
+        attr_reader wait_selector: String?
+        attr_reader extra_wait: Integer?
+        attr_reader proxy: ProxyConfig?
+    attr_reader block_url_patterns: Array[String]
+        attr_reader eval_script: String?
+        attr_reader robots_user_agent: String?
+        attr_reader capture_network_events: bool
+        attr_reader session_affinity: bool
+        attr_reader chrome_path: String?
+    attr_reader chrome_args: Array[String]
+
+def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String]) -> void
+    end
+
+    class BrowserExtras
+        attr_reader eval_result: String?
+    attr_reader network_events: Array[ResponseMeta]
+    attr_reader cookies: Array[CookieInfo]
+
+def initialize: (?eval_result: String, ?network_events: Array[ResponseMeta], ?cookies: Array[CookieInfo]) -> void
+    end
+
+    class CitationReference
+        attr_reader index: Integer
+        attr_reader url: String
+        attr_reader text: String
+
+    def initialize: (?index: Integer, ?url: String, ?text: String) -> void
+    end
+
+    class CitationResult
+        attr_reader content: String
+    attr_reader references: Array[CitationReference]
+
+def initialize: (?content: String, ?references: Array[CitationReference]) -> void
+    end
+
+    class ContentConfig
+        attr_reader output_format: String
+        attr_reader preprocessing_preset: String
+        attr_reader remove_navigation: bool
+        attr_reader remove_forms: bool
+    attr_reader strip_tags: Array[String]
+    attr_reader preserve_tags: Array[String]
+    attr_reader exclude_selectors: Array[String]
+        attr_reader skip_images: bool
+        attr_reader max_depth: Integer?
+        attr_reader wrap: bool
+        attr_reader wrap_width: Integer
+        attr_reader include_document_structure: bool
+        attr_reader extract_metadata: bool
+        attr_reader strip_newlines: bool
+        attr_reader bullets: String
+        attr_reader list_indent_width: Integer
+        attr_reader strong_em_symbol: String
+        attr_reader compact_tables: bool
+    attr_reader keep_inline_images_in: Array[String]
+        attr_reader keep_hidden: bool
+
+def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool, ?extract_metadata: bool, ?strip_newlines: bool, ?bullets: String, ?list_indent_width: Integer, ?strong_em_symbol: String, ?compact_tables: bool, ?keep_inline_images_in: Array[String], ?keep_hidden: bool) -> void
+    end
+
+    class CookieInfo
+        attr_reader name: String
+        attr_reader value: String
+        attr_reader domain: String?
+        attr_reader path: String?
+
+    def initialize: (?name: String, ?value: String, ?domain: String, ?path: String) -> void
+    end
+
+    class CrawlConfig
+        attr_reader max_depth: Integer?
+        attr_reader max_pages: Integer?
+        attr_reader max_links_per_page: Integer?
+        attr_reader max_concurrent: Integer?
+        attr_reader crawl_strategy: enum_CrawlStrategyKind
+        attr_reader content_filter: enum_ContentFilterKind?
+        attr_reader bm25_query: String?
+        attr_reader bm25_threshold: Float?
+        attr_reader respect_robots_txt: bool
+        attr_reader soft_http_errors: bool
+        attr_reader user_agent: String?
+        attr_reader stay_on_domain: bool
+        attr_reader allow_subdomains: bool
+    attr_reader include_paths: Array[String]
+    attr_reader exclude_paths: Array[String]
+        attr_reader path_patterns_match_query: bool
+        attr_reader path_patterns_match_url: bool
+        attr_reader dedup_include_query: bool
+        attr_reader strip_tracking_params: bool
+    attr_reader tracking_params: Array[String]
+    attr_reader custom_headers: Hash[String, String]
+        attr_reader request_timeout: Integer
+        attr_reader rate_limit_ms: Integer?
+        attr_reader max_redirects: Integer
+        attr_reader retry_count: Integer
+    attr_reader retry_codes: Array[Integer]
+        attr_reader retry_initial_delay_ms: Integer
+        attr_reader retry_max_delay_ms: Integer
+        attr_reader rate_limit_jitter_ratio: Float
+        attr_reader cookies_enabled: bool
+        attr_reader auth: AuthConfig?
+        attr_reader max_body_size: Integer?
+    attr_reader remove_tags: Array[String]
+        attr_reader content: ContentConfig
+        attr_reader map_limit: Integer?
+        attr_reader map_search: String?
+        attr_reader download_assets: bool
+    attr_reader asset_types: Array[enum_AssetCategory]
+        attr_reader max_asset_size: Integer?
+        attr_reader browser: BrowserConfig
+        attr_reader proxy: ProxyConfig?
+    attr_reader user_agents: Array[String]
+        attr_reader capture_screenshot: bool
+        attr_reader follow_document_urls: bool
+        attr_reader document_url_depth: Integer?
+        attr_reader download_documents: bool
+        attr_reader document_max_size: Integer?
+    attr_reader document_mime_types: Array[String]
+        attr_reader document_output_dir: String?
+        attr_reader document_content_encoding: enum_DocumentContentEncoding?
+        attr_reader warc_output: String?
+        attr_reader browser_profile: String?
+        attr_reader save_browser_profile: bool
+        attr_reader ssrf: SsrfPolicy
+        attr_reader ssrf_deny_private_explicit: bool?
+
+def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
+    def validate: () -> void
+    end
+
+    class CrawlEngineHandle
+def crawl_stream: (CrawlStreamRequest req) -> Enumerator[CrawlEvent]
+def batch_crawl_stream: (BatchCrawlStreamRequest req) -> Enumerator[CrawlEvent]
+    end
+
+    class CrawlPageResult
+        attr_reader url: String
+        attr_reader normalized_url: String
+        attr_reader status_code: Integer
+        attr_reader content_type: String
+        attr_reader html: String
+        attr_reader body_size: Integer
+        attr_reader metadata: PageMetadata
+    attr_reader links: Array[LinkInfo]
+    attr_reader images: Array[ImageInfo]
+    attr_reader feeds: Array[FeedInfo]
+    attr_reader json_ld: Array[JsonLdEntry]
+        attr_reader depth: Integer
+        attr_reader stayed_on_domain: bool
+        attr_reader was_skipped: bool
+        attr_reader is_pdf: bool
+        attr_reader detected_charset: String?
+        attr_reader markdown: MarkdownResult?
+        attr_reader extracted_data: String?
+        attr_reader extraction_meta: ExtractionMeta?
+        attr_reader downloaded_document: DownloadedDocument?
+        attr_reader browser_used: bool
+        attr_reader final_url: String
+        attr_reader redirect_count: Integer
+        attr_reader noindex_detected: bool
+        attr_reader nofollow_detected: bool
+    attr_reader ssrf_refused_urls: Array[String]
+        attr_reader original_url: String
+
+def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?depth: Integer, ?stayed_on_domain: bool, ?was_skipped: bool, ?is_pdf: bool, ?detected_charset: String, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser_used: bool, ?final_url: String, ?redirect_count: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?ssrf_refused_urls: Array[String], ?original_url: String) -> void
+    end
+
+    class CrawlResult
+    attr_reader pages: Array[CrawlPageResult]
+        attr_reader final_url: String
+        attr_reader redirect_count: Integer
+        attr_reader was_skipped: bool
+        attr_reader error: String?
+    attr_reader cookies: Array[CookieInfo]
+        attr_reader stayed_on_domain: bool
+        attr_reader browser_used: bool
+
+def initialize: (?pages: Array[CrawlPageResult], ?final_url: String, ?redirect_count: Integer, ?was_skipped: bool, ?error: String, ?cookies: Array[CookieInfo], ?stayed_on_domain: bool, ?browser_used: bool) -> void
+    def unique_normalized_urls: () -> Integer
+    end
+
+    class CrawlStreamRequest
+        attr_reader url: String
+
+    def initialize: (?url: String) -> void
+    end
+
+    class DownloadedAsset
+        attr_reader url: String
+        attr_reader content_hash: String
+        attr_reader mime_type: String?
+        attr_reader size: Integer
+        attr_reader asset_category: enum_AssetCategory
+        attr_reader html_tag: String?
+
+    def initialize: (?url: String, ?content_hash: String, ?mime_type: String, ?size: Integer, ?asset_category: enum_AssetCategory, ?html_tag: String) -> void
+    end
+
+    class DownloadedDocument
+        attr_reader url: String
+        attr_reader mime_type: String
+        attr_reader size: Integer
+        attr_reader filename: String?
+        attr_reader content_hash: String
+    attr_reader headers: Hash[String, String]
+        attr_reader truncated: bool
+        attr_reader content_path: String?
+        attr_reader content_base64: String?
+
+def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: String, ?content_hash: String, ?headers: Hash[String, String], ?truncated: bool, ?content_path: String, ?content_base64: String) -> void
+    end
+
+    class ExtractionMeta
+        attr_reader cost: Float?
+        attr_reader prompt_tokens: Integer?
+        attr_reader completion_tokens: Integer?
+        attr_reader model: String?
+        attr_reader chunks_processed: Integer
+
+    def initialize: (?cost: Float, ?prompt_tokens: Integer, ?completion_tokens: Integer, ?model: String, ?chunks_processed: Integer) -> void
+    end
+
+    class FaviconInfo
+        attr_reader url: String
+        attr_reader rel: String
+        attr_reader sizes: String?
+        attr_reader mime_type: String?
+
+    def initialize: (?url: String, ?rel: String, ?sizes: String, ?mime_type: String) -> void
+    end
+
+    class FeedInfo
+        attr_reader url: String
+        attr_reader title: String?
+        attr_reader feed_type: enum_FeedType
+
+    def initialize: (?url: String, ?title: String, ?feed_type: enum_FeedType) -> void
+    end
+
+    class HeadingInfo
+        attr_reader level: Integer
+        attr_reader text: String
+
+    def initialize: (?level: Integer, ?text: String) -> void
+    end
+
+    class HreflangEntry
+        attr_reader lang: String
+        attr_reader url: String
+
+    def initialize: (?lang: String, ?url: String) -> void
+    end
+
+    class ImageInfo
+        attr_reader url: String
+        attr_reader alt: String?
+        attr_reader width: Integer?
+        attr_reader height: Integer?
+        attr_reader source: enum_ImageSource
+
+    def initialize: (?url: String, ?alt: String, ?width: Integer, ?height: Integer, ?source: enum_ImageSource) -> void
+    end
+
+    class InteractionResult
+    attr_reader action_results: Array[ActionResult]
+        attr_reader final_html: String
+        attr_reader final_url: String
+        attr_reader screenshot_base64: String?
+    attr_reader ssrf_refused_urls: Array[String]
+
+def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?final_url: String, ?screenshot_base64: String, ?ssrf_refused_urls: Array[String]) -> void
+    end
+
+    class JsonLdEntry
+        attr_reader schema_type: String
+        attr_reader name: String?
+        attr_reader raw: String
+
+    def initialize: (?schema_type: String, ?name: String, ?raw: String) -> void
+    end
+
+    class LinkInfo
+        attr_reader url: String
+        attr_reader text: String
+        attr_reader link_type: enum_LinkType
+        attr_reader rel: String?
+        attr_reader nofollow: bool
+        attr_reader original_url: String
+
+    def initialize: (?url: String, ?text: String, ?link_type: enum_LinkType, ?rel: String, ?nofollow: bool, ?original_url: String) -> void
+    end
+
+    class MapResult
+    attr_reader urls: Array[SitemapUrl]
+
+def initialize: (?urls: Array[SitemapUrl]) -> void
+    end
+
+    class MarkdownResult
+        attr_reader content: String
+        attr_reader document_structure: String?
+    attr_reader tables: Array[String]
+    attr_reader warnings: Array[String]
+        attr_reader citations: bool
+        attr_reader fit_content: String?
+
+def initialize: (?content: String, ?document_structure: String, ?tables: Array[String], ?warnings: Array[String], ?citations: bool, ?fit_content: String) -> void
+    end
+
+    class PageMetadata
+        attr_reader title: String?
+        attr_reader description: String?
+        attr_reader canonical_url: String?
+        attr_reader keywords: String?
+        attr_reader author: String?
+        attr_reader viewport: String?
+        attr_reader theme_color: String?
+        attr_reader generator: String?
+        attr_reader robots: String?
+        attr_reader html_lang: String?
+        attr_reader html_dir: String?
+        attr_reader og_title: String?
+        attr_reader og_type: String?
+        attr_reader og_image: String?
+        attr_reader og_description: String?
+        attr_reader og_url: String?
+        attr_reader og_site_name: String?
+        attr_reader og_locale: String?
+        attr_reader og_video: String?
+        attr_reader og_audio: String?
+    attr_reader og_locale_alternates: Array[String]?
+        attr_reader twitter_card: String?
+        attr_reader twitter_title: String?
+        attr_reader twitter_description: String?
+        attr_reader twitter_image: String?
+        attr_reader twitter_site: String?
+        attr_reader twitter_creator: String?
+        attr_reader dc_title: String?
+        attr_reader dc_creator: String?
+        attr_reader dc_subject: String?
+        attr_reader dc_description: String?
+        attr_reader dc_publisher: String?
+        attr_reader dc_date: String?
+        attr_reader dc_type: String?
+        attr_reader dc_format: String?
+        attr_reader dc_identifier: String?
+        attr_reader dc_language: String?
+        attr_reader dc_rights: String?
+        attr_reader article: ArticleMetadata?
+    attr_reader hreflangs: Array[HreflangEntry]?
+    attr_reader favicons: Array[FaviconInfo]?
+    attr_reader headings: Array[HeadingInfo]?
+        attr_reader word_count: Integer?
+
+def initialize: (?title: String, ?description: String, ?canonical_url: String, ?keywords: String, ?author: String, ?viewport: String, ?theme_color: String, ?generator: String, ?robots: String, ?html_lang: String, ?html_dir: String, ?og_title: String, ?og_type: String, ?og_image: String, ?og_description: String, ?og_url: String, ?og_site_name: String, ?og_locale: String, ?og_video: String, ?og_audio: String, ?og_locale_alternates: Array[String], ?twitter_card: String, ?twitter_title: String, ?twitter_description: String, ?twitter_image: String, ?twitter_site: String, ?twitter_creator: String, ?dc_title: String, ?dc_creator: String, ?dc_subject: String, ?dc_description: String, ?dc_publisher: String, ?dc_date: String, ?dc_type: String, ?dc_format: String, ?dc_identifier: String, ?dc_language: String, ?dc_rights: String, ?article: ArticleMetadata, ?hreflangs: Array[HreflangEntry], ?favicons: Array[FaviconInfo], ?headings: Array[HeadingInfo], ?word_count: Integer) -> void
+    end
+
+    class ProxyConfig
+        attr_reader url: String
+        attr_reader username: String?
+        attr_reader password: String?
+
+    def initialize: (?url: String, ?username: String, ?password: String) -> void
+    end
+
+    class ResponseMeta
+        attr_reader etag: String?
+        attr_reader last_modified: String?
+        attr_reader cache_control: String?
+        attr_reader server: String?
+        attr_reader x_powered_by: String?
+        attr_reader content_language: String?
+        attr_reader content_encoding: String?
+
+    def initialize: (?etag: String, ?last_modified: String, ?cache_control: String, ?server: String, ?x_powered_by: String, ?content_language: String, ?content_encoding: String) -> void
+    end
+
+    class ScrapeResult
+        attr_reader status_code: Integer
+        attr_reader final_url: String
+        attr_reader content_type: String
+        attr_reader html: String
+        attr_reader body_size: Integer
+        attr_reader metadata: PageMetadata
+    attr_reader links: Array[LinkInfo]
+    attr_reader images: Array[ImageInfo]
+    attr_reader feeds: Array[FeedInfo]
+    attr_reader json_ld: Array[JsonLdEntry]
+        attr_reader is_allowed: bool
+        attr_reader crawl_delay: Integer?
+        attr_reader noindex_detected: bool
+        attr_reader nofollow_detected: bool
+        attr_reader x_robots_tag: String?
+        attr_reader is_pdf: bool
+        attr_reader was_skipped: bool
+        attr_reader detected_charset: String?
+        attr_reader auth_header_sent: bool
+        attr_reader response_meta: ResponseMeta?
+    attr_reader assets: Array[DownloadedAsset]
+        attr_reader js_render_hint: bool
+        attr_reader browser_used: bool
+        attr_reader markdown: MarkdownResult?
+        attr_reader extracted_data: String?
+        attr_reader extraction_meta: ExtractionMeta?
+        attr_reader screenshot_base64: String?
+        attr_reader downloaded_document: DownloadedDocument?
+        attr_reader browser: BrowserExtras?
+    attr_reader ssrf_refused_urls: Array[String]
+
+def initialize: (?status_code: Integer, ?final_url: String, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?is_allowed: bool, ?crawl_delay: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?x_robots_tag: String, ?is_pdf: bool, ?was_skipped: bool, ?detected_charset: String, ?auth_header_sent: bool, ?response_meta: ResponseMeta, ?assets: Array[DownloadedAsset], ?js_render_hint: bool, ?browser_used: bool, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?screenshot_base64: String, ?downloaded_document: DownloadedDocument, ?browser: BrowserExtras, ?ssrf_refused_urls: Array[String]) -> void
+    end
+
+    class SitemapUrl
+        attr_reader url: String
+        attr_reader lastmod: String?
+        attr_reader changefreq: String?
+        attr_reader priority: String?
+
+    def initialize: (?url: String, ?lastmod: String, ?changefreq: String, ?priority: String) -> void
+    end
+
+    class SsrfPolicy
+        attr_reader deny_private: bool
+    attr_reader allowlist: Array[HostMatcher]
+    attr_reader denylist: Array[HostMatcher]
+        attr_reader max_redirects: Integer
+    attr_reader scheme_allowlist: Array[String]
+
+def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
+    end
+
+    type enum_AssetCategory = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
+
+    class AuthConfig
+    end
+
+    type enum_BrowserBackend = :chromiumoxide | :native
+
+    type enum_BrowserMode = :auto | :always | :never | :stealth
+
+    type enum_BrowserWait = :network_idle | :selector | :fixed
+
+    type enum_ContentFilterKind = :bm25
+
+    class CrawlEvent
+    end
+
+    type enum_CrawlStrategyKind = :bfs | :dfs | :best_first | :adaptive
+
+    type enum_DocumentContentEncoding = :base64
+
+    type enum_FeedType = :rss | :atom | :json_feed
+
+    class HostMatcher
+    end
+
+    type enum_ImageSource = :img | :picture_source | :"og:image" | :"twitter:image"
+
+    type enum_LinkType = :internal | :external | :anchor | :document
+
+    class PageAction
+    end
+
+    type enum_ScrollDirection = :up | :down
+
+def self.batch_crawl: (CrawlEngineHandle engine, Array[String] urls) -> BatchCrawlResults
+
+def self.batch_scrape: (CrawlEngineHandle engine, Array[String] urls) -> BatchScrapeResults
+
+def self.crawl: (CrawlEngineHandle engine, String url) -> CrawlResult
+
+def self.create_engine: (?CrawlConfig config) -> CrawlEngineHandle
+
+def self.generate_citations: (String markdown) -> CitationResult
+
+def self.interact: (CrawlEngineHandle engine, String url, Array[PageAction] actions) -> InteractionResult
+
+def self.map_urls: (CrawlEngineHandle engine, String url) -> MapResult
+
+def self.scrape: (CrawlEngineHandle engine, String url) -> ScrapeResult
+
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: crawlberg
+version: !ruby/object:Gem::Version
+  version: 1.11.0
+platform: x86_64-linux
+authors:
+- Na'aman Hirschfeld
+- Tobias Perlstein
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-10-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.130
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.130
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: sorbet-runtime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+description: High-performance web crawling engine
+email:
+- naaman@xberg.io
+- tobias@xberg.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- Steepfile
+- ext/crawlberg_rb/native/Cargo.lock
+- ext/crawlberg_rb/native/Cargo.toml
+- ext/crawlberg_rb/native/extconf.rb
+- ext/crawlberg_rb/src/lib.rs
+- lib/crawlberg.rb
+- lib/crawlberg/native.rb
+- lib/crawlberg/version.rb
+- lib/crawlberg_rb/3.3.0/crawlberg_rb.so
+- sig/types.rbs
+homepage: https://github.com/xberg-io/crawlberg
+licenses:
+- MIT
+metadata:
+  keywords: crawl,scrape,spider,web
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.3'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: High-performance web crawling engine
+test_files: []