crawlberg 1.12.0 → 1.12.3

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:a2ea8938a47348aa37d91448784e4263e6b5f4df1fe3a7ff1f9a8bdd3bf04009
+# alef:hash:4cdc9eb4c9b67c7adb9baf33b70eb0c6ee54e28b0c0973121fbef2934ff8a431
 # To regenerate: alef generate
 # To verify freshness: alef verify
 
@@ -89,8 +89,9 @@ def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?com
     attr_reader chrome_args: Array[String]
         attr_reader scroll_containers: bool
         attr_reader dom_quiet_window: Integer
+        attr_reader dom_quiet_cap: Integer
 
-def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String], ?scroll_containers: bool, ?dom_quiet_window: Integer) -> void
+def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String], ?scroll_containers: bool, ?dom_quiet_window: Integer, ?dom_quiet_cap: Integer) -> void
     end
 
     class BrowserExtras
@@ -137,8 +138,12 @@ def initialize: (?content: String, ?references: Array[CitationReference]) -> voi
         attr_reader compact_tables: bool
     attr_reader keep_inline_images_in: Array[String]
         attr_reader keep_hidden: bool
+        attr_reader expand_abbreviations: bool
+        attr_reader escape_misc: bool
+        attr_reader escape_asterisks: bool
+        attr_reader include_icon_labels: bool
 
-def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool, ?extract_metadata: bool, ?strip_newlines: bool, ?bullets: String, ?list_indent_width: Integer, ?strong_em_symbol: String, ?compact_tables: bool, ?keep_inline_images_in: Array[String], ?keep_hidden: bool) -> void
+def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool, ?extract_metadata: bool, ?strip_newlines: bool, ?bullets: String, ?list_indent_width: Integer, ?strong_em_symbol: String, ?compact_tables: bool, ?keep_inline_images_in: Array[String], ?keep_hidden: bool, ?expand_abbreviations: bool, ?escape_misc: bool, ?escape_asterisks: bool, ?include_icon_labels: bool) -> void
     end
 
     class CookieInfo
@@ -207,8 +212,9 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader ssrf: SsrfPolicy
         attr_reader ssrf_deny_private_explicit: bool?
         attr_reader danger_accept_invalid_certs: bool
+    attr_reader subdomain_header_allowlist: Array[String]
 
-def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool, ?danger_accept_invalid_certs: bool) -> void
+def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool, ?danger_accept_invalid_certs: bool, ?subdomain_header_allowlist: Array[String]) -> void
     def validate: () -> void
     end
 
@@ -524,7 +530,46 @@ def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist:
 
     type enum_AssetCategory = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
 
-    class AuthConfig
+    module AuthConfig
+    def self.from_hash: (untyped hash) -> untyped
+    end
+
+    class AuthConfigBasic
+        include AuthConfig
+        attr_reader username: String
+        attr_reader password: String
+
+    def basic?: () -> bool
+    def bearer?: () -> bool
+    def header?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (username: String, password: String) -> AuthConfigBasic
+    end
+
+    class AuthConfigBearer
+        include AuthConfig
+        attr_reader token: String
+
+    def basic?: () -> bool
+    def bearer?: () -> bool
+    def header?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (token: String) -> AuthConfigBearer
+    end
+
+    class AuthConfigHeader
+        include AuthConfig
+        attr_reader name: String
+        attr_reader value: String
+
+    def basic?: () -> bool
+    def bearer?: () -> bool
+    def header?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (name: String, value: String) -> AuthConfigHeader
     end
 
     type enum_BrowserBackend = :chromiumoxide | :native
@@ -535,7 +580,45 @@ def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist:
 
     type enum_ContentFilterKind = :bm25
 
-    class CrawlEvent
+    module CrawlEvent
+    def self.from_hash: (untyped hash) -> untyped
+    end
+
+    class CrawlEventPage
+        include CrawlEvent
+        attr_reader result: CrawlPageResult
+
+    def page?: () -> bool
+    def error?: () -> bool
+    def complete?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (result: CrawlPageResult) -> CrawlEventPage
+    end
+
+    class CrawlEventError
+        include CrawlEvent
+        attr_reader url: String
+        attr_reader error: String
+
+    def page?: () -> bool
+    def error?: () -> bool
+    def complete?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (url: String, error: String) -> CrawlEventError
+    end
+
+    class CrawlEventComplete
+        include CrawlEvent
+        attr_reader pages_crawled: Integer
+
+    def page?: () -> bool
+    def error?: () -> bool
+    def complete?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (pages_crawled: Integer) -> CrawlEventComplete
     end
 
     type enum_CrawlStrategyKind = :bfs | :dfs | :best_first | :adaptive
@@ -544,14 +627,190 @@ def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist:
 
     type enum_FeedType = :rss | :atom | :json_feed
 
-    class HostMatcher
+    module HostMatcher
+    def self.from_hash: (untyped hash) -> untyped
+    end
+
+    class HostMatcherExact
+        include HostMatcher
+        attr_reader value: String
+
+    def exact?: () -> bool
+    def suffix?: () -> bool
+    def cidr?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (value: String) -> HostMatcherExact
+    end
+
+    class HostMatcherSuffix
+        include HostMatcher
+        attr_reader value: String
+
+    def exact?: () -> bool
+    def suffix?: () -> bool
+    def cidr?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (value: String) -> HostMatcherSuffix
+    end
+
+    class HostMatcherCidr
+        include HostMatcher
+        attr_reader value: String
+
+    def exact?: () -> bool
+    def suffix?: () -> bool
+    def cidr?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (value: String) -> HostMatcherCidr
     end
 
     type enum_ImageSource = :img | :picture_source | :"og:image" | :"twitter:image"
 
     type enum_LinkType = :internal | :external | :anchor | :document
 
-    class PageAction
+    module PageAction
+    def self.from_hash: (untyped hash) -> untyped
+    end
+
+    class PageActionClick
+        include PageAction
+        attr_reader selector: String
+
+    def click?: () -> bool
+    def type_text?: () -> bool
+    def press?: () -> bool
+    def scroll?: () -> bool
+    def wait?: () -> bool
+    def screenshot?: () -> bool
+    def execute_js?: () -> bool
+    def scrape?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (selector: String) -> PageActionClick
+    end
+
+    class PageActionTypeText
+        include PageAction
+        attr_reader selector: String
+        attr_reader text: String
+
+    def click?: () -> bool
+    def type_text?: () -> bool
+    def press?: () -> bool
+    def scroll?: () -> bool
+    def wait?: () -> bool
+    def screenshot?: () -> bool
+    def execute_js?: () -> bool
+    def scrape?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (selector: String, text: String) -> PageActionTypeText
+    end
+
+    class PageActionPress
+        include PageAction
+        attr_reader key: String
+
+    def click?: () -> bool
+    def type_text?: () -> bool
+    def press?: () -> bool
+    def scroll?: () -> bool
+    def wait?: () -> bool
+    def screenshot?: () -> bool
+    def execute_js?: () -> bool
+    def scrape?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (key: String) -> PageActionPress
+    end
+
+    class PageActionScroll
+        include PageAction
+        attr_reader direction: enum_ScrollDirection
+        attr_reader selector: String?
+        attr_reader amount: Integer?
+
+    def click?: () -> bool
+    def type_text?: () -> bool
+    def press?: () -> bool
+    def scroll?: () -> bool
+    def wait?: () -> bool
+    def screenshot?: () -> bool
+    def execute_js?: () -> bool
+    def scrape?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (direction: enum_ScrollDirection, selector: String, amount: Integer) -> PageActionScroll
+    end
+
+    class PageActionWait
+        include PageAction
+        attr_reader milliseconds: Integer?
+        attr_reader selector: String?
+
+    def click?: () -> bool
+    def type_text?: () -> bool
+    def press?: () -> bool
+    def scroll?: () -> bool
+    def wait?: () -> bool
+    def screenshot?: () -> bool
+    def execute_js?: () -> bool
+    def scrape?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (milliseconds: Integer, selector: String) -> PageActionWait
+    end
+
+    class PageActionScreenshot
+        include PageAction
+        attr_reader full_page: bool?
+
+    def click?: () -> bool
+    def type_text?: () -> bool
+    def press?: () -> bool
+    def scroll?: () -> bool
+    def wait?: () -> bool
+    def screenshot?: () -> bool
+    def execute_js?: () -> bool
+    def scrape?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (full_page: bool) -> PageActionScreenshot
+    end
+
+    class PageActionExecuteJs
+        include PageAction
+        attr_reader script: String
+
+    def click?: () -> bool
+    def type_text?: () -> bool
+    def press?: () -> bool
+    def scroll?: () -> bool
+    def wait?: () -> bool
+    def screenshot?: () -> bool
+    def execute_js?: () -> bool
+    def scrape?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+    def self.new: (script: String) -> PageActionExecuteJs
+    end
+
+    class PageActionScrape
+        include PageAction
+    def click?: () -> bool
+    def type_text?: () -> bool
+    def press?: () -> bool
+    def scroll?: () -> bool
+    def wait?: () -> bool
+    def screenshot?: () -> bool
+    def execute_js?: () -> bool
+    def scrape?: () -> bool
+
+    def self.from_hash: (untyped hash) -> untyped
+        def self.new:  -> PageActionScrape
     end
 
     type enum_ScrollDirection = :up | :down

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlberg
 version: !ruby/object:Gem::Version
-  version: 1.12.0
+  version: 1.12.3
 platform: ruby
 authors:
 - Na'aman Hirschfeld
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-22 00:00:00.000000000 Z
+date: 2026-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -18,9 +18,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.130
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,9 +25,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.130
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: sorbet-runtime
   requirement: !ruby/object:Gem::Requirement