crawlberg 1.7.1 → 1.10.1

data/lib/crawlberg/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:925532ba2f606ce36a0d3444d5c96407ca56a1c55bcd256372a1ea2a383d11e5
+# alef:hash:d323ed0aab9764791b48678b68c8fedcf8e7b3e1dbe50ec190a9023e68acc3bb
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
@@ -68,12 +68,10 @@ module Crawlberg
 
     # Username sent in the `Authorization: Basic` header.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def username = super
+    def username = Data.instance_method(:to_h).bind_call(self).fetch(:username)
     # Password sent in the `Authorization: Basic` header.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def password = super
+    def password = Data.instance_method(:to_h).bind_call(self).fetch(:password)
     sig { returns(T::Boolean) }
     def basic? = true
     sig { returns(T::Boolean) }
@@ -94,8 +92,7 @@ module Crawlberg
 
     # Token sent in the `Authorization: Bearer` header.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def token = super
+    def token = Data.instance_method(:to_h).bind_call(self).fetch(:token)
     sig { returns(T::Boolean) }
     def basic? = false
     sig { returns(T::Boolean) }
@@ -116,12 +113,10 @@ module Crawlberg
 
     # HTTP header name to set on each request.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def name = super
+    def name = Data.instance_method(:to_h).bind_call(self).fetch(:name)
     # HTTP header value to send.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def value = super
+    def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
     sig { returns(T::Boolean) }
     def basic? = false
     sig { returns(T::Boolean) }
@@ -175,8 +170,7 @@ module Crawlberg
 
     # The crawled page result.
     sig { returns(CrawlPageResult) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def result = super
+    def result = Data.instance_method(:to_h).bind_call(self).fetch(:result)
     sig { returns(T::Boolean) }
     def page? = true
     sig { returns(T::Boolean) }
@@ -197,12 +191,10 @@ module Crawlberg
 
     # The URL that failed.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def url = super
+    def url = Data.instance_method(:to_h).bind_call(self).fetch(:url)
     # The error message.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def error = super
+    def error = Data.instance_method(:to_h).bind_call(self).fetch(:error)
     sig { returns(T::Boolean) }
     def page? = false
     sig { returns(T::Boolean) }
@@ -223,8 +215,7 @@ module Crawlberg
 
     # Total number of pages crawled.
     sig { returns(Integer) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def pages_crawled = super
+    def pages_crawled = Data.instance_method(:to_h).bind_call(self).fetch(:pages_crawled)
     sig { returns(T::Boolean) }
     def page? = false
     sig { returns(T::Boolean) }
@@ -287,8 +278,7 @@ module Crawlberg
 
     # The hostname to match.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def value = super
+    def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
     sig { returns(T::Boolean) }
     def exact? = true
     sig { returns(T::Boolean) }
@@ -309,8 +299,7 @@ module Crawlberg
 
     # The dot-prefixed suffix to match. A leading dot is optional.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def value = super
+    def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
     sig { returns(T::Boolean) }
     def exact? = false
     sig { returns(T::Boolean) }
@@ -332,8 +321,7 @@ module Crawlberg
     # The CIDR block. Validated when built through [`HostMatcher::cidr`] or
     # deserialization.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def value = super
+    def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
     sig { returns(T::Boolean) }
     def exact? = false
     sig { returns(T::Boolean) }
@@ -395,8 +383,7 @@ module Crawlberg
 
     # CSS selector for the element to click.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def selector = super
+    def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
     sig { returns(T::Boolean) }
     def click? = true
     sig { returns(T::Boolean) }
@@ -427,12 +414,10 @@ module Crawlberg
 
     # CSS selector for the input element.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def selector = super
+    def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
     # Text to type into the element.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def text = super
+    def text = Data.instance_method(:to_h).bind_call(self).fetch(:text)
     sig { returns(T::Boolean) }
     def click? = false
     sig { returns(T::Boolean) }
@@ -463,8 +448,7 @@ module Crawlberg
 
     # Key name to press.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def key = super
+    def key = Data.instance_method(:to_h).bind_call(self).fetch(:key)
     sig { returns(T::Boolean) }
     def click? = false
     sig { returns(T::Boolean) }
@@ -495,16 +479,13 @@ module Crawlberg
 
     # Direction to scroll.
     sig { returns(ScrollDirection) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def direction = super
+    def direction = Data.instance_method(:to_h).bind_call(self).fetch(:direction)
     # Optional CSS selector for a scrollable element. Scrolls the page if absent.
     sig { returns(T.nilable(String)) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def selector = super
+    def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
     # Optional pixel amount to scroll. Uses a default if absent.
     sig { returns(T.nilable(Integer)) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def amount = super
+    def amount = Data.instance_method(:to_h).bind_call(self).fetch(:amount)
     sig { returns(T::Boolean) }
     def click? = false
     sig { returns(T::Boolean) }
@@ -539,12 +520,10 @@ module Crawlberg
 
     # Milliseconds to wait. Ignored if `selector` is provided.
     sig { returns(T.nilable(Integer)) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def milliseconds = super
+    def milliseconds = Data.instance_method(:to_h).bind_call(self).fetch(:milliseconds)
     # CSS selector to wait for.
     sig { returns(T.nilable(String)) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def selector = super
+    def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
     sig { returns(T::Boolean) }
     def click? = false
     sig { returns(T::Boolean) }
@@ -579,8 +558,7 @@ module Crawlberg
     # `full_page` (snake_case) alias so language bindings and fixtures can
     # use either convention without error.
     sig { returns(T.nilable(T::Boolean)) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def full_page = super
+    def full_page = Data.instance_method(:to_h).bind_call(self).fetch(:full_page)
     sig { returns(T::Boolean) }
     def click? = false
     sig { returns(T::Boolean) }
@@ -616,8 +594,7 @@ module Crawlberg
 
     # JavaScript source code to execute. Max 1 MB.
     sig { returns(String) }
-    # rubocop:disable Lint/UselessMethodDefinition
-    def script = super
+    def script = Data.instance_method(:to_h).bind_call(self).fetch(:script)
     sig { returns(T::Boolean) }
     def click? = false
     sig { returns(T::Boolean) }

data/lib/crawlberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:f45711d921997740dbd2259778d7ff9e5d0d75d2ce27ef835c9ce8916b1dc19d
+# alef:hash:233df82206c190592be07a69f614e55087625819abb77c2acf04112c5d8380ee
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
 
 module Crawlberg
   ## The version string for this package.
-  VERSION = "1.7.1"
+  VERSION = "1.10.1"
 end

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:1a2c1bf0fb6b53091ffe21fcecbee4a7e5f8214d544addbfe5c39f34e5aa7590
+# alef:hash:3beedc8e4020e7af03194ac67ead1f260c8777df76ad65e2d9682e60fc03dd3b
 # To regenerate: alef generate
 # To verify freshness: alef verify
 
@@ -70,11 +70,13 @@ def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?com
     end
 
     class BrowserConfig
-        attr_reader mode: BrowserMode
-        attr_reader backend: BrowserBackend
+        attr_reader mode: enum_BrowserMode
+        attr_reader backend: enum_BrowserBackend
         attr_reader endpoint: String?
         attr_reader timeout: Integer
-        attr_reader wait: BrowserWait
+        attr_reader overall_timeout: Integer
+        attr_reader shutdown_timeout: Integer
+        attr_reader wait: enum_BrowserWait
         attr_reader wait_selector: String?
         attr_reader extra_wait: Integer?
         attr_reader proxy: ProxyConfig?
@@ -83,8 +85,10 @@ def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?com
         attr_reader robots_user_agent: String?
         attr_reader capture_network_events: bool
         attr_reader session_affinity: bool
+        attr_reader chrome_path: String?
+    attr_reader chrome_args: Array[String]
 
-def initialize: (?mode: BrowserMode, ?backend: BrowserBackend, ?endpoint: String, ?timeout: Integer, ?wait: BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool) -> void
+def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String]) -> void
     end
 
     class BrowserExtras
@@ -123,8 +127,9 @@ def initialize: (?content: String, ?references: Array[CitationReference]) -> voi
         attr_reader wrap: bool
         attr_reader wrap_width: Integer
         attr_reader include_document_structure: bool
+        attr_reader extract_metadata: bool
 
-def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool) -> void
+def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool, ?extract_metadata: bool) -> void
     end
 
     class CookieInfo
@@ -141,8 +146,8 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader max_pages: Integer?
         attr_reader max_links_per_page: Integer?
         attr_reader max_concurrent: Integer?
-        attr_reader crawl_strategy: CrawlStrategyKind
-        attr_reader content_filter: ContentFilterKind?
+        attr_reader crawl_strategy: enum_CrawlStrategyKind
+        attr_reader content_filter: enum_ContentFilterKind?
         attr_reader bm25_query: String?
         attr_reader bm25_threshold: Float?
         attr_reader respect_robots_txt: bool
@@ -152,12 +157,20 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader allow_subdomains: bool
     attr_reader include_paths: Array[String]
     attr_reader exclude_paths: Array[String]
+        attr_reader path_patterns_match_query: bool
+        attr_reader path_patterns_match_url: bool
+        attr_reader dedup_include_query: bool
+        attr_reader strip_tracking_params: bool
+    attr_reader tracking_params: Array[String]
     attr_reader custom_headers: Hash[String, String]
         attr_reader request_timeout: Integer
         attr_reader rate_limit_ms: Integer?
         attr_reader max_redirects: Integer
         attr_reader retry_count: Integer
     attr_reader retry_codes: Array[Integer]
+        attr_reader retry_initial_delay_ms: Integer
+        attr_reader retry_max_delay_ms: Integer
+        attr_reader rate_limit_jitter_ratio: Float
         attr_reader cookies_enabled: bool
         attr_reader auth: AuthConfig?
         attr_reader max_body_size: Integer?
@@ -166,7 +179,7 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader map_limit: Integer?
         attr_reader map_search: String?
         attr_reader download_assets: bool
-    attr_reader asset_types: Array[AssetCategory]
+    attr_reader asset_types: Array[enum_AssetCategory]
         attr_reader max_asset_size: Integer?
         attr_reader browser: BrowserConfig
         attr_reader proxy: ProxyConfig?
@@ -178,14 +191,14 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader document_max_size: Integer?
     attr_reader document_mime_types: Array[String]
         attr_reader document_output_dir: String?
-        attr_reader document_content_encoding: DocumentContentEncoding?
+        attr_reader document_content_encoding: enum_DocumentContentEncoding?
         attr_reader warc_output: String?
         attr_reader browser_profile: String?
         attr_reader save_browser_profile: bool
         attr_reader ssrf: SsrfPolicy
         attr_reader ssrf_deny_private_explicit: bool?
 
-def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: CrawlStrategyKind, ?content_filter: ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
+def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
     def validate: () -> void
     end
 
@@ -216,8 +229,13 @@ def batch_crawl_stream: (BatchCrawlStreamRequest req) -> Enumerator[CrawlEvent]
         attr_reader extraction_meta: ExtractionMeta?
         attr_reader downloaded_document: DownloadedDocument?
         attr_reader browser_used: bool
+        attr_reader final_url: String
+        attr_reader redirect_count: Integer
+        attr_reader noindex_detected: bool
+        attr_reader nofollow_detected: bool
+    attr_reader ssrf_refused_urls: Array[String]
 
-def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?depth: Integer, ?stayed_on_domain: bool, ?was_skipped: bool, ?is_pdf: bool, ?detected_charset: String, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser_used: bool) -> void
+def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?depth: Integer, ?stayed_on_domain: bool, ?was_skipped: bool, ?is_pdf: bool, ?detected_charset: String, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser_used: bool, ?final_url: String, ?redirect_count: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?ssrf_refused_urls: Array[String]) -> void
     end
 
     class CrawlResult
@@ -245,10 +263,10 @@ def initialize: (?pages: Array[CrawlPageResult], ?final_url: String, ?redirect_c
         attr_reader content_hash: String
         attr_reader mime_type: String?
         attr_reader size: Integer
-        attr_reader asset_category: AssetCategory
+        attr_reader asset_category: enum_AssetCategory
         attr_reader html_tag: String?
 
-    def initialize: (?url: String, ?content_hash: String, ?mime_type: String, ?size: Integer, ?asset_category: AssetCategory, ?html_tag: String) -> void
+    def initialize: (?url: String, ?content_hash: String, ?mime_type: String, ?size: Integer, ?asset_category: enum_AssetCategory, ?html_tag: String) -> void
     end
 
     class DownloadedDocument
@@ -287,9 +305,9 @@ def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: St
     class FeedInfo
         attr_reader url: String
         attr_reader title: String?
-        attr_reader feed_type: FeedType
+        attr_reader feed_type: enum_FeedType
 
-    def initialize: (?url: String, ?title: String, ?feed_type: FeedType) -> void
+    def initialize: (?url: String, ?title: String, ?feed_type: enum_FeedType) -> void
     end
 
     class HeadingInfo
@@ -311,9 +329,9 @@ def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: St
         attr_reader alt: String?
         attr_reader width: Integer?
         attr_reader height: Integer?
-        attr_reader source: ImageSource
+        attr_reader source: enum_ImageSource
 
-    def initialize: (?url: String, ?alt: String, ?width: Integer, ?height: Integer, ?source: ImageSource) -> void
+    def initialize: (?url: String, ?alt: String, ?width: Integer, ?height: Integer, ?source: enum_ImageSource) -> void
     end
 
     class InteractionResult
@@ -321,8 +339,9 @@ def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: St
         attr_reader final_html: String
         attr_reader final_url: String
         attr_reader screenshot_base64: String?
+    attr_reader ssrf_refused_urls: Array[String]
 
-def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?final_url: String, ?screenshot_base64: String) -> void
+def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?final_url: String, ?screenshot_base64: String, ?ssrf_refused_urls: Array[String]) -> void
     end
 
     class JsonLdEntry
@@ -336,11 +355,11 @@ def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?fin
     class LinkInfo
         attr_reader url: String
         attr_reader text: String
-        attr_reader link_type: LinkType
+        attr_reader link_type: enum_LinkType
         attr_reader rel: String?
         attr_reader nofollow: bool
 
-    def initialize: (?url: String, ?text: String, ?link_type: LinkType, ?rel: String, ?nofollow: bool) -> void
+    def initialize: (?url: String, ?text: String, ?link_type: enum_LinkType, ?rel: String, ?nofollow: bool) -> void
     end
 
     class MapResult
@@ -458,8 +477,9 @@ def initialize: (?title: String, ?description: String, ?canonical_url: String, ?
         attr_reader screenshot_base64: String?
         attr_reader downloaded_document: DownloadedDocument?
         attr_reader browser: BrowserExtras?
+    attr_reader ssrf_refused_urls: Array[String]
 
-def initialize: (?status_code: Integer, ?final_url: String, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?is_allowed: bool, ?crawl_delay: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?x_robots_tag: String, ?is_pdf: bool, ?was_skipped: bool, ?detected_charset: String, ?auth_header_sent: bool, ?response_meta: ResponseMeta, ?assets: Array[DownloadedAsset], ?js_render_hint: bool, ?browser_used: bool, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?screenshot_base64: String, ?downloaded_document: DownloadedDocument, ?browser: BrowserExtras) -> void
+def initialize: (?status_code: Integer, ?final_url: String, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?is_allowed: bool, ?crawl_delay: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?x_robots_tag: String, ?is_pdf: bool, ?was_skipped: bool, ?detected_charset: String, ?auth_header_sent: bool, ?response_meta: ResponseMeta, ?assets: Array[DownloadedAsset], ?js_render_hint: bool, ?browser_used: bool, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?screenshot_base64: String, ?downloaded_document: DownloadedDocument, ?browser: BrowserExtras, ?ssrf_refused_urls: Array[String]) -> void
     end
 
     class SitemapUrl
@@ -474,67 +494,46 @@ def initialize: (?status_code: Integer, ?final_url: String, ?content_type: Strin
     class SsrfPolicy
         attr_reader deny_private: bool
     attr_reader allowlist: Array[HostMatcher]
+    attr_reader denylist: Array[HostMatcher]
         attr_reader max_redirects: Integer
     attr_reader scheme_allowlist: Array[String]
 
-def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
+def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
     end
 
-    class AssetCategory
-        type value = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
-    end
+    type enum_AssetCategory = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
 
     class AuthConfig
     end
 
-    class BrowserBackend
-        type value = :chromiumoxide | :native
-    end
+    type enum_BrowserBackend = :chromiumoxide | :native
 
-    class BrowserMode
-        type value = :auto | :always | :never | :stealth
-    end
+    type enum_BrowserMode = :auto | :always | :never | :stealth
 
-    class BrowserWait
-        type value = :network_idle | :selector | :fixed
-    end
+    type enum_BrowserWait = :network_idle | :selector | :fixed
 
-    class ContentFilterKind
-        type value = :bm25
-    end
+    type enum_ContentFilterKind = :bm25
 
     class CrawlEvent
     end
 
-    class CrawlStrategyKind
-        type value = :bfs | :dfs | :best_first | :adaptive
-    end
+    type enum_CrawlStrategyKind = :bfs | :dfs | :best_first | :adaptive
 
-    class DocumentContentEncoding
-        type value = :base64
-    end
+    type enum_DocumentContentEncoding = :base64
 
-    class FeedType
-        type value = :rss | :atom | :json_feed
-    end
+    type enum_FeedType = :rss | :atom | :json_feed
 
     class HostMatcher
     end
 
-    class ImageSource
-        type value = :img | :picture_source | :"og:image" | :"twitter:image"
-    end
+    type enum_ImageSource = :img | :picture_source | :"og:image" | :"twitter:image"
 
-    class LinkType
-        type value = :internal | :external | :anchor | :document
-    end
+    type enum_LinkType = :internal | :external | :anchor | :document
 
     class PageAction
     end
 
-    class ScrollDirection
-        type value = :up | :down
-    end
+    type enum_ScrollDirection = :up | :down
 
 def self.batch_crawl: (CrawlEngineHandle engine, Array[String] urls) -> BatchCrawlResults
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlberg
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.10.1
 platform: ruby
 authors:
 - Xberg Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-16 00:00:00.000000000 Z
+date: 2026-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys