crawlberg 1.11.1-aarch64-linux → 1.12.2-aarch64-linux

data/lib/crawlberg/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:d323ed0aab9764791b48678b68c8fedcf8e7b3e1dbe50ec190a9023e68acc3bb
+# alef:hash:60c8949f72e7c9412406a7c721b2968d8acafde7ead2534f00d423542e33bade
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
@@ -164,6 +164,10 @@ module Crawlberg
     end
   end
   ## A single page has been crawled.
+  ##
+  ## A link whose redirect lands on a URL the crawl already queued or visited gets no event of
+  ## its own: it is requested, then dropped as a duplicate of the destination, which is reported
+  ## once under its own URL. The request count can therefore exceed the event count.
   CrawlEventPage = Data.define(:result) do
     include CrawlEvent
     extend T::Sig

data/lib/crawlberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:0fcaeaf159ae13b7e3d56e201029880b4cf648beeb0bb8efc6d0ff175d94e894
+# alef:hash:b402d1b3bc989de04db5241785f4bc16c0b5e0f76714438d3687648c64d8c10f
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
 
 module Crawlberg
   ## The version string for this package.
-  VERSION = "1.11.1"
+  VERSION = "1.12.2"
 end

data/lib/crawlberg_rb/3.3.0/crawlberg_rb.so

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: data/lib/crawlberg_rb/3.3.0/crawlberg_rb.so
-size: 42845560 bytes
-sha256: 186ed69c575cd2ebc6874646740214c39484ee1f6e9526f67bce0d7a23096dca
+size: 43901016 bytes
+sha256: 12154e0f5bee2365c281ab54ac2c1f1873eff79f37654c206b593f5795af8d29

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:ad470f62b746634fbd629f163e7bae4a88fb011bbde28d998e38ade4877adf3f
+# alef:hash:9aca68325287aae22a855afd34597e3d8bd90c6b7774f3b305591fde4fb051cd
 # To regenerate: alef generate
 # To verify freshness: alef verify
 
@@ -87,8 +87,11 @@ def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?com
         attr_reader session_affinity: bool
         attr_reader chrome_path: String?
     attr_reader chrome_args: Array[String]
+        attr_reader scroll_containers: bool
+        attr_reader dom_quiet_window: Integer
+        attr_reader dom_quiet_cap: Integer
 
-def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String]) -> void
+def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String], ?scroll_containers: bool, ?dom_quiet_window: Integer, ?dom_quiet_cap: Integer) -> void
     end
 
     class BrowserExtras
@@ -135,8 +138,12 @@ def initialize: (?content: String, ?references: Array[CitationReference]) -> voi
         attr_reader compact_tables: bool
     attr_reader keep_inline_images_in: Array[String]
         attr_reader keep_hidden: bool
+        attr_reader expand_abbreviations: bool
+        attr_reader escape_misc: bool
+        attr_reader escape_asterisks: bool
+        attr_reader include_icon_labels: bool
 
-def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool, ?extract_metadata: bool, ?strip_newlines: bool, ?bullets: String, ?list_indent_width: Integer, ?strong_em_symbol: String, ?compact_tables: bool, ?keep_inline_images_in: Array[String], ?keep_hidden: bool) -> void
+def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool, ?extract_metadata: bool, ?strip_newlines: bool, ?bullets: String, ?list_indent_width: Integer, ?strong_em_symbol: String, ?compact_tables: bool, ?keep_inline_images_in: Array[String], ?keep_hidden: bool, ?expand_abbreviations: bool, ?escape_misc: bool, ?escape_asterisks: bool, ?include_icon_labels: bool) -> void
     end
 
     class CookieInfo
@@ -204,8 +211,10 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader save_browser_profile: bool
         attr_reader ssrf: SsrfPolicy
         attr_reader ssrf_deny_private_explicit: bool?
+        attr_reader danger_accept_invalid_certs: bool
+    attr_reader subdomain_header_allowlist: Array[String]
 
-def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
+def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool, ?danger_accept_invalid_certs: bool, ?subdomain_header_allowlist: Array[String]) -> void
     def validate: () -> void
     end
 
@@ -255,8 +264,9 @@ def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?
     attr_reader cookies: Array[CookieInfo]
         attr_reader stayed_on_domain: bool
         attr_reader browser_used: bool
+        attr_reader waf_block: WafBlockDetails?
 
-def initialize: (?pages: Array[CrawlPageResult], ?final_url: String, ?redirect_count: Integer, ?was_skipped: bool, ?error: String, ?cookies: Array[CookieInfo], ?stayed_on_domain: bool, ?browser_used: bool) -> void
+def initialize: (?pages: Array[CrawlPageResult], ?final_url: String, ?redirect_count: Integer, ?was_skipped: bool, ?error: String, ?cookies: Array[CookieInfo], ?stayed_on_domain: bool, ?browser_used: bool, ?waf_block: WafBlockDetails) -> void
     def unique_normalized_urls: () -> Integer
     end
 
@@ -510,6 +520,14 @@ def initialize: (?status_code: Integer, ?final_url: String, ?content_type: Strin
 def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
     end
 
+    class WafBlockDetails
+        attr_reader vendor: String
+        attr_reader status: Integer?
+        attr_reader body_prefix: String?
+
+    def initialize: (?vendor: String, ?status: Integer, ?body_prefix: String) -> void
+    end
+
     type enum_AssetCategory = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
 
     class AuthConfig

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlberg
 version: !ruby/object:Gem::Version
-  version: 1.11.1
+  version: 1.12.2
 platform: aarch64-linux
 authors:
 - Na'aman Hirschfeld
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-16 00:00:00.000000000 Z
+date: 2026-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -18,9 +18,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.130
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,9 +25,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.130
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: sorbet-runtime
   requirement: !ruby/object:Gem::Requirement