crawlberg 1.7.2 → 1.10.2

data/lib/crawlberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:fdaf5ebf5fb30639d0b25b5f62a7044ed59029769d49609135c3bfd37b6f18e8
+# alef:hash:700d0a4f554a82a4d199e9c3c841556663485b9338869650cdd2738506c08763
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
 
 module Crawlberg
   ## The version string for this package.
-  VERSION = "1.7.2"
+  VERSION = "1.10.2"
 end

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:ddb9c01b1ccd8b583329fc083873cbeaec5fed7790bc44b0db41df5ccc4b7d91
+# alef:hash:3beedc8e4020e7af03194ac67ead1f260c8777df76ad65e2d9682e60fc03dd3b
 # To regenerate: alef generate
 # To verify freshness: alef verify
 
@@ -74,6 +74,8 @@ def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?com
         attr_reader backend: enum_BrowserBackend
         attr_reader endpoint: String?
         attr_reader timeout: Integer
+        attr_reader overall_timeout: Integer
+        attr_reader shutdown_timeout: Integer
         attr_reader wait: enum_BrowserWait
         attr_reader wait_selector: String?
         attr_reader extra_wait: Integer?
@@ -83,8 +85,10 @@ def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?com
         attr_reader robots_user_agent: String?
         attr_reader capture_network_events: bool
         attr_reader session_affinity: bool
+        attr_reader chrome_path: String?
+    attr_reader chrome_args: Array[String]
 
-def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool) -> void
+def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String]) -> void
     end
 
     class BrowserExtras
@@ -123,8 +127,9 @@ def initialize: (?content: String, ?references: Array[CitationReference]) -> voi
         attr_reader wrap: bool
         attr_reader wrap_width: Integer
         attr_reader include_document_structure: bool
+        attr_reader extract_metadata: bool
 
-def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool) -> void
+def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_navigation: bool, ?remove_forms: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?exclude_selectors: Array[String], ?skip_images: bool, ?max_depth: Integer, ?wrap: bool, ?wrap_width: Integer, ?include_document_structure: bool, ?extract_metadata: bool) -> void
     end
 
     class CookieInfo
@@ -152,12 +157,20 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader allow_subdomains: bool
     attr_reader include_paths: Array[String]
     attr_reader exclude_paths: Array[String]
+        attr_reader path_patterns_match_query: bool
+        attr_reader path_patterns_match_url: bool
+        attr_reader dedup_include_query: bool
+        attr_reader strip_tracking_params: bool
+    attr_reader tracking_params: Array[String]
     attr_reader custom_headers: Hash[String, String]
         attr_reader request_timeout: Integer
         attr_reader rate_limit_ms: Integer?
         attr_reader max_redirects: Integer
         attr_reader retry_count: Integer
     attr_reader retry_codes: Array[Integer]
+        attr_reader retry_initial_delay_ms: Integer
+        attr_reader retry_max_delay_ms: Integer
+        attr_reader rate_limit_jitter_ratio: Float
         attr_reader cookies_enabled: bool
         attr_reader auth: AuthConfig?
         attr_reader max_body_size: Integer?
@@ -185,7 +198,7 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader ssrf: SsrfPolicy
         attr_reader ssrf_deny_private_explicit: bool?
 
-def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
+def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
     def validate: () -> void
     end
 
@@ -216,8 +229,13 @@ def batch_crawl_stream: (BatchCrawlStreamRequest req) -> Enumerator[CrawlEvent]
         attr_reader extraction_meta: ExtractionMeta?
         attr_reader downloaded_document: DownloadedDocument?
         attr_reader browser_used: bool
+        attr_reader final_url: String
+        attr_reader redirect_count: Integer
+        attr_reader noindex_detected: bool
+        attr_reader nofollow_detected: bool
+    attr_reader ssrf_refused_urls: Array[String]
 
-def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?depth: Integer, ?stayed_on_domain: bool, ?was_skipped: bool, ?is_pdf: bool, ?detected_charset: String, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser_used: bool) -> void
+def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?depth: Integer, ?stayed_on_domain: bool, ?was_skipped: bool, ?is_pdf: bool, ?detected_charset: String, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser_used: bool, ?final_url: String, ?redirect_count: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?ssrf_refused_urls: Array[String]) -> void
     end
 
     class CrawlResult
@@ -321,8 +339,9 @@ def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: St
         attr_reader final_html: String
         attr_reader final_url: String
         attr_reader screenshot_base64: String?
+    attr_reader ssrf_refused_urls: Array[String]
 
-def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?final_url: String, ?screenshot_base64: String) -> void
+def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?final_url: String, ?screenshot_base64: String, ?ssrf_refused_urls: Array[String]) -> void
     end
 
     class JsonLdEntry
@@ -458,8 +477,9 @@ def initialize: (?title: String, ?description: String, ?canonical_url: String, ?
         attr_reader screenshot_base64: String?
         attr_reader downloaded_document: DownloadedDocument?
         attr_reader browser: BrowserExtras?
+    attr_reader ssrf_refused_urls: Array[String]
 
-def initialize: (?status_code: Integer, ?final_url: String, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?is_allowed: bool, ?crawl_delay: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?x_robots_tag: String, ?is_pdf: bool, ?was_skipped: bool, ?detected_charset: String, ?auth_header_sent: bool, ?response_meta: ResponseMeta, ?assets: Array[DownloadedAsset], ?js_render_hint: bool, ?browser_used: bool, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?screenshot_base64: String, ?downloaded_document: DownloadedDocument, ?browser: BrowserExtras) -> void
+def initialize: (?status_code: Integer, ?final_url: String, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?is_allowed: bool, ?crawl_delay: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?x_robots_tag: String, ?is_pdf: bool, ?was_skipped: bool, ?detected_charset: String, ?auth_header_sent: bool, ?response_meta: ResponseMeta, ?assets: Array[DownloadedAsset], ?js_render_hint: bool, ?browser_used: bool, ?markdown: MarkdownResult, ?extracted_data: String, ?extraction_meta: ExtractionMeta, ?screenshot_base64: String, ?downloaded_document: DownloadedDocument, ?browser: BrowserExtras, ?ssrf_refused_urls: Array[String]) -> void
     end
 
     class SitemapUrl
@@ -474,10 +494,11 @@ def initialize: (?status_code: Integer, ?final_url: String, ?content_type: Strin
     class SsrfPolicy
         attr_reader deny_private: bool
     attr_reader allowlist: Array[HostMatcher]
+    attr_reader denylist: Array[HostMatcher]
         attr_reader max_redirects: Integer
     attr_reader scheme_allowlist: Array[String]
 
-def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
+def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
     end
 
     type enum_AssetCategory = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlberg
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 1.10.2
 platform: ruby
 authors:
 - Xberg Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-24 00:00:00.000000000 Z
+date: 2026-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys