crawlberg 1.1.4 → 1.2.1

data/lib/crawlberg/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:899a9b0b3964b8cc10628ac686b74140e934aa910533d6b9333c2acf97f2773b
+# alef:hash:0ce4d753fdb4854e44358639dcbaebee3449a4afa142dbc4f0a72aa72c214648
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
@@ -561,3 +561,112 @@ module Crawlberg
     end
   end
 end
+
+module Crawlberg
+  # Hostname/IP allowlist matcher for SSRF policy.
+  #
+  # Serializes as an internally-tagged object so each variant is distinguishable on the
+  # wire and round-trips losslessly:
+  #
+  # ```json
+  # {"type": "exact",  "value": "api.example.com"}
+  # {"type": "suffix", "value": ".example.com"}
+  # {"type": "cidr",   "value": "10.0.0.0/8"}
+  # ```
+  #
+  # A bare JSON string is still accepted on deserialization and resolves to [`Exact`],
+  # preserving configs written against the previous untagged representation.
+  #
+  # [`Exact`]: HostMatcher::Exact
+  module HostMatcher
+    extend T::Helpers
+    extend T::Sig
+
+    interface!
+
+    # Dispatch from a Hash to the appropriate variant constructor.
+    # @param hash [Hash] with discriminator field and variant-specific fields
+    # @return [variant_class] an instance of the appropriate variant
+    sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
+    def self.from_hash(hash)
+      discriminator = hash[:type] || hash["type"]
+      case discriminator
+      when "exact"
+        HostMatcherExact.from_hash(hash)
+      when "suffix"
+        HostMatcherSuffix.from_hash(hash)
+      when "cidr"
+        HostMatcherCidr.from_hash(hash)
+      else
+        raise "Unknown discriminator: #{discriminator}"
+      end
+    end
+  end
+  ## Exact hostname match (case-insensitive).
+  HostMatcherExact = Data.define(:value) do
+    include HostMatcher
+    extend T::Sig
+
+    # The hostname to match.
+    sig { returns(String) }
+    # rubocop:disable Lint/UselessMethodDefinition
+    def value = super
+    sig { returns(T::Boolean) }
+    def exact? = true
+    sig { returns(T::Boolean) }
+    def suffix? = false
+    sig { returns(T::Boolean) }
+    def cidr? = false
+    # @param hash [Hash] deserialized from the native extension
+    # @return [self]
+    sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
+    def self.from_hash(hash)
+      new(value: hash[:value] || hash["value"])
+    end
+  end
+  ## Suffix match: ".xberg.io" matches "api.xberg.io" and "xberg.io".
+  HostMatcherSuffix = Data.define(:value) do
+    include HostMatcher
+    extend T::Sig
+
+    # The dot-prefixed suffix to match. A leading dot is optional.
+    sig { returns(String) }
+    # rubocop:disable Lint/UselessMethodDefinition
+    def value = super
+    sig { returns(T::Boolean) }
+    def exact? = false
+    sig { returns(T::Boolean) }
+    def suffix? = true
+    sig { returns(T::Boolean) }
+    def cidr? = false
+    # @param hash [Hash] deserialized from the native extension
+    # @return [self]
+    sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
+    def self.from_hash(hash)
+      new(value: hash[:value] || hash["value"])
+    end
+  end
+  ## CIDR match: "10.0.0.0/8" matches IP addresses in that range.
+  HostMatcherCidr = Data.define(:value) do
+    include HostMatcher
+    extend T::Sig
+
+    # The CIDR block. Validated when built through [`HostMatcher::cidr`] or
+    # deserialization.
+    sig { returns(String) }
+    # rubocop:disable Lint/UselessMethodDefinition
+    def value = super
+    sig { returns(T::Boolean) }
+    def exact? = false
+    sig { returns(T::Boolean) }
+    def suffix? = false
+    sig { returns(T::Boolean) }
+    def cidr? = true
+    # @param hash [Hash] deserialized from the native extension
+    # @return [self]
+    sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
+    def self.from_hash(hash)
+      new(value: hash[:value] || hash["value"])
+    end
+  end
+end

data/lib/crawlberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:899a9b0b3964b8cc10628ac686b74140e934aa910533d6b9333c2acf97f2773b
+# alef:hash:0ce4d753fdb4854e44358639dcbaebee3449a4afa142dbc4f0a72aa72c214648
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
 
 module Crawlberg
   ## The version string for this package.
-  VERSION = "1.1.4"
+  VERSION = "1.2.1"
 end

data/lib/crawlberg.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:899a9b0b3964b8cc10628ac686b74140e934aa910533d6b9333c2acf97f2773b
+# alef:hash:0ce4d753fdb4854e44358639dcbaebee3449a4afa142dbc4f0a72aa72c214648
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:899a9b0b3964b8cc10628ac686b74140e934aa910533d6b9333c2acf97f2773b
+# alef:hash:0ce4d753fdb4854e44358639dcbaebee3449a4afa142dbc4f0a72aa72c214648
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 
@@ -67,6 +67,7 @@ def initialize: (?mode: BrowserMode, ?backend: BrowserBackend, ?endpoint: String
     class CrawlConfig
         attr_accessor max_depth: Integer?
         attr_accessor max_pages: Integer?
+        attr_accessor max_links_per_page: Integer?
         attr_accessor max_concurrent: Integer?
         attr_accessor respect_robots_txt: bool?
         attr_accessor soft_http_errors: bool?
@@ -100,12 +101,15 @@ def initialize: (?mode: BrowserMode, ?backend: BrowserBackend, ?endpoint: String
         attr_accessor download_documents: bool?
         attr_accessor document_max_size: Integer?
     attr_accessor document_mime_types: Array[String]?
+        attr_accessor document_output_dir: String?
+        attr_accessor document_content_encoding: DocumentContentEncoding?
         attr_accessor warc_output: String?
         attr_accessor browser_profile: String?
         attr_accessor save_browser_profile: bool?
         attr_accessor ssrf: SsrfPolicy?
+        attr_accessor ssrf_deny_private_explicit: bool?
 
-def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_concurrent: Integer, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy) -> void
+def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
     def validate: () -> void
     def self.default: () -> CrawlConfig
     end
@@ -125,16 +129,20 @@ def initialize: (?eval_result: json_value, ?network_events: Array[ResponseMeta],
         attr_accessor filename: String?
         attr_accessor content_hash: String?
     attr_accessor headers: Hash[String, String]?
+        attr_accessor truncated: bool?
+        attr_accessor content_path: String?
+        attr_accessor content_base64: String?
 
-def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: String, ?content_hash: String, ?headers: Hash[String, String]) -> void
+def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: String, ?content_hash: String, ?headers: Hash[String, String], ?truncated: bool, ?content_path: String, ?content_base64: String) -> void
     end
 
     class InteractionResult
     attr_accessor action_results: Array[ActionResult]?
         attr_accessor final_html: String?
         attr_accessor final_url: String?
+        attr_accessor screenshot_base64: String?
 
-def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?final_url: String) -> void
+def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?final_url: String, ?screenshot_base64: String) -> void
     end
 
     class ActionResult
@@ -174,10 +182,11 @@ def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?fin
         attr_accessor markdown: MarkdownResult?
         attr_accessor extracted_data: json_value?
         attr_accessor extraction_meta: ExtractionMeta?
+        attr_accessor screenshot_base64: String?
         attr_accessor downloaded_document: DownloadedDocument?
         attr_accessor browser: BrowserExtras?
 
-def initialize: (?status_code: Integer, ?final_url: String, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?is_allowed: bool, ?crawl_delay: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?x_robots_tag: String, ?is_pdf: bool, ?was_skipped: bool, ?detected_charset: String, ?auth_header_sent: bool, ?response_meta: ResponseMeta, ?assets: Array[DownloadedAsset], ?js_render_hint: bool, ?browser_used: bool, ?markdown: MarkdownResult, ?extracted_data: json_value, ?extraction_meta: ExtractionMeta, ?downloaded_document: DownloadedDocument, ?browser: BrowserExtras) -> void
+def initialize: (?status_code: Integer, ?final_url: String, ?content_type: String, ?html: String, ?body_size: Integer, ?metadata: PageMetadata, ?links: Array[LinkInfo], ?images: Array[ImageInfo], ?feeds: Array[FeedInfo], ?json_ld: Array[JsonLdEntry], ?is_allowed: bool, ?crawl_delay: Integer, ?noindex_detected: bool, ?nofollow_detected: bool, ?x_robots_tag: String, ?is_pdf: bool, ?was_skipped: bool, ?detected_charset: String, ?auth_header_sent: bool, ?response_meta: ResponseMeta, ?assets: Array[DownloadedAsset], ?js_render_hint: bool, ?browser_used: bool, ?markdown: MarkdownResult, ?extracted_data: json_value, ?extraction_meta: ExtractionMeta, ?screenshot_base64: String, ?downloaded_document: DownloadedDocument, ?browser: BrowserExtras) -> void
     end
 
     class CrawlPageResult
@@ -463,9 +472,10 @@ def initialize: (?results: Array[BatchCrawlResult], ?total_count: Integer, ?comp
 
     class SsrfPolicy
         attr_accessor deny_private: bool?
+    attr_accessor allowlist: Array[HostMatcher]?
         attr_accessor max_redirects: Integer?
 
-    def initialize: (?deny_private: bool, ?max_redirects: Integer) -> void
+def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?max_redirects: Integer) -> void
     def self.default: () -> SsrfPolicy
     def self.from_env: () -> SsrfPolicy
     end
@@ -482,6 +492,10 @@ def initialize: (?results: Array[BatchCrawlResult], ?total_count: Integer, ?comp
         type value = :chromiumoxide | :native
     end
 
+    class DocumentContentEncoding
+        type value = :base64
+    end
+
     class AuthConfig
     end
 
@@ -511,6 +525,9 @@ def initialize: (?results: Array[BatchCrawlResult], ?total_count: Integer, ?comp
         type value = :up | :down
     end
 
+    class HostMatcher
+    end
+
 def self.generate_citations: (String markdown) -> CitationResult
 
 def self.create_engine: (?CrawlConfig config) -> CrawlEngineHandle

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlberg
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.2.1
 platform: ruby
 authors:
 - Xberg Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-05 00:00:00.000000000 Z
+date: 2026-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys