scrapetor 0.2.0

data/lib/scrapetor/entities.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Scrapetor
+  # HTML named-entity decoder.
+  #
+  # The C native engine handles the minimal set (`& < >
+  # " '   &#N; &#xH;`) inline during extraction. This
+  # module is the broader Ruby table — useful when post-processing raw
+  # strings, or when running the Ruby fallback path against HTML with
+  # uncommon entities.
+  #
+  # The table here covers the ~140 most frequent named entities from
+  # the HTML5 spec — enough to handle real-world text.
+  module Entities
+    TABLE = {
+      "amp"     => "&",  "lt"      => "<",  "gt"      => ">",
+      "quot"    => '"',  "apos"    => "'",  "nbsp"    => " ",
+      "copy"    => "©",  "reg"     => "®",  "trade"   => "™",
+      "mdash"   => "—",  "ndash"   => "–",  "hellip"  => "…",
+      "ldquo"   => "“",  "rdquo"   => "”",  "lsquo"   => "‘",
+      "rsquo"   => "’",  "laquo"   => "«",  "raquo"   => "»",
+      "lsaquo"  => "‹",  "rsaquo"  => "›",  "sbquo"   => "‚",
+      "bdquo"   => "„",  "times"   => "×",  "divide"  => "÷",
+      "plusmn"  => "±",  "deg"     => "°",  "sect"    => "§",
+      "para"    => "¶",  "middot"  => "·",  "bull"    => "•",
+      "dagger"  => "†",  "Dagger"  => "‡",  "permil"  => "‰",
+      "prime"   => "′",  "Prime"   => "″",  "ne"      => "≠",
+      "le"      => "≤",  "ge"      => "≥",  "asymp"   => "≈",
+      "equiv"   => "≡",  "infin"   => "∞",  "sum"     => "∑",
+      "prod"    => "∏",  "int"     => "∫",  "radic"   => "√",
+      "part"    => "∂",  "nabla"   => "∇",  "minus"   => "−",
+      "plus"    => "+",  "lowast"  => "∗",  "frasl"   => "⁄",
+      "larr"    => "←",  "rarr"    => "→",  "uarr"    => "↑",
+      "darr"    => "↓",  "harr"    => "↔",  "crarr"   => "↵",
+      "lArr"    => "⇐",  "rArr"    => "⇒",  "uArr"    => "⇑",
+      "dArr"    => "⇓",  "hArr"    => "⇔",  "spades"  => "♠",
+      "clubs"   => "♣",  "hearts"  => "♥",  "diams"   => "♦",
+      "loz"     => "◊",  "Aacute"  => "Á",  "aacute"  => "á",
+      "Acirc"   => "Â",  "acirc"   => "â",  "Agrave"  => "À",
+      "agrave"  => "à",  "Aring"   => "Å",  "aring"   => "å",
+      "Atilde"  => "Ã",  "atilde"  => "ã",  "Auml"    => "Ä",
+      "auml"    => "ä",  "AElig"   => "Æ",  "aelig"   => "æ",
+      "Ccedil"  => "Ç",  "ccedil"  => "ç",  "Eacute"  => "É",
+      "eacute"  => "é",  "Ecirc"   => "Ê",  "ecirc"   => "ê",
+      "Egrave"  => "È",  "egrave"  => "è",  "Euml"    => "Ë",
+      "euml"    => "ë",  "Iacute"  => "Í",  "iacute"  => "í",
+      "Icirc"   => "Î",  "icirc"   => "î",  "Igrave"  => "Ì",
+      "igrave"  => "ì",  "Iuml"    => "Ï",  "iuml"    => "ï",
+      "Ntilde"  => "Ñ",  "ntilde"  => "ñ",  "Oacute"  => "Ó",
+      "oacute"  => "ó",  "Ocirc"   => "Ô",  "ocirc"   => "ô",
+      "Ograve"  => "Ò",  "ograve"  => "ò",  "Oslash"  => "Ø",
+      "oslash"  => "ø",  "Otilde"  => "Õ",  "otilde"  => "õ",
+      "Ouml"    => "Ö",  "ouml"    => "ö",  "Uacute"  => "Ú",
+      "uacute"  => "ú",  "Ucirc"   => "Û",  "ucirc"   => "û",
+      "Ugrave"  => "Ù",  "ugrave"  => "ù",  "Uuml"    => "Ü",
+      "uuml"    => "ü",  "Yacute"  => "Ý",  "yacute"  => "ý",
+      "yuml"    => "ÿ",  "szlig"   => "ß",
+      "iexcl"   => "¡",  "iquest"  => "¿",  "cent"    => "¢",
+      "pound"   => "£",  "yen"     => "¥",  "euro"    => "€",
+      "curren"  => "¤",  "shy"     => "­",
+      "frac12"  => "½",  "frac14"  => "¼",  "frac34"  => "¾",
+      "alpha"   => "α",  "beta"    => "β",  "gamma"   => "γ",
+      "delta"   => "δ",  "epsilon" => "ε",  "pi"      => "π",
+      "sigma"   => "σ",  "omega"   => "ω",
+      "ensp"    => " ",  "emsp"    => " ",  "thinsp"  => " ",
+      "zwnj"    => "‌", "zwj"     => "‍"
+    }.freeze
+
+    ENTITY_RE = /&(?:#(?:x([0-9A-Fa-f]+)|(\d+))|([a-zA-Z][a-zA-Z0-9]+));/.freeze
+
+    # Decode a string containing HTML entities into plain UTF-8 text.
+    def self.decode(s)
+      return s if s.nil? || s.empty?
+      s.to_s.gsub(ENTITY_RE) do
+        hex = Regexp.last_match(1)
+        dec = Regexp.last_match(2)
+        named = Regexp.last_match(3)
+        if hex
+          [hex.to_i(16)].pack("U")
+        elsif dec
+          [dec.to_i].pack("U")
+        elsif named
+          TABLE[named] || Regexp.last_match(0)
+        else
+          Regexp.last_match(0)
+        end
+      end
+    end
+  end
+end

data/lib/scrapetor/errors.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Scrapetor
+  # Base for all Scrapetor errors.
+  class Error < StandardError; end
+
+  # Raised when a required field in the schema can't be found.
+  class ExtractionError < Error; end
+
+  # Raised when a schema descriptor isn't valid for the native engine.
+  class SchemaError < Error; end
+end

data/lib/scrapetor/extractor.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Scrapetor
+  # Schema execution. The hot path operates on raw Nokolexbor nodes and
+  # inlines coercion — no Scrapetor::Node allocations per emitted field.
+  module Extractor
+    def self.run(doc, scope, schema)
+      result = {}
+      base_url = doc.respond_to?(:base_url) ? doc.base_url : nil
+      schema.fields.each do |f|
+        result[f.name] = extract_field(scope, f, base_url)
+      end
+      schema.groups.each do |g|
+        result[g.name] = run_group(scope, g, base_url)
+      end
+      result
+    end
+
+    def self.run_group(scope, group, base_url)
+      out = []
+      scope.css(group.selector).each do |sub|
+        inner = {}
+        group.fields.each { |f| inner[f.name] = extract_field(sub, f, base_url) }
+        group.groups.each { |gg| inner[gg.name] = run_group(sub, gg, base_url) }
+        out << inner
+      end
+      out
+    end
+
+    def self.extract_field(scope, f, base_url)
+      selectors = f.selector.is_a?(Array) ? f.selector : [f.selector]
+      value =
+        if f.multi
+          extract_multi(scope, selectors, f, base_url)
+        else
+          extract_single(scope, selectors, f, base_url)
+        end
+
+      # default + required
+      missing = value.nil? || (f.multi && value.empty?)
+      value = f.default if missing && !f.default.nil?
+
+      if f.required && (value.nil? || (f.multi && value.respond_to?(:empty?) && value.empty?))
+        raise ExtractionError, "required field `#{f.name}` not found"
+      end
+
+      # transform last (after coerce + default)
+      value = f.transform.call(value) if f.transform && !value.nil?
+      value
+    end
+
+    def self.extract_single(scope, selectors, f, base_url)
+      selectors.each do |sel|
+        n = sel ? scope.at_css(sel) : scope
+        next if n.nil?
+        raw = extract_raw(n, f)
+        next if raw.nil?
+        return coerce(raw, f, base_url, n)
+      end
+      nil
+    end
+
+    def self.extract_multi(scope, selectors, f, base_url)
+      out = []
+      selectors.each do |sel|
+        nodes = sel ? scope.css(sel) : [scope]
+        nodes.each do |n|
+          raw = extract_raw(n, f)
+          next if raw.nil?
+          v = coerce(raw, f, base_url, n)
+          out << v unless v.nil?
+        end
+        break unless out.empty?
+      end
+      out
+    end
+
+    def self.extract_raw(node, f)
+      return node.inner_html if f.type == :html && f.attr_str.nil?
+      if f.attr_str
+        node[f.attr_str]
+      else
+        node.text
+      end
+    end
+
+    def self.coerce(raw, f, base_url, _node)
+      return nil if raw.nil?
+      v = raw
+      v = Cleaner.clean(v) if f.clean
+      case f.type
+      when :text     then v
+      when :integer  then int_coerce(v)
+      when :float    then float_coerce(v)
+      when :money    then Money.parse(v)
+      when :url      then f.normalize_url ? URL.absolute(v, base_url) : v
+      when :date     then date_coerce(v)
+      when :json     then json_coerce(v)
+      when :boolean  then bool_coerce(v)
+      when :list     then list_coerce(v, f.delimiter)
+      when :html     then v # already inner_html
+      else v
+      end
+    end
+
+    def self.int_coerce(v)
+      s = v.to_s.gsub(/[^\d\-]/, "")
+      s.empty? || s == "-" ? nil : s.to_i
+    end
+
+    def self.float_coerce(v)
+      s = v.to_s.gsub(/[^\d.\-]/, "")
+      s.empty? || s == "-" || s == "." ? nil : s.to_f
+    end
+
+    def self.date_coerce(v)
+      require "date"
+      ::Date.parse(v.to_s)
+    rescue ::ArgumentError, ::TypeError
+      nil
+    end
+
+    def self.json_coerce(v)
+      JSON.parse(v.to_s)
+    rescue JSON::ParserError
+      nil
+    end
+
+    TRUTHY_STRINGS = %w[true yes 1 on enabled].freeze
+    FALSY_STRINGS  = %w[false no 0 off disabled].freeze
+    private_constant :TRUTHY_STRINGS, :FALSY_STRINGS
+
+    def self.bool_coerce(v)
+      s = v.to_s.strip.downcase
+      return true  if TRUTHY_STRINGS.include?(s)
+      return false if FALSY_STRINGS.include?(s)
+      return true  if s == "yes"
+      nil
+    end
+
+    def self.list_coerce(v, delimiter)
+      v.to_s.split(delimiter).map(&:strip).reject(&:empty?)
+    end
+  end
+end

data/lib/scrapetor/fetcher.rb

@@ -0,0 +1,390 @@
+# frozen_string_literal: true
+
+module Scrapetor
+  # Native HTTP/2-capable fetch layer. Wraps the libcurl-backed
+  # Scrapetor::Native::Http module. Distinct from Scrapetor::HTTP /
+  # Scrapetor.fetch, which is the Net::HTTP-based fallback used by
+  # tests, the CLI, and environments without libcurl.
+  #
+  # Capabilities depend on the libcurl your gem links against. Inspect
+  # Scrapetor::Fetcher.features to see what's actually wired up
+  # (HTTP/2 / brotli / zstd / libz). HTTP/2 + gzip is the typical
+  # baseline on macOS; brotli/zstd need a libcurl rebuilt with them.
+  #
+  # The connection cache lives on a per-OS-thread libcurl easy handle —
+  # repeated fetches to the same host inside a single thread reuse the
+  # TLS session and HTTP/2 stream. The fetch itself drops the GVL, so
+  # background Ruby threads keep running during the round-trip.
+  #
+  #   resp = Scrapetor::Fetcher.get("https://api.example.com/items")
+  #   resp[:status]        # => 200
+  #   resp[:http_version]  # => "2"
+  #   resp[:headers]       # => {"content-type" => "application/json", ...}
+  #   resp[:body]          # => "..."
+  #
+  #   doc = Scrapetor::Fetcher.fetch("https://example.com/")
+  #   # => Scrapetor::Document parsed from the response body, base_url
+  #   #    set to the final URL after redirects.
+  module Fetcher
+    class NotAvailableError < StandardError; end
+    class FetchError < StandardError
+      attr_reader :status, :response
+      def initialize(msg, status: nil, response: nil)
+        super(msg)
+        @status = status
+        @response = response
+      end
+    end
+
+    DEFAULT_USER_AGENT = "scrapetor/#{Scrapetor::VERSION} (libcurl)"
+
+    # Status codes worth retrying. 408 (timeout), 425 (early), 429
+    # (rate-limit), 500–504 (transient upstream). 5xx >= 505 are
+    # protocol-level rejections; they don't usually heal on retry.
+    DEFAULT_RETRY_STATUSES = [408, 425, 429, 500, 502, 503, 504].freeze
+
+    # Compute the backoff delay for attempt N (1-indexed). Exponential
+    # with full jitter — the AWS-style 'random between 0 and 2^n * base'
+    # variant — capped at max_backoff.
+    def self.backoff_for(attempt, base: 0.3, max: 10.0, retry_after: nil)
+      return [retry_after.to_f, max].min if retry_after && retry_after.to_f > 0
+      hi = [base * (2.0**(attempt - 1)), max].min
+      rand * hi
+    end
+
+    def self.retryable_response?(resp, retry_statuses)
+      retry_statuses.any? { |s| s == resp[:status] }
+    end
+
+    def self.parse_retry_after(headers)
+      v = headers && (headers["retry-after"] || headers["Retry-After"])
+      return nil unless v
+      # Either integer seconds or HTTP-date. We only honour the
+      # integer form (the date form is rare and parsing it adds a
+      # dependency on Time.httpdate that the caller may not need).
+      v.to_s.strip.match?(/\A\d+\z/) ? v.to_i : nil
+    end
+
+    def self.available?
+      defined?(Scrapetor::Native::Http::AVAILABLE) &&
+        Scrapetor::Native::Http::AVAILABLE
+    end
+
+    def self.features
+      ensure_available!
+      Scrapetor::Native::Http.features
+    end
+
+    # Single GET with optional retry + exponential backoff.
+    #
+    #   retry: 0     - try once and return whatever happens (default).
+    #   retry: N     - retry up to N times on transient failure.
+    #   backoff:     - base backoff in seconds (default 0.3).
+    #   max_backoff: - cap on a single sleep (default 10.0).
+    #   retry_on:    - statuses to retry (default [408, 425, 429, 500..504]).
+    #
+    # Network failures (IOError from libcurl: connect refused, DNS,
+    # TLS, timeout) are also retried. The wait between attempts honours
+    # Retry-After response headers (numeric form) when present and
+    # otherwise uses exponential backoff with full jitter.
+    def self.get(url, **opts)
+      ensure_available!
+      retries        = opts.delete(:retry) || 0
+      base           = opts.delete(:backoff) || 0.3
+      max_backoff    = opts.delete(:max_backoff) || 10.0
+      retry_on       = opts.delete(:retry_on) || DEFAULT_RETRY_STATUSES
+      opts[:user_agent] ||= DEFAULT_USER_AGENT
+      attempt = 0
+      last_err = nil
+      loop do
+        begin
+          resp = Scrapetor::Native::Http.get(url.to_s, opts)
+          return resp unless retries > attempt && retryable_response?(resp, retry_on)
+          ra = parse_retry_after(resp[:headers])
+          sleep backoff_for(attempt + 1, base: base, max: max_backoff,
+                            retry_after: ra)
+        rescue IOError => e
+          last_err = e
+          raise e unless retries > attempt
+          sleep backoff_for(attempt + 1, base: base, max: max_backoff)
+        end
+        attempt += 1
+      end
+    rescue IOError
+      raise last_err if last_err
+      raise
+    end
+
+    # Fetch + parse. Raises FetchError on non-2xx status by default;
+    # pass raise_for_status: false to inspect non-2xx responses.
+    def self.fetch(url, raise_for_status: true, **opts)
+      resp = get(url, **opts)
+      if raise_for_status && (resp[:status] < 200 || resp[:status] >= 400)
+        raise FetchError.new(
+          "Scrapetor::Fetcher.fetch #{url} -> HTTP #{resp[:status]}",
+          status: resp[:status], response: resp
+        )
+      end
+      Scrapetor.parse(resp[:body], base_url: resp[:final_url])
+    end
+
+    def self.ensure_available!
+      return if available?
+      raise NotAvailableError,
+            "Scrapetor::Fetcher requires libcurl at build time. " \
+            "Reinstall after `brew install curl` / `apt-get install libcurl4-openssl-dev`."
+    end
+
+    # N concurrent GETs across pthread workers, each with a persistent
+    # libcurl handle (per-OS-thread connection cache). The full batch
+    # runs under one GVL release — other Ruby threads stay live
+    # throughout.
+    #
+    #   results = Scrapetor::Fetcher.parallel_get(urls, threads: 8,
+    #                                              timeout_ms: 5_000)
+    #   # results is Array<Hash>; successful entries carry
+    #   #   :status, :headers, :body, :final_url, :http_version
+    #   # failed entries carry { error: { url:, error: } } only.
+    # N concurrent GETs with optional retry. The native batch returns
+    # all results in one pass; failed entries (transient status or
+    # network error) get a second batch dispatch under retry, with
+    # the previous-attempt's wait honoured globally — i.e. one sleep
+    # between attempts rather than per-URL — so the pool keeps moving.
+    #
+    # Per-URL Retry-After headers are read on retryable responses and
+    # the maximum of them governs the next inter-attempt sleep, so a
+    # rate-limited host pulls the whole pool to its backoff rather
+    # than thrashing the rest in parallel.
+    def self.parallel_get(urls, **opts)
+      ensure_available!
+      urls = Array(urls).map(&:to_s)
+      return [] if urls.empty?
+
+      retries     = opts.delete(:retry) || 0
+      base        = opts.delete(:backoff) || 0.3
+      max_backoff = opts.delete(:max_backoff) || 10.0
+      retry_on    = opts.delete(:retry_on) || DEFAULT_RETRY_STATUSES
+      opts[:user_agent] ||= DEFAULT_USER_AGENT
+
+      results = Array.new(urls.size)
+      pending = (0...urls.size).to_a
+      attempt = 0
+      loop do
+        batch = pending.map { |i| urls[i] }
+        batch_res = Scrapetor::Native::Http.parallel_fetch(batch, opts)
+        next_pending = []
+        next_retry_after = nil
+        pending.each_with_index do |orig_i, pos|
+          r = batch_res[pos]
+          if attempt < retries && retry_eligible?(r, retry_on)
+            ra = r[:headers] ? parse_retry_after(r[:headers]) : nil
+            next_retry_after = ra if ra && (next_retry_after.nil? || ra > next_retry_after)
+            next_pending << orig_i
+          else
+            results[orig_i] = r
+          end
+        end
+        break if next_pending.empty?
+        attempt += 1
+        sleep backoff_for(attempt, base: base, max: max_backoff,
+                          retry_after: next_retry_after)
+        pending = next_pending
+      end
+      results
+    end
+
+    def self.retry_eligible?(r, retry_on)
+      return true if r[:error]                       # network-level
+      r[:status] && retry_on.any? { |s| s == r[:status] }
+    end
+
+    # Single-thread curl_multi bulk fetch — one driver thread, one
+    # multi handle, N concurrent transfers multiplexed via
+    # curl_multi_perform. Complements parallel_get:
+    #
+    #   parallel_get  - N pthread workers, each running blocking easy.
+    #                   Best when each transfer has CPU work after the
+    #                   fetch (decode + parse) since the GVL is released
+    #                   across the full batch and CPU scales with cores.
+    #
+    #   multi_get     - one driver thread, N concurrent in-flight.
+    #                   Best for I/O-dominated high-fan-out (hundreds of
+    #                   URLs across many hosts) where pthread setup
+    #                   overhead outweighs the in-flight count.
+    #
+    # Both share the same global CURLSH so connections / DNS / TLS
+    # sessions pool across them.
+    def self.multi_get(urls, **opts)
+      ensure_available!
+      urls = Array(urls).map(&:to_s)
+      return [] if urls.empty?
+      opts[:user_agent] ||= DEFAULT_USER_AGENT
+      Scrapetor::Native::Http.multi_fetch(urls, opts)
+    end
+
+    # Bulk-revalidate cached entries. Issues HEAD with
+    # If-None-Match / If-Modified-Since for every URL whose cache
+    # entry exists; the server's 304 / 200 verdict classifies each:
+    #
+    #   :fresh    - server said 304; cache still valid.
+    #   :changed  - server returned a new 2xx; cache rewritten.
+    #   :missing  - server returned 4xx (gone / not found).
+    #   :error    - transport failure.
+    #
+    # Returns a Hash[url => Symbol]. Optimal for crawls of N pages
+    # over moderate intervals: HEAD round-trips are cheap and run
+    # all-concurrent under curl_multi.
+    def self.revalidate(urls, cache_dir:, **opts)
+      ensure_available!
+      urls = Array(urls).map(&:to_s)
+      return {} if urls.empty?
+      opts[:user_agent] ||= DEFAULT_USER_AGENT
+      results = Scrapetor::Native::Http.multi_fetch(urls,
+        opts.merge(method: :head, cache_dir: cache_dir))
+      out = {}
+      results.each_with_index do |r, i|
+        url = urls[i]
+        out[url] =
+          if r[:error]
+            :error
+          elsif r[:headers] && r[:headers]["x-scrapetor-cache"] == "hit"
+            :fresh
+          elsif r[:status] && r[:status] >= 400
+            :missing
+          else
+            :changed
+          end
+      end
+      out
+    end
+
+    # multi_get + per-response parse, all under one no-GVL window.
+    # Returns Array<Scrapetor::Document | nil>, in input order. Failed
+    # entries are nil. Best for high-fan-out crawls where you want
+    # parsed Documents back and the I/O outweighs the per-page
+    # CPU cost.
+    def self.multi_fetch(urls, **opts)
+      urls = Array(urls).map(&:to_s)
+      return [] if urls.empty?
+      ensure_available!
+      opts[:user_agent] ||= DEFAULT_USER_AGENT
+      results = Scrapetor::Native::Http.multi_fetch(urls, opts.merge(parse: true))
+      results.map do |r|
+        next nil if r[:error]
+        native = r[:document]
+        next Scrapetor.parse(r[:body], base_url: r[:final_url]) unless native
+        Scrapetor::Document.new("", base_url: r[:final_url], native: native)
+      end
+    end
+
+    # Streaming variant of multi_get: yields each response as it
+    # completes (in completion order, not input order), so the caller
+    # can start processing while other transfers are still in flight.
+    # Pass parse: true to also parse the body in the worker thread.
+    #
+    #   Scrapetor::Fetcher.multi_each(urls, threads: 8) do |r|
+    #     puts r[:final_url], r[:status]
+    #     # later transfers may still be on the wire here
+    #   end
+    def self.multi_each(urls, **opts)
+      return enum_for(:multi_each, urls, **opts) unless block_given?
+      ensure_available!
+      urls = Array(urls).map(&:to_s)
+      return if urls.empty?
+      opts[:user_agent] ||= DEFAULT_USER_AGENT
+      batch = Scrapetor::Native::Http::MultiBatch.new(urls, opts)
+      while (r = batch.next)
+        yield r
+      end
+    end
+
+    # Method shorthands. Each is just a `.get` invocation with the
+    # corresponding method, plus the body sugar that POST/PUT/PATCH
+    # almost always need.
+    def self.post(url, body: nil, form: nil, json: nil, multipart: nil, **opts)
+      if multipart
+        opts[:multipart] = multipart
+        get(url, **opts.merge(method: :post))
+      else
+        body, opts = build_body(body, form, json, opts)
+        get(url, **opts.merge(method: :post, body: body))
+      end
+    end
+
+    # Convenience constructors for multipart values.
+    def self.upload_file(path, filename: nil, content_type: nil)
+      h = { path: path.to_s }
+      h[:filename] = filename if filename
+      h[:content_type] = content_type if content_type
+      h
+    end
+
+    def self.upload_bytes(bytes, filename:, content_type: "application/octet-stream")
+      { data: bytes, filename: filename, content_type: content_type }
+    end
+
+    def self.put(url, body: nil, form: nil, json: nil, **opts)
+      body, opts = build_body(body, form, json, opts)
+      get(url, **opts.merge(method: :put, body: body))
+    end
+
+    def self.patch(url, body: nil, form: nil, json: nil, **opts)
+      body, opts = build_body(body, form, json, opts)
+      get(url, **opts.merge(method: :patch, body: body))
+    end
+
+    def self.delete(url, **opts)
+      get(url, **opts.merge(method: :delete))
+    end
+
+    def self.head(url, **opts)
+      get(url, **opts.merge(method: :head))
+    end
+
+    # Build the request body from one of :body / :form / :json.
+    # Returns [body_string, opts_with_content_type_header_set].
+    def self.build_body(body, form, json, opts)
+      headers = (opts[:headers] || {}).dup
+      if json
+        require "json"
+        body = JSON.generate(json)
+        headers["Content-Type"] ||= "application/json"
+      elsif form
+        require "uri"
+        body = URI.encode_www_form(form)
+        headers["Content-Type"] ||= "application/x-www-form-urlencoded"
+      end
+      opts[:headers] = headers unless headers.empty?
+      [body, opts]
+    end
+
+    # Convenience: parallel_get + parse each successful response into
+    # a Scrapetor::Document. Failed entries return nil.
+    #
+    # The parse runs INSIDE the same no-GVL pthread worker that did
+    # the fetch (parse: true on the C side), so the whole batch —
+    # network + decode + transcode + dom_parse + index build — runs
+    # multi-core under a single GVL release. The main thread only
+    # wraps already-parsed documents.
+    def self.parallel_fetch(urls, **opts)
+      urls = Array(urls).map(&:to_s)
+      return [] if urls.empty?
+      ensure_available!
+      opts[:user_agent] ||= DEFAULT_USER_AGENT
+      results = Scrapetor::Native::Http.parallel_fetch(urls, opts.merge(parse: true))
+      results.map do |r|
+        next nil if r[:error]
+        native = r[:document]
+        next Scrapetor.parse(r[:body], base_url: r[:final_url]) unless native
+        Scrapetor::Document.new("", base_url: r[:final_url], native: native)
+      end
+    end
+  end
+
+  # Top-level shorthand for the libcurl path. Distinct from
+  # Scrapetor.fetch (Net::HTTP) so callers can opt-in to HTTP/2 +
+  # connection reuse where it's actually available.
+  def self.fetch_http2(url, **opts)
+    Fetcher.fetch(url, **opts)
+  end
+end

data/lib/scrapetor/fingerprint.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Scrapetor
+  # Structural fingerprint of a DOM subtree.
+  # Phase 1: tag-bigram rolling hash over the top `depth` levels.
+  # Phase 2+: tag bigrams + attribute-presence hash + child-shape hash.
+  module Fingerprint
+    MASK = 0xFFFFFFFFFFFFFFFF
+
+    def self.structural(node, depth: 4)
+      backing = node.respond_to?(:backing_node) ? node.backing_node : node
+      h = 0
+      walk(backing, depth) do |tag|
+        h = (h * 1_315_423_911 + tag.hash) & MASK
+      end
+      h
+    end
+
+    def self.walk(nlx, depth, &block)
+      return if depth <= 0
+      return unless nlx.respond_to?(:children)
+      nlx.children.each do |c|
+        next unless c.respond_to?(:element?) && c.element?
+        block.call(c.name)
+        walk(c, depth - 1, &block)
+      end
+    end
+  end
+end