scrapetor 0.2.0

data/lib/scrapetor/builder.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Scrapetor
+  # Pure-Ruby HTML construction DSL. No external dependency.
+  #
+  # Two usage patterns:
+  #
+  #   # 1. Block with explicit receiver:
+  #   html = Scrapetor::Builder.build do |b|
+  #     b.html do
+  #       b.head { b.title "My Page" }
+  #       b.body do
+  #         b.h1 "Hello", class: "hdr"
+  #         b.p "world", id: "lead"
+  #         b.a("More", href: "/x")
+  #       end
+  #     end
+  #   end
+  #
+  #   # 2. Direct instance:
+  #   b = Scrapetor::Builder.new
+  #   b.div(class: "card") { b.h1 "Title" }
+  #   b.to_html
+  class Builder
+    VOID = %w[
+      area base br col embed hr img input link meta source track wbr
+    ].freeze
+    private_constant :VOID
+
+    def self.build(&block)
+      new(&block).to_html
+    end
+
+    def initialize(&block)
+      @stack = []
+      @root  = []
+      if block
+        if block.arity == 1
+          block.call(self)
+        else
+          instance_eval(&block)
+        end
+      end
+    end
+
+    # Inject a raw text node at the current position.
+    def text(s)
+      append(s.to_s)
+      self
+    end
+
+    # Inject pre-escaped raw HTML.
+    def raw(s)
+      append(RawHTML.new(s.to_s))
+      self
+    end
+
+    # Inject an HTML comment.
+    def comment(s)
+      append(Comment.new(s.to_s))
+      self
+    end
+
+    # Inject a doctype.
+    def doctype(name = "html")
+      append(Doctype.new(name.to_s))
+      self
+    end
+
+    # Method-missing dispatch: any unknown method becomes a tag.
+    #
+    #   b.div("hi", class: "card") { b.span "x" }
+    #     ->  <div class="card">hi<span>x</span></div>
+    def method_missing(name, *args, &block)
+      content = nil
+      attrs   = {}
+      args.each do |a|
+        case a
+        when Hash   then attrs = attrs.merge(a)
+        when String then content ||= a
+        else             content ||= a.to_s
+        end
+      end
+      element = Element.new(name.to_s, attrs, [])
+      append(element)
+      @stack.push(element)
+      element.children << content unless content.nil?
+      if block
+        if block.arity == 1
+          block.call(self)
+        else
+          instance_eval(&block)
+        end
+      end
+      @stack.pop
+      self
+    end
+
+    def respond_to_missing?(_name, _include_private = false)
+      true
+    end
+
+    def to_html
+      @root.map { |n| serialize(n) }.join
+    end
+    alias to_s to_html
+
+    private
+
+    def append(node)
+      if @stack.empty?
+        @root << node
+      else
+        @stack.last.children << node
+      end
+    end
+
+    def serialize(node)
+      case node
+      when String
+        escape_text(node)
+      when RawHTML
+        node.body
+      when Comment
+        "<!--#{node.body}-->"
+      when Doctype
+        "<!DOCTYPE #{node.body}>"
+      when Element
+        attr_str = node.attrs.map { |k, v| %( #{k}="#{escape_attr(v)}") }.join
+        if VOID.include?(node.name) && node.children.empty?
+          "<#{node.name}#{attr_str}>"
+        else
+          inner = node.children.map { |c| serialize(c) }.join
+          "<#{node.name}#{attr_str}>#{inner}</#{node.name}>"
+        end
+      end
+    end
+
+    def escape_text(s)
+      s.to_s.gsub(/[&<>]/, "&" => "&amp;", "<" => "&lt;", ">" => "&gt;")
+    end
+
+    def escape_attr(s)
+      s.to_s.gsub(/[&<>"']/,
+                  "&" => "&amp;",
+                  "<" => "&lt;",
+                  ">" => "&gt;",
+                  '"' => "&quot;",
+                  "'" => "&#39;")
+    end
+
+    Element = Struct.new(:name, :attrs, :children)
+    RawHTML = Struct.new(:body)
+    Comment = Struct.new(:body)
+    Doctype = Struct.new(:body)
+    private_constant :Element, :RawHTML, :Comment, :Doctype
+  end
+end

data/lib/scrapetor/cleaner.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Scrapetor
+  module Cleaner
+    def self.clean(s)
+      return nil if s.nil?
+      s.to_s.gsub(/\s+/, " ").strip
+    end
+  end
+end

data/lib/scrapetor/comment_node.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Scrapetor
+  # Result type for XPath `comment()` queries (`//comment()`,
+  # `child::comment()`, etc.). Carries the comment's text payload and
+  # implements the Node-shape predicate methods so duck-typing checks
+  # (`n.comment?`, `n.element?`, `n.name == "#comment"`) match what
+  # Nokogiri would return.
+  #
+  # The constructor accepts a String (extracted by the native engine's
+  # `node_comment_text`) or a Dom::Comment (Ruby fallback path); in
+  # either case `#text` / `#content` returns the payload between
+  # `<!--` and `-->`.
+  class CommentNode
+    attr_reader :document
+
+    def initialize(document, payload)
+      @document = document
+      @text = payload.is_a?(String) ? payload :
+              (payload.respond_to?(:content) ? payload.content.to_s : payload.to_s)
+    end
+
+    def text;        @text; end
+    alias content     text
+    alias inner_text  text
+
+    def to_s;        @text; end
+    def to_html;     "<!--#{@text}-->"; end
+    alias outer_html to_html
+    alias inner_html to_html
+
+    def name;        "#comment"; end
+    alias node_name  name
+
+    def comment?;    true;  end
+    def element?;    false; end
+    def text?;       false; end
+    def document?;   false; end
+    def cdata?;      false; end
+    def node_type;   8;     end
+
+    # Node-shape probes that scraping code occasionally fires against
+    # mixed result sets. Returning a benign default keeps a stray
+    # `.css(...)` or `.attributes` from raising NoMethodError when a
+    # caller iterates over an Array<Element + CommentNode>.
+    def attributes;  {};    end
+    def attribute_nodes; []; end
+    def attribute(_); nil; end
+    def keys;        [];    end
+    def values;      [];    end
+    def children;    [];    end
+    def element_children; []; end
+    def classes;     [];    end
+    def has_class?(_); false; end
+    def [](*_args);  nil;   end
+    def css(_);      [];    end
+    def at_css(_);   nil;   end
+    def at(_);       nil;   end
+    def search(_);   [];    end
+    def xpath(*_);   [];    end
+    def at_xpath(*_); nil;  end
+
+    def inspect
+      "#<Scrapetor::CommentNode #{@text.inspect}>"
+    end
+  end
+end

data/lib/scrapetor/document.rb

@@ -0,0 +1,457 @@
+# frozen_string_literal: true
+
+module Scrapetor
+  class Document
+    attr_reader :base_url, :encoding
+
+    def initialize(html, base_url: nil, build_indexes: false, encoding: :auto, native: nil)
+      @base_url = base_url
+      raw = html.to_s
+      if encoding == :auto
+        @encoding = Scrapetor::Encoding.detect(raw)
+        @html_str = Scrapetor::Encoding.to_utf8(raw)
+      else
+        @encoding = encoding.to_s
+        @html_str = raw.dup.force_encoding(@encoding).encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
+      end
+      @backing = nil # parsed lazily; native extract bypasses this entirely
+      @selector_cache = {}
+      @indexes_built = false
+      @class_index = nil
+      @id_index = nil
+      @tag_index = nil
+      # Hot-path slots (populated by backing()): keeping these
+      # initialised silences "instance variable not initialized" and
+      # makes the fast-path test a simple nil check.
+      @native_doc     = nil
+      @native_wrapper = nil
+      @plan_cache     = nil
+      @lazy_ids       = nil
+      # If a pre-parsed native handle was passed in (persistent-cache
+      # hit), wrap it directly and skip the lazy-parse path.
+      @prebuilt_native = native
+      build_indexes! if build_indexes
+    end
+
+    def html_str
+      @html_str
+    end
+
+    # CSS query entry point. Inlined hot path for the >95% case: a
+    # selector with no `::` pseudo-element and a cache-hit native plan.
+    # That bypasses backing.lazy_css, peel_pseudo_element, and the
+    # method dispatch chain, dropping the per-call Ruby overhead to a
+    # single Hash#[] + Struct.new + NodeSet.new.
+    def css(*selectors)
+      # Nokogiri-compat: `doc.css(sel1, sel2, ...)` accepts multiple
+      # selectors and returns the union of matches across all of them.
+      # Drop trailing non-string arguments (Nokogiri also accepts an
+      # XPath namespaces hash here — that's a no-op for CSS).
+      selectors = selectors.reject { |a| !a.is_a?(String) }
+      raise ArgumentError, "Document#css requires at least one selector" if selectors.empty?
+      return css_single(selectors.first) if selectors.size == 1
+
+      seen = {}
+      union = []
+      string_result = nil
+      selectors.each do |sel|
+        result = css_single(sel)
+        if result.is_a?(Array)
+          string_result = true
+          result.each { |s| union << s }
+        else
+          # NodeSet — pull backing items and dedupe.
+          string_result = false if string_result.nil?
+          result.each do |node|
+            bk = node.respond_to?(:backing_node) ? node.backing_node : node
+            key = bk.object_id
+            next if seen[key]
+            seen[key] = true
+            union << bk
+          end
+        end
+      end
+      string_result ? union : NodeSet.new(self, union)
+    end
+
+    def css_single(selector)
+      # Fast path: native backing, no mutations applied yet, plain
+      # String selector, no pseudo-element. One Hash lookup + one C
+      # call + two allocations. After any mutation the wrapper flips
+      # into dom_mode and we route through the slow path so reads see
+      # the user's edits — checking @native_wrapper.dom_mode? is one
+      # ivar read, negligible vs the saving when we stay native.
+      if @native_doc && !@native_wrapper.dom_mode? && selector.is_a?(String) && !selector.include?("::")
+        plan = @plan_cache[selector]
+        if plan
+          return NodeSet.new(self, @lazy_ids.new(@native_wrapper, @native_doc, @native_doc.run_chain(plan, nil)))
+        elsif !@plan_cache.key?(selector)
+          plan = Scrapetor::Native.compile_selector_chain(selector)
+          @plan_cache[selector] = plan || false
+          if plan
+            return NodeSet.new(self, @lazy_ids.new(@native_wrapper, @native_doc, @native_doc.run_chain(plan, nil)))
+          end
+        end
+      end
+      # Slow path: pseudo-element, comma, fallback, post-mutation, or
+      # non-native backing.
+      bk = backing
+      result = bk.respond_to?(:lazy_css) ? bk.lazy_css(selector) : bk.css(selector)
+      if result.is_a?(Array) && (result.first.is_a?(String) || (result.empty? && pseudo_element?(selector)))
+        return result
+      end
+      if @lazy_ids && result.is_a?(@lazy_ids)
+        return NodeSet.new(self, result)
+      end
+      NodeSet.new(self, result.to_a)
+    end
+
+    # Run an array of CSS selectors in ONE Ruby/C boundary crossing.
+    # On selector-heavy workloads (SERP-style pages with ~30
+    # selectors per scrape) this amortises the per-query Ruby overhead
+    # across all of them — N selectors cost roughly one selector
+    # worth of Ruby dispatch, not N. Returns an Array of NodeSets (or
+    # Arrays-of-strings, for `::text` / `::attr(name)` selectors)
+    # parallel to the input.
+    #
+    #   title_ns, price_strs, hrefs = doc.batch_css(
+    #     ["h1.title", ".price::text", "a::attr(href)"]
+    #   )
+    def batch_css(selectors)
+      bk = backing
+      unless bk.respond_to?(:batch_css)
+        # Pure-Ruby Dom fallback — no native engine. Loop manually.
+        return selectors.map { |s| css(s) }
+      end
+      bk.batch_css(self, selectors)
+    end
+
+    # Hash form: `{ name => selector, ... }` -> `{ name => result, ... }`.
+    # The classic scrape pattern in two lines. Same one-boundary cost
+    # as batch_css.
+    def extract_css(map)
+      keys = map.keys
+      selectors = map.values
+      results = batch_css(selectors)
+      out = {}
+      keys.each_with_index { |k, i| out[k] = results[i] }
+      out
+    end
+
+    # Single-result extract on the document scope. One C call covers
+    # field compilation, plan lookup, and result assembly.
+    def extract(map)
+      bk = backing
+      if defined?(Scrapetor::Native::DocumentWrapper) &&
+         bk.is_a?(Scrapetor::Native::DocumentWrapper) && !bk.dom_mode?
+        r = bk.native.extract_one_h(nil, map, bk)
+        return r unless r.equal?(true)
+      end
+      out = {}
+      map.each_pair { |k, sel| out[k] = at_css(sel) }
+      out
+    end
+
+    # Iterate matches of `outer_selector` across the whole document
+    # and build a Hash per match using `fields` (a {key => selector}
+    # map). Returns Array<Hash>. The inner selectors run scoped to
+    # each match, so a `result.at_css(field)`-style parser becomes:
+    #
+    #   doc.extract_each(".result", {
+    #     title: ".title::text",
+    #     price: ".price::text",
+    #     href:  "a::attr(href)",
+    #   })
+    #
+    # When the document is native-backed and every selector compiles
+    # cleanly, the whole iteration runs in a single C call — one outer
+    # plan + N inner plans times M matches, zero Ruby↔C round-trips on
+    # the hot path. Falls back to the per-row Ruby loop only when a
+    # selector compiles to nil (rare; the engine covers nearly every
+    # CSS Selectors L4 shape natively after the audit-driven coverage
+    # work).
+    def extract_each(outer_selector, fields)
+      bk = backing
+      if defined?(Scrapetor::Native::DocumentWrapper) &&
+         bk.is_a?(Scrapetor::Native::DocumentWrapper) && !bk.dom_mode?
+        outer_str = outer_selector.is_a?(String) ? outer_selector : outer_selector.to_s
+        r = bk.native.extract_each_h(outer_str, nil, fields, bk)
+        return r unless r.equal?(true)
+      end
+      css(outer_selector).map { |node| node.extract(fields) }
+    end
+
+    # Accepts the Nokogiri-compatible signature `doc.at(sel, ns_or_handler)`.
+    # The extra args (namespace prefix, handler) only matter for XPath
+    # land — CSS selectors ignore them — so we accept varargs and
+    # discard everything past the selector. Without this, callers that
+    # pass `doc.at(sel, namespaces_hash)` (or similar Bing-style
+    # patterns) hit `ArgumentError: wrong number of arguments`.
+    def at(selector, *_extra)
+      result = backing.at_css(selector)
+      return nil if result.nil?
+      return result if result.is_a?(String)
+      Node.new(self, result)
+    end
+    alias at_css at
+    alias search css
+
+    # Evaluate an XPath expression against this document. Implements
+    # the common XPath 1.0 subset via Scrapetor::XPath (descendant /
+    # child / parent axes, tag / @attr / text() node tests, position +
+    # attr-presence + attr-equality + contains() + starts-with() +
+    # text() predicates). Returns an Array of Scrapetor::Node when the
+    # expression ends at element nodes, or an Array of String for
+    # `/@attr` and `/text()` terminations. See lib/scrapetor/xpath.rb
+    # for the full supported grammar.
+    def xpath(expr)
+      Scrapetor::XPath.evaluate(self, expr)
+    end
+
+    def at_xpath(expr)
+      result = xpath(expr)
+      result.is_a?(Array) ? result.first : result
+    end
+
+    def traverse(&block)
+      return enum_for(:traverse) unless block_given?
+      backing.traverse { |n| yield(n.respond_to?(:element?) ? Node.new(self, n) : n) } if backing.respond_to?(:traverse)
+      self
+    end
+
+    private
+
+    def pseudo_element?(selector)
+      selector.to_s =~ /::(text|attr\([^)]+\)|first-letter|first-line|before|after)\s*\z/i
+    end
+
+    public
+
+    def root
+      el = backing.at_css("html") || backing
+      Node.new(self, el)
+    end
+
+    def text
+      backing.text
+    end
+    alias content    text
+    alias inner_text text
+
+    def title
+      n = backing.at_css("title")
+      n && n.text
+    end
+
+    def body
+      n = backing.at_css("body")
+      n && Node.new(self, n)
+    end
+
+    def head
+      n = backing.at_css("head")
+      n && Node.new(self, n)
+    end
+
+    def html
+      n = backing.at_css("html") || backing
+      Node.new(self, n)
+    end
+
+    def to_html
+      backing.to_html
+    end
+    alias to_s to_html
+
+    # Nokogiri-compat predicates.
+    def errors
+      []
+    end
+
+    def html?
+      true
+    end
+
+    def xml?
+      false
+    end
+
+    # Structured-data extractors — for SEO/RAG/structured-content pipelines.
+
+    def json_ld
+      Scrapetor::StructuredData.json_ld(self)
+    end
+
+    def opengraph
+      Scrapetor::StructuredData.opengraph(self)
+    end
+
+    def twitter_card
+      Scrapetor::StructuredData.twitter_card(self)
+    end
+
+    def schema_org(type: nil)
+      Scrapetor::StructuredData.schema_org(self, type: type)
+    end
+
+    def microdata
+      Scrapetor::Microdata.extract(self)
+    end
+
+    def rdfa
+      Scrapetor::RDFa.extract(self)
+    end
+
+    def page_type
+      Scrapetor::PageType.detect(self)
+    end
+
+    def extract(schema = nil, &block)
+      schema ||= Schema.build(&block)
+      if Native.available?
+        result = extract_via_native(schema)
+        return result unless result.nil?
+      end
+      Extractor.run(self, backing, schema)
+    end
+
+    private
+
+    # Try the native path. Returns the result Hash on success, nil if the
+    # schema can't compile (caller falls back to Ruby).
+    #
+    # Schemas with both top-level fields AND repeated groups run two
+    # native passes — the engine supports one active record at a time, so
+    # a synthetic <html>-bound root for top-level fields can't co-exist
+    # with a repeated group inside the same scan. Two-pass cost is a
+    # second 65μs scan; still ~10× ahead of Nokolexbor at this size.
+    def extract_via_native(schema)
+      has_fields = schema.fields.any?
+      has_groups = schema.groups.any?
+      return nil unless has_fields || has_groups
+
+      # Common case: only repeated groups — one pass, no schema split.
+      if has_groups && !has_fields
+        desc = Native.compile_descriptor(schema)
+        return nil unless desc
+        return Native.extract(@html_str, desc, @base_url)
+      end
+
+      # Top-level fields only — one pass via synthetic root.
+      if has_fields && !has_groups
+        desc = Native.compile_descriptor(schema)
+        return nil unless desc
+        raw = Native.extract(@html_str, desc, @base_url)
+        root_records = raw[Native::SYNTHETIC_ROOT]
+        return nil if !root_records.is_a?(Array) || root_records.empty?
+        return root_records[0]
+      end
+
+      # Mixed: two-pass. The C engine handles one active record at a
+      # time, so a synthetic root for top-level fields can't run in
+      # the same scan as a repeated group. We split the schema into
+      # two sub-descriptors and run extract twice. Both sub-descriptors
+      # are memoised on the original schema so the split itself only
+      # allocates on the first call.
+      groups_desc = Native.split_descriptor(schema, :groups)
+      return nil unless groups_desc
+      result = Native.extract(@html_str, groups_desc, @base_url)
+
+      fields_desc = Native.split_descriptor(schema, :fields)
+      return nil unless fields_desc
+      raw = Native.extract(@html_str, fields_desc, @base_url)
+      root_records = raw[Native::SYNTHETIC_ROOT]
+      return nil if !root_records.is_a?(Array) || root_records.empty?
+
+      root_records[0].merge(result)
+    end
+
+    public
+
+    def stats
+      {
+        classes: @class_index ? @class_index.size : 0,
+        ids: @id_index ? @id_index.size : 0,
+        tags: @tag_index ? @tag_index.size : 0,
+        selector_cache_size: @selector_cache.size,
+        indexes_built: @indexes_built
+      }
+    end
+
+    def backing
+      return @backing if @backing
+      @backing =
+        if defined?(Scrapetor::Native::DocumentWrapper) && Scrapetor::Native::AVAILABLE_DOM
+          native = @prebuilt_native || Scrapetor::Native::Document.parse(@html_str)
+          @prebuilt_native = nil
+          Scrapetor::Native::DocumentWrapper.new(native)
+        else
+          Dom::Parser.parse(@html_str)
+        end
+      # Cache the hot-path slots so Document#css can skip the indirection.
+      if defined?(Scrapetor::Native::DocumentWrapper) &&
+         @backing.is_a?(Scrapetor::Native::DocumentWrapper)
+        @native_doc     = @backing.native
+        @native_wrapper = @backing
+        @plan_cache     = @backing.instance_variable_get(:@compile_cache)
+        @lazy_ids       = Scrapetor::Native::DocumentWrapper::LazyIds
+      end
+      @backing
+    end
+
+    # Phase-2 hooks: structural indexes. Built on demand. The native
+    # backend will replace these with arena-resident indexes.
+    def class_index
+      build_indexes! unless @indexes_built
+      @class_index
+    end
+
+    def id_index
+      build_indexes! unless @indexes_built
+      @id_index
+    end
+
+    def tag_index
+      build_indexes! unless @indexes_built
+      @tag_index
+    end
+
+    def all_elements
+      build_indexes! unless @indexes_built
+      @all_elements
+    end
+
+    def run_selector(selector, scope)
+      plan = @selector_cache[selector] ||= Selector.compile(selector)
+      Selector.execute(self, plan, scope)
+    end
+
+    def cache_selector(selector)
+      @selector_cache[selector] ||= Selector.compile(selector)
+    end
+
+    def selector_cache_size
+      @selector_cache.size
+    end
+
+    private
+
+    def build_indexes!
+      return if @indexes_built
+      @class_index = Hash.new { |h, k| h[k] = [] }
+      @id_index = {}
+      @tag_index = Hash.new { |h, k| h[k] = [] }
+      @all_elements = backing.css("*").to_a
+      @all_elements.each do |el|
+        @tag_index[el.name.to_sym] << el
+        id = el["id"]
+        @id_index[id] ||= el if id
+        cls = el["class"]
+        if cls
+          cls.split(/\s+/).each { |c| @class_index[c] << el unless c.empty? }
+        end
+      end
+      @indexes_built = true
+    end
+  end
+end