RubyGems - scrapetor - Versions diffs - 0.2.0 - Mend

scrapetor 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (51) hide show

checksums.yaml +7 -0
data/CHANGELOG.md +242 -0
data/LICENSE +21 -0
data/README.md +440 -0
data/bin/scrapetor +190 -0
data/bin/scrapetor-bench +5 -0
data/ext/scrapetor/README.md +53 -0
data/ext/scrapetor/native/extconf.rb +67 -0
data/ext/scrapetor/native/scrapetor_dom.c +6346 -0
data/ext/scrapetor/native/scrapetor_http.c +2591 -0
data/ext/scrapetor/native/scrapetor_native.c +1156 -0
data/lib/scrapetor/builder.rb +158 -0
data/lib/scrapetor/cleaner.rb +10 -0
data/lib/scrapetor/comment_node.rb +67 -0
data/lib/scrapetor/document.rb +457 -0
data/lib/scrapetor/dom/parser.rb +69 -0
data/lib/scrapetor/dom/selectors.rb +208 -0
data/lib/scrapetor/dom.rb +563 -0
data/lib/scrapetor/encoding.rb +85 -0
data/lib/scrapetor/entities.rb +90 -0
data/lib/scrapetor/errors.rb +12 -0
data/lib/scrapetor/extractor.rb +147 -0
data/lib/scrapetor/fetcher.rb +390 -0
data/lib/scrapetor/fingerprint.rb +29 -0
data/lib/scrapetor/form.rb +141 -0
data/lib/scrapetor/http.rb +114 -0
data/lib/scrapetor/microdata.rb +132 -0
data/lib/scrapetor/money.rb +30 -0
data/lib/scrapetor/native.rb +291 -0
data/lib/scrapetor/native_dom.rb +2258 -0
data/lib/scrapetor/node.rb +539 -0
data/lib/scrapetor/node_set.rb +301 -0
data/lib/scrapetor/page_type.rb +95 -0
data/lib/scrapetor/pagination.rb +109 -0
data/lib/scrapetor/persistent_cache.rb +130 -0
data/lib/scrapetor/robots.rb +159 -0
data/lib/scrapetor/sax.rb +285 -0
data/lib/scrapetor/schema.rb +144 -0
data/lib/scrapetor/selector.rb +576 -0
data/lib/scrapetor/session.rb +141 -0
data/lib/scrapetor/sitemap.rb +52 -0
data/lib/scrapetor/stream.rb +111 -0
data/lib/scrapetor/structured_data.rb +74 -0
data/lib/scrapetor/template_registry.rb +24 -0
data/lib/scrapetor/text_node.rb +101 -0
data/lib/scrapetor/url.rb +21 -0
data/lib/scrapetor/version.rb +5 -0
data/lib/scrapetor/xpath.rb +1603 -0
data/lib/scrapetor.rb +167 -0
data/scrapetor.gemspec +77 -0
metadata +200 -0

data/lib/scrapetor/native.rb ADDED Viewed

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+module Scrapetor
+  # Bridge to the native streaming extraction engine.
+  #
+  # If the C extension is loadable, Native.available? is true and
+  # Native.compile_descriptor turns a Schema into the flat format the
+  # C side consumes. Schemas using features outside the native
+  # fast-path subset (combinators, pseudo-classes, nested repeated
+  # groups, top-level fields without a repeated context) compile to
+  # nil, and the Extractor falls back to the Ruby path.
+  module Native
+    begin
+      require "scrapetor/scrapetor_native"
+      AVAILABLE  = true
+      LOAD_ERROR = nil
+    rescue LoadError => e
+      AVAILABLE  = false
+      LOAD_ERROR = e
+    end
+    def self.available?
+      AVAILABLE
+    end
+    # Compile a Schema into the descriptor format the C side consumes.
+    #
+    #   desc   = [group, group, ...]
+    #   group  = [name_sym, sel, fields_array]
+    #   field  = [name_sym, sel, attr_str_or_nil, type_sym, clean_bool,
+    #             normalize_url_bool, multi_bool]
+    #   sel    = [tag_or_nil, classes_array, id_or_nil, attrs_array]
+    #   attrs_array = [[name_str, op_str_or_nil, val_str_or_nil], ...]
+    #
+    # Returns nil if the schema uses features the native path doesn't
+    # support yet.
+    SYNTHETIC_ROOT = :__scrapetor_root__
+    HTML_ROOT_SEL  = ["html", [], nil, []].freeze
+    # Memoised on the Schema instance — the descriptor Array tree is
+    # identical for every call against the same schema, so rebuilding
+    # it on each extract was just GC pressure. Both successful
+    # descriptors and the "can't compile" outcome are cached.
+    def self.compile_descriptor(schema)
+      cached = schema.instance_variable_get(:@__scrapetor_native_desc)
+      unless cached.nil?
+        return cached == false ? nil : cached
+      end
+      desc = build_descriptor(schema)
+      schema.instance_variable_set(:@__scrapetor_native_desc, desc.nil? ? false : desc)
+      desc
+    end
+    # For "mixed" schemas (top-level fields + at least one repeated
+    # group) the C engine needs two passes — one for the groups, one
+    # for the synthetic root holding the fields. We split the schema
+    # here, memoise the result on the original Schema instance so the
+    # allocations only happen once, and let callers run the two
+    # extractions back-to-back.
+    def self.split_descriptor(schema, kind)
+      ivar = (kind == :groups ? :@__scrapetor_split_groups : :@__scrapetor_split_fields)
+      cached = schema.instance_variable_get(ivar)
+      unless cached.nil?
+        return cached == false ? nil : cached
+      end
+      sub = Schema.new
+      if kind == :groups
+        schema.groups.each { |g| sub.groups << g }
+      else
+        schema.fields.each { |f| sub.fields << f }
+      end
+      desc = build_descriptor(sub)
+      schema.instance_variable_set(ivar, desc.nil? ? false : desc)
+      desc
+    end
+    def self.build_descriptor(schema)
+      groups = []
+      # Top-level fields become a synthetic group bound to the <html>
+      # element. The Document layer unwraps the single result back into
+      # the top of the response hash. Fragments without <html> fall back
+      # to the Ruby path.
+      if schema.fields.any?
+        field_descs = schema.fields.map { |f| compile_field(f) }
+        return nil if field_descs.any?(&:nil?)
+        groups << [SYNTHETIC_ROOT, HTML_ROOT_SEL, field_descs]
+      end
+      schema.groups.each do |g|
+        gd = compile_group(g)
+        return nil unless gd
+        groups << gd
+      end
+      return nil if groups.empty?
+      groups
+    end
+    def self.compile_group(group)
+      sel = parse_selector(group.selector)
+      return nil unless sel
+      return nil unless group.groups.empty? # nested groups: Ruby fallback
+      fields = []
+      group.fields.each do |f|
+        fd = compile_field(f)
+        return nil unless fd
+        fields << fd
+      end
+      [group.name, sel, fields]
+    end
+    def self.compile_field(field)
+      # Features the native engine doesn't yet support — fall back to Ruby.
+      return nil if field.selector.is_a?(Array)
+      return nil if field.transform
+      return nil unless field.default.nil?
+      return nil if field.required
+      return nil if %i[html list json boolean].include?(field.type)
+      # Try simple selector first.
+      simple = parse_selector(field.selector)
+      if simple
+        return [field.name, simple, field.attr_str, field.type,
+                !!field.clean, !!field.normalize_url, !!field.multi,
+                nil, nil]
+      end
+      # Try combinator selector.
+      combo = parse_selector_with_combinator(field.selector)
+      if combo
+        primary, combinator, context = combo
+        return [field.name, primary, field.attr_str, field.type,
+                !!field.clean, !!field.normalize_url, !!field.multi,
+                context, combinator]
+      end
+      nil
+    end
+    # Parse a CSS selector with at most one combinator (`A B` or `A > B`).
+    # Returns [primary_sel, combinator_str, context_sel] or nil if the
+    # input has multiple combinators or other unsupported syntax.
+    def self.parse_selector_with_combinator(selector)
+      s = selector.to_s.strip
+      return nil if s.empty?
+      # Split on first combinator at top level (outside [...] groups).
+      split = split_at_combinator(s)
+      return nil unless split
+      left_str, combinator, right_str = split
+      left  = parse_selector(left_str)
+      right = parse_selector(right_str)
+      return nil unless left && right
+      [right, combinator, left]
+    end
+    def self.split_at_combinator(s)
+      depth = 0
+      i = 0
+      while i < s.length
+        ch = s[i]
+        if ch == "["
+          depth += 1
+        elsif ch == "]"
+          depth -= 1 if depth.positive?
+        elsif depth.zero?
+          if ch == ">"
+            left = s[0...i].strip
+            right = s[(i + 1)..].strip
+            return nil if left.empty? || right.empty?
+            # Reject if there are further combinators in either half.
+            return nil if has_combinator?(left) || has_combinator?(right)
+            return [left, "child", right]
+          elsif ch == " " || ch == "\t" || ch == "\n"
+            left = s[0...i].strip
+            rest = s[(i + 1)..].lstrip
+            next i += 1 if rest.empty?
+            # The next non-whitespace char must not be > / + / ~ — those
+            # are picked up on their own iteration.
+            if !left.empty? && !"<>+~,".include?(rest[0] || "")
+              right = rest
+              return nil if has_combinator?(left) || has_combinator?(right)
+              return [left, "descendant", right]
+            end
+          end
+        end
+        i += 1
+      end
+      nil
+    end
+    def self.has_combinator?(s)
+      depth = 0
+      s.each_char do |ch|
+        if ch == "["       then depth += 1
+        elsif ch == "]"    then depth -= 1 if depth.positive?
+        elsif depth.zero?
+          return true if [" ", "\t", "\n", ">", "+", "~"].include?(ch)
+        end
+      end
+      false
+    end
+    # Parse a simple CSS selector into the [tag, classes, id, attrs] form
+    # that the C engine accepts. Returns nil if the selector uses
+    # combinators or pseudo-classes (those force the Ruby fallback).
+    #
+    # Supported:
+    #   tag                    div
+    #   .class                 .product-card
+    #   tag.class.other        span.price.big
+    #   #id                    #main
+    #   tag#id                 article#main
+    #   [attr]                 [data-sku]
+    #   [attr=val]             [data-sku="A1"]
+    #   [attr*=val]            [class*=card]
+    #   [attr^=val]            [href^=https]
+    #   [attr$=val]            [href$=.pdf]
+    #   [attr~=val]            [class~=primary]
+    #   [attr|=val]            [lang|=en]
+    #   ... and combinations
+    def self.parse_selector(selector)
+      return nil unless selector
+      s = selector.to_s.strip
+      return nil if s.empty?
+      # Check for combinators / unsupported syntax outside [...] brackets,
+      # since `*` and `~` are valid inside attribute operators.
+      outside = s.gsub(/\[[^\]]*\]/, "")
+      return nil if outside =~ /[\s>+~,*]/
+      tag      = nil
+      classes  = []
+      id       = nil
+      attrs    = []
+      i = 0
+      if (m = s[i..].match(/\A([a-zA-Z][\w-]*)/))
+        tag = m[1].downcase
+        i += m[0].length
+      end
+      while i < s.length
+        case s[i]
+        when "."
+          m = s[i..].match(/\A\.([\w-]+)/)
+          return nil unless m
+          classes << m[1]
+          i += m[0].length
+        when "#"
+          m = s[i..].match(/\A#([\w-]+)/)
+          return nil unless m
+          return nil if id # only one id allowed
+          id = m[1]
+          i += m[0].length
+        when "["
+          # Mirror Scrapetor::Selector::ATTR_RE — same quote-style-aware
+          # value extraction so an attribute like `[class*="L'appareil"]`
+          # parses without choking on the embedded apostrophe.
+          m = s[i..].match(/
+            \A\[
+              ([\w:\-\u{0080}-\u{10FFFF}]+)
+              (?:
+                ([*^$~|]?=)
+                (?:
+                  "((?:[^"\\]|\\.)*)"
+                | '((?:[^'\\]|\\.)*)'
+                | ([^\]\s]+)
+                )
+              )?
+            \]
+          /x)
+          return nil unless m
+          attrs << [m[1], m[2], (m[3] || m[4] || m[5])]
+          i += m[0].length
+        else
+          return nil
+        end
+      end
+      return nil if tag.nil? && classes.empty? && id.nil? && attrs.empty?
+      return nil if classes.size > 8 || attrs.size > 8
+      [tag, classes, id, attrs]
+    end
+  end
+end