scrapetor 0.2.0

data/lib/scrapetor.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "scrapetor/version"
+require "scrapetor/errors"
+require "scrapetor/encoding"
+require "scrapetor/cleaner"
+require "scrapetor/money"
+require "scrapetor/url"
+require "scrapetor/fingerprint"
+require "scrapetor/text_node"
+require "scrapetor/comment_node"
+require "scrapetor/selector"
+require "scrapetor/sax"
+require "scrapetor/dom"
+require "scrapetor/dom/parser"
+require "scrapetor/dom/selectors"
+require "scrapetor/node"
+require "scrapetor/node_set"
+require "scrapetor/schema"
+require "scrapetor/extractor"
+require "scrapetor/document"
+require "scrapetor/template_registry"
+require "scrapetor/structured_data"
+require "scrapetor/microdata"
+require "scrapetor/page_type"
+require "scrapetor/entities"
+require "scrapetor/builder"
+require "scrapetor/http"
+require "scrapetor/native"
+require "scrapetor/native_dom"
+require "scrapetor/persistent_cache"
+require "scrapetor/stream"
+require "scrapetor/fetcher"
+require "scrapetor/session"
+require "scrapetor/robots"
+require "scrapetor/sitemap"
+require "scrapetor/pagination"
+require "scrapetor/form"
+require "scrapetor/xpath"
+
+module Scrapetor
+  # ----- Parsing entry points -----
+
+  def self.parse(html, base_url: nil, build_indexes: false)
+    if PersistentCache.enabled? && html.is_a?(String) && !html.empty?
+      cached = PersistentCache.load(html)
+      if cached
+        doc = Document.new(html, base_url: base_url,
+                           build_indexes: build_indexes, native: cached)
+        return doc
+      end
+    end
+    doc = Document.new(html, base_url: base_url, build_indexes: build_indexes)
+    if PersistentCache.enabled? && html.is_a?(String) && !html.empty?
+      PersistentCache.store(html, doc.backing.native) rescue nil
+    end
+    doc
+  end
+
+  # `Scrapetor::HTML(html)` — capital-H convenience method.
+  def self.HTML(html, base_url = nil)
+    parse(html, base_url: base_url)
+  end
+
+  def self.parse_html(html, base_url: nil)
+    parse(html, base_url: base_url)
+  end
+
+  def self.parse_fragment(html, base_url: nil)
+    parse(html, base_url: base_url)
+  end
+
+  # Parse from an arbitrary IO-like (responds to `read`) or a file path.
+  def self.parse_io(io, base_url: nil)
+    parse(io.read, base_url: base_url)
+  end
+
+  def self.parse_file(path, base_url: nil)
+    parse(File.read(path), base_url: base_url)
+  end
+
+  # Parse N documents in parallel via native pthread workers, releasing
+  # the GVL for the duration. Returns Array<Scrapetor::Document> in the
+  # same order as the input. Skips the in-memory parse cache (which is
+  # GVL-bound); use single-document Scrapetor.parse for cache-friendly
+  # workloads.
+  #
+  # Use this for batch jobs over distinct documents where parsing
+  # dominates: pre-warming a fixture corpus, indexing a crawl, A/B
+  # comparing parsed shapes. Falls through to a serial parse when only
+  # one document is provided.
+  def self.parallel_parse(htmls, threads: nil)
+    htmls = Array(htmls)
+    return [] if htmls.empty?
+    return [parse(htmls.first)] if htmls.size == 1
+    n = threads || default_parallel_threads(htmls.size)
+    natives = Native::Document.parallel_parse(htmls, n)
+    natives.each_with_index.map do |native, i|
+      Document.new(htmls[i], native: native)
+    end
+  end
+
+  def self.default_parallel_threads(n_items)
+    cpu = begin
+      require "etc"
+      Etc.nprocessors
+    rescue StandardError
+      4
+    end
+    [n_items, cpu].min
+  end
+
+  # Run an extraction schema directly against a file or IO.
+  def self.extract_file(path, schema, base_url: nil)
+    extract(File.read(path), schema, base_url: base_url)
+  end
+
+  # `Scrapetor::HTML5(html)` — same parser, alternate name.
+  def self.HTML5(*args, &block)
+    parse(*args, &block)
+  end
+
+  # `Scrapetor::HTML.parse` / `.fragment` namespace.
+  module HTML
+    def self.parse(*args, &block)
+      Scrapetor.parse(*args, &block)
+    end
+
+    def self.fragment(*args, &block)
+      Scrapetor.parse_fragment(*args, &block)
+    end
+  end
+
+  module HTML5
+    def self.parse(*args, &block)
+      Scrapetor.parse(*args, &block)
+    end
+
+    def self.fragment(*args, &block)
+      Scrapetor.parse_fragment(*args, &block)
+    end
+  end
+
+  # ----- Extraction DSL -----
+
+  def self.schema(&block)
+    Schema.build(&block)
+  end
+
+  def self.extract(html, schema = nil, base_url: nil, &block)
+    parse(html, base_url: base_url).extract(schema, &block)
+  end
+
+  # Force the native streaming path. Raises if the schema can't compile.
+  def self.extract_native(html, schema, base_url: nil)
+    raise Error, "native extension not loaded" unless Native.available?
+    desc = Native.compile_descriptor(schema)
+    raise Error, "schema not native-compilable" unless desc
+    Native.extract(html.to_s, desc, base_url)
+  end
+
+  # Force the Ruby reference path. Useful for parity tests + benchmarks.
+  def self.extract_ruby(html, schema, base_url: nil)
+    doc = parse(html, base_url: base_url)
+    Extractor.run(doc, doc.backing, schema)
+  end
+end

data/scrapetor.gemspec

@@ -0,0 +1,77 @@
+require_relative "lib/scrapetor/version"
+
+Gem::Specification.new do |spec|
+  spec.name        = "scrapetor"
+  spec.version     = Scrapetor::VERSION
+  spec.authors     = ["Alaa Abdulridha"]
+  spec.email       = ["alaa@serpapi.com"]
+
+  spec.summary     = "Production HTML parser + scraping toolkit. Native arena DOM, HTTP/2 fetch layer, streaming extraction."
+  spec.description =
+    "Scrapetor is a Ruby HTML parsing + scraping toolkit. The parser is a " \
+    "native C arena DOM with structural indexes built at parse time and " \
+    "NEON SIMD scanners in the SAX hot loop. A streaming extraction engine " \
+    "compiles the schema DSL into a single forward pass — no DOM " \
+    "materialised, one Ruby boundary crossing per document. " \
+    "On builds where libcurl is available, Scrapetor::Fetcher adds an " \
+    "HTTP/2-capable fetch layer with per-thread connection cache, shared " \
+    "DNS + TLS session pool, in-process gzip / deflate / brotli / zstd " \
+    "decoding, iconv charset transcoding, retry + exponential backoff, " \
+    "ETag / Last-Modified disk cache with bulk revalidation, per-host " \
+    "throttle, cookie jar, basic + bearer auth, proxy, and three bulk " \
+    "concurrency models (parallel_fetch / multi_fetch / streaming " \
+    "multi_each). Scrapetor::Session ties the cookie / auth / throttle / " \
+    "retry policies together. Also ships robots.txt + sitemap.xml " \
+    "parsers, a bounded-memory streaming HTML parser, and structured-data " \
+    "extractors (JSON-LD, OpenGraph, Schema.org, Microdata, RDFa, Twitter " \
+    "Cards). The Net::HTTP-based Scrapetor.fetch is preserved as the " \
+    "no-libcurl fallback."
+
+  spec.homepage    = "https://scrapetor.org"
+  spec.license     = "MIT"
+  spec.required_ruby_version     = ">= 2.7.0"
+  spec.required_rubygems_version = ">= 3.0.0"
+
+  spec.metadata = {
+    "homepage_uri"          => "http://scrapetor.org",
+    "source_code_uri"       => "https://github.com/Alaa-abdulridha/scrapetor",
+    "bug_tracker_uri"       => "https://github.com/Alaa-abdulridha/scrapetor/issues",
+    "changelog_uri"         => "https://github.com/Alaa-abdulridha/scrapetor/blob/main/CHANGELOG.md",
+    "documentation_uri"     => "http://scrapetor.org/docs",
+    "wiki_uri"              => "https://github.com/Alaa-abdulridha/scrapetor/wiki",
+    "rubygems_mfa_required" => "true"
+  }
+
+  spec.files = Dir[
+    "lib/**/*.rb",
+    "ext/**/*.{rb,c,h}",
+    "ext/**/README.md",
+    "bin/*",
+    "CHANGELOG.md",
+    "LICENSE",
+    "README.md",
+    "scrapetor.gemspec"
+  ]
+
+  spec.bindir        = "bin"
+  spec.executables   = ["scrapetor", "scrapetor-bench"]
+  spec.require_paths = ["lib"]
+  spec.extensions    = ["ext/scrapetor/native/extconf.rb"]
+
+  # No runtime gem dependencies. Scrapetor is self-contained: pure Ruby
+  # plus a single C99 extension. The extension compiles at install time
+  # via the standard mkmf path; only a working C compiler is required.
+
+  spec.add_development_dependency "minitest",      "~> 5.0"
+  spec.add_development_dependency "benchmark-ips", "~> 2.0"
+  spec.add_development_dependency "rake",          "~> 13.0"
+  # webrick was bundled with Ruby 2.7 / earlier; removed from stdlib
+  # in 3.0. The Fetcher + Session test suites spin up local HTTP
+  # servers via it.
+  spec.add_development_dependency "webrick",       "~> 1.7"
+
+  # Comparison oracles used by the benchmark scripts only. Not loaded by
+  # production code.
+  spec.add_development_dependency "nokogiri",      ">= 1.13"
+  spec.add_development_dependency "nokolexbor",    ">= 0.6"
+end

metadata

@@ -0,0 +1,200 @@
+--- !ruby/object:Gem::Specification
+name: scrapetor
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Alaa Abdulridha
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: nokolexbor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.6'
+description: Scrapetor is a Ruby HTML parsing + scraping toolkit. The parser is a
+  native C arena DOM with structural indexes built at parse time and NEON SIMD scanners
+  in the SAX hot loop. A streaming extraction engine compiles the schema DSL into
+  a single forward pass — no DOM materialised, one Ruby boundary crossing per document.
+  On builds where libcurl is available, Scrapetor::Fetcher adds an HTTP/2-capable
+  fetch layer with per-thread connection cache, shared DNS + TLS session pool, in-process
+  gzip / deflate / brotli / zstd decoding, iconv charset transcoding, retry + exponential
+  backoff, ETag / Last-Modified disk cache with bulk revalidation, per-host throttle,
+  cookie jar, basic + bearer auth, proxy, and three bulk concurrency models (parallel_fetch
+  / multi_fetch / streaming multi_each). Scrapetor::Session ties the cookie / auth
+  / throttle / retry policies together. Also ships robots.txt + sitemap.xml parsers,
+  a bounded-memory streaming HTML parser, and structured-data extractors (JSON-LD,
+  OpenGraph, Schema.org, Microdata, RDFa, Twitter Cards). The Net::HTTP-based Scrapetor.fetch
+  is preserved as the no-libcurl fallback.
+email:
+- alaa@serpapi.com
+executables:
+- scrapetor
+- scrapetor-bench
+extensions:
+- ext/scrapetor/native/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- bin/scrapetor
+- bin/scrapetor-bench
+- ext/scrapetor/README.md
+- ext/scrapetor/native/extconf.rb
+- ext/scrapetor/native/scrapetor_dom.c
+- ext/scrapetor/native/scrapetor_http.c
+- ext/scrapetor/native/scrapetor_native.c
+- lib/scrapetor.rb
+- lib/scrapetor/builder.rb
+- lib/scrapetor/cleaner.rb
+- lib/scrapetor/comment_node.rb
+- lib/scrapetor/document.rb
+- lib/scrapetor/dom.rb
+- lib/scrapetor/dom/parser.rb
+- lib/scrapetor/dom/selectors.rb
+- lib/scrapetor/encoding.rb
+- lib/scrapetor/entities.rb
+- lib/scrapetor/errors.rb
+- lib/scrapetor/extractor.rb
+- lib/scrapetor/fetcher.rb
+- lib/scrapetor/fingerprint.rb
+- lib/scrapetor/form.rb
+- lib/scrapetor/http.rb
+- lib/scrapetor/microdata.rb
+- lib/scrapetor/money.rb
+- lib/scrapetor/native.rb
+- lib/scrapetor/native_dom.rb
+- lib/scrapetor/node.rb
+- lib/scrapetor/node_set.rb
+- lib/scrapetor/page_type.rb
+- lib/scrapetor/pagination.rb
+- lib/scrapetor/persistent_cache.rb
+- lib/scrapetor/robots.rb
+- lib/scrapetor/sax.rb
+- lib/scrapetor/schema.rb
+- lib/scrapetor/selector.rb
+- lib/scrapetor/session.rb
+- lib/scrapetor/sitemap.rb
+- lib/scrapetor/stream.rb
+- lib/scrapetor/structured_data.rb
+- lib/scrapetor/template_registry.rb
+- lib/scrapetor/text_node.rb
+- lib/scrapetor/url.rb
+- lib/scrapetor/version.rb
+- lib/scrapetor/xpath.rb
+- scrapetor.gemspec
+homepage: https://scrapetor.org
+licenses:
+- MIT
+metadata:
+  homepage_uri: http://scrapetor.org
+  source_code_uri: https://github.com/Alaa-abdulridha/scrapetor
+  bug_tracker_uri: https://github.com/Alaa-abdulridha/scrapetor/issues
+  changelog_uri: https://github.com/Alaa-abdulridha/scrapetor/blob/main/CHANGELOG.md
+  documentation_uri: http://scrapetor.org/docs
+  wiki_uri: https://github.com/Alaa-abdulridha/scrapetor/wiki
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Production HTML parser + scraping toolkit. Native arena DOM, HTTP/2 fetch
+  layer, streaming extraction.
+test_files: []