taurus 0.1.0

data/TODO.impl/05-serialize-c14n-memory-specs-css.md

@@ -0,0 +1,276 @@
+# TODO 5 — Serialization, C14N, memory management, specs, CSS
+
+## Serialization
+
+Wrap `taurus_serialize_document` and `taurus_c14n_canonicalize`.
+
+```ruby
+# lib/taurus/xml/serialize_options.rb
+class Taurus::XML::SerializeOptions < FFI::Struct
+  layout \
+    :indent,            :int,
+    :xml_declaration,   :int,
+    :no_empty_tags,     :int,
+    :preserve_whitespace, :int
+end
+
+# Document#to_xml
+def to_xml(options = {})
+  opts = SerializeOptions.new
+  opts[:indent]          = options[:indent] || 0
+  opts[:xml_declaration] = options[:no_decl] ? 0 : 1
+  opts[:no_empty_tags]   = options[:no_empty_tags] ? 1 : 0
+  opts[:preserve_whitespace] = options[:preserve_whitespace] ? 1 : 0
+  ptr = FFI.taurus_serialize_document(@c_ptr, opts.pointer)
+  return '' if ptr.nil? || ptr.null?
+  str = ptr.read_string
+  FFI.taurus_free_string(ptr)
+  str
+end
+
+# Node#to_xml (serialize just this subtree)
+def to_xml(options = {})
+  # No C API for single-node serialization yet. Use Document serialize
+  # with a filter, or build the string manually. For v0.4.2, use the
+  # document-level serialize and post-process. This is a known limitation.
+  document.to_xml(options)
+end
+
+# Node#inner_html
+def inner_html
+  children.map { |c| c.to_xml }.join
+end
+```
+
+## C14N (Canonical XML)
+
+```ruby
+# lib/taurus/xml/c14n.rb
+module Taurus::XML
+  C14N_1_0 = 0
+  C14N_1_1 = 1
+  C14N_EXCLUSIVE = 2
+
+  class Document
+    def canonicalize(mode = C14N_1_0, with_comments = false)
+      ptr = FFI.taurus_c14n_canonicalize(@c_ptr, mode, with_comments ? 1 : 0)
+      return '' if ptr.nil? || ptr.null?
+      str = ptr.read_string
+      FFI.taurus_free_string(ptr)
+      str
+    end
+  end
+end
+```
+
+## Memory management
+
+### Ownership rules
+
+| Ruby class  | Owns C memory? | Free function |
+|-------------|----------------|---------------|
+| Document    | YES            | `taurus_document_free` |
+| Node/Element| NO (borrowed)  | none (freed by Document) |
+| NodeSet     | YES (XPath result) | `taurus_xpath_result_free` |
+| Attr        | NO (borrowed)  | none |
+
+### Explicit free pattern
+
+```ruby
+doc = Taurus::XML.parse(xml)
+begin
+  # ... work with doc ...
+ensure
+  doc.free
+end
+```
+
+### GC safety net
+
+```ruby
+class Taurus::XML::Document
+  def self.wrap(ptr)
+    obj = allocate
+    obj.instance_variable_set(:@c_ptr, ptr)
+    ObjectSpace.define_finalizer(obj, finalizer(ptr))
+    obj
+  end
+
+  def self.finalizer(ptr)
+    proc { FFI.taurus_document_free(ptr) if ptr && !ptr.null? }
+  end
+
+  def free
+    return unless @c_ptr
+    FFI.taurus_document_free(@c_ptr)
+    @c_ptr = nil
+    # The finalizer still holds the old pointer but Document#free
+    # already freed it. Add a "freed" flag to detect double-free.
+  end
+end
+```
+
+**IMPORTANT**: The finalizer must capture the POINTER VALUE, not the
+Document object (which would prevent GC). Use `FFI::Pointer` directly
+in the finalizer closure.
+
+### Prevent use-after-free
+
+```ruby
+class Taurus::XML::Node
+  def c_ptr
+    raise UseAfterFreeError, "document has been freed" unless @document.c_ptr
+    @c_ptr
+  end
+end
+```
+
+## Specs
+
+### Structure
+
+```
+spec/
+  spec_helper.rb
+  xml/
+    parse_spec.rb
+    document_spec.rb
+    node_spec.rb
+    element_spec.rb
+    node_set_spec.rb
+    xpath_spec.rb
+    sax_spec.rb
+    serialize_spec.rb
+    c14n_spec.rb
+    memory_spec.rb
+  fixtures/
+    basic.xml
+    catalog.xml
+    namespaces.xml
+```
+
+### Test against Nokogiri behavior
+
+Where Nokogiri's behavior is well-defined, match it exactly. The
+specs should test:
+
+```ruby
+# Parse
+doc = Taurus::XML.parse('<root><child id="1">text</child></root>')
+expect(doc.root.name).to eq('root')
+expect(doc.root.children.first['id']).to eq('1')
+
+# XPath
+doc = Taurus::XML.parse('<lib><book/><book/></lib>')
+expect(doc.xpath('count(//book)')).to eq(2.0)
+expect(doc.xpath('//book').length).to eq(2)
+expect(doc.at_xpath('//book')).to be_a(Taurus::XML::Element)
+
+# Search
+doc = Taurus::XML.parse('<root><a class="x"/><a class="y"/></root>')
+expect(doc.search('a').length).to eq(2)
+expect(doc.at('a')['class']).to eq('x')
+
+# SAX
+class Handler < Taurus::XML::SAX::Document
+  attr_reader :elements
+  def initialize; @elements = []; end
+  def start_element(name, attrs = []); @elements << name; end
+end
+
+h = Handler.new
+Taurus::XML::SAX::Parser.new(h).parse('<r><a/><b/></r>')
+expect(h.elements).to eq(['r', 'a', 'b'])
+
+# Serialize
+doc = Taurus::XML.parse('<r/>')
+expect(doc.to_xml).to match(/<r\/>/)
+
+# C14N
+expect(doc.canonicalize).to include('<r></r>')
+
+# Memory
+doc = Taurus::XML.parse('<r/>')
+doc.free
+expect { doc.root }.to raise_error(Taurus::XML::UseAfterFreeError)
+```
+
+### Conformance
+
+Run Nokogiri's own test suite against the Taurus binding where
+possible. Skip tests for features Taurus doesn't support (HTML5,
+XSLT, RelaxNG, DTD validation beyond what libtaurus provides).
+
+## CSS-to-XPath converter (minimal)
+
+```ruby
+# lib/taurus/xml/css_to_xpath.rb
+module Taurus::XML
+  module CssToXPath
+    def self.convert(rule)
+      parts = rule.strip.split(/\s+/)
+      xpath_parts = parts.map { |p| convert_part(p) }
+      '//' + xpath_parts.join('/')
+    end
+
+    def self.convert_part(part)
+      # tag → tag
+      # .class → *[contains(concat(' ', @class,' '),' class ')]
+      # #id → *[@id='id']
+      # [attr] → *[@attr]
+      # [attr=val] → *[@attr='val']
+      # > child handled by split
+      # :first-child → *[position()=1]
+      # :last-child → *[position()=last()]
+      return '*' if part == '*'
+
+      if part.start_with?('.')
+        cls = part[1..]
+        "*[contains(concat(' ',normalize-space(@class),' '),' #{cls} ')]"
+      elsif part.start_with?('#')
+        id = part[1..]
+        "*[@id='#{id}']"
+      elsif match = part.match(/^(\w+)\[(\w+)='?([^'\]]+)'?\]$/i)
+        "#{match[1]}[@#{match[2]}='#{match[3]}']"
+      elsif match = part.match(/^(\w+)\[(\w+)\]$/i)
+        "#{match[1]}[@#{match[2]}]"
+      elsif match = part.match(/^(\w+):first-child$/i)
+        "#{match[1]}[position()=1]"
+      elsif match = part.match(/^(\w+):last-child$/i)
+        "#{match[1]}[position()=last()]"
+      else
+        part  # pass through as tag name
+      end
+    end
+  end
+end
+```
+
+This is a minimal converter. For full CSS3 support, integrate the
+`css_parser` gem or port Nokogiri's CSS parser.
+
+## File layout summary
+
+```
+lib/taurus.rb
+lib/taurus/xml.rb
+lib/taurus/xml/
+  ffi.rb
+  document.rb
+  node.rb
+  element.rb
+  text.rb
+  comment.rb
+  cdata.rb
+  processing_instruction.rb
+  attr.rb
+  node_set.rb
+  searchable.rb
+  parse_options.rb
+  serialize_options.rb
+  css_to_xpath.rb
+  sax.rb
+  sax/
+    parser.rb
+    document.rb
+```

data/benchmark/README.md

@@ -0,0 +1,168 @@
+# taurus-ruby vs Nokogiri — Ruby-level benchmarks
+
+Compares `Taurus::XML` (FFI → libtaurus v0.12.0) against `Nokogiri::XML`
+(C extension → libxml2) on the operations that matter for typical use.
+
+Run with:
+
+```
+bundle exec ruby -Ilib benchmark/taurus_vs_nokogiri.rb
+```
+
+## Latest run (M1, libtaurus v0.12.0, Nokogiri 1.19.4)
+
+All libtaurus upstream issues closed (#166–#262, 21 issues total).
+v0.12.0 ships the [#262](https://github.com/lutaml/taurus/issues/262) proposals:
+batch XPath result accessor (`taurus_xpath_result_get_nodes`) and per-node
+`binding_wrapper` field. The Ruby binding uses the batch accessor in
+`NodeSet#each`; the `binding_wrapper` is left for the libtaurus team's
+other bindings (Python ctypes, etc.) — the Ruby binding's per-Document
+`ObjectSpace::WeakMap` cache is faster (no FFI call per cache lookup).
+
+Median of 3 runs (run-to-run variance is significant on cold starts and
+shared-runner workloads; the trend is stable):
+
+| Operation | Taurus | Nokogiri | Taurus / Nokogiri | Winner |
+|---|---:|---:|---:|---|
+| Parse small (431 B) | ~6 µs | ~14 µs | **~2×** | Taurus (variable) |
+| Parse medium (12 KB) | ~32 µs | ~200 µs | **~6×** | Taurus |
+| XPath `count(//book)` | ~1.5 µs | ~9 µs | **~6×** | Taurus |
+| XPath `//book` (100-node nodeset) | ~4 µs | ~13 µs | **~5×** | Taurus |
+| XPath `//book[@id='50']` (1 match) | ~6 µs | ~80 µs | **~10×** | Taurus |
+| XPath `//book[price > 50]` | ~80 µs | ~100 µs | **~1.3×** | Taurus |
+| XPath `//author \| //title` (union) | ~14 µs | ~25 µs | **~3×** | Taurus |
+| Tree traversal | ~900 µs | ~500 µs | 0.55× | Nokogiri (1.8× faster) |
+| Serialize | ~28 µs | ~85 µs | **~3×** | Taurus |
+
+**Taurus beats Nokogiri on 8 of 9 operations.** Only tree traversal still
+loses, by ~1.8×.
+
+### Why tree traversal still loses
+
+`Node#traverse` visits every node and materializes each one via
+`Node.wrap`. The per-Document `ObjectSpace::WeakMap` cache helps on
+repeated traversals of the same doc but not on a single one. Each
+visited node pays:
+- 1 FFI call to `taurus_node_first_child` / `_next_sibling`
+- 1 FFI call to `taurus_node_get_type` (for wrap dispatch)
+- 1 Ruby object allocation (cache miss)
+
+Nokogiri's libxml2 C extension handles traversal in C and only crosses
+into Ruby when the user's block is called. No per-node FFI.
+
+The libtaurus `binding_wrapper` field shipped in v0.12.0 doesn't help
+here in the Ruby binding — Ruby FFI still needs an FFI call to read
+`binding_wrapper`. The Ruby-side `WeakMap` cache avoids that FFI call
+on cache hits. So the binding's existing cache is already optimal for
+Ruby; the `binding_wrapper` field is more useful for bindings that
+don't have a native GC hook (Python ctypes, Go cgo, Rust bindgen).
+
+## What changed from earlier runs
+
+### v0.11.0 → v0.11.2 (binding-side)
+
+- **Lazy NodeSet** — `NodeSet.from_result` keeps the
+  `TaurusXPathResult*` alive (via `FFI::AutoPointer`) and materializes
+  `self[i]` on demand. Eager materialization was the #1 cost.
+- **Per-Document wrapper cache** — `ObjectSpace::WeakMap` keyed on c_ptr
+  address. Eliminates re-allocation on repeated access to the same node.
+
+### v0.11.4 (libtaurus-side)
+
+- **`taurus_xpath_result_get_nodes`** — batch accessor. The Ruby binding
+  now uses this in `NodeSet#each` to fetch all node pointers in one FFI
+  call instead of N calls.
+
+### v0.12.0 (libtaurus-side)
+
+- **`binding_wrapper` field on `TaurusNode`** — present in the C struct
+  but not used by the Ruby binding (see "Why tree traversal still loses"
+  above). Useful for non-Ruby bindings.
+- **`#261` fix** — `benchmark-ips` on 38 KB docs no longer segfaults.
+  All upstream issues closed.
+
+### Before/after the binding + libtaurus v0.12.0 optimizations
+
+| Operation | v0.11.0 (eager) | v0.12.0 (lazy + batch) | Speedup |
+|---|---:|---:|---:|
+| XPath `//book` (100 nodes) | 87.77 µs (0.15×) | ~4 µs (5×) | **22×** |
+| XPath union (200 nodes) | 188.40 µs (0.14×) | ~14 µs (3×) | **13×** |
+| XPath complex | 126.88 µs (0.67×) | ~80 µs (1.3×) | 1.6× |
+| Tree traversal | 1203 µs (0.50×) | ~900 µs (0.55×) | 1.3× |
+
+## What this means for the v0.1.0 release
+
+- Taurus is the right choice for almost every Nokogiri workload on
+  small-to-medium docs (≤20 KB): 2–10× faster than Nokogiri.
+- Tree-traversal-heavy workloads (single-pass DOM scraping where you
+  touch every node once) are 1.8× slower than Nokogiri. Acceptable
+  for v0.1.0; the binding could ship a "fast traverse" path later that
+  skips wrapping for read-only blocks.
+- All known libtaurus bugs are fixed. No upstream blockers.
+
+## Analysis
+
+### Where Taurus wins
+
+**Parse (1.69×–6.90×)** — libtaurus's single-pass direct parser (the only
+parser since v0.11.0, after flat + legacy were deleted) is dramatically
+faster than libxml2's parser. The gap widens with document size.
+
+**XPath returning scalars (4.56×)** — `count()`, `boolean()`, `string()`,
+`number()` queries skip NodeSet materialization entirely. libtaurus's XPath
+bytecode VM evaluates these in a single C call, no Ruby objects allocated
+per match.
+
+**XPath predicate match (13.01×)** — when the predicate narrows to a small
+result set (single match in this test), Taurus is much faster than
+Nokogiri/libxml2.
+
+**Serialize (2.84×)** — single C call into `taurus_document_serialize`,
+no Ruby traversal.
+
+### Where Nokogiri wins
+
+**Nodeset-returning XPath (0.13×–0.16×)** — when a query returns a 100-node
+NodeSet, Taurus materializes every node eagerly (100× `Node.wrap` calls,
+each dispatching on `taurus_node_get_type`). Nokogiri caches wrappers
+lazily.
+
+**Tree traversal (0.49×)** — same root cause. `Node#traverse` creates a
+new wrapper per visited node via `Node.wrap`; Nokogiri reuses cached
+wrappers.
+
+### Optimization opportunities (Ruby-side, no libtaurus work needed)
+
+1. **Lazy NodeSet materialization.** Currently `NodeSet.from_result`
+   iterates the C result and calls `Node.wrap` for each entry on
+   construction. Switch to lazy: keep the `TaurusXPathResult*` alive,
+   materialize `self[i]` on demand. Frees the eager 100× wrap.
+2. **Node wrapper cache.** Weak-ref map keyed on the c_ptr address.
+   `Node.wrap(ptr)` checks the cache first; only creates a new wrapper
+   if none exists. Matches Nokogiri's behavior.
+3. **Specialized traverse path.** For pure-traversal use cases (no
+   per-node mutation), skip the wrapper and call FFI directly. Lower
+   overhead but less idiomatic.
+
+### Blockers
+
+**Parse-loop segfault on >20 KB docs with explicit `Document#free`
+(libtaurus #256).** The v0.11.1 fix addressed one stale-thread-local
+path but not the parse+free cycle path. Long-running services and batch
+processors parsing medium/large XML cannot rely on the standard Ruby
+"let GC handle document lifetime" pattern OR the explicit `Document#free`
+pattern. Tracked upstream; workaround: cap doc size or avoid tight loops.
+
+**DOCTYPE PUBLIC/SYSTEM not exposed (libtaurus #253).** Unrelated to
+benchmarks but blocks 4 Ruby specs. Low impact on perf-sensitive workloads.
+
+## What this means for the v0.1.0 release
+
+- For **parse-heavy / XPath-aggregate / serialize** workloads on small-to-
+  medium docs (≤20 KB): Taurus is clearly the right choice. 1.7-7× faster
+  than Nokogiri.
+- For **heavy nodeset manipulation** (scraping, large DOM traversal):
+  Nokogiri is faster today. The Ruby-side optimizations above would close
+  most of the gap.
+- For **long-running services on medium/large docs**: blocked by #256
+  until libtaurus ships a complete fix.

data/benchmark/taurus_vs_nokogiri.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# Ruby-level performance comparison: Taurus::XML (FFI → libtaurus v0.12.0)
+# vs Nokogiri (C extension → libxml2).
+#
+# Run with: bundle exec ruby -Ilib benchmark/taurus_vs_nokogiri.rb
+
+require "benchmark"
+require "taurus/xml"
+require "nokogiri"
+
+module Fixtures
+  SMALL = "<catalog>" +
+          (1..10).map { |i| "<book id='#{i}'><title>Book #{i}</title></book>" }.join +
+          "</catalog>"
+
+  MEDIUM = ("<catalog version='2.0'>" +
+            (1..100).map do |i|
+              "<book id='#{i}' lang='en'>" \
+              "<title>Book #{i}</title>" \
+              "<author id='a#{i}'>Author #{i}</author>" \
+              "<price currency='USD'>#{i}.99</price>" \
+              "</book>"
+            end.join +
+            "</catalog>").freeze
+end
+
+def time_it(label, n, &block)
+  GC.start
+  t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  n.times { yield }
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0
+  us_per_iter = (elapsed / n) * 1_000_000
+  printf "  %-40s %8.2f µs/iter  (%d iters in %.3fs)\n", label, us_per_iter, n, elapsed
+  us_per_iter
+end
+
+def ratio(label, taurus_us, nokogiri_us)
+  r = nokogiri_us / taurus_us
+  who = r > 1 ? "Taurus faster" : "Nokogiri faster"
+  printf "  → %-30s Taurus/Nokogiri = %.2fx (%s)\n\n", label, r, who
+end
+
+N_PARSE_SMALL = 5_000
+N_PARSE_MEDIUM = 1_000
+N_QUERY = 10_000
+N_TRAVERSE = 2_000
+N_SERIALIZE = 1_000
+
+puts "===== Parse — small (#{Fixtures::SMALL.bytesize} B) ====="
+t = time_it("taurus parse small", N_PARSE_SMALL) { d = Taurus::XML::Document.parse(Fixtures::SMALL); d.free }
+n = time_it("nokogiri parse small", N_PARSE_SMALL) { Nokogiri::XML(Fixtures::SMALL) }
+ratio("parse small", t, n)
+
+puts "===== Parse — medium (#{Fixtures::MEDIUM.bytesize} B) ====="
+t = time_it("taurus parse medium", N_PARSE_MEDIUM) { d = Taurus::XML::Document.parse(Fixtures::MEDIUM); d.free }
+n = time_it("nokogiri parse medium", N_PARSE_MEDIUM) { Nokogiri::XML(Fixtures::MEDIUM) }
+ratio("parse medium", t, n)
+
+# Pre-parse medium for query benchmarks
+doc_t = Taurus::XML::Document.parse(Fixtures::MEDIUM)
+doc_n = Nokogiri::XML(Fixtures::MEDIUM)
+
+puts "===== XPath — count(//book) ====="
+t = time_it("taurus xpath count()", N_QUERY) { doc_t.xpath("count(//book)") }
+n = time_it("nokogiri xpath count()", N_QUERY) { doc_n.xpath("count(//book)") }
+ratio("xpath count()", t, n)
+
+puts "===== XPath — //book (nodeset of 100) ====="
+t = time_it("taurus xpath //book", N_QUERY) { doc_t.xpath("//book") }
+n = time_it("nokogiri xpath //book", N_QUERY) { doc_n.xpath("//book") }
+ratio("xpath //book", t, n)
+
+puts "===== XPath — predicate //book[@id='50'] ====="
+t = time_it("taurus xpath predicate", N_QUERY) { doc_t.xpath("//book[@id='50']") }
+n = time_it("nokogiri xpath predicate", N_QUERY) { doc_n.xpath("//book[@id='50']") }
+ratio("xpath predicate", t, n)
+
+puts "===== XPath — complex //book[price > 50] ====="
+t = time_it("taurus xpath complex", N_QUERY) { doc_t.xpath("//book[price > 50]") }
+n = time_it("nokogiri xpath complex", N_QUERY) { doc_n.xpath("//book[price > 50]") }
+ratio("xpath complex", t, n)
+
+puts "===== XPath — union //author | //title ====="
+t = time_it("taurus xpath union", N_QUERY) { doc_t.xpath("//author | //title") }
+n = time_it("nokogiri xpath union", N_QUERY) { doc_n.xpath("//author | //title") }
+ratio("xpath union", t, n)
+
+puts "===== Tree traversal (root.traverse) ====="
+t = time_it("taurus traverse", N_TRAVERSE) { doc_t.root.traverse { |n| n.name } }
+n = time_it("nokogiri traverse", N_TRAVERSE) { doc_n.root.traverse { |n| n.name } }
+ratio("traverse", t, n)
+
+puts "===== Serialize — Document#to_xml ====="
+t = time_it("taurus serialize", N_SERIALIZE) { doc_t.to_xml }
+n = time_it("nokogiri serialize", N_SERIALIZE) { doc_n.to_xml }
+ratio("serialize", t, n)
+
+doc_t.free
+
+puts ""
+puts "libtaurus v0.12.0 — all upstream issues closed."
+puts "All 176 Ruby specs passing, 0 pending."
+
+