RubyGems - makiri - Versions diffs - 0.2.0 → 0.4.0 - Mend

makiri 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (130) hide show

checksums.yaml +4 -4
data/.github/workflows/conformance.yml +22 -0
data/.github/workflows/libfuzzer.yml +83 -0
data/.github/workflows/release.yml +12 -7
data/.github/workflows/security.yml +88 -3
data/.github/workflows/valgrind.yml +135 -0
data/CHANGELOG.md +152 -15
data/README.md +183 -13
data/Rakefile +294 -7
data/ext/makiri/bridge/bridge.h +28 -0
data/ext/makiri/bridge/ruby_string.c +282 -12
data/ext/makiri/core/mkr_alloc.c +40 -3
data/ext/makiri/core/mkr_alloc.h +28 -5
data/ext/makiri/core/mkr_buf.c +47 -3
data/ext/makiri/core/mkr_buf.h +112 -3
data/ext/makiri/core/mkr_core.c +143 -0
data/ext/makiri/core/mkr_core.h +11 -2
data/ext/makiri/core/mkr_hash.h +1 -1
data/ext/makiri/core/mkr_span.h +186 -0
data/ext/makiri/core/mkr_text.h +8 -8
data/ext/makiri/core/mkr_utf8.c +101 -0
data/ext/makiri/core/mkr_utf8.h +88 -0
data/ext/makiri/extconf.rb +123 -10
data/ext/makiri/fuzz/Makefile +95 -0
data/ext/makiri/fuzz/check_fuzzer.cc +4 -0
data/ext/makiri/fuzz/xml_fuzz.c +24 -0
data/ext/makiri/fuzz/xpath_fuzz.c +109 -0
data/ext/makiri/glue/glue.h +55 -11
data/ext/makiri/glue/ruby_doc.c +129 -59
data/ext/makiri/glue/ruby_html_css.c +292 -0
data/ext/makiri/glue/{ruby_mutate.c → ruby_html_mutate.c} +248 -52
data/ext/makiri/glue/ruby_html_node.c +859 -0
data/ext/makiri/glue/ruby_html_serialize.c +154 -0
data/ext/makiri/glue/ruby_node.c +74 -729
data/ext/makiri/glue/ruby_node_set.c +167 -32
data/ext/makiri/glue/ruby_xml.c +602 -0
data/ext/makiri/glue/ruby_xml_node.c +1373 -0
data/ext/makiri/glue/ruby_xpath.c +63 -30
data/ext/makiri/glue/ruby_xpath.h +19 -0
data/ext/makiri/lexbor_compat/compat.h +42 -9
data/ext/makiri/lexbor_compat/compat_internal.h +1 -1
data/ext/makiri/lexbor_compat/dom_index.c +2 -2
data/ext/makiri/lexbor_compat/post_parse.c +100 -10
data/ext/makiri/lexbor_compat/source_loc.c +15 -13
data/ext/makiri/lexbor_compat/text_index.c +14 -8
data/ext/makiri/lexbor_compat/utf8_input.c +19 -33
data/ext/makiri/makiri.c +184 -6
data/ext/makiri/makiri.h +43 -2
data/ext/makiri/xml/mkr_xml.h +125 -0
data/ext/makiri/xml/mkr_xml_chars.c +195 -0
data/ext/makiri/xml/mkr_xml_index.c +169 -0
data/ext/makiri/xml/mkr_xml_index.h +48 -0
data/ext/makiri/xml/mkr_xml_mutate.c +817 -0
data/ext/makiri/xml/mkr_xml_mutate.h +139 -0
data/ext/makiri/xml/mkr_xml_node.c +399 -0
data/ext/makiri/xml/mkr_xml_node.h +184 -0
data/ext/makiri/xml/mkr_xml_tree.c +1515 -0
data/ext/makiri/xpath/mkr_css.c +1023 -0
data/ext/makiri/xpath/mkr_css.h +65 -0
data/ext/makiri/xpath/mkr_xpath.c +96 -32
data/ext/makiri/xpath/mkr_xpath.h +109 -4
data/ext/makiri/xpath/mkr_xpath_engine_html.c +17 -0
data/ext/makiri/xpath/mkr_xpath_engine_xml.c +12 -0
data/ext/makiri/xpath/{mkr_xpath_eval.c → mkr_xpath_eval_body.h} +551 -241
data/ext/makiri/xpath/{mkr_xpath_funcs.c → mkr_xpath_funcs_body.h} +318 -276
data/ext/makiri/xpath/mkr_xpath_internal.h +177 -206
data/ext/makiri/xpath/mkr_xpath_lex.c +95 -125
data/ext/makiri/xpath/mkr_xpath_node_access_html.h +138 -0
data/ext/makiri/xpath/mkr_xpath_node_access_xml.h +145 -0
data/ext/makiri/xpath/mkr_xpath_number.c +109 -0
data/ext/makiri/xpath/mkr_xpath_parse.c +83 -94
data/ext/makiri/xpath/mkr_xpath_prelude_html.h +30 -0
data/ext/makiri/xpath/mkr_xpath_prelude_xml.h +28 -0
data/ext/makiri/xpath/mkr_xpath_shared.c +609 -0
data/ext/makiri/xpath/mkr_xpath_value_body.h +801 -0
data/ext/makiri/xpath/mkr_xpath_xml_selftest.c +76 -0
data/lib/makiri/{attribute.rb → attr.rb} +7 -3
data/lib/makiri/cdata_section.rb +19 -0
data/lib/makiri/comment.rb +10 -0
data/lib/makiri/compat_aliases.rb +30 -0
data/lib/makiri/document.rb +9 -73
data/lib/makiri/document_fragment.rb +14 -9
data/lib/makiri/element.rb +4 -4
data/lib/makiri/html/document.rb +106 -0
data/lib/makiri/html/node_methods.rb +19 -0
data/lib/makiri/html.rb +12 -0
data/lib/makiri/node.rb +58 -15
data/lib/makiri/node_set.rb +8 -0
data/lib/makiri/processing_instruction.rb +10 -0
data/lib/makiri/text.rb +1 -1
data/lib/makiri/version.rb +1 -1
data/lib/makiri/xml/builder.rb +263 -0
data/lib/makiri/xml/document.rb +24 -0
data/lib/makiri/xml/node_methods.rb +84 -0
data/lib/makiri/xml.rb +10 -0
data/lib/makiri/xpath_context.rb +1 -1
data/lib/makiri.rb +24 -5
data/script/build_native_gem.rb +2 -2
data/script/check_alloc_failures.rb +266 -0
data/script/check_c_safety.rb +77 -2
data/script/check_c_safety_allowlist.yml +102 -0
data/script/check_leaks.rb +64 -0
data/script/leaks_harness.rb +64 -0
data/vendor/lexbor/CMakeLists.txt +6 -0
data/vendor/lexbor/README.md +12 -0
data/vendor/lexbor/config.cmake +1 -1
data/vendor/lexbor/source/lexbor/core/base.h +1 -1
data/vendor/lexbor/source/lexbor/core/config.cmake +9 -1
data/vendor/lexbor/source/lexbor/css/selectors/pseudo_state.c +2 -3
data/vendor/lexbor/source/lexbor/css/selectors/state.c +3 -0
data/vendor/lexbor/source/lexbor/dom/interfaces/element.c +21 -0
data/vendor/lexbor/source/lexbor/dom/interfaces/element.h +5 -0
data/vendor/lexbor/source/lexbor/encoding/decode.c +33 -4
data/vendor/lexbor/source/lexbor/html/base.h +1 -1
data/vendor/lexbor/source/lexbor/html/interfaces/select_element.c +4 -0
data/vendor/lexbor/source/lexbor/html/serialize.c +545 -41
data/vendor/lexbor/source/lexbor/html/serialize.h +2 -1
data/vendor/lexbor/source/lexbor/html/tokenizer.h +2 -2
data/vendor/lexbor/source/lexbor/html/tree/insertion_mode/in_body.c +1 -1
data/vendor/lexbor/source/lexbor/html/tree.c +6 -6
data/vendor/lexbor/source/lexbor/selectors/selectors.c +12 -3
data/vendor/lexbor/source/lexbor/url/base.h +1 -1
data/vendor/lexbor/source/lexbor/url/url.c +5 -2
data/vendor/lexbor/source/lexbor/url/url.h +9 -0
data/vendor/lexbor/version +1 -1
metadata +53 -9
data/ext/makiri/glue/ruby_css.c +0 -185
data/ext/makiri/glue/ruby_serialize.c +0 -92
data/ext/makiri/xpath/mkr_xpath_value.c +0 -1286
data/lib/makiri/cdata.rb +0 -6

data/README.md CHANGED Viewed

@@ -1,7 +1,9 @@
 # Makiri
-Standards-oriented HTML5 parsing, CSS selector querying, and XPath 1.0
-querying for Ruby, powered by Lexbor and a native XPath engine.
+Makiri is a Ruby library for parsing and querying HTML and XML documents.
+It uses [Lexbor](https://lexbor.com/) for HTML parsing and CSS selector matching, and includes a built-in native XPath 1.0 engine and XML 1.0 parser.
+Makiri does not depend on libxml2.
 > [!WARNING]
 > Status: early release. APIs and behavior may change before v1.0.
@@ -13,13 +15,17 @@ XPath 1.0 evaluation in its own native engine, with no libxml2 dependency.
 * HTML5 parsing via [Lexbor](https://lexbor.com)
   * Makiri uses Lexbor as the parsing backend and provides a Ruby-facing DOM/query layer.
-  * Lexbor-specific behavior is isolated in a thin compatibility layer
-    (`ext/makiri/lexbor_compat/`).
 * CSS selector support via Lexbor
   * Supports Lexbor-backed standard CSS selector querying, including `:is`/`:where`/`:has`
 * Native XPath 1.0 engine
   * XPath is parsed and evaluated by Makiri's own engine, written from scratch.
   * Makiri does not depend on libxml2 for parsing, DOM representation, or XPath evaluation.
+* Native XML 1.0 parser
+  * A strict, non-validating, fail-closed parser with its own node arena (not
+    Lexbor's HTML DOM), queried through the same native XPath engine, with
+    in-place tree edits (attributes, content, rename, remove).
+  * Conformance is held by the W3C XML Conformance Test Suite, an XPath
+    differential, and property-based testing vs Nokogiri (see below).
 * Bounded, fail-closed execution
   * XPath evaluation is bounded by per-evaluation limits on work, memory, and recursion.
   * Ownership and borrowing are kept explicit across layers, with owned/borrowed
@@ -46,7 +52,7 @@ HTML
 doc.css("a").map { |a| a["href"] }      # => ["/a", "/b"]
 doc.at_css("p.lead").text               # => "Hello"
-# XPath 1.0 (native engine — no libxml2)
+# XPath 1.0 (native engine - no libxml2)
 doc.xpath("//a").length                 # => 2
 doc.xpath("count(//a)")                 # => 2.0
 doc.at_xpath('//*[@id="main"]/p').text  # => "Hello"
@@ -72,16 +78,120 @@ ctx.register_variable("cls", "lead")
 ctx.evaluate('//p[@class=$cls]').first.text   # => "Hello"
 ```
+### XML (with in-place editing)
+```ruby
+doc = Makiri::XML(<<~XML)
+  <feed xmlns="http://www.w3.org/2005/Atom">
+    <entry><title>Hello</title></entry>
+    <entry><title>World</title></entry>
+  </feed>
+XML
+# Namespace matching is strict, so a default namespace needs a registered prefix.
+ns = { "a" => "http://www.w3.org/2005/Atom" }
+doc.xpath("//entry").length                    # => 0  (default namespace)
+doc.xpath("//a:entry", ns).length              # => 2
+doc.at_xpath("//a:entry/a:title", ns).text     # => "Hello"
+# Or reuse a context (caches registrations + compiled expressions):
+ctx = Makiri::XPathContext.new(doc.root)
+ctx.register_namespace("a", "http://www.w3.org/2005/Atom")
+ctx.evaluate("//a:entry").length               # => 2
+el = doc.at_xpath("//a:entry", ns)
+el.local_name                                  # => "entry"
+el.namespace_uri                               # => "http://www.w3.org/2005/Atom"
+# CSS selectors work too (lowered to the native XPath engine): a bare type
+# selector binds to the document's default namespace, so this just works.
+doc.css("entry").length                        # => 2
+doc.css("feed > entry").map { |e| e.at_css("title").text }  # => ["Hello", "World"]
+# Serialize back to XML
+doc.to_xml                                 # => "<?xml version=\"1.0\"?>\n<feed ...>...</feed>\n"
+doc.at_xpath("//a:entry", ns).to_xml       # => "<entry><title>Hello</title></entry>" (no declaration)
+doc.to_xml(pretty: true)                   # indented, element-only content
+# DOCTYPE is recognized but the DTD is not processed (no entities, no I/O):
+dtd = Makiri::XML(%(<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0//EN" "x.dtd"><html/>))
+        .internal_subset
+dtd.name         # => "html"
+dtd.external_id  # => "-//W3C//DTD XHTML 1.0//EN"  (alias: #public_id)
+dtd.system_id    # => "x.dtd"
+```
+The tree supports in-place mutation.
+```ruby
+doc = Makiri::XML(%(<feed xmlns:dc="urn:dc"><entry id="1">Hi</entry><draft/></feed>))
+e   = doc.at_xpath("//entry")
+e["id"]   = "9"            # add or replace an attribute (value escaped on output)
+e["dc:k"] = "v"           # a prefixed name resolves against the in-scope xmlns
+e.content = "Bye"         # replace an element's children with text
+e.name    = "post"        # rename in place (identity + namespace re-resolved)
+e.delete("id")            # remove an attribute
+doc.at_xpath("//draft").remove
+doc.root.to_xml           # => "<feed xmlns:dc=\"urn:dc\"><post dc:k=\"v\">Bye</post></feed>"
+```
+XML subtrees can be built with `Document#create_element` and related node factory methods,
+then inserted with `#add_child`, `#before`, `#after`, or `#replace`;
+namespaces are resolved at insertion time, and cross-document nodes are deep-copied.
+```ruby
+doc   = Makiri::XML(%(<feed xmlns="urn:a" xmlns:dc="urn:dc"/>))
+entry = doc.create_element("entry")
+entry["dc:id"] = "42"                       # prefixed attr resolves on insertion
+entry.add_child(doc.create_element("title", "Hello"))
+doc.root.add_child(entry)
+doc.to_xml   # => "...<entry dc:id=\"42\"><title>Hello</title></entry>..."
+```
+`Makiri::XML::Builder` is the Nokogiri-compatible DSL over those factories.
+```ruby
+builder = Makiri::XML::Builder.new do |xml|
+  xml.feed("xmlns" => "http://www.w3.org/2005/Atom", "xmlns:dc" => "urn:dc") do
+    xml.title("Example Feed")
+    xml.entry("dc:id" => "1") do
+      xml.title("First")
+      xml.summary { xml.cdata("raw <b>html</b>") }
+    end
+  end
+end
+builder.to_xml                 # the whole document (with XML declaration)
+builder.doc                    # the Makiri::XML::Document being built
+```
+XML parsing is bounded by an arena memory limit, 256 MiB by default,
+and unusually large documents can raise it with `max_bytes:`.
+```ruby
+Makiri::XML(huge_xml, max_bytes: 512 * 1024 * 1024)   # also Makiri::XML::Document.parse(..., max_bytes:)
+```
 ## Non-goals (v1.0)
-* XML parsing (HTML only).
+* Passing a raw markup string straight to an insertion method
+  (`node.add_child("<x/>")`); parse it into a fragment first
+  (`Document#fragment` / `DocumentFragment.parse`). (Building XML from scratch
+  (`XML::Document.new` + `#root=`), the node factories - `Document#create_element`
+  etc. - fragments, node insertion (`#add_child` / `#before` / `#after` /
+  `#replace`), and `#to_xml` serialization ARE supported.)
 * XSLT, DTD / Schema / RelaxNG validation, XPointer, XInclude.
 * Streaming / SAX parsing.
 * Drop-in replacement for every Nokogiri method. Makiri covers the common
   HTML-scraping and manipulation surface. Deliberately not provided:
-  - XML/XHTML serialization variants (`to_xml`, `to_xhtml`, `write_xml_to`)
+  - XHTML serialization variants (`to_xhtml`, `write_xml_to`); `#to_xml` is supported
   - XML/DTD construction (`create_internal_subset`, `external_subset`)
-  - namespace introspection beyond `namespace-uri()` (`namespace_definitions`, `add_namespace`, `collect_namespaces`)
+  - namespace *mutation* (`add_namespace_definition`); read introspection
+    (`#namespace`, `#namespace_definitions`, `#namespaces`, `#collect_namespaces`)
+    is supported on `Makiri::XML` nodes
   - Nokogiri internals (`decorate`, `slop!`, `validate`).
 ## Differences from Nokogiri
@@ -103,16 +213,67 @@ Detailed, test-backed notes live in `spec/conformance/README.md`.
 * `namespace-uri()` of an HTML element returns the XHTML URI (DOM-correct, as browsers report)
   * `Nokogiri::HTML5` returns `""`.
+### XML
+* `Makiri::XML` is **XML 1.0 only and non-validating**.
+  * A `version="1.1"` declaration is rejected; Nokogiri parses XML 1.1.
+  * The DTD is recognized but not processed: DTD-defined entities are not
+    expanded and DTD default attributes are not applied (Nokogiri/libxml2 can do
+    both). External entities/subsets are never fetched (no I/O).
+  * Mutation supports in-place edits, the node factories, fragments
+    (`Document#fragment` / `DocumentFragment.parse`), node insertion, and building
+    a document from scratch (`XML::Document.new` + `#root=`); only handing a raw
+    markup string straight to `#add_child` is unsupported (parse it into a fragment
+    first). (`#to_xml` serialization is supported; HTML serialization - `to_html`
+    / `inner_html` / `outer_html` - is not.)
+* Otherwise the parsed tree is byte-identical to `Nokogiri::XML`'s (verified by
+  the property-based differential), including namespaces, prolog/epilog comments
+  and PIs, and adjacent-CDATA coalescing.
 ### CSS
-* jQuery/Nokogiri CSS extensions are not supported (`:contains`, `:gt`, `:lt`, `:eq`, `:first`, …)
-  * Makiri uses Lexbor's standards-only selector engine.
-    Use XPath (`xpath("//p[contains(., 'x')]")`) or Enumerable (`css('li')[1]`).
+* Most jQuery/Nokogiri CSS extensions are not supported (`:gt`, `:lt`, `:eq`, `:first`, ...)
+  * Makiri uses Lexbor's selector engine, which is standards-based apart from one
+    text-containment extension. Use XPath (`xpath("//p[contains(., 'x')]")`) or
+    Enumerable (`css('li')[1]`) for the rest.
     Standard Level-4 selectors (`:is` / `:where` / `:has`) are supported; some of which Nokogiri rejects.
+  * `:lexbor-contains("text")` **is** supported (on both HTML and XML) - Lexbor's
+    spelling of the jQuery `:contains()` substring filter, matching an element
+    whose text contains the string; append ` i` (`:lexbor-contains("text" i)`)
+    for an ASCII case-insensitive match. (Nokogiri's name `:contains` is not an
+    alias.) Like Lexbor's matcher, it tests the element's **immediate child text
+    nodes** (not the deep string-value), so HTML and XML agree; on XML it lowers
+    to XPath `child::text()[contains(., "text")]`.
+* Untyped `:*-of-type` (`:first-of-type`, `:nth-of-type(an+b)`, ... with no type
+  selector) is supported and correct on both HTML and XML - the "type" is the
+  element's own expanded name.
+  * Nokogiri (XML and HTML5) mistranslates these to first-/only-child
+    (`//*[position()=1]` / `//*[last()=1]`), so it under-matches; Makiri matches
+    Lexbor's HTML matcher.
 * Type selectors are ASCII case-insensitive (CSS-correct for HTML; `LI` matches `<li>`)
   * `Nokogiri::HTML5` is case-sensitive there.
-* Class/ID selectors are matched case-insensitively regardless of quirks mode (a Lexbor behaviour)
-  * In a no-quirks document browsers and `Nokogiri::HTML5` match them case-sensitively.
+## Conformance
+The XPath engine and XML parser are original code, so their correctness is held by
+differential and standards harnesses in `spec/conformance/`.
+The HTML XPath and CSS suites are differentials against **`Nokogiri::HTML5`**
+(Gumbo / WHATWG, never libxml2's non-conformant HTML4 parser): both sides parse
+HTML5, so the DOM is isomorphic and results are compared node-for-node. HTML
+parsing itself is checked against the WHATWG html5lib-tests corpus, and
+XPath-over-HTML semantics additionally against browsers via a WPT port.
+See also [`spec/conformance/README.md`](spec/conformance/README.md).
+| Suite | Input | Oracle | `rake` task |
+|---|---|---|---|
+| HTML parsing | HTML | WHATWG html5lib-tests (expected-tree corpus) | `conformance:html5` |
+| XPath 1.0 | HTML | `Nokogiri::HTML5` (libxml2 XPath) — differential | `conformance:xpath` |
+| XPath over HTML | HTML | browsers (WPT `domxpath`, hand-ported; runs under `rake spec`) | — |
+| CSS selectors | HTML | `Nokogiri::HTML5#css` — differential | `conformance:css` |
+| Well-formedness | XML | W3C XML Conformance Test Suite | `conformance:xmlconf` |
+| XPath 1.0 | XML | `Nokogiri::XML` — differential | `conformance:xpath_xml` |
+| Parsed tree (property-based) | XML | `Nokogiri::XML` — differential | `conformance:xml_pbt` |
+| CSS selectors | XML | `Nokogiri::XML` — differential | `conformance:css_xml` |
 ## Requirements
@@ -129,6 +290,15 @@ bundle exec rake compile
 bundle exec rake spec
 ```
+### Vendored Lexbor version
+`vendor/lexbor` is pinned to `3a2d595` (`v3.0.0-25`), an untagged `master`
+commit, for fixes that v3.0.0 lacks: two upstreamed CSS-selector fixes (class/ID
+case-sensitivity in quirks mode, and prefix-less type-selector namespacing), a
+heap-overflow fix in the `:lexbor-contains()` parser, and other post-v3.0.0
+bugfixes. Lexbor stays vanilla; we return to a release tag once one ships after
+v3.0.0. See `CLAUDE.md` for details.
 ## License
 Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).

data/Rakefile CHANGED Viewed

@@ -4,9 +4,28 @@ require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 require "rake/extensiontask"
 require "shellwords"
+require "tmpdir"
 GEMSPEC = Gem::Specification.load("makiri.gemspec")
+# Replace bundler/gem_tasks' `release` (which builds a source-only gem and
+# `gem push`es it from the dev machine) with a tag push: it hands the build,
+# GitHub Release, and the approval-gated RubyGems publish off to CI
+# (.github/workflows/release.yml). Nothing is pushed to RubyGems locally.
+Rake::Task["release"].clear
+desc "Tag v#{GEMSPEC.version} and push it; CI builds, releases, and publishes"
+task release: %w[release:guard_clean release:source_control_push] do
+  puts <<~MSG
+    Pushed tag v#{GEMSPEC.version}. GitHub Actions (release.yml) will now:
+      1. build the source gem + precompiled native gems,
+      2. create the GitHub Release and attach them, then
+      3. publish to RubyGems via OIDC - after the `rubygems` environment approval.
+    Approve the pending deployment in the Actions run to publish; nothing is
+    pushed to RubyGems from this machine.
+  MSG
+end
 Rake::ExtensionTask.new("makiri", GEMSPEC) do |ext|
   ext.lib_dir       = "lib/makiri"
   ext.ext_dir       = "ext/makiri"
@@ -17,6 +36,59 @@ RSpec::Core::RakeTask.new(:spec)
 task default: %i[compile spec]
+# `rake spec:valgrind` - run the spec suite under Valgrind memcheck via
+# ruby_memcheck (Linux CI; see .github/workflows/valgrind.yml). The gem ships
+# Ruby's own Valgrind suppression files (matched by Ruby version) and filters
+# the report down to errors whose stack touches our extension, so we no longer
+# have to fetch ruby.supp from ruby/ruby (that path was removed upstream).
+#
+# We keep this job's historical contract: catch *use of uninitialised values*
+# and *invalid reads/writes* (incl. intra-arena overflows) - NOT leaks (leak
+# detection stays with `rake leaks`). So we override ruby_memcheck's defaults,
+# which disable undef-value errors and turn on full leak-check.
+#
+# `filter_all_errors: true` is essential: by default ruby_memcheck only applies
+# its "stack must touch the makiri binary" filter to *leak*-kind errors
+# (`ValgrindError#should_filter? = filter_all_errors? || kind_leak?`), so every
+# uninitialised-value report is surfaced regardless of where it comes from. Ruby's
+# conservative GC (machine-context scan, RVALUE flag aging, free-at-exit teardown)
+# legitimately reads uninitialised words, and the bundled ruby.supp does not cover
+# the free-at-exit / subprocess stacks the `:isolated` specs spin up under
+# `--trace-children=yes` - which buried the run in ~3500 pure-Ruby false positives.
+# Filtering all error kinds by the same binary-touch rule keeps the gate scoped to
+# *our* code: a real uninit/invalid access in mkr_*/Lexbor still has a makiri frame
+# and is still reported.
+#
+# Guarded: ruby_memcheck lives in the optional :valgrind bundler group, so a
+# normal `bundle exec rake` (without that group) must not fail to load.
+begin
+  require "ruby_memcheck"
+  require "ruby_memcheck/rspec/rake_task"
+  RubyMemcheck.config(
+    binary_name: "makiri",
+    filter_all_errors: true,    # apply the binary-touch filter to ALL error kinds,
+                                # not just leaks (see note above) - drops Ruby's own
+                                # GC uninitialised-value noise, keeps mkr_* reports
+    valgrind_options: [
+      "--num-callers=50",
+      "--error-limit=no",
+      "--trace-children=yes",   # spec processes may fork
+      "--undef-value-errors=yes", # the point of this job (ruby_memcheck defaults to =no)
+      "--track-origins=yes",    # report where an uninitialised value came from
+      "--leak-check=no",        # leaks are `rake leaks`' job, not this one
+    ],
+  )
+  namespace :spec do
+    desc "Run the spec suite under Valgrind memcheck (ruby_memcheck; needs the " \
+         ":valgrind bundler group and the valgrind binary)"
+    RubyMemcheck::RSpec::RakeTask.new(valgrind: :compile)
+  end
+rescue LoadError
+  # ruby_memcheck not installed (optional :valgrind group absent) - skip the task.
+end
 namespace :security do
   desc "Run mechanical C safety lint over ext/makiri"
   task :clint do
@@ -26,7 +98,7 @@ end
 # `rake clean` (from rake-compiler) removes the ext build dir under tmp/,
 # including the generated Makefile. The next `rake compile` re-runs extconf,
-# so newly-added .c files are picked up — without this, a stale Makefile omits
+# so newly-added .c files are picked up - without this, a stale Makefile omits
 # new sources and macOS's -undefined dynamic_lookup turns the missing symbols
 # into runtime NULL calls. The vendored Lexbor build is deliberately NOT wiped
 # here (it is slow to rebuild and rarely changes); use `rake clean:lexbor` for
@@ -63,6 +135,28 @@ def asan_runtime_path
   nil
 end
+def libfuzzer_available?
+  cxx = ENV["CXX"].to_s.empty? ? "clang++" : ENV["CXX"]
+  Dir.mktmpdir("makiri-libfuzzer-check") do |dir|
+    src = File.join(dir, "check.cc")
+    exe = File.join(dir, "check")
+    File.write(src, "extern \"C\" int LLVMFuzzerTestOneInput(const unsigned char*, unsigned long){return 0;}\n")
+    return system(cxx, "-fsanitize=fuzzer,address,undefined", src, "-o", exe,
+                  out: File::NULL, err: File::NULL)
+  end
+end
+# The compiled extension, and whether it carries sanitizer instrumentation, so
+# `fuzz:sanitize SKIP_BUILD=1` can refuse to run a plain (non-ASan) build.
+def ext_bundle_path
+  Dir["lib/makiri/makiri.{bundle,so}"].first
+end
+def ext_sanitized?
+  bundle = ext_bundle_path or return false
+  !(`nm "#{bundle}" 2>/dev/null` =~ /asan|ubsan/i).nil?
+end
 desc "Build the extension with sanitizers (MAKIRI_SANITIZE, default " \
      "address,undefined) and run the spec suite under them"
 task :sanitize do
@@ -87,11 +181,99 @@ task :sanitize do
   sh(env, "#{FileUtils::RUBY} -S rspec")
 end
+desc "Measure C coverage of OUR sources (clang source-based) over the spec suite. " \
+     "Prints an llvm-cov region+branch report (excludes vendored Lexbor) and writes " \
+     "a line-level detail file to tmp/coverage/show.txt."
+task :coverage do
+  require "fileutils"
+  dir = File.expand_path("tmp/coverage")
+  FileUtils.rm_rf(dir)
+  FileUtils.mkdir_p(dir)
+  # Instrument only our sources (Lexbor is built separately, uninstrumented).
+  sh({ "MAKIRI_COVERAGE" => "1" }, "#{FileUtils::RUBY} -S rake clean compile")
+  # %p -> PID, so any forked spec process gets its own raw profile.
+  sh({ "LLVM_PROFILE_FILE" => File.join(dir, "makiri-%p.profraw") }, "#{FileUtils::RUBY} -S rspec")
+  profdata = File.join(dir, "makiri.profdata")
+  bundle   = "lib/makiri/makiri.bundle"
+  ignore   = "(vendor/lexbor|/usr/|/Library/|ruby/|rubygems)"
+  sh "xcrun llvm-profdata merge -sparse #{dir}/*.profraw -o #{profdata}"
+  sh "xcrun llvm-cov report #{bundle} -instr-profile=#{profdata} " \
+     "-ignore-filename-regex='#{ignore}' -show-branch-summary"
+  show = File.join(dir, "show.txt")
+  sh "xcrun llvm-cov show #{bundle} -instr-profile=#{profdata} " \
+     "-ignore-filename-regex='#{ignore}' -show-branches=count -show-line-counts-or-regions > #{show}"
+  puts "\ncoverage line/branch detail: #{show}"
+  puts "(coverage build left in place; run `rake clean compile` to restore a normal build)"
+end
+desc "Like :sanitize but also builds the vendored Lexbor under ASan, so overflows " \
+     "INSIDE Lexbor's mraw arena are caught (slow: full Lexbor rebuild). Runs the " \
+     "spec suite, or FUZZ_ARGS via the fuzzer when set."
+task "sanitize:lexbor" do
+  sanitize = ENV["MAKIRI_SANITIZE"] || "address,undefined"
+  sanitize.include?("address") or
+    abort "sanitize:lexbor needs an address build (MAKIRI_SANITIZE must include 'address')"
+  # MAKIRI_SANITIZE_LEXBOR makes extconf build Lexbor with -DLEXBOR_BUILD_WITH_ASAN
+  # (enabling its mraw poisoning); the build-mode stamp auto-rebuilds Lexbor on the
+  # plain<->asan switch, so no manual clean:lexbor is needed before or after.
+  build_env = { "MAKIRI_SANITIZE" => sanitize, "MAKIRI_SANITIZE_LEXBOR" => "1" }
+  sh(build_env, "#{FileUtils::RUBY} -S rake clean compile")
+  env = {
+    "ASAN_OPTIONS"  => "detect_leaks=0:detect_container_overflow=0:" \
+                       "detect_odr_violation=0:abort_on_error=1:halt_on_error=1",
+    "UBSAN_OPTIONS" => "print_stacktrace=1:halt_on_error=1",
+  }
+  runtime = asan_runtime_path or
+    abort "sanitize:lexbor: could not locate the ASan runtime for #{RbConfig::CONFIG['CC']}"
+  preload = RbConfig::CONFIG["target_os"] =~ /darwin/ ? "DYLD_INSERT_LIBRARIES" : "LD_PRELOAD"
+  env[preload] = runtime
+  puts "sanitize:lexbor: preloading #{runtime} via #{preload}"
+  if ENV["FUZZ_ARGS"]
+    sh(env, "#{FileUtils::RUBY} -Ilib spec/fuzz/run.rb #{ENV['FUZZ_ARGS']}")
+  else
+    sh(env, "#{FileUtils::RUBY} -S rspec")
+  end
+end
 desc "Run the robustness fuzzer (override options via FUZZ_ARGS)"
 task fuzz: :compile do
   sh "#{FileUtils::RUBY} -Ilib spec/fuzz/run.rb #{ENV['FUZZ_ARGS']}"
 end
+desc "Fuzz the XML parser (hostile/mutated documents; override via FUZZ_ARGS)"
+task "fuzz:xml": :compile do
+  sh "#{FileUtils::RUBY} -Ilib spec/fuzz/run.rb --target xml #{ENV['FUZZ_ARGS']}"
+end
+desc "Fuzz the XML mutation surface (random edit sequences + invariants; override via FUZZ_ARGS)"
+task "fuzz:mutate": :compile do
+  sh "#{FileUtils::RUBY} -Ilib spec/fuzz/run.rb --target mutate #{ENV['FUZZ_ARGS']}"
+end
+desc "Malloc-leak gate (macOS `leaks`): fails on per-call leak stacks through the ext"
+task leaks: :compile do
+  # ASan runs with detect_leaks=0 (Ruby/Lexbor are uninstrumented), so plain
+  # leaks are otherwise never machine-checked; see script/check_leaks.rb.
+  sh "#{FileUtils::RUBY} script/check_leaks.rb"
+end
+desc "OOM-injection gate: rebuild with MAKIRI_ALLOC_INJECT=1 and sweep every core " \
+     "allocation site, verifying each failure fails closed (clean raise or " \
+     "baseline-identical result, never truncated output)"
+task :oom do
+  # The hook is compiled in only under MAKIRI_ALLOC_INJECT=1 (zero overhead in
+  # a normal build), so this needs its own rebuild; see
+  # script/check_alloc_failures.rb for the protocol and the property gated.
+  sh({ "MAKIRI_ALLOC_INJECT" => "1" }, "#{FileUtils::RUBY} -S rake clean compile")
+  sh "#{FileUtils::RUBY} -Ilib script/check_alloc_failures.rb"
+  puts "(injection build left in place; run `rake clean compile` to restore a normal build)"
+end
 desc "Run the performance benchmark (Makiri vs Nokogiri reference)"
 task bench: :compile do
   # Run outside the bundle so the bench-only gems (nokogiri, benchmark-ips)
@@ -101,6 +283,13 @@ task bench: :compile do
   end
 end
+desc "Run the XML reader benchmark (Makiri::XML vs Nokogiri::XML reference)"
+task "bench:xml" => :compile do
+  Bundler.with_unbundled_env do
+    sh "#{FileUtils::RUBY} -Ilib bench/bench_xml.rb"
+  end
+end
 namespace :conformance do
   desc "WHATWG HTML5 parsing conformance: run html5lib-tests through Makiri"
   task html5: :compile do
@@ -116,22 +305,77 @@ namespace :conformance do
     end
   end
+  desc "XML XPath 1.0 differential conformance: Makiri::XML vs Nokogiri::XML"
+  task xpath_xml: :compile do
+    Bundler.with_unbundled_env do
+      sh "#{FileUtils::RUBY} -Ilib spec/conformance/xml_xpath_diff.rb #{ENV['XPATH_ARGS']}"
+    end
+  end
+  desc "W3C XML Conformance Test Suite: well-formedness through Makiri::XML"
+  task xmlconf: :compile do
+    # Nokogiri (bench-only) parses the manifests, so run outside the bundle.
+    Bundler.with_unbundled_env do
+      sh "#{FileUtils::RUBY} -Ilib spec/conformance/xmlconf_runner.rb #{ENV['XMLCONF_ARGS']}"
+    end
+  end
+  desc "Property-based XML differential: generated documents, Makiri vs Nokogiri tree"
+  task xml_pbt: :compile do
+    Bundler.with_unbundled_env do
+      sh "#{FileUtils::RUBY} -Ilib spec/conformance/xml_pbt_diff.rb #{ENV['PBT_ARGS']}"
+    end
+  end
   desc "CSS Selectors differential conformance vs Nokogiri::HTML5"
   task css: :compile do
     Bundler.with_unbundled_env do
       sh "#{FileUtils::RUBY} -Ilib spec/conformance/css_diff.rb #{ENV['CSS_ARGS']}"
     end
   end
+  desc "XML CSS-selector differential conformance: Makiri::XML vs Nokogiri::XML"
+  task css_xml: :compile do
+    Bundler.with_unbundled_env do
+      sh "#{FileUtils::RUBY} -Ilib spec/conformance/xml_css_diff.rb #{ENV['CSS_XML_ARGS']}"
+    end
+  end
+  desc "XML Builder differential conformance: Makiri::XML::Builder vs Nokogiri::XML::Builder"
+  task builder: :compile do
+    Bundler.with_unbundled_env do
+      sh "#{FileUtils::RUBY} -Ilib spec/conformance/builder_diff.rb #{ENV['BUILDER_ARGS']}"
+    end
+  end
 end
-desc "Run all conformance suites (html5lib-tests + XPath & CSS differentials)"
-task conformance: %w[conformance:html5 conformance:xpath conformance:css]
+desc "Run all conformance suites"
+task conformance: %w[conformance:html5 conformance:xpath conformance:css
+                     conformance:xmlconf conformance:xpath_xml conformance:css_xml
+                     conformance:builder]
 namespace :fuzz do
-  desc "Run the fuzzer under AddressSanitizer (rebuilds the ext; --isolated)"
+  # Run the fuzzer under the sanitizer. Toggles (all via env):
+  #   FAST=1        run the surfaces NON-isolated (one process, no fork-per-query).
+  #                 Far higher throughput; ASan still aborts on a memory error
+  #                 (halt_on_error). The default (isolated) is the complete net:
+  #                 it also survives + attributes a genuine segfault and catches a
+  #                 hang via the per-query timeout, at much lower throughput.
+  #   SKIP_BUILD=1  reuse the current build instead of rebuilding (refuses to run
+  #                 if it is not a sanitizer build, so you never fuzz a plain ext).
+  #   FUZZ_TIME=N   seconds per surface (default 90).
+  #   FUZZ_ARGS=... run a single custom invocation instead of the three surfaces.
+  desc "Run the fuzzer under AddressSanitizer (FAST=1 non-isolated, SKIP_BUILD=1 reuse build)"
   task :sanitize do
     sanitize = ENV["MAKIRI_SANITIZE"] || "address,undefined"
-    sh({ "MAKIRI_SANITIZE" => sanitize }, "#{FileUtils::RUBY} -S rake clean compile")
+    if %w[1 true yes].include?(ENV["SKIP_BUILD"].to_s.downcase)
+      ext_sanitized? or
+        abort "fuzz:sanitize: SKIP_BUILD set but lib/makiri is not a sanitizer build; " \
+              "drop SKIP_BUILD to rebuild with MAKIRI_SANITIZE"
+      puts "fuzz:sanitize: reusing the existing sanitizer build (SKIP_BUILD)"
+    else
+      sh({ "MAKIRI_SANITIZE" => sanitize }, "#{FileUtils::RUBY} -S rake clean compile")
+    end
     env = {
       "ASAN_OPTIONS"  => "detect_leaks=0:detect_container_overflow=0:" \
@@ -144,7 +388,50 @@ namespace :fuzz do
       preload = RbConfig::CONFIG["target_os"] =~ /darwin/ ? "DYLD_INSERT_LIBRARIES" : "LD_PRELOAD"
       env[preload] = runtime
     end
-    args = ENV["FUZZ_ARGS"] || "--isolated --time 120"
-    sh(env, "#{FileUtils::RUBY} -Ilib spec/fuzz/run.rb #{args}")
+    if ENV["FUZZ_ARGS"]
+      sh(env, "#{FileUtils::RUBY} -Ilib spec/fuzz/run.rb #{ENV['FUZZ_ARGS']}")
+    else
+      iso  = %w[1 true yes].include?(ENV["ISOLATED"].to_s.downcase) ? "--isolated" : ""
+      secs = ENV["FUZZ_TIME"] || "90"
+      # Cover every surface under the sanitizer: the query engine (XPath/CSS over
+      # parsed fixtures), the XML parser (hostile documents), and the XML mutation
+      # surface (random edit sequences + invariants).
+      ["", "--target xml", "--target mutate", "--target xmlcss"].each do |surface|
+        sh(env, "#{FileUtils::RUBY} -Ilib spec/fuzz/run.rb #{surface} #{iso} --time #{secs}".squeeze(" ").strip)
+      end
+    end
   end
+  # Coverage-guided libFuzzer harnesses for the pure-C surfaces (XML parser and
+  # XPath compile+eval).  These are Ruby-free standalone binaries, so they run
+  # directly under clang's libFuzzer driver without the Ruby interpreter.
+  # They complement the Ruby-based robustness fuzzer by providing coverage
+  # feedback and 2-3 orders of magnitude faster execution for the C core.
+  desc "Build the libFuzzer harnesses (requires clang with libFuzzer support)"
+  task :libfuzzer_build => :compile do
+    libfuzzer_available? or
+      abort "fuzz:libfuzzer_build: #{ENV['CXX'] || 'clang++'} cannot link libFuzzer. " \
+            "Install an LLVM clang with libFuzzer support and run with " \
+            "CLANG=/path/to/clang CXX=/path/to/clang++."
+    Dir.chdir("ext/makiri/fuzz") do
+      sh "make clean"
+      sh "make all"
+    end
+  end
+  desc "Run the libFuzzer coverage-guided harnesses (default: 60s per target)"
+  task :libfuzzer => :libfuzzer_build do
+    time = ENV["FUZZ_TIME"] || "60"
+    Dir.chdir("ext/makiri/fuzz") do
+      sh "mkdir -p corpus/xml corpus/xpath"
+      sh "./xml_fuzz -max_total_time=#{time} -max_len=4096 corpus/xml"
+      sh "./xpath_fuzz -max_total_time=#{time} -max_len=4096 corpus/xpath"
+    end
+  end
+end
+desc "Show code statistics"
+task :stats do
+  sh "tokei lib ext spec script --exclude tmp --exclude vendor --exclude docs"
 end

data/ext/makiri/bridge/bridge.h CHANGED Viewed

@@ -46,6 +46,34 @@ mkr_ruby_borrowed_bytes_t mkr_ruby_bytes_view(VALUE in);
  * for an empty input), suitable for use while the GVL is released. */
 int mkr_ruby_copy_bytes(VALUE in, mkr_owned_bytes_t *out);
+/* Return a UTF-8 Ruby String for `str`, honouring its declared encoding: UTF-8 /
+ * US-ASCII / ASCII-8BIT are returned unchanged (the parser handles their bytes
+ * directly); any other encoding is transcoded to UTF-8 (invalid/undef -> U+FFFD)
+ * so its content is preserved rather than read as raw UTF-8. The UTF-8 common
+ * case is a single encoding comparison. */
+VALUE mkr_ruby_to_utf8(VALUE str);
+/* STRICT decode for XML (§2.1): like mkr_ruby_to_utf8 it honours the String's
+ * declared encoding (UTF-8 / US-ASCII / ASCII-8BIT pass through; any other
+ * encoding is transcoded to UTF-8) - but FAIL-CLOSED, never lenient: a non-UTF-8
+ * byte that can't be converted, invalid UTF-8, or an embedded NUL all raise
+ * Makiri::XML::SyntaxError (no U+FFFD replacement). Returns a validated,
+ * UTF-8-tagged Ruby String. (The HTML replace path mkr_ruby_to_utf8 itself is
+ * NOT reused for the conversion - only its encoding-judgment rule is shared.)
+ *
+ * +max_bytes+ bounds the decoded UTF-8 length: an input that already exceeds the
+ * parser's arena byte budget is rejected here with Makiri::XML::LimitExceeded,
+ * before the validation copy and the caller's GVL-release copy (so a hostile
+ * oversized document is not copied twice for a doomed parse). 0 disables the
+ * check (decode-only callers that build no arena). */
+VALUE mkr_xml_decode_input(VALUE str, size_t max_bytes);
+/* True if `str` is *already known* to be valid UTF-8 - pure ASCII, or valid in
+ * the UTF-8 encoding - from its cached coderange, WITHOUT forcing a scan. Lets
+ * the parse skip mkr_utf8_sanitize's validation pass for input Ruby has already
+ * classified (an unknown/broken coderange returns false: sanitize handles it). */
+bool mkr_ruby_str_known_valid_utf8(VALUE str);
 /* Validate a Ruby String for use as an XPath engine string: valid UTF-8,
  * no interior NUL, and at most +max_bytes+. Returns NULL on success and fills
  * +out+; otherwise returns a static reason string. +sv+ must be a String. */