moxml 0.5.147 → 0.5.149

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e5797a825682b992d84b5ab9155d74f81fdeca24293b40ad7b9e58860c407f0
-  data.tar.gz: 11accb2da9da7cf656d28ab7e128da698f1702120332d94d06b4a20ea39ad112
+  metadata.gz: 3af023098a8dde5521f0297f240a1c6fa71625cc2b3cd6924eba638724e12517
+  data.tar.gz: 9fb194d059475ed0c4aea3471b9048195ce7ea0bef349c25bdc52f991944ca5f
 SHA512:
-  metadata.gz: 2b8980eb872815020ed2b6d582172ffd62c08a426c2fa2104bd891cd5e2f1e69392fd82d9573215383399087cb686852c47fb10e1d93e812c1641f8eb5e4eeef
-  data.tar.gz: 924b76844411af051e0a9b02ea55788dbc4f5d0e6ec0be31ade4ded2c17c6c6e58356f687ef42e2d7934a4a8f684720f77b0d7688e108437aa64c68a41b68d06
+  metadata.gz: 860ac49f2684d504273e22dcbc9c6ed43106ebf67d9ca093af74543ab284e89c7be5c02d3991421973674837ece981127ede74329bfce9a352236bf9b3c094ff
+  data.tar.gz: 24b50dee831eb6856dc3f24b7759e32aead60c98896427ed7776277cd69aca0db01f3521841cbc556ccfaaee8fca56a071f2f219276f0553fe919b712c87ef2b

data/.github/workflows/nightly-fuzz.yml

@@ -0,0 +1,80 @@
+name: nightly fuzz
+
+# TODO.sota/02's nightly leg: the property suites swept at random
+# seeds and larger case counts, with findings filed as issues carrying
+# a minimal repro. PR legs run the same suites at a fixed, bounded
+# seed (MOXML_FUZZ/CI defaults in the spec); this leg is the breadth
+# pass, not a gate.
+
+on:
+  schedule:
+    - cron: "23 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  fuzz:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: "recursive"
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.4"
+          bundler-cache: false
+
+      - name: Install ragel
+        run: sudo apt-get update && sudo apt-get install -y ragel
+
+      - name: Install ruby-ll (for rake vendor:prepare)
+        run: gem install ruby-ll --no-document
+
+      - name: Generate ragel / ruby-ll outputs in vendored forks
+        run: rake vendor:prepare
+
+      - name: Install dependencies
+        run: bundle install
+
+      - name: Compile liboga / libll native extensions
+        run: rake vendor:compile
+
+      - name: Pick fuzz seeds
+        run: |
+          SEEDS=$(ruby -e 'srand; puts Array.new(3) { rand(1_000_000) }.join(",")')
+          echo "SOTA_FUZZ_SEEDS=$SEEDS" >> "$GITHUB_ENV"
+          echo "SOTA_FUZZ_CASES=1000" >> "$GITHUB_ENV"
+
+      - name: Fuzz (properties + structural suite)
+        run: |
+          bundle exec rspec \
+            spec/consistency/metamorphic_properties_spec.rb \
+            spec/consistency/metamorphic_spec.rb 2>&1 | tee fuzz.log
+
+      - name: File an issue on findings
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          {
+            echo "The nightly fuzz leg found findings at seeds \`$SOTA_FUZZ_SEEDS\` (\`SOTA_FUZZ_CASES=$SOTA_FUZZ_CASES\`)."
+            echo
+            echo "Reproduce:"
+            echo
+            echo '```'
+            echo "SOTA_FUZZ_SEEDS=$SOTA_FUZZ_SEEDS SOTA_FUZZ_CASES=$SOTA_FUZZ_CASES \\"
+            echo "  bundle exec rspec spec/consistency/metamorphic_properties_spec.rb spec/consistency/metamorphic_spec.rb"
+            echo '```'
+            echo
+            echo '```'
+            grep -E "Failure/Error|gen [0-9]+/|rspec \./" fuzz.log | head -40
+            echo '```'
+          } > fuzz-issue.md
+          gh issue create \
+            --title "nightly fuzz findings (seeds $SOTA_FUZZ_SEEDS)" \
+            --body-file fuzz-issue.md

data/docs/_pages/divergences.adoc

@@ -87,4 +87,62 @@ attribute-value predicates, namespace/default-namespace tests,
 `position()`/`count()`, `text()`, boolean expressions, `child::`/
 `descendant::` axes, or unions. Pinned in
 `spec/moxml/xpath_capabilities_spec.rb` and `spec/moxml/node_spec.rb`.
+
+| D-13
+| ox, headed_ox
+| Inter-element whitespace-only text nodes are dropped at parse
+(node counts and leading/trailing text differ from the other
+adapters). Surfaced by the taxonomy differential. Pinned in
+`spec/consistency/differential_taxonomy_spec.rb`.
+
+| D-14
+| oga
+| Processing-instruction content leaks into text reads (oga's own
+`text` folds PIs; the model says only text nodes contribute).
+Surfaced by the taxonomy differential. Pinned in
+`spec/consistency/differential_taxonomy_spec.rb`.
+
+| D-15
+| rexml
+| The whitespace text node between a pre-root PI and the root is
+retained (other adapters drop it). Surfaced by the taxonomy
+differential. Pinned in `spec/consistency/differential_taxonomy_spec.rb`.
+
+| D-16
+| rexml
+| Entity-expansion guard caps huge documents (~10k expansions) and
+raised a bare engine error — the facade now names it a ParseError;
+the cap itself remains (nokogiri's HUGE lane has no such cap).
+Surfaced by the taxonomy differential. Pinned in
+`spec/consistency/differential_taxonomy_spec.rb`.
+
+| D-17
+| oga
+| The strict parse lane does not validate nesting well-formedness
+(oga's strict is lexer-strict, not well-formedness-strict); bad
+nesting parses where other adapters raise. Surfaced by the taxonomy
+differential. Pinned in
+`spec/consistency/differential_taxonomy_spec.rb`.
+
+| D-18
+| leptris, oga, ox, headed_ox
+| Strict parse accepts invalid UTF-8 byte sequences (no encoding
+validation at parse; the bytes flow through as binary). The
+validating adapters (nokogiri, rexml, libxml) raise the named
+ParseError. On leptris the facade raises the named ParseError in the
+strict lane (default-UTF-8 rule; prolog-declared non-UTF-8 encodings
+own their validity) pending engine-side validation —
+leptris-ruby#514. Surfaced by the taxonomy differential. Pinned in
+`spec/consistency/differential_taxonomy_spec.rb`.
+
+| D-19
+| nokogiri, libxml
+| Adjacent text-like leaves (CDATA next to CDATA or text) coalesce
+into a single text node on the libxml2 lane; oga, rexml, ox,
+headed_ox, and leptris keep them as distinct nodes. The `text`
+answer is identical — the node count and kinds differ. The model is
+silent on text-node coalescing (an open decision). Surfaced by the
+generated metamorphic differential (M5). Pinned in
+`spec/consistency/metamorphic_properties_spec.rb` through
+`InputGenerator::FEATURE_EXEMPT`.
 |===

data/docs/_references/processing-model.adoc

@@ -18,9 +18,20 @@ skip with their entry ID — an undocumented divergence is a bug.
 === Status and precedence
 
 1. This document — the model.
-2. `spec/consistency/` — the model pinned as executable expectations.
+2. `spec/consistency/` — the model pinned as executable expectations,
+   with `lib/moxml/reference/` as the **engine-free executable
+   reference** (a pure-Ruby reader, writer, and normalizer). Adapters
+   are differentially tested against the reference
+   (`spec/consistency/reference_differential_spec.rb`), not only
+   against each other; the reference is pure Ruby and transpiles to
+   the Opal lane.
 3. Adapter implementations and capability gates — representations.
 
+A pin cites the model section it enforces
+(`# Processing model §<section> (normative)`); the coverage audit
+asserts both directions of that linkage, so removing any of the
+document, a pin, or the reference fails CI.
+
 A semantics change amends this document first (see
 `TODO.model/05-model-evolution-protocol.md`).
 
@@ -48,6 +59,12 @@ the same underlying node; node identity is a property of the
 underlying tree, not of wrapper object identity. Comparisons use node
 equality semantics, never `equal?` on wrappers.
 
+Kind:: Every node answers `node_type` — one of `:document`,
+`:element`, `:attribute`, `:text`, `:comment`, `:cdata`,
+`:processing_instruction`, `:doctype`, `:declaration`,
+`:entity_reference`, `:namespace` — the predicates (`element?`,
+`text?`, …) are its derivatives.
+
 Ownership:: The `Document` owns the tree's lifetime; nodes are valid
 within it. Deterministic release is available via `Document#free`.
 
@@ -61,6 +78,10 @@ listings or serialization child walks.
 * Walks (`each_node`, `materialize`) visit depth-first, pre-order.
 * `Document#children` and `#root` never include the declaration;
   doctype exposure follows the Nokogiri contract.
+* Document-level parts (`Declaration`, `Doctype`) are state, not
+  nodes: walks (`traverse`, `each_node`, `materialize`) never yield
+  them, and walks over a DOCTYPE-bearing document are well-defined on
+  every adapter.
 * `NodeSet` preserves document order for axis results; unions preserve
   first-encounter document order.
 
@@ -86,7 +107,9 @@ content; entity references restored when the document is
 entity-bearing.
 
 `content`:: The node's content per its kind; on text-like nodes, with
-entity-marker restoration; memoized against mutation.
+entity-marker restoration; memoized against mutation. A processing
+instruction's content is its data **without** the separating
+whitespace after the target (uniform across adapters).
 
 `raw_content`:: The backing text **without** entity restoration.
 
@@ -128,6 +151,9 @@ with an unambiguous local name.
   `xmlns:` default-namespace matching; attribute terminals in every
   position (`//a/@id`, `//@id`, `r/a/@id`); symbol- and string-keyed
   bindings resolve identically.
+* `at_xpath` is sugar over `xpath`: it answers the first node of a
+  node-set and passes scalar results (`count()`, `string()`,
+  `boolean()`) through unwrapped, uniformly across adapters.
 
 === Serialization
 
@@ -137,7 +163,8 @@ with an unambiguous local name.
 
 | `to_xml`
 | Declaration per configuration; empty elements per `expand_empty`;
-  namespace declarations preserved.
+  namespace declarations preserved; a parsed DOCTYPE is emitted with
+  its external identifiers and internal subset.
 
 | `to_html`
 | HTML method: void elements unclosed (`<br>`), raw text

data/lib/compat/opal/moxml_boot.rb

@@ -62,3 +62,7 @@ require "moxml/xpath/errors"
 require "moxml/xpath/ast/node"
 require "moxml/xpath/ruby/node"
 require "moxml/xpath/ruby/generator"
+require "moxml/reference"
+require "moxml/reference/parser"
+require "moxml/reference/writer"
+require "moxml/reference/normalizer"

data/lib/moxml/adapter/leptris.rb

@@ -105,6 +105,9 @@ module Moxml
         ::Leptris::XML::Document.method(:parse)
           .parameters.any? { |_, key| key == :strict }
 
+      ADD_CHILD_NATIVE =
+        NATIVE_READ_LAYER && NN.method_defined?(:add_child_native)
+
       if NATIVE_READ_LAYER
         NN_ATTRIBUTE = NN.instance_method(:attribute)
         NN_NAME = NN.instance_method(:name)
@@ -345,9 +348,13 @@ module Moxml
 
         def xsd_content_valid?(xsd_handle, element_name, children)
           # The binding reads each child's name + effective
-          # namespace through its own node API — pass the natives.
+          # namespace through its own node API (c_ptr) — bridge to
+          # the binding's nodes. 1.9.374+ mints registered klasses
+          # at create, so children arrive as Identity wrappers whose
+          # .native is self (no c_ptr); to_binding hands back the
+          # binding node whatever the wrapper flavor.
           xsd_handle.content_valid?(
-            element_name, children.map(&:native)
+            element_name, children.map { |c| to_binding(c.native) }
           )
         end
 
@@ -522,12 +529,18 @@ module Moxml
               include ::Moxml::Node
               include ::Moxml::ProcessingInstruction
 
-              # Same nil bridge as Comment (leptris-ruby#344).
+              # Same nil bridge as Comment (leptris-ruby#344); the
+              # separating whitespace after the target is not content
+              # (uniform PI semantics, TODO.sota/02).
               def content
                 content = native_content
-                return ::Moxml::Adapter::Leptris.to_binding(self).content if content.nil?
+                if content.nil?
+                  return ::Moxml::Adapter::Leptris.to_binding(self)
+                      .content.to_s
+                      .sub(/\A\s+/, "")
+                end
 
-                content
+                content.sub(/\A\s+/, "")
               end
             end
 
@@ -624,6 +637,31 @@ module Moxml
               end
               TEXT.include(TextContractReads)
             end
+
+            # leptris-ruby#507: children_read joins the alias set. The
+            # generation-checked memo STAYS — re-walks of the same
+            # document are the common harness/consumer shape and the
+            # memo serves them allocation-free (dropping it regressed
+            # the walk 2x in GC). Only the MISS gets cheaper: the
+            # klass-children face plain-dispatches and returns the
+            # already-minted wrappers, so the adapter crossing
+            # (bind_call + to_a + keep-refs probe) drops out, and
+            # entity-marker documents take super for the split.
+            if NN.method_defined?(:children_read)
+              module ChildReads
+                def children
+                  @children = nil if @children_gen != context.children_generation
+                  @children_gen = context.children_generation
+                  @children ||= if doc_entity_bearing?
+                                  super
+                                else
+                                  NodeSet.new(children_read, context, self)
+                                end
+                end
+              end
+              ELEMENT.include(ChildReads)
+              TEXT.include(ChildReads)
+            end
           end
 
           # leptris-ruby#246 klass-propagating reads: the read
@@ -1146,8 +1184,36 @@ module Moxml
           true
         end
 
+        # UTF-8 is the default effective encoding; a prolog declaring
+        # another encoding hands byte validity to that encoding's lane.
+        DECLARED_ENCODING_RE =
+          /<\?xml\s[^>]*?encoding\s*=\s*["']([^"']+)["']/i
+        private_constant :DECLARED_ENCODING_RE
+
+        def encoding_lane_owns_validity?(xml_string)
+          # Binary-tagged head: the prolog is ASCII, and matching
+          # never raises on binary-tagged bytes (invalid-UTF-8 input
+          # is exactly the case this check exists for).
+          declared = xml_string.b[0, 256].match(DECLARED_ENCODING_RE)
+          declared &&
+            !declared[1].match?(/\A(?:utf-?8|us-ascii|ascii)\z/i)
+        end
+
         def parse(xml, options = {}, _context = nil)
           xml_string = xml.is_a?(IO) || xml.is_a?(StringIO) ? xml.read : xml.to_s
+          # Strict parse raises on bytes invalid in the effective
+          # encoding (default UTF-8; declared non-UTF-8 encodings own
+          # their validity) — the validating engines do this at load
+          # (libxml2: "Invalid bytes in character encoding"), the
+          # leptris engine lane does not yet (leptris-ruby#514), so
+          # the facade names it (taxonomy differential, D-18).
+          if options[:strict] && !xml_string.valid_encoding? &&
+              !encoding_lane_owns_validity?(xml_string)
+            raise Moxml::ParseError.new(
+              "Invalid bytes in character encoding (#{xml_string.encoding})",
+              source: xml_string[0, 100],
+            )
+          end
           # The marker flag rides preprocess's own `&` scan — no
           # second full-buffer probe (issue #132 parse-side note).
           entity_mode_keep =
@@ -1185,7 +1251,11 @@ module Moxml
               options: parse_flags(options, recover: recover,
                                             context: _context),
             )
-          rescue ::Leptris::XML::ParseError => e
+          rescue ::Leptris::XML::Error => e
+            # Invalid encoding bytes surface as the base engine
+            # error class, not ParseError — parse failures are all
+            # named by the facade (taxonomy differential, malformed
+            # class).
             raise Moxml::ParseError.new(e.message) if options[:strict]
 
             # Engines below the 1.9.344 recover floor (or the
@@ -1355,7 +1425,11 @@ module Moxml
             else
               ::Leptris::XML.parse_html(html_string)
             end
-          rescue ::Leptris::XML::ParseError => e
+          rescue ::Leptris::XML::Error => e
+            # ParseError is the common class; other engine errors
+            # (invalid encoding bytes surface as the base class) are
+            # parse failures too — the facade names them all
+            # (taxonomy differential, malformed class).
             raise Moxml::ParseError.new(e.message)
           end
           attachments.set(native_doc, :entity_markers, false)
@@ -1464,6 +1538,14 @@ module Moxml
           count = ffi.leptris_document_parse_diag_count(doc)
           return [] if count.zero?
 
+          # leptris#1771: one face carries the diag's machine-readable
+          # code plus line/column with the entry (nil when the engine
+          # has no token code — pre-#1771 records). The face probe
+          # keeps older bindings on kind+message (issue #408).
+          lint_code = if ffi.respond_to?(:leptris_document_parse_diag_code)
+                        ffi.method(:leptris_document_parse_diag_code)
+                      end
+
           kind_ptr = ::FFI::MemoryPointer.new(:int)
           diagnostics = []
           count.times do |i|
@@ -1473,7 +1555,19 @@ module Moxml
             ) == 1
 
             kind = DIAG_KINDS[kind_ptr.read_int] || :unknown
-            diagnostics << { kind: kind, message: buffer.read_string }
+            diag = { kind: kind, message: buffer.read_string }
+            if lint_code
+              code_buf = ::FFI::MemoryPointer.new(:char, 128)
+              line_ptr = ::FFI::MemoryPointer.new(:int)
+              column_ptr = ::FFI::MemoryPointer.new(:int)
+              lint_code.call(doc, i, code_buf, 128, line_ptr, column_ptr)
+              code = code_buf.read_string
+              diag[:code] = code.empty? ? nil : code
+              diag[:line] = line_ptr.read_int.zero? ? nil : line_ptr.read_int
+              diag[:column] =
+                column_ptr.read_int.zero? ? nil : column_ptr.read_int
+            end
+            diagnostics << diag
           end
           diagnostics
         end
@@ -2217,9 +2311,19 @@ module Moxml
         def attribute_pairs(element)
           binding_element = to_binding(element)
           return super unless binding_element.respond_to?(:attribute_pairs)
-          return binding_element.attributes_snapshot.to_a if ELEMENT_ATTR_SNAPSHOT
 
-          binding_element.attribute_pairs
+          pairs = if ELEMENT_ATTR_SNAPSHOT
+                    binding_element.attributes_snapshot.to_a
+                  else
+                    binding_element.attribute_pairs
+                  end
+          # The engine exposes qualified names; the facade's listing
+          # face is bare (Attribute#name), uniform with every adapter.
+          pairs.map { |(name, value)| [local_name(name), value] }
+        end
+
+        def local_name(name)
+          name.include?(":") ? name.split(":", 2)[1] : name
         end
 
         def attribute_element(attr)
@@ -2496,6 +2600,27 @@ module Moxml
         end
 
         def add_child(parent, child)
+          # Same-document Identity append — the builder hot path. One
+          # document pair decides the verdict: the pin, the absorb
+          # checks and the binding bridges only exist for
+          # cross-document work, and the plain C append (version bump
+          # inside) is the whole job. bind_call, not dispatch — the
+          # contract modules shadow add_child on Identity receivers
+          # (moxml#2560 note below).
+          if NATIVE_MUTATIONS_COHERENT && parent.is_a?(NN) && child.is_a?(NN) &&
+              !child.is_a?(CustomizedLeptris::EntityReference)
+            parent_doc = NN_DOCUMENT.bind_call(parent)
+            if parent_doc
+              child_doc = NN_DOCUMENT.bind_call(child)
+              if child_doc && parent_doc.equal?(child_doc)
+                # add_child_native (leptris-ruby#509) plain-dispatches
+                # on Identity receivers — one frame, no bind_call.
+                return parent.add_child_native(child) if ADD_CHILD_NATIVE
+
+                return NN_ADD_CHILD.bind_call(parent, child)
+              end
+            end
+          end
           # 1.9.311: binding-family children (created PIs, texts...) —
           # a DIRECT absorb makes the binding add below move the node
           # by reference. A source already owned elsewhere would need
@@ -2890,16 +3015,20 @@ module Moxml
         end
 
         def processing_instruction_content(node)
+          # The model's PI content excludes the separating whitespace
+          # after the target (Nokogiri semantics; uniform across
+          # adapters — TODO.sota/02).
           if NATIVE_READ_LAYER && node.is_a?(NN)
             content = NN_CONTENT.bind_call(node)
             # NativeNode#content answers nil for PI-kind nodes too
             # (leptris-ruby#344 covers comments); bridge for the data.
-            return to_binding(node).content if content.nil?
+            return to_binding(node).content.to_s.sub(/\A\s+/, "") if
+              content.nil?
 
-            return content
+            return content.sub(/\A\s+/, "")
           end
 
-          node.content
+          node.content.to_s.sub(/\A\s+/, "")
         end
 
         def set_processing_instruction_content(node, content)
@@ -3035,6 +3164,12 @@ module Moxml
           # namespace-bound queries to the Ruby engine until the
           # engine grows correct prefix resolution.
           return nil if namespaces && !namespaces.empty?
+          # Reverse axes answer reverse-ordered from the native
+          # engine; route them to the Ruby engine (document order).
+          return nil if REVERSE_AXIS_CACHE.get_or_set(expression) do
+            expression_uses_reverse_axis?(expression)
+          end
+
           # The binding reports the root element parentless while moxml
           # roots it at the document, so parent-axis queries from the
           # root element keep the Ruby engine.
@@ -3121,6 +3256,14 @@ module Moxml
             node.is_a?(::Leptris::XML::Element)
         end
 
+        # Reverse axes answer in reverse document order from the
+        # native engine; XPath path results are document-ordered (the
+        # model's order contract — TODO.sota/02, M3), so these
+        # expressions route to the Ruby engine, which orders them.
+        REVERSE_AXES = %w[ancestor ancestor-or-self preceding
+                          preceding-sibling parent].freeze
+        REVERSE_AXIS_CACHE = XPath::Cache.new(200)
+
         def expression_uses_parent_axis?(expression)
           ast = XPath::Parser.parse_with_cache(expression)
           contains_parent_axis?(ast)
@@ -3128,6 +3271,24 @@ module Moxml
           true
         end
 
+        def expression_uses_reverse_axis?(expression)
+          ast = XPath::Parser.parse_with_cache(expression)
+          contains_reverse_axis?(ast)
+        rescue XPath::SyntaxError
+          false
+        end
+
+        def contains_reverse_axis?(ast)
+          return true if ast.type == :parent
+          if ast.type == :axis && REVERSE_AXES.include?(ast.children.first)
+            return true
+          end
+
+          ast.children.any? do |child|
+            child.is_a?(XPath::AST::Node) && contains_reverse_axis?(child)
+          end
+        end
+
         def contains_parent_axis?(ast)
           return true if ast.type == :parent
           return true if ast.type == :axis && ast.children.first == "parent"
@@ -3297,13 +3458,12 @@ module Moxml
 
         def record_source_declaration(native_doc, xml_string)
           head = xml_string[0, 256]
-          had = if head.match?(SOURCE_DECLARATION_RE)
-                  true
-                elsif head.match?(/\A\s*\z/)
-                  xml_string.match?(SOURCE_DECLARATION_RE)
-                else
-                  false
-                end
+          # invalid bytes never reach regexes: named parse errors own
+          # malformed input (taxonomy differential, malformed class)
+          had = xml_string.valid_encoding? &&
+            (head.match?(SOURCE_DECLARATION_RE) ||
+             (head.match?(/\A\s*\z/) &&
+              xml_string.match?(SOURCE_DECLARATION_RE)))
           attachments.set(native_doc, :had_source_declaration, had)
         end
       end