moxml 0.5.148 → 0.5.149

data/lib/moxml/adapter/libxml.rb

@@ -7,23 +7,26 @@ require "libxml-ruby"
 module Moxml
   module Adapter
     class Libxml < Base
-      # DOCTYPE prolog sniffer; see parse for the index-then-match
-      # strategy that keeps this off the hot path.
-      DOCTYPE_RE = /<!DOCTYPE\s+([^\s>]+)(?:\s+PUBLIC\s+"([^"]+)"\s+"([^"]+)"|\s+SYSTEM\s+"([^"]+)")?\s*>/i
-      private_constant :DOCTYPE_RE
+      # DOCTYPE prolog scanning is manual (index/char scan): a regex
+      # over hostile prolog text backtracks polynomially on inputs
+      # with repeated <!DOCTYPE prefixes or unterminated subsets
+      # (CodeQL rb/polynomial-redos). See doctype_parts.
+      DOCTYPE_OPEN_SIZE = 9
+      QUOTES = ['"', "'"].freeze
 
       autoload :EntityRefRegistry, "moxml/adapter/libxml/entity_ref_registry"
 
       # Wrapper class to store DOCTYPE information
       class DoctypeWrapper
         attr_reader :native_doc
-        attr_accessor :name, :external_id, :system_id
+        attr_accessor :name, :external_id, :system_id, :internal_subset
 
-        def initialize(doc, name, external_id, system_id)
+        def initialize(doc, name, external_id, system_id, internal_subset = nil)
           @native_doc = doc
           @name = name
           @external_id = external_id
           @system_id = system_id
+          @internal_subset = internal_subset
         end
 
         # Provide native method to match adapter pattern
@@ -32,7 +35,8 @@ module Moxml
         end
 
         def to_xml
-          XmlEmitter.doctype_xml(@name, @external_id, @system_id)
+          XmlEmitter.doctype_xml(@name, @external_id, @system_id,
+                                 @internal_subset)
         end
       end
 
@@ -83,6 +87,104 @@ module Moxml
           false
         end
 
+        # Linear DOCTYPE prolog scanner. Returns
+        # {name:, external_id:, system_id:, internal_subset:} or nil.
+        # Single pass, quote- and bracket-aware: no backtracking, so
+        # hostile prolog text cannot make parsing superlinear.
+        def doctype_parts(source, from = 0)
+          start = source.index("<!DOCTYPE", from)
+          return unless start
+
+          pos = skip_doctype_space(source, start + DOCTYPE_OPEN_SIZE)
+          name_start = pos
+          pos += 1 while pos < source.length &&
+              !" \t\r\n>".include?(source[pos])
+          name = source[name_start...pos]
+          return if name.empty?
+
+          pos = skip_doctype_space(source, pos)
+          external_id = nil
+          system_id = nil
+          keyword = source[pos, 6].to_s.upcase
+          if keyword == "PUBLIC"
+            pos = skip_doctype_space(source, pos + 6)
+            pair = scan_doctype_quoted(source, pos)
+            return unless pair
+
+            external_id, pos = pair
+            pos = skip_doctype_space(source, pos)
+            pair = scan_doctype_quoted(source, pos)
+            return unless pair
+
+            system_id, pos = pair
+          elsif keyword == "SYSTEM"
+            pos = skip_doctype_space(source, pos + 6)
+            pair = scan_doctype_quoted(source, pos)
+            return unless pair
+
+            system_id, pos = pair
+          end
+
+          pos = skip_doctype_space(source, pos)
+          internal_subset = nil
+          if source[pos] == "["
+            subset_start = pos + 1
+            pos = matching_subset_end(source, pos + 1)
+            return unless pos
+
+            internal_subset = source[subset_start...pos]
+            pos += 1
+          end
+
+          pos = skip_doctype_space(source, pos)
+          return unless source[pos] == ">"
+
+          {
+            name: name,
+            external_id: external_id,
+            system_id: system_id,
+            internal_subset: internal_subset,
+          }
+        end
+
+        def skip_doctype_space(source, pos)
+          pos += 1 while pos < source.length &&
+              " \t\r\n".include?(source[pos])
+          pos
+        end
+
+        def scan_doctype_quoted(source, pos)
+          quote = source[pos]
+          return unless QUOTES.include?(quote)
+
+          stop = source.index(quote, pos + 1)
+          return unless stop
+
+          [source[(pos + 1)...stop], stop + 1]
+        end
+
+        # Index just past the closing bracket at nesting depth zero,
+        # or nil when the subset is unterminated.
+        def matching_subset_end(source, pos)
+          depth = 1
+          quote = nil
+          while pos < source.length
+            char = source[pos]
+            if quote
+              quote = nil if char == quote
+            elsif QUOTES.include?(char)
+              quote = char
+            elsif char == "["
+              depth += 1
+            elsif char == "]"
+              depth -= 1
+              return pos if depth.zero?
+            end
+            pos += 1
+          end
+          nil
+        end
+
         def parse(xml, options = {}, _context = nil)
           # LibXML doesn't preserve DOCTYPE during parsing, so we need to extract it manually
           xml_string = if xml.is_a?(String)
@@ -99,19 +201,24 @@ module Moxml
           xml_string = preprocess_entities(xml_string)
 
           # Extract DOCTYPE before parsing. Locate the literal first:
-          # a plain memscan is ~2x cheaper than the regex's literal
-          # prefix scan on doctype-less documents (the common case),
-          # and matching from the found offset keeps captures
-          # identical.
+          # a plain memscan keeps doctype-less documents (the common
+          # case) off the scanner path; the scan itself is linear.
           doctype_idx = xml_string.index("<!DOCTYPE")
-          doctype_match = if doctype_idx
-                            xml_string.match(DOCTYPE_RE, doctype_idx)
-                          end
+          doctype = doctype_idx && doctype_parts(xml_string, doctype_idx)
 
           native_doc = begin
             # Handle both string and file inputs
             parser = ::LibXML::XML::Parser.string(xml_string)
             parser.parse
+          rescue ::ArgumentError => e
+            # Invalid byte sequences raise ArgumentError from Ruby's
+            # encoding checks before libxml sees the bytes; the
+            # facade names them (taxonomy differential, malformed
+            # class).
+            raise Moxml::ParseError.new(
+              e.message,
+              source: xml.is_a?(String) ? xml[0..100] : nil,
+            )
           rescue ::LibXML::XML::Error => e
             if options[:strict]
               line = e.line
@@ -127,16 +234,17 @@ module Moxml
           end
 
           # Store DOCTYPE if found
-          if doctype_match
-            name = doctype_match[1]
-            external_id = doctype_match[2]
-            system_id = doctype_match[3] || doctype_match[4]
+          if doctype
+            name = doctype[:name]
+            external_id = doctype[:external_id]
+            system_id = doctype[:system_id]
 
             doctype_wrapper = DoctypeWrapper.new(
               native_doc,
               name,
               external_id,
               system_id,
+              doctype[:internal_subset],
             )
             attachments.set(native_doc, :doctype, doctype_wrapper)
           end
@@ -855,7 +963,8 @@ module Moxml
 
         def processing_instruction_content(node)
           # XML 1.0 §2.6: PI content is verbatim — no entity resolution.
-          unpatch_node(node)&.content
+          # Empty data is the empty string, never nil (uniform Kind face).
+          unpatch_node(node)&.content.to_s
         end
 
         def set_processing_instruction_content(node, content)
@@ -936,12 +1045,12 @@ module Moxml
 
         # libxml-ruby 5.x exposes no external/system ID accessor on
         # the DTD node — the DOCTYPE source text is the only source,
-        # so re-derive from it (same DOCTYPE_RE as the parse path).
+        # so re-derive from it (same linear scanner as the parse path).
         def doctype_external_id(native)
           if native.is_a?(DoctypeWrapper)
             native.external_id
           else
-            native.to_s[DOCTYPE_RE, 2]
+            Libxml.doctype_parts(native.to_s)&.dig(:external_id)
           end
         end
 
@@ -949,7 +1058,7 @@ module Moxml
           if native.is_a?(DoctypeWrapper)
             native.system_id
           else
-            native.to_s[DOCTYPE_RE, 3] || native.to_s[DOCTYPE_RE, 4]
+            Libxml.doctype_parts(native.to_s)&.dig(:system_id)
           end
         end
 
@@ -962,8 +1071,10 @@ module Moxml
           # expression from the document node is equivalent to the
           # absolute form — the document's only name-testable child
           # is the root — so rewrite instead of losing the match.
+          # Function calls and unions are not name-test paths: a
+          # "/count(...)" rewrite is invalid XPath (scalar arm).
           if native_node.is_a?(::LibXML::XML::Document) &&
-              expression.match?(/\A[A-Za-z_*@]/)
+              relative_name_path?(expression)
             expression = "/#{expression}"
           end
 
@@ -997,6 +1108,16 @@ module Moxml
           results&.first
         end
 
+        # A relative name-test path (the only shape the document-context
+        # rewrite is for): a name/`*`/`@` start, no function call (the
+        # scalar arm), no union.
+        def relative_name_path?(expression)
+          return false unless expression.match?(/\A[A-Za-z_*@]/)
+          return false if expression.match?(/\A[A-Za-z][\w.-]*\s*\(/)
+
+          !expression.include?("|")
+        end
+
         # Shallow duplication: copies the node itself (name, attrs, namespaces)
         # but NOT its descendants (deep duplication composes this).
         # walks the source tree and re-adds children one at a time via
@@ -1019,6 +1140,7 @@ module Moxml
                 node.name,
                 node.external_id,
                 node.system_id,
+                node.internal_subset,
               )
             else
               node

data/lib/moxml/adapter/oga.rb

@@ -40,6 +40,18 @@ module Moxml
             )
           end
 
+          # Rootless parses (empty or declaration-only input) are
+          # malformed: XML requires exactly one root. Oga returns a
+          # bare document where nokogiri/leptris raise — the facade
+          # names it instead (loudness differential, TODO.sota/01).
+          if options[:strict] && native_doc.respond_to?(:root_node) &&
+              native_doc.children.none?(::Oga::XML::Element)
+            raise Moxml::ParseError.new(
+              "document has no root element",
+              source: xml.is_a?(String) ? xml[0..100] : nil,
+            )
+          end
+
           # Oga parses multiple root elements leniently; the moxml
           # contract requires a single root. to_a because the vendored
           # fork's NodeSet#count ignores its block.
@@ -499,7 +511,13 @@ module Moxml
         end
 
         def processing_instruction_content(node)
-          node.text
+          # Oga keeps the target/content separator space inside the
+          # PI's text; the model's content is the data AFTER the
+          # separator (D-04). Separator whitespace is parser syntax, and
+          # leaving it in makes canonicalization non-idempotent (the
+          # serializer re-adds the separator each round — TODO.sota/02,
+          # M2).
+          node.text.to_s.sub(/\A\s+/, "")
         end
 
         def set_processing_instruction_content(node, content)
@@ -549,8 +567,20 @@ module Moxml
         def xpath(node, expression, namespaces = nil)
           result = node.xpath(expression, {},
                               namespaces: namespaces&.transform_keys(&:to_s))
-          # Adapter contract: Array<native> | scalar
-          result.is_a?(Enumerable) ? result.to_a : result
+          # Adapter contract: Array<native> | scalar. Oga's engine
+          # answers node-sets in evaluation order, not document order
+          # (//* on <r><a><b/></a><c/></r> answers a,c,b); path results
+          # are document-ordered by the model — re-order here
+          # (TODO.sota/02, M3).
+          if result.is_a?(Enumerable)
+            nodes = result.to_a
+            ctx = Context.new(:oga)
+            wrapped = nodes.map { |n| Moxml::Node.wrap(n, ctx) }
+            ordered = Moxml::XPath::Ordering.document_order(wrapped)
+            ordered.map(&:native)
+          else
+            result
+          end
         rescue ::LL::ParserError => e
           raise Moxml::XPathError.new(
             e.message,

data/lib/moxml/adapter/ox.rb

@@ -60,13 +60,28 @@ module Moxml
               assign_parents(doc)
               doc
             end
-          rescue ::Ox::ParseError => e
+          rescue ::Ox::ParseError, ::RuntimeError => e
+            # Ox leaks raw RuntimeErrors from some malformed shapes
+            # (e.g. declaration-only input reaches Element#<<); the
+            # facade names every parse-time engine failure.
             raise Moxml::ParseError.new(
               e.message,
               source: xml.is_a?(String) ? xml[0..100] : nil,
             )
           end
 
+          # Rootless parses (empty or declaration-only input) are
+          # malformed: XML requires exactly one root. Ox answers a
+          # bare document — the facade names it (uniform with the
+          # rexml/oga arms; loudness differential, TODO.sota/01).
+          if options[:strict] && native_doc.is_a?(::Ox::Document) &&
+              native_doc.nodes.none?(::Ox::Element)
+            raise Moxml::ParseError.new(
+              "document has no root element",
+              source: xml.is_a?(String) ? xml[0..100] : nil,
+            )
+          end
+
           ctx = _context || Context.new(context_adapter_name)
           Wrappers::Document.new(native_doc, ctx)
         end
@@ -627,7 +642,11 @@ module Moxml
         end
 
         def processing_instruction_content(node)
-          node.content.to_s
+          # Ox keeps the target/content separator space inside the
+          # PI content; the model's content is the data AFTER the
+          # separator. Leaving it in makes canonicalization
+          # non-idempotent (TODO.sota/02, M2).
+          node.content.to_s.sub(/\A\s+/, "")
         end
 
         def set_processing_instruction_content(node, content)
@@ -708,6 +727,11 @@ module Moxml
             end)
           when NodeSet
             dedupe_natives(result.to_a.map(&:native))
+          when Moxml::Node
+            # The engine answers a lone node for the context node
+            # (`.`); the adapter contract is a node-set for node
+            # results.
+            [result.native]
           else
             result
           end

data/lib/moxml/adapter/rexml.rb

@@ -46,6 +46,29 @@ module Moxml
               )
             end
             create_document
+          rescue ::RuntimeError => e
+            # REXML's entity-expansion cap (and any engine-level
+            # guard) surfaces as a bare RuntimeError; the facade
+            # names it (issue: taxonomy differential, huge-entity
+            # class; the cap itself is D-16).
+            raise Moxml::ParseError.new(
+              e.message,
+              source: xml.is_a?(String) ? xml[0..100] : nil,
+            )
+          end
+
+          # Rootless parses (empty or declaration-only input) are
+          # malformed: XML requires exactly one root. Nokogiri and
+          # leptris raise at the engine; rexml returns a bare
+          # document — the facade names it instead (loudness
+          # differential, TODO.sota/01). Recover mode keeps the
+          # rootless-document fallback with its diagnostics (#147).
+          if options[:strict] && native_doc.respond_to?(:root) &&
+              native_doc.root.nil?
+            raise Moxml::ParseError.new(
+              "document has no root element",
+              source: xml.is_a?(String) ? xml[0..100] : nil,
+            )
           end
 
           ctx = _context || Context.new(:rexml)
@@ -435,7 +458,7 @@ module Moxml
         end
 
         def processing_instruction_content(node)
-          node.content
+          node.content.to_s
         end
 
         def set_processing_instruction_content(node, content)
@@ -458,17 +481,26 @@ module Moxml
           return "" unless element
 
           text = ""
-          element.children.each do |child|
-            case child
-            when ::REXML::Text
-              # Preserve original spacing from text nodes exactly including newlines and all whitespace
-              text += child.value
-            when ::REXML::Element
-              # Extract text recursively from child element
-              child_text = extract_text_recursively(child)
-              # Concatenate directly like other adapters - NO SPACE INSERTION
-              text += child_text
+          begin
+            element.children.each do |child|
+              case child
+              when ::REXML::Text
+                # Preserve original spacing from text nodes exactly including newlines and all whitespace
+                text += child.value
+              when ::REXML::Element
+                # Extract text recursively from child element
+                child_text = extract_text_recursively(child)
+                # Concatenate directly like other adapters - NO SPACE INSERTION
+                text += child_text
+              end
             end
+          rescue ::RuntimeError => e
+            # rexml expands entities lazily: the parse accepts, the
+            # read hits the expansion guard (D-16). The facade names
+            # it instead of leaking the bare RuntimeError.
+            raise Moxml::Error,
+                  "entity expansion limit exceeded (rexml guard, D-16): " \
+                  "#{e.message}"
           end
           # Don't strip - preserve original spacing including newlines
           text
@@ -612,11 +644,29 @@ module Moxml
           # namespaced elements never match (issue #395).
           namespaces = namespaces.transform_keys(&:to_s) if
             namespaces.is_a?(Hash)
-          if namespaces && !namespaces.empty?
-            ::REXML::XPath.match(node, expression, namespaces).to_a
-          else
-            ::REXML::XPath.match(node, expression).to_a
+          results = if namespaces && !namespaces.empty?
+                      ::REXML::XPath.match(node, expression, namespaces).to_a
+                    else
+                      ::REXML::XPath.match(node, expression).to_a
+                    end
+          # REXML answers unions in expression order (//a | //b on
+          # <r><b/><a/></r> answers a,a,b); path results are
+          # document-ordered by the model (TODO.sota/02, M3) —
+          # re-order here, the oga lane's pattern.
+          return results if results.empty?
+
+          # Scalar arm: count()/string()/boolean()/arithmetic answer a
+          # one-element array holding the value; the adapter contract
+          # passes scalars through unwrapped (numbers as Float,
+          # uniform with the other adapters).
+          unless results.first.is_a?(::REXML::Node)
+            value = results.first
+            return value.is_a?(Numeric) ? value.to_f : value
           end
+
+          ctx = Context.new(:rexml)
+          wrapped = results.map { |n| Moxml::Node.wrap(n, ctx) }
+          Moxml::XPath::Ordering.document_order(wrapped).map(&:native)
         rescue ::REXML::ParseException => e
           raise Moxml::XPathError.new(
             e.message,

data/lib/moxml/context.rb

@@ -119,7 +119,11 @@ module Moxml
                    end
       # Allocation-free declaration sniff: String#strip would copy the
       # whole buffer just to test its prefix.
-      has_declaration = xml_string.match?(/\A[ \t\r\n\f\v\0]*<\?xml/)
+      # Invalid byte sequences would raise from the regex before any
+      # adapter sees them; the adapters' named parse errors own
+      # malformed input (taxonomy differential, malformed class).
+      has_declaration = xml_string.valid_encoding? &&
+        xml_string.match?(/\A[ \t\r\n\f\v\0]*<\?xml/)
 
       # Parse with adapter, passing self (context) so adapter can use our config
       parsed_options = default_options.merge(options)

data/lib/moxml/document.rb

@@ -272,7 +272,7 @@ module Moxml
       # - NodeSet: already wrapped, return directly
       # - Array / LazyNodeSet: wrap in NodeSet
       case result
-      when NodeSet, Float, String, TrueClass, FalseClass, NilClass
+      when NodeSet, Numeric, String, TrueClass, FalseClass, NilClass
         result
       when Array, LazyNodeSet
         NodeSet.new(result, context)
@@ -282,12 +282,12 @@ module Moxml
       end
     end
 
+    # Sugar over xpath: the first node of a node-set; scalar results
+    # (count(), string(), boolean()) pass through unwrapped and
+    # uniform across adapters.
     def at_xpath(expression, namespaces = nil)
-      expression = lax_document_path(expression) if relative_name_test?(expression)
-
-      if (native_node = adapter.at_xpath(@native, expression, namespaces))
-        Moxml::Node.wrap(native_node, context)
-      end
+      result = xpath(expression, namespaces)
+      result.is_a?(NodeSet) ? result.first : result
     end
 
     # Quick element creation and addition

data/lib/moxml/node.rb

@@ -367,9 +367,12 @@ module Moxml
       Materializer.materialize_fields(self, &block)
     end
 
+    # Sugar over xpath: the first node of a node-set; scalar results
+    # (count(), string(), boolean()) pass through unwrapped and
+    # uniform across adapters.
     def at_xpath(expression, namespaces = {})
-      Moxml::Node.wrap(adapter.at_xpath(@native, expression, namespaces),
-                       context)
+      result = xpath(expression, namespaces)
+      result.is_a?(NodeSet) ? result.first : result
     end
 
     # Nokogiri-compatible sugar: at/search/at_css/css. Nokogiri's
@@ -410,10 +413,16 @@ module Moxml
     end
 
     # Nokogiri-compatible depth-first traversal: yields self, then
-    # children recursively (document-order visitor).
+    # children recursively (document-order visitor). Document-level
+    # parts (Declaration, Doctype) are state, not nodes: they are
+    # never yielded, and their (adapter-specific, sometimes missing)
+    # child faces are never consulted (issue-free by the model's
+    # object-model rule; found by the taxonomy differential).
     def traverse(&block)
       return to_enum(:traverse) unless block
 
+      return self if is_a?(Moxml::Declaration) || is_a?(Moxml::Doctype)
+
       yield self
       children.each { |child| child.traverse(&block) }
       self
@@ -682,9 +691,16 @@ module Moxml
       end
     end
 
-    # The adapter's type probe is an FFI call per invocation; the
-    # type is fixed for a node's lifetime, so the first answer is
-    # memoized on the wrapper.
+    # The node's kind (the model's Kind face): :document, :element,
+    # :text, :comment, :cdata, :processing_instruction, :doctype,
+    # :declaration, :entity_reference, :namespace, :attribute. The
+    # adapter's type probe is an FFI call per invocation; the type is
+    # fixed for a node's lifetime, so the first answer is memoized on
+    # the wrapper.
+    def node_type
+      node_type_cached
+    end
+
     def node_type_cached
       @node_type_cached ||= adapter.node_type(@native)
     end

data/lib/moxml/node_set.rb

@@ -119,6 +119,17 @@ module Moxml
 
     # Nokogiri-compatible: Array#flatten and multiple-assignment
     # treat a NodeSet as its wrapped-element array via to_ary.
+    # The same node-set in document order (XPath path results are
+    # conventionally document-ordered; the compiler batches multi-step
+    # evaluation and must re-order). Wrappers are ordered, then the
+    # internal natives are re-paired — no re-wrapping.
+    def document_order
+      return self if native_nodes.size < 2
+
+      ordered = Moxml::XPath::Ordering.document_order(to_a)
+      NodeSet.new(ordered.map(&:native), @context, @parent_node)
+    end
+
     def to_ary
       to_a
     end

data/lib/moxml/reference/normalizer.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Moxml
+  module Reference
+    # Pure-Ruby normalization faces: XML 1.0 §2.11 line endings and
+    # entity expansion. These are the model's normative answers with
+    # no engine involved; adapters are differentially tested against
+    # them.
+    module Normalizer
+      BUILTIN_ENTITIES = {
+        "amp" => "&", "lt" => "<", "gt" => ">",
+        "quot" => '"', "apos" => "'"
+      }.freeze
+      ENTITY_REFERENCE = /&([^;&<>]*);/
+
+      module_function
+
+      # CRLF → LF; bare CR → LF. Character references are untouched
+      # (&#xD; keeps a literal CR).
+      def normalize_eol(text)
+        text.to_s.gsub("\r\n", "\n").gsub("\r", "\n")
+      end
+
+      # Attribute-value normalization: literal tab/newline/CR become a
+      # space (character references keep their value).
+      def normalize_attribute_literal(text)
+        text.to_s.gsub(/[\t\n\r]/, " ")
+      end
+
+      # Expand built-in and numeric references; custom entities come
+      # from the provided table. An undefined reference raises a named
+      # ParseError (the model has no silently unresolved entities).
+      def expand_entities(text, entities = {})
+        text.to_s.gsub(ENTITY_REFERENCE) do
+          reference = Regexp.last_match(1)
+          resolve(reference, BUILTIN_ENTITIES.merge(entities))
+        end
+      end
+
+      def resolve(reference, entities)
+        if reference.start_with?("#x")
+          numeric(reference[2..], 16)
+        elsif reference.start_with?("#")
+          numeric(reference[1..], 10)
+        elsif BUILTIN_ENTITIES.key?(reference)
+          BUILTIN_ENTITIES[reference]
+        else
+          value = entities[reference]
+          raise ParseError, "undefined entity &#{reference};" unless value
+
+          expand_entities(value, entities)
+        end
+      end
+
+      def numeric(digits, base)
+        pattern = base == 16 ? /\A[0-9A-Fa-f]+\z/ : /\A[0-9]+\z/
+        raise ParseError, "malformed character reference" unless digits.match?(pattern)
+
+        [digits.to_i(base)].pack("U")
+      end
+    end
+  end
+end