moxml 0.1.9 → 0.1.11

data/lib/moxml/adapter/oga.rb

@@ -9,17 +9,24 @@ module Moxml
   module Adapter
     class Oga < Base
       class << self
+        # Standard XML entities handled natively by parsers
+        STANDARD_XML_ENTITIES = %w[amp lt gt quot apos].freeze
+
         def set_root(doc, element)
           # Clear existing root element if any - Oga's NodeSet needs special handling
           # We need to manually remove elements since NodeSet doesn't support clear or delete_if
-          elements_to_remove = doc.children.select { |child| child.is_a?(::Oga::XML::Element) }
+          elements_to_remove = doc.children.grep(::Oga::XML::Element)
           elements_to_remove.each { |elem| doc.children.delete(elem) }
           doc.children << element
         end
 
-        def parse(xml, options = {})
+        def parse(xml, options = {}, _context = nil)
+          # Pre-process XML to convert named entities to marker form (\x01name;).
+          # Oga drops named entity references like &nbsp; during parsing.
+          processed_xml = preprocess_named_entities(xml)
+
           native_doc = begin
-            ::Oga.parse_xml(xml, strict: options[:strict])
+            ::Oga.parse_xml(processed_xml, strict: options[:strict])
           rescue LL::ParserError => e
             raise Moxml::ParseError.new(
               e.message,
@@ -27,7 +34,8 @@ module Moxml
             )
           end
 
-          DocumentBuilder.new(Context.new(:oga)).build(native_doc)
+          ctx = _context || Context.new(:oga)
+          DocumentBuilder.new(ctx).build(native_doc)
         end
 
         # SAX parsing implementation for Oga
@@ -61,7 +69,7 @@ module Moxml
         end
 
         def create_native_text(content)
-          ::Oga::XML::Text.new(text: content)
+          ::Oga::XML::Text.new(text: encode_entity_markers(content))
         end
 
         def create_native_cdata(content)
@@ -74,7 +82,8 @@ module Moxml
 
         def create_native_doctype(name, external_id, system_id)
           ::Oga::XML::Doctype.new(
-            name: name, public_id: external_id, system_id: system_id, type: "PUBLIC",
+            name: name, public_id: external_id, system_id: system_id,
+            type: external_id ? "PUBLIC" : "SYSTEM"
           )
         end
 
@@ -224,7 +233,7 @@ module Moxml
           attr = ::Oga::XML::Attribute.new(
             name: name.to_s,
             namespace_name: namespace_name,
-            value: value.to_s,
+            value: encode_entity_markers(value.to_s),
           )
           element.add_attribute(attr)
         end
@@ -234,7 +243,7 @@ module Moxml
         end
 
         def get_attribute_value(element, name)
-          element[name.to_s]
+          restore_entity_markers(element[name.to_s])
         end
 
         def remove_attribute(element, name)
@@ -303,24 +312,25 @@ module Moxml
         end
 
         def text_content(node)
-          node.text
+          restore_entity_markers(node.text)
         end
 
         def inner_text(node)
-          if node.respond_to?(:inner_text)
-            node.inner_text
-          else
-            # Oga::XML::Text node for example
-            node.text
-          end
+          text = if node.respond_to?(:inner_text)
+                   node.inner_text
+                 else
+                   # Oga::XML::Text node for example
+                   node.text
+                 end
+          restore_entity_markers(text)
         end
 
         def set_text_content(node, content)
+          encoded = encode_entity_markers(content)
           if node.respond_to?(:inner_text=)
-            node.inner_text = content
+            node.inner_text = encoded
           else
-            # Oga::XML::Text node for example
-            node.text = content
+            node.text = encoded
           end
         end
 
@@ -365,6 +375,19 @@ module Moxml
           node.namespaces.values
         end
 
+        # Doctype accessor methods
+        def doctype_name(native)
+          native.name
+        end
+
+        def doctype_external_id(native)
+          native.public_id
+        end
+
+        def doctype_system_id(native)
+          native.system_id
+        end
+
         def xpath(node, expression, namespaces = nil)
           node.xpath(expression, {},
                      namespaces: namespaces&.transform_keys(&:to_s)).to_a
@@ -389,6 +412,53 @@ module Moxml
         end
 
         def serialize(node, options = {})
+          output = serialize_without_entity_processing(node, options)
+          # Post-process: convert entity markers back to entity references
+          output.gsub(ENTITY_MARKER_REGEX, '&\1;')
+        end
+
+        # Shared entity name pattern (W3C: 2-31 chars, starts with alpha)
+        ENTITY_PATTERN = "([a-zA-Z][a-zA-Z0-9]{1,30})"
+
+        # Marker character for entity preservation through Oga's parser.
+        # U+0001 is preserved literally by Oga through parse/serialize cycle.
+        ENTITY_MARKER = "\x01"
+
+        # Regular expression for entity marker post-processing
+        ENTITY_MARKER_REGEX = /#{ENTITY_MARKER}#{ENTITY_PATTERN};/
+
+        # Simple entity-only regex with no nested quantifiers
+        ENTITY_REF_REGEX = /&#{ENTITY_PATTERN};/
+
+        private
+
+        # Convert &entity; back to \x01entity; for Oga text storage.
+        # Used when setting text content programmatically (not from parsing).
+        def encode_entity_markers(text)
+          return text unless text&.include?("&")
+
+          text.gsub(ENTITY_REF_REGEX) do
+            name = ::Regexp.last_match(1)
+
+            next ::Regexp.last_match(0) if STANDARD_XML_ENTITIES.include?(name)
+
+            codepoint = Moxml::EntityRegistry.default.codepoint_for_name(name)
+            if codepoint
+              "#{ENTITY_MARKER}#{name};"
+            else
+              ::Regexp.last_match(0)
+            end
+          end
+        end
+
+        # Convert \x01entity; back to &entity; for text accessors.
+        def restore_entity_markers(text)
+          return text unless text
+
+          text.gsub(ENTITY_MARKER_REGEX, '&\1;')
+        end
+
+        def serialize_without_entity_processing(node, options = {})
           # Oga's XmlGenerator doesn't support options directly
           # We need to handle declaration options ourselves for Document nodes
           if node.is_a?(::Oga::XML::Document)
@@ -403,7 +473,11 @@ module Moxml
                                     node.xml_declaration ? true : false
                                   end
 
-            if should_include_decl && !node.xml_declaration
+            # Fix: Check if declaration already exists in children
+            # This prevents duplicate declarations when document already has one
+            has_existing_declaration = node.children.any?(::Oga::XML::XmlDeclaration)
+
+            if should_include_decl && !node.xml_declaration && !has_existing_declaration
               # Need to add declaration - create default one
               output = +""
               output << '<?xml version="1.0" encoding="UTF-8"?>'
@@ -437,7 +511,50 @@ module Moxml
           end
 
           # Default: use XmlGenerator
-          ::Moxml::Adapter::CustomizedOga::XmlGenerator.new(node).to_xml
+          # But first check if we need to handle declaration specially
+          if node.is_a?(::Oga::XML::Document) && node.xml_declaration
+            # Document has declaration - use custom handling to avoid duplicates
+            output = +""
+
+            # Serialize children, but skip XmlDeclaration if it would cause duplication
+            node.children.each do |child|
+              # Check if this would cause duplication by seeing if we already have one in output
+              if child.is_a?(::Oga::XML::XmlDeclaration) && output.include?("<?xml")
+                next # Skip duplicate declaration
+              end
+
+              output << ::Moxml::Adapter::CustomizedOga::XmlGenerator.new(child).to_xml
+            end
+
+            output
+          else
+            # Normal case - use XmlGenerator directly
+            ::Moxml::Adapter::CustomizedOga::XmlGenerator.new(node).to_xml
+          end
+        end
+
+        # Pre-process XML to convert named entities to marker format.
+        # Oga drops named entity references like &nbsp; but preserves control chars.
+        # By converting known named entities to marker form (\x01name;), we can
+        # reconstruct them during serialization.
+        #
+        # @param xml [String, #to_s] The XML string to process
+        # @return [String] The XML with known named entities converted to marker form
+        def preprocess_named_entities(xml)
+          return xml unless xml.is_a?(String)
+
+          xml.gsub(ENTITY_REF_REGEX) do
+            name = Regexp.last_match(1)
+
+            next Regexp.last_match(0) if STANDARD_XML_ENTITIES.include?(name)
+
+            codepoint = Moxml::EntityRegistry.default.codepoint_for_name(name)
+            if codepoint
+              "#{ENTITY_MARKER}#{name};"
+            else
+              Regexp.last_match(0)
+            end
+          end
         end
       end
     end

data/lib/moxml/adapter/ox.rb

@@ -17,7 +17,7 @@ module Moxml
           replace_children(doc, [element])
         end
 
-        def parse(xml, _options = {})
+        def parse(xml, _options = {}, _context = nil)
           native_doc = begin
             result = ::Ox.parse(xml)
 
@@ -36,7 +36,8 @@ module Moxml
             )
           end
 
-          DocumentBuilder.new(Context.new(:ox)).build(native_doc)
+          ctx = _context || Context.new(:ox)
+          DocumentBuilder.new(ctx).build(native_doc)
         end
 
         # SAX parsing implementation for Ox
@@ -452,7 +453,7 @@ module Moxml
         def inner_text(node)
           return "" unless node.respond_to?(:nodes)
 
-          node.nodes.select { _1.is_a?(String) }.join
+          node.nodes.grep(String).join
         end
 
         def set_text_content(node, content)
@@ -510,6 +511,31 @@ module Moxml
           end.values
         end
 
+        # Doctype accessor methods
+        # Ox stores DOCTYPE as a string, so we parse it
+        def doctype_name(native)
+          # Parse: "name PUBLIC \"external_id\" \"system_id\"" or "name SYSTEM \"system_id\""
+          value = native.value.to_s.strip
+          # Extract the first word (the name)
+          value.split(/\s+/).first
+        end
+
+        def doctype_external_id(native)
+          value = native.value.to_s
+          # Match PUBLIC "external_id"
+          match = value.match(/PUBLIC\s+"([^"]*)"/)
+          match ? match[1] : nil
+        end
+
+        def doctype_system_id(native)
+          value = native.value.to_s
+          # Match the last quoted string (system_id)
+          # For PUBLIC: "name PUBLIC \"external_id\" \"system_id\""
+          # For SYSTEM: "name SYSTEM \"system_id\""
+          matches = value.scan(/"([^"]*)"/)
+          matches.last&.first
+        end
+
         def xpath(node, expression, namespaces = {})
           # Translate common XPath patterns to Ox locate() syntax
           locate_expr = translate_xpath_to_locate(expression, namespaces)

data/lib/moxml/adapter/rexml.rb

@@ -10,9 +10,16 @@ module Moxml
   module Adapter
     class Rexml < Base
       class << self
-        def parse(xml, options = {})
+        def parse(xml, options = {}, _context = nil)
+          # Handle frozen strings by creating a mutable copy
+          processed_xml = if xml.frozen?
+                            xml.dup.force_encoding("UTF-8").encode("UTF-8")
+                          else
+                            xml.force_encoding("UTF-8").encode("UTF-8")
+                          end
+
           native_doc = begin
-            ::REXML::Document.new(xml)
+            ::REXML::Document.new(processed_xml)
           rescue ::REXML::ParseException => e
             if options[:strict]
               raise Moxml::ParseError.new(
@@ -24,7 +31,15 @@ module Moxml
             create_document
           end
 
-          DocumentBuilder.new(Context.new(:rexml)).build(native_doc)
+          ctx = _context || Context.new(:rexml)
+          DocumentBuilder.new(ctx).build(native_doc)
+        end
+
+        def extract_encoding_from_xml(xml)
+          # Match XML declaration pattern: <?xml version="..." encoding="..."?>
+          # Use atomic group (?>) to prevent polynomial backtracking ReDoS
+          match = xml.match(/<\?xml(?>[^>]*)\bencoding\s*=\s*["']([^"']+)["']/i)
+          match ? match[1] : "UTF-8"
         end
 
         # SAX parsing implementation for REXML
@@ -359,16 +374,35 @@ module Moxml
           when ::REXML::Text, ::REXML::CData
             node.value.to_s
           when ::REXML::Element
-            # Get all text nodes, filter out duplicates, and join
-            text_nodes = node.texts.uniq(&:object_id)
-            text_nodes.map(&:value).join
+            # Extract text recursively from all children to match other adapters
+            extract_text_recursively(node)
+          end
+        end
+
+        def extract_text_recursively(element)
+          return "" unless element
+
+          text = ""
+          element.children.each do |child|
+            case child
+            when ::REXML::Text
+              # Preserve original spacing from text nodes exactly including newlines and all whitespace
+              text += child.value
+            when ::REXML::Element
+              # Extract text recursively from child element
+              child_text = extract_text_recursively(child)
+              # Concatenate directly like other adapters - NO SPACE INSERTION
+              text += child_text
+            end
           end
+          # Don't strip - preserve original spacing including newlines
+          text
         end
 
         def inner_text(node)
           # Get direct text children only, filter duplicates
           text_children = node.children
-            .select { _1.is_a?(::REXML::Text) }
+            .grep(::REXML::Text)
             .uniq(&:object_id)
           text_children.map(&:value).join
         end
@@ -426,6 +460,19 @@ module Moxml
           end
         end
 
+        # Doctype accessor methods
+        def doctype_name(native)
+          native.name
+        end
+
+        def doctype_external_id(native)
+          native.public
+        end
+
+        def doctype_system_id(native)
+          native.system
+        end
+
         # not used at the moment
         # but may be useful when the xpath is upgraded to work with namespaces
         def prepare_xpath_namespaces(node)

data/lib/moxml/attribute.rb

@@ -10,6 +10,12 @@ module Moxml
       adapter.set_attribute_name(@native, new_name)
     end
 
+    # Returns the primary identifier for this attribute (its name)
+    # @return [String] the attribute name
+    def identifier
+      name
+    end
+
     def value
       @native.value
     end

data/lib/moxml/builder.rb

@@ -2,6 +2,8 @@
 
 module Moxml
   class Builder
+    attr_reader :document
+
     def initialize(context)
       @context = context
       @current = @document = context.create_document
@@ -60,6 +62,10 @@ module Moxml
       @current.add_child(@document.create_comment(content))
     end
 
+    def entity_reference(name)
+      @current.add_child(@document.create_entity_reference(name))
+    end
+
     def processing_instruction(target, content)
       @current.add_child(
         @document.create_processing_instruction(target, content),

data/lib/moxml/config.rb

@@ -5,6 +5,13 @@ module Moxml
     VALID_ADAPTERS = %i[nokogiri oga rexml ox headed_ox libxml].freeze
     DEFAULT_ADAPTER = VALID_ADAPTERS.first
 
+    # Entity loading modes:
+    # - :required - Must load entities, raise error if unavailable (default)
+    # - :optional - Try to load, continue silently if unavailable
+    # - :disabled - Don't load entities, use empty registry
+    # - :custom - Use custom entity provider via entity_provider callback
+    ENTITY_LOAD_MODES = %i[required optional disabled custom].freeze
+
     class << self
       attr_writer :default_adapter
 
@@ -17,11 +24,18 @@ module Moxml
       end
     end
 
+    NAMESPACE_URI_MODES = %i[strict lenient].freeze
+
     attr_reader :adapter_name
     attr_accessor :strict_parsing,
                   :default_encoding,
                   :entity_encoding,
-                  :default_indent
+                  :default_indent,
+                  :restore_entities,
+                  :preload_entity_sets,
+                  :entity_load_mode,
+                  :entity_provider,
+                  :namespace_uri_mode
 
     def initialize(adapter_name = nil, strict_parsing = nil,
                    default_encoding = nil)
@@ -31,6 +45,11 @@ module Moxml
       # reserved for future use
       @default_indent = 2
       @entity_encoding = :basic
+      @restore_entities = false
+      @preload_entity_sets = []
+      @entity_load_mode = :required
+      @entity_provider = nil
+      @namespace_uri_mode = :strict
     end
 
     def adapter=(name)
@@ -57,5 +76,37 @@ module Moxml
     def adapter
       @adapter ||= Adapter.load(@adapter_name)
     end
+
+    def entity_load_mode=(mode)
+      unless ENTITY_LOAD_MODES.include?(mode)
+        raise ArgumentError,
+              "Invalid entity_load_mode: #{mode}. Must be one of: #{ENTITY_LOAD_MODES.join(', ')}"
+      end
+
+      @entity_load_mode = mode
+    end
+
+    def namespace_uri_mode=(mode)
+      mode = mode.to_sym
+      unless NAMESPACE_URI_MODES.include?(mode)
+        raise ArgumentError,
+              "Invalid namespace_uri_mode: #{mode}. Must be one of: #{NAMESPACE_URI_MODES.join(', ')}"
+      end
+
+      @namespace_uri_mode = mode
+    end
+
+    # Backward compatibility: convert old boolean to new symbol
+    def load_external_entities=(value)
+      @entity_load_mode = case value
+                          when true then :required
+                          when false then :disabled
+                          else value
+                          end
+    end
+
+    def load_external_entities
+      @entity_load_mode == :required
+    end
   end
 end

data/lib/moxml/context.rb

@@ -8,6 +8,10 @@ module Moxml
       @config = Config.new(adapter)
     end
 
+    def entity_registry
+      @entity_registry ||= build_entity_registry
+    end
+
     def create_document(native_doc = nil)
       Document.new(config.adapter.create_document(native_doc), self)
     end
@@ -23,9 +27,9 @@ module Moxml
                    end
       has_declaration = xml_string.strip.start_with?("<?xml")
 
-      # Parse with adapter (without declaration info - adapters don't need it)
+      # Parse with adapter, passing self (context) so adapter can use our config
       parsed_options = default_options.merge(options)
-      doc = config.adapter.parse(xml_string, parsed_options)
+      doc = config.adapter.parse(xml_string, parsed_options, self)
 
       # Set declaration flag on Document wrapper (proper OOP)
       doc.has_xml_declaration = has_declaration if doc.is_a?(Document)
@@ -73,6 +77,21 @@ module Moxml
 
     private
 
+    def build_entity_registry
+      registry = EntityRegistry.new(
+        mode: config.entity_load_mode,
+        entity_provider: config.entity_provider,
+      )
+      config.preload_entity_sets.each do |set_name|
+        case set_name
+        when :html5 then registry.load_html5
+        when :mathml then registry.load_mathml
+        when :iso then registry.load_iso
+        end
+      end
+      registry
+    end
+
     def default_options
       {
         encoding: config.default_encoding,

data/lib/moxml/doctype.rb

@@ -1,17 +1,50 @@
 # frozen_string_literal: true
 
 module Moxml
+  # Represents an XML DOCTYPE declaration
+  #
+  # @note Doctype accessor methods are not fully implemented across all adapters.
+  #   The availability of #name, #external_id, and #system_id depends on whether
+  #   the specific adapter implements the corresponding adapter methods:
+  #   - adapter.doctype_name(native)
+  #   - adapter.doctype_external_id(native)
+  #   - adapter.doctype_system_id(native)
+  #
+  #   Most adapters do not currently implement these methods. If you need DOCTYPE
+  #   information, consider using adapter-specific methods or parsing the serialized
+  #   XML manually.
   class Doctype < Node
+    # Returns the DOCTYPE name (root element name)
+    #
+    # @return [String, nil] the DOCTYPE name
+    # @raise [NotImplementedError] if the adapter doesn't implement doctype_name
     def name
       adapter.doctype_name(@native)
     end
 
+    # Returns the DOCTYPE external ID
+    #
+    # @return [String, nil] the external ID
+    # @raise [NotImplementedError] if the adapter doesn't implement doctype_external_id
     def external_id
       adapter.doctype_external_id(@native)
     end
 
+    # Returns the DOCTYPE system ID
+    #
+    # @return [String, nil] the system ID
+    # @raise [NotImplementedError] if the adapter doesn't implement doctype_system_id
     def system_id
       adapter.doctype_system_id(@native)
     end
+
+    # Returns the primary identifier for this doctype
+    # Since DOCTYPE information is not reliably available across adapters,
+    # this returns nil.
+    #
+    # @return [nil]
+    def identifier
+      name
+    end
   end
 end

data/lib/moxml/document.rb

@@ -9,6 +9,7 @@ require_relative "processing_instruction"
 require_relative "declaration"
 require_relative "namespace"
 require_relative "doctype"
+require_relative "entity_reference"
 
 module Moxml
   class Document < Node
@@ -68,6 +69,10 @@ module Moxml
       Declaration.new(decl, context)
     end
 
+    def create_entity_reference(name)
+      EntityReference.new(adapter.create_entity_reference(name), context)
+    end
+
     def add_child(node)
       node = prepare_node(node)
 
@@ -109,7 +114,7 @@ module Moxml
 
     def at_xpath(expression, namespaces = nil)
       if (native_node = adapter.at_xpath(@native, expression, namespaces))
-        Node.wrap(native_node, context)
+        Moxml::Node.wrap(native_node, context)
       end
     end