moxml 0.1.10 → 0.1.12

data/lib/moxml/adapter/rexml.rb

@@ -10,9 +10,16 @@ module Moxml
   module Adapter
     class Rexml < Base
       class << self
-        def parse(xml, options = {})
+        def parse(xml, options = {}, _context = nil)
+          # Handle frozen strings by creating a mutable copy
+          processed_xml = if xml.frozen?
+                            xml.dup.force_encoding("UTF-8").encode("UTF-8")
+                          else
+                            xml.force_encoding("UTF-8").encode("UTF-8")
+                          end
+
           native_doc = begin
-            ::REXML::Document.new(xml)
+            ::REXML::Document.new(processed_xml)
           rescue ::REXML::ParseException => e
             if options[:strict]
               raise Moxml::ParseError.new(
@@ -24,7 +31,15 @@ module Moxml
             create_document
           end
 
-          DocumentBuilder.new(Context.new(:rexml)).build(native_doc)
+          ctx = _context || Context.new(:rexml)
+          DocumentBuilder.new(ctx).build(native_doc)
+        end
+
+        def extract_encoding_from_xml(xml)
+          # Match XML declaration pattern: <?xml version="..." encoding="..."?>
+          # Use atomic group (?>) to prevent polynomial backtracking ReDoS
+          match = xml.match(/<\?xml(?>[^>]*)\bencoding\s*=\s*["']([^"']+)["']/i)
+          match ? match[1] : "UTF-8"
         end
 
         # SAX parsing implementation for REXML
@@ -359,16 +374,35 @@ module Moxml
           when ::REXML::Text, ::REXML::CData
             node.value.to_s
           when ::REXML::Element
-            # Get all text nodes, filter out duplicates, and join
-            text_nodes = node.texts.uniq(&:object_id)
-            text_nodes.map(&:value).join
+            # Extract text recursively from all children to match other adapters
+            extract_text_recursively(node)
+          end
+        end
+
+        def extract_text_recursively(element)
+          return "" unless element
+
+          text = ""
+          element.children.each do |child|
+            case child
+            when ::REXML::Text
+              # Preserve original spacing from text nodes exactly including newlines and all whitespace
+              text += child.value
+            when ::REXML::Element
+              # Extract text recursively from child element
+              child_text = extract_text_recursively(child)
+              # Concatenate directly like other adapters - NO SPACE INSERTION
+              text += child_text
+            end
           end
+          # Don't strip - preserve original spacing including newlines
+          text
         end
 
         def inner_text(node)
           # Get direct text children only, filter duplicates
           text_children = node.children
-            .select { _1.is_a?(::REXML::Text) }
+            .grep(::REXML::Text)
             .uniq(&:object_id)
           text_children.map(&:value).join
         end

data/lib/moxml/builder.rb

@@ -2,6 +2,8 @@
 
 module Moxml
   class Builder
+    attr_reader :document
+
     def initialize(context)
       @context = context
       @current = @document = context.create_document
@@ -60,6 +62,10 @@ module Moxml
       @current.add_child(@document.create_comment(content))
     end
 
+    def entity_reference(name)
+      @current.add_child(@document.create_entity_reference(name))
+    end
+
     def processing_instruction(target, content)
       @current.add_child(
         @document.create_processing_instruction(target, content),

data/lib/moxml/config.rb

@@ -5,6 +5,13 @@ module Moxml
     VALID_ADAPTERS = %i[nokogiri oga rexml ox headed_ox libxml].freeze
     DEFAULT_ADAPTER = VALID_ADAPTERS.first
 
+    # Entity loading modes:
+    # - :required - Must load entities, raise error if unavailable (default)
+    # - :optional - Try to load, continue silently if unavailable
+    # - :disabled - Don't load entities, use empty registry
+    # - :custom - Use custom entity provider via entity_provider callback
+    ENTITY_LOAD_MODES = %i[required optional disabled custom].freeze
+
     class << self
       attr_writer :default_adapter
 
@@ -17,11 +24,18 @@ module Moxml
       end
     end
 
+    NAMESPACE_URI_MODES = %i[strict lenient].freeze
+
     attr_reader :adapter_name
     attr_accessor :strict_parsing,
                   :default_encoding,
                   :entity_encoding,
-                  :default_indent
+                  :default_indent,
+                  :restore_entities,
+                  :preload_entity_sets,
+                  :entity_load_mode,
+                  :entity_provider,
+                  :namespace_uri_mode
 
     def initialize(adapter_name = nil, strict_parsing = nil,
                    default_encoding = nil)
@@ -31,6 +45,11 @@ module Moxml
       # reserved for future use
       @default_indent = 2
       @entity_encoding = :basic
+      @restore_entities = false
+      @preload_entity_sets = []
+      @entity_load_mode = :required
+      @entity_provider = nil
+      @namespace_uri_mode = :strict
     end
 
     def adapter=(name)
@@ -57,5 +76,37 @@ module Moxml
     def adapter
       @adapter ||= Adapter.load(@adapter_name)
     end
+
+    def entity_load_mode=(mode)
+      unless ENTITY_LOAD_MODES.include?(mode)
+        raise ArgumentError,
+              "Invalid entity_load_mode: #{mode}. Must be one of: #{ENTITY_LOAD_MODES.join(', ')}"
+      end
+
+      @entity_load_mode = mode
+    end
+
+    def namespace_uri_mode=(mode)
+      mode = mode.to_sym
+      unless NAMESPACE_URI_MODES.include?(mode)
+        raise ArgumentError,
+              "Invalid namespace_uri_mode: #{mode}. Must be one of: #{NAMESPACE_URI_MODES.join(', ')}"
+      end
+
+      @namespace_uri_mode = mode
+    end
+
+    # Backward compatibility: convert old boolean to new symbol
+    def load_external_entities=(value)
+      @entity_load_mode = case value
+                          when true then :required
+                          when false then :disabled
+                          else value
+                          end
+    end
+
+    def load_external_entities
+      @entity_load_mode == :required
+    end
   end
 end

data/lib/moxml/context.rb

@@ -8,6 +8,10 @@ module Moxml
       @config = Config.new(adapter)
     end
 
+    def entity_registry
+      @entity_registry ||= build_entity_registry
+    end
+
     def create_document(native_doc = nil)
       Document.new(config.adapter.create_document(native_doc), self)
     end
@@ -23,9 +27,9 @@ module Moxml
                    end
       has_declaration = xml_string.strip.start_with?("<?xml")
 
-      # Parse with adapter (without declaration info - adapters don't need it)
+      # Parse with adapter, passing self (context) so adapter can use our config
       parsed_options = default_options.merge(options)
-      doc = config.adapter.parse(xml_string, parsed_options)
+      doc = config.adapter.parse(xml_string, parsed_options, self)
 
       # Set declaration flag on Document wrapper (proper OOP)
       doc.has_xml_declaration = has_declaration if doc.is_a?(Document)
@@ -73,6 +77,21 @@ module Moxml
 
     private
 
+    def build_entity_registry
+      registry = EntityRegistry.new(
+        mode: config.entity_load_mode,
+        entity_provider: config.entity_provider,
+      )
+      config.preload_entity_sets.each do |set_name|
+        case set_name
+        when :html5 then registry.load_html5
+        when :mathml then registry.load_mathml
+        when :iso then registry.load_iso
+        end
+      end
+      registry
+    end
+
     def default_options
       {
         encoding: config.default_encoding,

data/lib/moxml/document.rb

@@ -9,6 +9,7 @@ require_relative "processing_instruction"
 require_relative "declaration"
 require_relative "namespace"
 require_relative "doctype"
+require_relative "entity_reference"
 
 module Moxml
   class Document < Node
@@ -68,6 +69,10 @@ module Moxml
       Declaration.new(decl, context)
     end
 
+    def create_entity_reference(name)
+      EntityReference.new(adapter.create_entity_reference(name), context)
+    end
+
     def add_child(node)
       node = prepare_node(node)
 
@@ -109,7 +114,7 @@ module Moxml
 
     def at_xpath(expression, namespaces = nil)
       if (native_node = adapter.at_xpath(@native, expression, namespaces))
-        Node.wrap(native_node, context)
+        Moxml::Node.wrap(native_node, context)
       end
     end
 

data/lib/moxml/document_builder.rb

@@ -67,7 +67,46 @@ module Moxml
     def visit_text(node)
       # Prepare node for new document before wrapping
       prepared = adapter.prepare_for_new_document(node, @current_doc.native)
-      @node_stack.last&.add_child(Text.new(prepared, context))
+      content = adapter.text_content(node)
+
+      # Check if we should restore entity references for this text
+      if context.config.restore_entities && content.to_s =~ /[<>&"']/
+        restore_entities_in_text(content)
+      else
+        @node_stack.last&.add_child(Text.new(prepared, context))
+      end
+    end
+
+    def restore_entities_in_text(content)
+      parent = @node_stack.last
+      return unless parent
+
+      # Characters that should potentially be entity-encoded
+      # Per W3C XML spec, these characters have special meaning
+      entity_chars = {
+        "<" => "lt",
+        ">" => "gt",
+        "&" => "amp",
+        '"' => "quot",
+        "'" => "apos",
+      }
+
+      # Process character by character
+      chars = content.to_s.chars
+      chars.each do |char|
+        codepoint = char.ord
+        entity_name = context.entity_registry.primary_name_for_codepoint(codepoint)
+
+        if entity_name && entity_chars.value?(entity_name)
+          # This character should be an entity reference
+          entity_node = adapter.create_entity_reference(entity_name)
+          parent.add_child(EntityReference.new(entity_node, context))
+        else
+          # Regular character
+          text_node = adapter.create_text(char)
+          parent.add_child(Text.new(text_node, context))
+        end
+      end
     end
 
     def visit_cdata(node)
@@ -90,6 +129,11 @@ module Moxml
       @node_stack.last&.add_child(Doctype.new(prepared, context))
     end
 
+    def visit_entity_reference(node)
+      prepared = adapter.prepare_for_new_document(node, @current_doc.native)
+      @node_stack.last&.add_child(EntityReference.new(prepared, context))
+    end
+
     def visit_children(node)
       node_children = children(node).dup
       node_children.each do |child|

data/lib/moxml/element.rb

@@ -75,8 +75,8 @@ module Moxml
     end
 
     def add_namespace(prefix, uri)
-      validate_uri(uri)
-      adapter.create_native_namespace(@native, prefix, uri)
+      adapter.create_namespace(@native, prefix, uri,
+                               namespace_uri_mode: context.config.namespace_uri_mode)
       self
     rescue ValidationError => e
       # Re-raise as NamespaceError, provide attributes for error context
@@ -102,7 +102,8 @@ module Moxml
       if ns_or_hash.is_a?(Hash)
         adapter.set_namespace(
           @native,
-          adapter.create_namespace(@native, *ns_or_hash.to_a.first),
+          adapter.create_namespace(@native, *ns_or_hash.to_a.first,
+                                   namespace_uri_mode: context.config.namespace_uri_mode),
         )
       else
         adapter.set_namespace(@native, ns_or_hash&.native)

data/lib/moxml/entity_reference.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Moxml
+  class EntityReference < Node
+    def content
+      ""
+    end
+
+    def text
+      ""
+    end
+
+    def name
+      adapter.entity_reference_name(@native)
+    end
+
+    def to_xml(*)
+      "&#{name};"
+    end
+
+    def ==(other)
+      self.class == other.class && @native == other.native
+    end
+
+    def identifier
+      name
+    end
+  end
+end

data/lib/moxml/entity_registry.rb

@@ -0,0 +1,278 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Moxml
+  # EntityRegistry maintains a knowledge base of XML entity definitions.
+  #
+  # Data source: W3C XML Core WG Character Entities (bundled)
+  # https://www.w3.org/2003/entities/2007/htmlmathml
+  #
+  # The W3C entity data is bundled in data/w3c_entities.json and loaded
+  # from the gem's data directory. For development, MOXML_ENTITY_DEFINITIONS_PATH
+  # can be set to an external copy.
+  #
+  # Per W3C XML Core WG guidance:
+  # - Character entities are XML internal general entities providing a name for a single Unicode character
+  # - Standard XML entities (amp, lt, gt, quot, apos) are implicitly declared per XML specification
+  # - External entity sets (like HTML, MathML) can be referenced via DTD parameter entities
+  #
+  # @example Basic usage
+  #   registry = EntityRegistry.new
+  #   registry.declared?("amp")  # => true
+  #   registry.codepoint_for_name("amp")  # => 38
+  #
+  class EntityRegistry
+    # W3C entity data file name
+    ENTITY_DATA_FILE = "w3c_entities.json"
+
+    class << self
+      # Get the raw entity data from the bundled JSON source
+      # @return [Hash{String => String}] entity name to character mapping
+      def entity_data
+        @entity_data ||= load_entity_data
+      end
+
+      # Get the default registry instance (lazy loaded)
+      # @return [EntityRegistry]
+      def default
+        @default ||= new
+      end
+
+      # Reset the default registry (mainly for testing)
+      # @return [void]
+      def reset
+        @default = nil
+        @entity_data = nil
+      end
+
+      private
+
+      # Load entity data from bundled gem data or local file
+      # @return [Hash{String => String}]
+      def load_entity_data
+        # Try multiple paths in order of priority
+        paths_to_try = []
+
+        # 1. Environment variable override (for development/custom setups)
+        if ENV["MOXML_ENTITY_DEFINITIONS_PATH"]
+          paths_to_try << ENV["MOXML_ENTITY_DEFINITIONS_PATH"]
+        end
+
+        # 2. Relative to moxml lib directory (for development/installation)
+        # __dir__ is lib/moxml/entity_registry.rb
+        # So ../../data/ goes to project_root/data/
+        paths_to_try << File.expand_path(
+          "../../data/#{ENTITY_DATA_FILE}",
+          __dir__,
+        )
+
+        # 3. External xml-entities sibling directory (common development setup)
+        paths_to_try << File.expand_path(
+          "../../external/xml-entities/docs/2007/htmlmathml.json",
+          __dir__,
+        )
+
+        data = nil
+        paths_to_try.uniq.each do |path|
+          next unless path && File.exist?(path)
+
+          begin
+            data = File.read(path)
+            break
+          rescue StandardError
+            # Try next path
+          end
+        end
+
+        unless data
+          raise EntityDataError,
+                "Entity data not found. Set MOXML_ENTITY_DEFINITIONS_PATH or ensure data/#{ENTITY_DATA_FILE} exists."
+        end
+
+        JSON.parse(data)["characters"]
+      rescue StandardError => e
+        raise EntityDataError, "Failed to load entity definitions: #{e.message}"
+      end
+    end
+
+    # Error raised when entity data cannot be loaded
+    class EntityDataError < StandardError; end
+
+    # @return [Hash{String => Integer}] entity name to codepoint mapping
+    attr_reader :by_name
+
+    # @return [Hash{Integer => Array<String>}] codepoint to entity names mapping
+    attr_reader :by_codepoint
+
+    # @param mode [Symbol] Loading mode: :required, :optional, :disabled, :custom
+    # @param entity_provider [Proc, nil] Custom entity provider proc/lambda
+    def initialize(mode: :required, entity_provider: nil)
+      @by_name = {}
+      @by_codepoint = Hash.new { |h, k| h[k] = [] }
+      @mode = mode
+      @entity_provider = entity_provider
+
+      case mode
+      when :required
+        load_from_entity_data
+      when :optional
+        load_from_entity_data_optional
+      when :custom
+        load_custom_entities
+      when :disabled
+        # Don't load anything - empty registry
+      end
+    end
+
+    # Check if an entity name is declared
+    # @param name [String] entity name (e.g., "amp", "nbsp")
+    # @return [Boolean]
+    def declared?(name)
+      @by_name.key?(name)
+    end
+
+    # Get the Unicode codepoint for an entity name
+    # @param name [String] entity name
+    # @return [Integer, nil] codepoint or nil if not found
+    def codepoint_for_name(name)
+      @by_name[name]
+    end
+
+    # Get all entity names for a codepoint
+    # @param codepoint [Integer] Unicode codepoint
+    # @return [Array<String>] entity names mapping to this codepoint
+    def names_for_codepoint(codepoint)
+      @by_codepoint[codepoint]
+    end
+
+    # Get the primary (preferred) entity name for a codepoint
+    # @param codepoint [Integer] Unicode codepoint
+    # @return [String, nil] primary entity name or nil
+    def primary_name_for_codepoint(codepoint)
+      @by_codepoint[codepoint]&.first
+    end
+
+    # Register additional entities
+    # @param entities [Hash{String => Integer}] name => codepoint mapping
+    # @return [self]
+    def register(entities)
+      entities.each do |name, codepoint|
+        @by_name[name] = codepoint
+        @by_codepoint[codepoint] ||= []
+        @by_codepoint[codepoint] << name unless @by_codepoint[codepoint].include?(name)
+      end
+      self
+    end
+
+    # Load all entities from the W3C HTMLMathML entity set
+    # This is called automatically by initialize
+    # @return [self]
+    def load_html5
+      # All entities are loaded by default from initialize
+      self
+    end
+
+    # Load MathML entity set (included in HTMLMathML)
+    # @return [self]
+    def load_mathml
+      # All entities are loaded by default from initialize
+      self
+    end
+
+    # Load ISO entity sets (included in HTMLMathML)
+    # @param _set_name [Symbol] (ignored, all loaded together)
+    # @return [self]
+    def load_iso(_set_name = :iso8879)
+      # All entities are loaded by default from initialize
+      self
+    end
+
+    # Load all standard entity sets
+    # @return [self]
+    def load_all
+      # All entities are loaded by default from initialize
+      self
+    end
+
+    # Clear all entities (reset to empty)
+    # @return [self]
+    def clear!
+      @by_name = {}
+      @by_codepoint = Hash.new { |h, k| h[k] = [] }
+      self
+    end
+
+    private
+
+    # Load entities from the centralized JSON data source
+    # @raise [EntityDataError] if entity data is required but cannot be loaded
+    # @return [void]
+    def load_from_entity_data
+      data = self.class.entity_data
+
+      if data.nil?
+        raise EntityDataError,
+              "Entity data is not available. Set entity_load_mode to :optional or :disabled to skip entity loading."
+      end
+
+      data.each do |name, char|
+        codepoint = parse_codepoint(char)
+        next unless codepoint
+
+        @by_name[name] = codepoint
+        @by_codepoint[codepoint] ||= []
+        @by_codepoint[codepoint] << name unless @by_codepoint[codepoint].include?(name)
+      end
+    end
+
+    # Load entities from the centralized JSON data source (optional mode)
+    # Silently continues if entity data cannot be loaded
+    # @return [void]
+    def load_from_entity_data_optional
+      data = self.class.entity_data
+      return unless data
+
+      data.each do |name, char|
+        codepoint = parse_codepoint(char)
+        next unless codepoint
+
+        @by_name[name] = codepoint
+        @by_codepoint[codepoint] ||= []
+        @by_codepoint[codepoint] << name unless @by_codepoint[codepoint].include?(name)
+      end
+    rescue EntityDataError
+      # Silently ignore - optional mode
+    end
+
+    # Load custom entities from the provided entity provider
+    # @return [void]
+    def load_custom_entities
+      return unless @entity_provider
+
+      entities = @entity_provider.call
+      return unless entities
+
+      entities.each do |name, codepoint|
+        @by_name[name] = codepoint
+        @by_codepoint[codepoint] ||= []
+        @by_codepoint[codepoint] << name unless @by_codepoint[codepoint].include?(name)
+      end
+    end
+
+    # Parse a Unicode character escape to codepoint
+    # @param char [String] character or escape sequence
+    # @return [Integer, nil]
+    def parse_codepoint(char)
+      if char.start_with?("\\u")
+        # Handle \uXXXX format
+        char.unicode_normalize(:nfc)[2..].to_i(16)
+      else
+        # Single character - get its ord
+        char.ord
+      end
+    rescue StandardError
+      nil
+    end
+  end
+end