rng 0.1.2 → 0.3.4

data/lib/rng/grammar.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "lutaml/model"
-
 module Rng
   # This represents the RNG schema
   class Grammar < Lutaml::Model::Serializable
@@ -10,27 +8,31 @@ module Rng
     attribute :datatypeLibrary, :string
     attribute :start, Start, collection: true
     attribute :define, Define, collection: true, initialize_empty: true
-    attribute :element, Element, collection: true
+    attribute :element, Element, collection: true, initialize_empty: true
     attribute :include, Include, collection: true
+    attribute :div, Div, collection: true, initialize_empty: true
 
     xml do
-      root "grammar", ordered: true
-      namespace "http://relaxng.org/ns/structure/1.0"
+      element 'grammar'
+      ordered
+
+      namespace ::Rng::Namespaces::RngNamespace
 
-      map_attribute "datatypeLibrary", to: :datatypeLibrary, value_map: {
+      map_attribute 'datatypeLibrary', to: :datatypeLibrary, value_map: {
         from: { empty: :empty, omitted: :omitted, nil: :nil },
         to: { empty: :empty, omitted: :omitted, nil: :nil }
       }
-      map_attribute "ns", to: :ns, value_map: {
+      map_attribute 'ns', to: :ns, value_map: {
         from: { empty: :empty, omitted: :omitted, nil: :nil },
         to: { empty: :empty, omitted: :omitted, nil: :nil }
       }
-      map_attribute "id", to: :id
+      map_attribute 'id', to: :id
 
-      map_element "start", to: :start
-      map_element "define", to: :define
-      map_element "element", to: :element
-      map_element "include", to: :include
+      map_element 'start', to: :start
+      map_element 'define', to: :define
+      map_element 'element', to: :element
+      map_element 'include', to: :include
+      map_element 'div', to: :div
     end
   end
 end

data/lib/rng/group.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "lutaml/model"
-
 module Rng
   class Group < Lutaml::Model::Serializable
     attribute :id, :string
@@ -24,37 +22,41 @@ module Rng
     attribute :data, Data, collection: true, initialize_empty: true
     attribute :list, List, collection: true, initialize_empty: true
     attribute :notAllowed, NotAllowed, collection: true, initialize_empty: true
+    attribute :base, Lutaml::Xml::W3c::XmlBaseType
 
     xml do
-      root "group", ordered: true
-      namespace "http://relaxng.org/ns/structure/1.0"
+      element 'group'
+      ordered
+
+      namespace ::Rng::Namespaces::RngNamespace
 
-      map_attribute "id", to: :id
-      map_attribute "ns", to: :ns, value_map: {
+      map_attribute 'id', to: :id
+      w3c_attributes :base
+      map_attribute 'ns', to: :ns, value_map: {
         from: { empty: :empty, omitted: :omitted, nil: :nil },
         to: { empty: :empty, omitted: :omitted, nil: :nil }
       }
-      map_attribute "datatypeLibrary", to: :datatypeLibrary, value_map: {
+      map_attribute 'datatypeLibrary', to: :datatypeLibrary, value_map: {
         from: { empty: :empty, omitted: :omitted, nil: :nil },
         to: { empty: :empty, omitted: :omitted, nil: :nil }
       }
-      map_element "element", to: :element
-      map_element "attribute", to: :attribute
-      map_element "ref", to: :ref
-      map_element "choice", to: :choice
-      map_element "group", to: :group
-      map_element "interleave", to: :interleave
-      map_element "mixed", to: :mixed
-      map_element "optional", to: :optional
-      map_element "zeroOrMore", to: :zeroOrMore
-      map_element "oneOrMore", to: :oneOrMore
-      map_element "text", to: :text
-      map_element "empty", to: :empty
-      map_element "value", to: :value
-      map_element "data", to: :data
-      map_element "list", to: :list
-      map_element "notAllowed", to: :notAllowed
-      map_element "externalRef", to: :externalRef
+      map_element 'element', to: :element
+      map_element 'attribute', to: :attribute
+      map_element 'ref', to: :ref
+      map_element 'choice', to: :choice
+      map_element 'group', to: :group
+      map_element 'interleave', to: :interleave
+      map_element 'mixed', to: :mixed
+      map_element 'optional', to: :optional
+      map_element 'zeroOrMore', to: :zeroOrMore
+      map_element 'oneOrMore', to: :oneOrMore
+      map_element 'text', to: :text
+      map_element 'empty', to: :empty
+      map_element 'value', to: :value
+      map_element 'data', to: :data
+      map_element 'list', to: :list
+      map_element 'notAllowed', to: :notAllowed
+      map_element 'externalRef', to: :externalRef
     end
   end
 end

data/lib/rng/include.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "lutaml/model"
-
 module Rng
   class Include < Lutaml::Model::Serializable
     attribute :href, :string
@@ -10,15 +8,16 @@ module Rng
     attribute :grammar, Grammar
 
     xml do
-      root "include"
+      element 'include'
+      namespace ::Rng::Namespaces::RngNamespace
 
-      map_attribute "href", to: :href
-      map_attribute "ns", to: :ns, value_map: {
+      map_attribute 'href', to: :href
+      map_attribute 'ns', to: :ns, value_map: {
         from: { empty: :empty, omitted: :omitted, nil: :nil },
         to: { empty: :empty, omitted: :omitted, nil: :nil }
       }
       map_content to: :grammar
-      map_element "define", to: :define
+      map_element 'define', to: :define
     end
   end
 end

data/lib/rng/include_processor.rb

@@ -0,0 +1,461 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Rng
+  # Handles RNC file inclusion and grammar merging
+  #
+  # This class processes include directives in RNC files, resolving them
+  # recursively while preventing circular includes. It supports both:
+  # - Grammar-level includes (inside grammar blocks)
+  # - Top-level includes (Metanorma-style schemas)
+  #
+  # @example Parse a file with includes
+  #   processor = Rng::IncludeProcessor.new
+  #   grammar = processor.parse_file("schema.rnc")
+  #
+  class IncludeProcessor
+    # Initialize with optional converter
+    #
+    # @param converter [RncToRngConverter] Converter for parse tree to RNG XML
+    def initialize(converter = RncToRngConverter.new)
+      @converter = converter
+    end
+
+    # Parse a file with include resolution
+    #
+    # @param file_path [String] Path to RNC file
+    # @param base_dir [String, nil] Base directory for resolving relative paths
+    # @param visited_files [Set] Set of already visited file paths (for circular detection)
+    # @return [Grammar] Parsed grammar object
+    def parse_file(file_path, base_dir = nil, visited_files = Set.new)
+      tree = parse_file_to_tree(file_path, base_dir, visited_files)
+
+      # Process raw_grammar/raw_override/raw_patterns first (before include resolution)
+      # The include processor needs parsed content in the tree
+      process_raw_nodes!(tree)
+
+      # Process any includes in the tree (top-level or grammar-level)
+      process_includes(tree, base_dir || File.dirname(File.expand_path(file_path)),
+                       visited_files)
+
+      # Extract namespace from wrapper level if present
+      namespace = tree[:namespace]
+
+      # Build grammar tree from different structures
+      grammar_tree = build_grammar_tree(tree)
+      grammar_tree[:namespace] = namespace if namespace
+
+      # Convert to RNG XML and then to Grammar object
+      rng_xml = @converter.convert(grammar_tree)
+      Grammar.from_xml(rng_xml)
+    end
+
+    private
+
+    # Parse file to parse tree (not Grammar object)
+    #
+    # @param file_path [String] Path to RNC file
+    # @param base_dir [String, nil] Base directory for resolving relative paths
+    # @param visited_files [Set] Set of already visited file paths
+    # @return [Hash] Parse tree
+    def parse_file_to_tree(file_path, base_dir = nil, visited_files = Set.new)
+      # Resolve absolute path to prevent circular includes
+      abs_path = File.expand_path(file_path, base_dir)
+
+      # Check for circular includes
+      raise "Circular include detected: #{abs_path}" if visited_files.include?(abs_path)
+
+      # Mark file as visited
+      visited_files.add(abs_path)
+
+      # Read file content
+      raise "Include file not found: #{abs_path}" unless File.exist?(abs_path)
+
+      content = File.read(abs_path)
+
+      # Parse with includes, passing the directory for relative path resolution
+      parse_with_includes(content, File.dirname(abs_path), visited_files)
+    end
+
+    # Parse RNC content and resolve includes
+    #
+    # @param content [String] RNC content
+    # @param base_dir [String, nil] Base directory for resolving relative paths
+    # @param visited_files [Set] Set of already visited file paths
+    # @return [Hash] Parse tree with includes resolved
+    def parse_with_includes(content, base_dir = nil, visited_files = Set.new)
+      parser = RncParser.new
+      tree = parser.parse(content.strip)
+
+      # Process includes in the tree
+      process_includes(tree, base_dir, visited_files)
+
+      tree
+    end
+
+    # Process include directives by recursively parsing included files
+    #
+    # @param tree [Hash] Parse tree
+    # @param base_dir [String] Base directory for resolving relative paths
+    # @param visited_files [Set] Set of already visited file paths
+    def process_includes(tree, base_dir, visited_files)
+      # Handle top-level includes first (Metanorma-style schemas)
+      if tree.key?(:top_includes)
+        process_top_level_includes(tree, base_dir, visited_files)
+        return
+      end
+
+      return if tree[:includes] && tree[:includes].empty?
+
+      # Handle grammar-level includes (existing logic)
+      grammar_tree = extract_grammar_tree(tree)
+
+      return unless grammar_tree[:includes] && !grammar_tree[:includes].empty?
+
+      # Process each include
+      grammar_tree[:includes].each do |include_item|
+        process_single_include(grammar_tree, include_item, base_dir,
+                               visited_files)
+      end
+
+      # Remove includes array after processing (no longer needed in conversion)
+      grammar_tree.delete(:includes)
+    end
+
+    # Process top-level includes (Metanorma-style schemas)
+    #
+    # @param tree [Hash] Parse tree
+    # @param base_dir [String] Base directory for resolving relative paths
+    # @param visited_files [Set] Set of already visited file paths
+    def process_top_level_includes(tree, base_dir, visited_files)
+      # Create a temporary grammar_tree to hold merged content
+      grammar_tree = {
+        start: nil,
+        definitions: []
+      }
+
+      # Process each top-level include
+      tree[:top_includes].each do |include_item|
+        href = extract_string_literal(include_item[:href])
+        override = parse_override(include_item[:override])
+
+        # Resolve file path relative to base_dir
+        included_file_path = base_dir ? File.join(base_dir, href) : href
+
+        # Parse included file recursively
+        included_tree = parse_file_to_tree(included_file_path, base_dir,
+                                           visited_files.dup)
+
+        # Process raw grammar/override/patterns before extracting
+        process_raw_nodes!(included_tree)
+
+        # Extract grammar from included tree
+        included_grammar = extract_grammar_tree(included_tree)
+
+        # Recursively process includes in the included file
+        if included_grammar[:includes] && !included_grammar[:includes].empty?
+          process_includes(included_tree, File.dirname(included_file_path),
+                           visited_files.dup)
+
+          # Re-extract grammar after processing its includes
+          included_grammar = extract_grammar_tree(included_tree)
+        end
+
+        # Merge included definitions into temporary grammar_tree
+        merge_included_grammar(grammar_tree, included_grammar, override)
+      end
+
+      # Merge the resolved grammar_tree back into the main tree
+      tree[:start] = grammar_tree[:start] if grammar_tree[:start]
+      if grammar_tree[:definitions]
+        tree[:definitions] =
+          grammar_tree[:definitions]
+      end
+
+      # Process raw_trailing if present (named patterns after includes)
+      process_raw_trailing!(tree, grammar_tree) if tree[:raw_trailing]
+
+      # Clean up - remove top_includes key as it's been processed
+      tree.delete(:top_includes)
+    end
+
+    # Process a single include directive
+    #
+    # @param grammar_tree [Hash] Grammar tree to merge into
+    # @param include_item [Hash] Include item from parse tree
+    # @param base_dir [String] Base directory for resolving relative paths
+    # @param visited_files [Set] Set of already visited file paths
+    def process_single_include(grammar_tree, include_item, base_dir,
+                               visited_files)
+      href = extract_string_literal(include_item[:href])
+      override = parse_override(include_item[:override])
+
+      # Resolve file path relative to base_dir
+      included_file_path = base_dir ? File.join(base_dir, href) : href
+
+      # Parse included file recursively
+      included_tree = parse_file_to_tree(included_file_path, base_dir,
+                                         visited_files.dup)
+
+      # Process raw grammar/override/patterns before extracting
+      process_raw_nodes!(included_tree)
+
+      # Extract grammar from included tree
+      included_grammar = extract_grammar_tree(included_tree)
+
+      # Recursively process includes in the included file
+      if included_grammar[:includes] && !included_grammar[:includes].empty?
+        # Process includes in the included file
+        process_includes(included_tree, File.dirname(included_file_path),
+                         visited_files.dup)
+
+        # Re-extract grammar after processing its includes
+        included_grammar = extract_grammar_tree(included_tree)
+      end
+
+      # Merge included definitions into current tree
+      merge_included_grammar(grammar_tree, included_grammar, override)
+    end
+
+    # Extract grammar tree from parse tree
+    #
+    # @param tree [Hash] Parse tree
+    # @return [Hash] Grammar tree
+    def extract_grammar_tree(tree)
+      if tree.key?(:inner_grammar)
+        tree[:inner_grammar]
+      else
+        tree
+      end
+    end
+
+    # Process raw_grammar/raw_override/raw_patterns nodes in-place
+    #
+    # This is needed when included files contain grammar blocks or
+    # overrides that are captured as raw text by the parser.
+    #
+    # @param tree [Hash] Parse tree to process in-place
+    def process_raw_nodes!(tree)
+      ParseTreeProcessor.new(tree).send(:process_raw_overrides!, tree)
+    end
+
+    # Process raw_trailing content (named patterns after top-level includes)
+    #
+    # For schemas like ietf.rnc where include directives are followed by
+    # named pattern definitions, the trailing content is captured as raw_trailing.
+    # This method parses that content and adds definitions to grammar_tree.
+    #
+    # @param tree [Hash] Parse tree containing raw_trailing
+    # @param grammar_tree [Hash] Grammar tree to merge definitions into
+    def process_raw_trailing!(tree, grammar_tree)
+      raw = tree[:raw_trailing]
+      return unless raw
+
+      text = if raw.is_a?(Array)
+               raw.map { |r| r.respond_to?(:str) ? r.str : r.to_s }.join
+             else
+               (raw.respond_to?(:str) ? raw.str : raw.to_s)
+             end
+      return if text.strip.empty?
+
+      # Parse raw_trailing as a grammar (which handles named patterns)
+      parser = Rng::RncParser.new
+      begin
+        parsed = parser.grammar.parse(text.strip)
+        patterns = parsed[:patterns] || []
+        grammar_tree[:definitions] ||= []
+        grammar_tree[:definitions].concat(patterns)
+      rescue Parslet::ParseFailed => e
+        warn "Warning: Failed to parse trailing content: #{e.message}" if ENV['RNG_VERBOSE']
+      ensure
+        tree.delete(:raw_trailing)
+      end
+    end
+
+    # Build grammar tree from different tree structures
+    #
+    # Handles:
+    # - Top-level includes (Metanorma style)
+    # - Grammar block wrapper
+    # - Flat grammar
+    #
+    # @param tree [Hash] Parse tree
+    # @return [Hash] Normalized grammar tree
+    def build_grammar_tree(tree)
+      ParseTreeProcessor.new(tree).normalize.grammar_tree
+    end
+
+    # Merge included grammar into current grammar, applying overrides
+    #
+    # @param target_tree [Hash] Target grammar tree to merge into
+    # @param source_tree [Hash] Source grammar tree to merge from
+    # @param override [Hash, nil] Override definitions from include directive
+    def merge_included_grammar(target_tree, source_tree, override)
+      # Merge datatype library if not already set
+      if source_tree[:datatype_library] && !target_tree[:datatype_library]
+        target_tree[:datatype_library] =
+          source_tree[:datatype_library]
+      end
+
+      # Merge start pattern if not overridden
+      if source_tree[:start] && !override
+        # If target has no start, use source start
+        target_tree[:start] ||= source_tree[:start]
+      elsif override && override[:start]
+        # Use override start
+        target_tree[:start] = override[:start]
+      elsif source_tree[:start] && !target_tree[:start]
+        target_tree[:start] = source_tree[:start]
+      end
+
+      # Initialize definitions array if needed
+      target_tree[:definitions] ||= []
+
+      # Merge definitions from source
+      source_tree[:definitions]&.each do |source_def|
+        # Check if this definition is overridden
+        overridden = false
+
+        if override && override[:definitions]
+          override[:definitions].each do |override_def|
+            # Check if names match
+            next unless source_def[:name] && override_def[:name] &&
+                        extract_string(source_def[:name][:identifier]) == extract_string(override_def[:name][:identifier])
+
+            # Use override instead of source
+            target_tree[:definitions] << override_def
+            overridden = true
+            break
+          end
+        end
+
+        # If not overridden, add source definition
+        target_tree[:definitions] << source_def unless overridden
+      end
+
+      # Add any override definitions not matched with source
+      return unless override && override[:definitions]
+
+      override[:definitions].each do |override_def|
+        # Check if this override matched any source definition
+        matched = false
+
+        source_tree[:definitions]&.each do |source_def|
+          next unless source_def[:name] && override_def[:name] &&
+                      extract_string(source_def[:name][:identifier]) == extract_string(override_def[:name][:identifier])
+
+          matched = true
+          break
+        end
+
+        # If no match, this is a new definition - add it
+        target_tree[:definitions] << override_def unless matched
+      end
+    end
+
+    # Helper method to extract clean string without Parslet position markers
+    #
+    # @param obj [Object] Parslet::Slice or String
+    # @return [String] Clean string
+    def extract_string(obj)
+      RncParser.extract_string(obj)
+    end
+
+    # Parse raw override string into structured override hash
+    #
+    # @param override [Hash, nil] Override hash potentially containing :raw_override
+    # @return [Hash, nil] Parsed override with :start and :definitions, or nil
+    def parse_override(override)
+      return nil unless override
+      return override unless override[:raw_override]
+
+      raw = extract_string(override[:raw_override])
+      return nil if raw.nil? || raw.strip.empty?
+
+      # Parse the raw override content as RNC directly (top-level style)
+      parser = RncParser.new
+      begin
+        parsed = parser.parse(raw.strip)
+        # Normalize the parsed override
+        processor = ParseTreeProcessor.new(parsed)
+        normalized = processor.normalize
+        tree = normalized.grammar_tree
+        result = {}
+        result[:start] = tree[:start] if tree[:start]
+        result[:definitions] = tree[:definitions] if tree[:definitions] && !tree[:definitions].empty?
+        result.empty? ? nil : result
+      rescue Parslet::ParseFailed
+        nil
+      end
+    end
+
+    # Helper method to extract string literal with concatenations
+    #
+    # @param lit [Hash] String literal with :string_parts and :concatenations
+    # @return [String] Extracted string
+    def extract_string_literal(lit)
+      return '' unless lit
+
+      # Extract main string parts
+      result = extract_string_parts(lit[:string_parts])
+
+      # Handle concatenations if present
+      if lit[:concatenations].is_a?(Array)
+        lit[:concatenations].each do |concat|
+          result += extract_string_parts(concat[:concat_string_parts])
+        end
+      end
+
+      result
+    end
+
+    # Extract string from string_parts array
+    #
+    # @param parts [Array, String] String parts
+    # @return [String] Extracted string
+    def extract_string_parts(parts)
+      return '' unless parts
+      return parts if parts.is_a?(String)
+      return parts.str if parts.respond_to?(:str)
+
+      return '' unless parts.is_a?(Array)
+
+      parts.map do |part|
+        if part.is_a?(String)
+          part
+        elsif part.respond_to?(:str)
+          part.str
+        elsif part[:hex_escape]
+          # Handle \x{HEX}
+          hex_str = part[:hex_escape][:hex]
+          hex_str = hex_str.str if hex_str.respond_to?(:str)
+          [hex_str.to_i(16)].pack('U')
+        elsif part[:char_escape]
+          # Handle \", \\, \n, \r, \t, and RELAX NG class escapes \i, \c, \d, \w
+          char = part[:char_escape][:char]
+          char = char.str if char.respond_to?(:str)
+          case char
+          when '"' then '"'
+          when '\\' then '\\'
+          when 'n' then "\n"
+          when 'r' then "\r"
+          when 't' then "\t"
+          when 'i' then '\\i'
+          when 'c' then '\\c'
+          when 'd' then '\\d'
+          when 'w' then '\\w'
+          else char
+          end
+        elsif part[:char]
+          # Regular character (plain char in string literal)
+          c = part[:char]
+          c = c.str if c.respond_to?(:str)
+          c.to_s
+        else
+          part.to_s
+        end
+      end.join
+    end
+  end
+end

data/lib/rng/interleave.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "lutaml/model"
-
 module Rng
   class Interleave < Lutaml::Model::Serializable
     attribute :id, :string
@@ -25,34 +23,36 @@ module Rng
     attribute :notAllowed, NotAllowed, collection: true, initialize_empty: true
 
     xml do
-      root "interleave", ordered: true
-      namespace "http://relaxng.org/ns/structure/1.0"
+      element 'interleave'
+      ordered
+
+      namespace ::Rng::Namespaces::RngNamespace
 
-      map_attribute "id", to: :id
-      map_attribute "ns", to: :ns, value_map: {
+      map_attribute 'id', to: :id
+      map_attribute 'ns', to: :ns, value_map: {
         from: { empty: :empty, omitted: :omitted, nil: :nil },
         to: { empty: :empty, omitted: :omitted, nil: :nil }
       }
-      map_attribute "datatypeLibrary", to: :datatypeLibrary, value_map: {
+      map_attribute 'datatypeLibrary', to: :datatypeLibrary, value_map: {
         from: { empty: :empty, omitted: :omitted, nil: :nil },
         to: { empty: :empty, omitted: :omitted, nil: :nil }
       }
-      map_element "element", to: :element
-      map_element "attribute", to: :attribute
-      map_element "ref", to: :ref
-      map_element "choice", to: :choice
-      map_element "group", to: :group
-      map_element "interleave", to: :interleave
-      map_element "mixed", to: :mixed
-      map_element "optional", to: :optional
-      map_element "zeroOrMore", to: :zeroOrMore
-      map_element "oneOrMore", to: :oneOrMore
-      map_element "text", to: :text
-      map_element "empty", to: :empty
-      map_element "value", to: :value
-      map_element "data", to: :data
-      map_element "list", to: :list
-      map_element "notAllowed", to: :notAllowed
+      map_element 'element', to: :element
+      map_element 'attribute', to: :attribute
+      map_element 'ref', to: :ref
+      map_element 'choice', to: :choice
+      map_element 'group', to: :group
+      map_element 'interleave', to: :interleave
+      map_element 'mixed', to: :mixed
+      map_element 'optional', to: :optional
+      map_element 'zeroOrMore', to: :zeroOrMore
+      map_element 'oneOrMore', to: :oneOrMore
+      map_element 'text', to: :text
+      map_element 'empty', to: :empty
+      map_element 'value', to: :value
+      map_element 'data', to: :data
+      map_element 'list', to: :list
+      map_element 'notAllowed', to: :notAllowed
     end
   end
 end