rng 0.3.7 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64d22a32278b59bf3f6913561453c616ac39c259f4ffded1b200efddc50461af
-  data.tar.gz: 1fc2453376a3f7f65aa717beb3707e54ff1bdc751ae528ce8132e3f18720cf2b
+  metadata.gz: 8e689f7178c2c8a2e691e1327e99c1b74e7cea638ec80a12d5d557ad0c7a8003
+  data.tar.gz: 2d768848f695fab05f1f8808e44124178a5498b81bc12b277cc40b787fb862b4
 SHA512:
-  metadata.gz: dc8735cc2a8a3ae46635100a30ee39e114c762b9bace21955e2d361ad8bcbf65e0a0f10102f81085fea19a13d377ad93e1ac5313c8ae56a3237171712de7f965
-  data.tar.gz: 3ac117095690e1065a10921f4093072f151c84454cc0439fcd4a21764978654b8e5963f548969d92d76efb70c9ecca674984aa38fdc3a67cd3bb72c0924d394c
+  metadata.gz: b982725dbada931908431e146118e403f750ed9232d477491f568455a674b2f3d4efaeda73c3bbbd5b4fc0de49e1fab886f20fad6daf74dcdd283d20ff305569
+  data.tar.gz: 78f9da1bc7d6b3335af3e251954172e3d6d548f2a875c7d49b4f6c8cfe7a5cbd98dbe3f62d61b1effb2fd44088cd0f993a3b845fa014aaa87f2904f60be92e3e

data/lib/rng/external_ref_resolver.rb

@@ -22,6 +22,41 @@ module Rng
       end
     end
 
+    # Scan a grammar for unresolved external references.
+    #
+    # Walks the object graph forward-only via lutaml attribute reflection
+    # and collects a descriptive message for every Include, ExternalRef, and
+    # ParentRef found, including its href/name so callers can match on it.
+    #
+    # @param grammar [Grammar] The grammar to scan
+    # @return [Array<String>] Descriptive messages for unresolved references
+    def self.unresolved_ref_hrefs(grammar)
+      messages = []
+      collect_unresolved_refs(grammar, messages, [])
+      messages
+    end
+
+    # @api private
+    def self.collect_unresolved_refs(obj, messages, seen)
+      case obj
+      when Include
+        messages << "Unresolved include href: #{obj.href}" if obj.href
+      when ExternalRef
+        messages << "Unresolved externalRef href: #{obj.href}" if obj.href
+      when ParentRef
+        messages << "Unresolved parentRef name: #{obj.name}" if obj.name
+      end
+
+      return unless obj.is_a?(Lutaml::Model::Serializable)
+      return if seen.include?(obj.object_id)
+
+      seen << obj.object_id
+      obj.class.attributes.each_key do |attr_name|
+        value = obj.public_send(attr_name)
+        Array(value).each { |child| collect_unresolved_refs(child, messages, seen) }
+      end
+    end
+
     # Initialize the resolver
     #
     # @param grammar [Grammar] The grammar to resolve external refs in
@@ -83,18 +118,14 @@ module Rng
       # Copy start pattern
       if grammar.start && !grammar.start.empty?
         new_grammar.start = grammar.start.filter_map do |s|
-          resolved = resolve_pattern(deep_dup(s), base_dir, visited_files)
-          clear_element_order!(resolved)
-          resolved
+          resolve_pattern(deep_dup(s), base_dir, visited_files)
         end
       end
 
       # Copy define patterns
       if grammar.define && !grammar.define.empty?
         new_grammar.define = grammar.define.filter_map do |d|
-          resolved = resolve_pattern(deep_dup(d), base_dir, visited_files)
-          clear_element_order!(resolved)
-          resolved
+          resolve_pattern(deep_dup(d), base_dir, visited_files)
         end
       end
 
@@ -102,68 +133,10 @@ module Rng
       return unless grammar.div && !grammar.div.empty?
 
       new_grammar.div = grammar.div.filter_map do |div|
-        resolved_div = resolve_div(deep_dup(div), base_dir, visited_files)
-        clear_element_order!(resolved_div)
-        resolved_div
+        resolve_div(deep_dup(div), base_dir, visited_files)
       end
     end
 
-    # Recursively clear element_order on an object and its children
-    # This forces to_xml to use Ruby attributes instead of stale XML nodes
-    #
-    # @param obj [Object] Object to clear element_order on
-    def clear_element_order!(obj)
-      return obj unless obj
-
-      obj.instance_variable_set(:@element_order, nil)
-
-      # Recursively clear on children based on type
-      case obj
-      when Start
-        %i[element choice group interleave mixed optional zeroOrMore
-           oneOrMore text empty value data list parentRef notAllowed grammar].each do |attr|
-          children = obj.send(attr)
-          next if children.nil? || (children.respond_to?(:empty?) && children.empty?)
-
-          Array(children).each { |c| clear_element_order!(c) }
-        end
-      when Define
-        %i[ref element choice group interleave mixed optional zeroOrMore
-           oneOrMore text empty value data list notAllowed attribute grammar].each do |attr|
-          children = obj.send(attr)
-          next if children.nil? || (children.respond_to?(:empty?) && children.empty?)
-
-          Array(children).each { |c| clear_element_order!(c) }
-        end
-      when Element
-        %i[attribute ref choice group interleave mixed optional zeroOrMore
-           oneOrMore anyName text empty value data list notAllowed element grammar].each do |attr|
-          children = obj.send(attr)
-          next if children.nil? || (children.respond_to?(:empty?) && children.empty?)
-
-          Array(children).each { |c| clear_element_order!(c) }
-        end
-      when Group
-        %i[attribute ref choice group interleave mixed optional zeroOrMore
-           oneOrMore text empty value data list notAllowed].each do |attr|
-          children = obj.send(attr)
-          next if children.nil? || (children.respond_to?(:empty?) && children.empty?)
-
-          Array(children).each { |c| clear_element_order!(c) }
-        end
-      when Div
-        obj.div&.each { |d| clear_element_order!(d) }
-        %i[start define].each do |attr|
-          children = obj.send(attr)
-          next if children.nil? || (children.respond_to?(:empty?) && children.empty?)
-
-          Array(children).each { |c| clear_element_order!(c) }
-        end
-      end
-
-      obj
-    end
-
     # Resolve includes and return array of resolved grammars/content
     #
     # @param grammar [Grammar] Grammar containing includes
@@ -230,6 +203,7 @@ module Rng
           value = obj.public_send(attr_name)
           result.public_send(:"#{attr_name}=", deep_dup(value))
         end
+        copy_order_ivars!(obj, result)
         result
       when Array
         obj.map { |o| deep_dup(o) }
@@ -243,6 +217,21 @@ module Rng
       end
     end
 
+    # Carry lutaml's document-order ivars onto a deep-dup'd object so to_xml
+    # emits children in source order. These hold Lutaml::Xml::Element value
+    # objects (name/type), not live Nokogiri nodes.
+    #
+    # @param source [Object] Original object
+    # @param result [Object] Deep-dup'd copy
+    def copy_order_ivars!(source, result)
+      %i[@element_order @attribute_order].each do |ivar|
+        next unless source.instance_variable_defined?(ivar)
+
+        order = source.instance_variable_get(ivar)
+        result.instance_variable_set(ivar, order&.dup)
+      end
+    end
+
     # Resolve a single include directive
     #
     # @param include_directive [Include] Include element with href
@@ -310,32 +299,41 @@ module Rng
     def resolve_pattern(pattern, base_dir, visited_files)
       return unless pattern
 
-      # Handle Element pattern
       case pattern
       when Element
         resolve_element_external_ref!(pattern, base_dir, visited_files)
-
-        # Recursively resolve in Element's children
-        resolve_element_children!(pattern, base_dir, visited_files)
-      # Handle Group pattern
+        resolve_child_patterns!(pattern, base_dir, visited_files)
       when Group
         resolve_group_external_ref!(pattern, base_dir, visited_files)
-
-        # Recursively resolve in Group's children
-        resolve_group_children!(pattern, base_dir, visited_files)
-      # Handle Define pattern
-      when Define
-        resolve_define_children!(pattern, base_dir, visited_files)
-      # Handle Start pattern
-      when Start
-        resolve_start_children!(pattern, base_dir, visited_files)
-      when Start
-        resolve_pattern_children!(pattern, base_dir, visited_files)
+        resolve_child_patterns!(pattern, base_dir, visited_files)
+      when Define, Start
+        resolve_child_patterns!(pattern, base_dir, visited_files)
       end
 
       pattern
     end
 
+    # Recursively resolve external refs in a pattern's child patterns.
+    #
+    # Walks forward via lutaml attribute reflection (same approach as
+    # collect_unresolved_refs) instead of curated per-type attribute lists.
+    # Scalars are skipped by the Serializable guard. Only Element/Group/Define/
+    # Start dispatch into externalRef handling in resolve_pattern, so every
+    # other Serializable child is a no-op and the recursion stays bounded.
+    #
+    # @param pattern [Object] Pattern whose children to resolve
+    # @param base_dir [String] Base directory for href resolution
+    # @param visited_files [Set] Set of visited file paths
+    def resolve_child_patterns!(pattern, base_dir, visited_files)
+      pattern.class.attributes.each_key do |attr_name|
+        Array(pattern.public_send(attr_name)).each do |child|
+          next unless child.is_a?(Lutaml::Model::Serializable)
+
+          resolve_pattern(child, base_dir, visited_files)
+        end
+      end
+    end
+
     # Resolve external ref in an Element
     #
     # @param element [Element] Element with external_ref
@@ -386,9 +384,16 @@ module Rng
 
     # Copy pattern content from source to target
     #
+    # Rebuilds target's content from attributes, so its own document-order
+    # ivars no longer describe what it holds (e.g. an externalRef entry was
+    # replaced). Clear them on target only; children keep their own order.
+    #
     # @param target [Object] Target pattern (Element, Group, etc.)
     # @param source [Object] Source pattern
     def copy_pattern_content(target, source)
+      target.instance_variable_set(:@element_order, nil)
+      target.instance_variable_set(:@attribute_order, nil)
+
       case source
       when Start
         # Start pattern - copy its content (element, choice, group, etc.)
@@ -407,8 +412,6 @@ module Rng
                                          value data list notAllowed externalRef])
       when Choice
         target.choice = source.choice if source.choice
-      when Group
-        target.group = source.group if source.group
       when Interleave
         target.interleave = source.interleave if source.interleave
       when Optional
@@ -491,80 +494,6 @@ module Rng
       copy_pattern_content(group, start_pattern)
     end
 
-    # Recursively resolve children of an Element
-    #
-    # @param element [Element] Element to resolve children in
-    # @param base_dir [String] Base directory
-    # @param visited_files [Set] Set of visited file paths
-    def resolve_element_children!(element, base_dir, visited_files)
-      %i[attribute ref choice group interleave mixed optional zeroOrMore
-         oneOrMore anyName text empty value data list notAllowed element grammar].each do |attr|
-        children = element.send(attr)
-        next if children.nil? || (children.respond_to?(:empty?) && children.empty?)
-
-        Array(children).each do |child|
-          resolve_pattern(child, base_dir, visited_files)
-        end
-      end
-    end
-
-    # Recursively resolve children of a Group
-    #
-    # @param group [Group] Group to resolve children in
-    # @param base_dir [String] Base directory
-    # @param visited_files [Set] Set of visited file paths
-    def resolve_group_children!(group, base_dir, visited_files)
-      %i[attribute ref choice group interleave mixed optional zeroOrMore
-         oneOrMore text empty value data list notAllowed].each do |attr|
-        children = group.send(attr)
-        next if children.nil? || (children.respond_to?(:empty?) && children.empty?)
-
-        Array(children).each do |child|
-          resolve_pattern(child, base_dir, visited_files)
-        end
-      end
-    end
-
-    # Recursively resolve children of a Start pattern
-    #
-    # @param start [Start] Start pattern to resolve children in
-    # @param base_dir [String] Base directory
-    # @param visited_files [Set] Set of visited file paths
-    def resolve_start_children!(start, base_dir, visited_files)
-      # Start has: element, choice, group, interleave, mixed, optional,
-      #            zeroOrMore, oneOrMore, text, empty, value, data, list,
-      #            parentRef, notAllowed, grammar
-      %i[element choice group interleave mixed optional zeroOrMore
-         oneOrMore text empty value data list parentRef notAllowed grammar].each do |attr|
-        children = start.send(attr)
-        next if children.nil? || (children.respond_to?(:empty?) && children.empty?)
-
-        Array(children).each do |child|
-          resolve_pattern(child, base_dir, visited_files)
-        end
-      end
-    end
-
-    # Recursively resolve children of a Define
-    #
-    # @param define [Define] Define to resolve children in
-    # @param base_dir [String] Base directory
-    # @param visited_files [Set] Set of visited file paths
-    def resolve_define_children!(define, base_dir, visited_files)
-      # Define has: ref, element, choice, group, interleave, mixed, optional,
-      #            zeroOrMore, oneOrMore, text, empty, value, data, list,
-      #            notAllowed, attribute, grammar
-      %i[ref element choice group interleave mixed optional zeroOrMore
-         oneOrMore text empty value data list notAllowed attribute grammar].each do |attr|
-        children = define.send(attr)
-        next if children.nil? || (children.respond_to?(:empty?) && children.empty?)
-
-        Array(children).each do |child|
-          resolve_pattern(child, base_dir, visited_files)
-        end
-      end
-    end
-
     # Resolve href to absolute path with cycle detection
     #
     # @param href [String] Relative or absolute href

data/lib/rng/grammar.rb

@@ -3,6 +3,14 @@
 module Rng
   # This represents the RNG schema
   class Grammar < Lutaml::Model::Serializable
+    # Non-serialized diagnostics, e.g. unresolved external-reference hrefs
+    # collected when parsing without resolve_external.
+    attr_writer :warnings
+
+    def warnings
+      @warnings ||= []
+    end
+
     attribute :id, :string
     attribute :ns, :string
     attribute :datatypeLibrary, :string

data/lib/rng/include_processor.rb

@@ -29,69 +29,74 @@ module Rng
     # @param visited_files [Set] Set of already visited file paths (for circular detection)
     # @return [Grammar] Parsed grammar object
     def parse_file(file_path, base_dir = nil, visited_files = Set.new)
-      tree = parse_file_to_tree(file_path, base_dir, visited_files)
+      abs_path = track_visited_file(file_path, base_dir, visited_files)
 
-      # Process raw_grammar/raw_override/raw_patterns first (before include resolution)
-      # The include processor needs parsed content in the tree
-      process_raw_nodes!(tree)
-
-      # Process any includes in the tree (top-level or grammar-level)
-      process_includes(tree, base_dir || File.dirname(File.expand_path(file_path)),
-                       visited_files)
-
-      # Extract namespace from wrapper level if present
-      namespace = tree[:namespace]
+      # Read file content
+      raise "Include file not found: #{abs_path}" unless File.exist?(abs_path)
 
-      # Build grammar tree from different structures
-      grammar_tree = build_grammar_tree(tree)
-      grammar_tree[:namespace] = namespace if namespace
+      parse_tree_to_grammar(File.read(abs_path), File.dirname(abs_path), visited_files)
+    end
 
-      # Convert to RNG XML and then to Grammar object
-      rng_xml = @converter.convert(grammar_tree)
-      Grammar.from_xml(rng_xml)
+    # Parse in-memory RNC content with optional include resolution.
+    #
+    # When +location+ is nil, includes are not resolved and the pipeline matches
+    # {RncParser.parse} exactly. When +location+ names an existing directory it
+    # is used directly as the base directory for relative +include+ directives.
+    # Otherwise +location+ is treated as a source file path and includes are
+    # resolved against that file's directory.
+    #
+    # @param content [String] RNC content
+    # @param location [String, nil] Base directory or source file path used to resolve relative includes
+    # @param visited_files [Set] Set of already visited file paths (for circular detection)
+    # @return [Grammar] Parsed grammar object
+    def parse_content(content, location: nil, visited_files: Set.new)
+      parse_tree_to_grammar(content, resolve_base_dir(location, visited_files), visited_files)
     end
 
     private
 
-    # Parse file to parse tree (not Grammar object)
-    #
-    # @param file_path [String] Path to RNC file
-    # @param base_dir [String, nil] Base directory for resolving relative paths
-    # @param visited_files [Set] Set of already visited file paths
-    # @return [Hash] Parse tree
-    def parse_file_to_tree(file_path, base_dir = nil, visited_files = Set.new)
-      # Resolve absolute path to prevent circular includes
-      abs_path = File.expand_path(file_path, base_dir)
+    def resolve_base_dir(location, visited_files)
+      return nil if location.nil?
+      return location if File.directory?(location)
 
-      # Check for circular includes
-      raise "Circular include detected: #{abs_path}" if visited_files.include?(abs_path)
+      File.dirname(track_visited_file(location, nil, visited_files))
+    end
 
-      # Mark file as visited
-      visited_files.add(abs_path)
+    def parse_tree_to_grammar(content, base_dir, visited_files)
+      tree = parse_content_to_tree(content)
 
-      # Read file content
-      raise "Include file not found: #{abs_path}" unless File.exist?(abs_path)
+      if base_dir
+        # process_includes runs before normalize and needs raw_override /
+        # raw_grammar / raw_patterns expanded so it can walk the parsed
+        # subtrees. normalize would do this on the grammar_tree later, but
+        # by then process_includes has already executed.
+        process_raw_nodes!(tree)
+        process_includes(tree, base_dir, visited_files)
+      end
+
+      rng_xml = @converter.convert(build_grammar_tree(tree))
+      Grammar.from_xml(rng_xml)
+    end
 
-      content = File.read(abs_path)
+    def track_visited_file(file_path, base_dir, visited_files)
+      abs_path = File.expand_path(file_path, base_dir)
+      raise "Circular include detected: #{abs_path}" if visited_files.include?(abs_path)
 
-      # Parse with includes, passing the directory for relative path resolution
-      parse_with_includes(content, File.dirname(abs_path), visited_files)
+      visited_files.add(abs_path)
+      abs_path
     end
 
-    # Parse RNC content and resolve includes
+    # Parse RNC content into a raw parse tree.
+    #
+    # Applies hex-escape preprocessing so callers get the same parse tree
+    # they would from {RncParser.parse}. Preprocessing is idempotent on input
+    # that contains no hex escapes outside string literals, so it is safe to
+    # apply to file content as well.
     #
     # @param content [String] RNC content
-    # @param base_dir [String, nil] Base directory for resolving relative paths
-    # @param visited_files [Set] Set of already visited file paths
-    # @return [Hash] Parse tree with includes resolved
-    def parse_with_includes(content, base_dir = nil, visited_files = Set.new)
-      parser = RncParser.new
-      tree = parser.parse(content.strip)
-
-      # Process includes in the tree
-      process_includes(tree, base_dir, visited_files)
-
-      tree
+    # @return [Hash] Raw parse tree
+    def parse_content_to_tree(content)
+      RncParser.new.parse(RncParser.preprocess_hex_escapes(content.strip))
     end
 
     # Process include directives by recursively parsing included files
@@ -140,27 +145,7 @@ module Rng
         href = extract_string_literal(include_item[:href])
         override = parse_override(include_item[:override])
 
-        # Resolve file path relative to base_dir
-        included_file_path = base_dir ? File.join(base_dir, href) : href
-
-        # Parse included file recursively
-        included_tree = parse_file_to_tree(included_file_path, base_dir,
-                                           visited_files.dup)
-
-        # Process raw grammar/override/patterns before extracting
-        process_raw_nodes!(included_tree)
-
-        # Extract grammar from included tree
-        included_grammar = extract_grammar_tree(included_tree)
-
-        # Recursively process includes in the included file
-        if included_grammar[:includes] && !included_grammar[:includes].empty?
-          process_includes(included_tree, File.dirname(included_file_path),
-                           visited_files.dup)
-
-          # Re-extract grammar after processing its includes
-          included_grammar = extract_grammar_tree(included_tree)
-        end
+        included_grammar = resolve_included_grammar(href, base_dir, visited_files.dup)
 
         # Merge included definitions into temporary grammar_tree
         merge_included_grammar(grammar_tree, included_grammar, override)
@@ -191,28 +176,7 @@ module Rng
       href = extract_string_literal(include_item[:href])
       override = parse_override(include_item[:override])
 
-      # Resolve file path relative to base_dir
-      included_file_path = base_dir ? File.join(base_dir, href) : href
-
-      # Parse included file recursively
-      included_tree = parse_file_to_tree(included_file_path, base_dir,
-                                         visited_files.dup)
-
-      # Process raw grammar/override/patterns before extracting
-      process_raw_nodes!(included_tree)
-
-      # Extract grammar from included tree
-      included_grammar = extract_grammar_tree(included_tree)
-
-      # Recursively process includes in the included file
-      if included_grammar[:includes] && !included_grammar[:includes].empty?
-        # Process includes in the included file
-        process_includes(included_tree, File.dirname(included_file_path),
-                         visited_files.dup)
-
-        # Re-extract grammar after processing its includes
-        included_grammar = extract_grammar_tree(included_tree)
-      end
+      included_grammar = resolve_included_grammar(href, base_dir, visited_files.dup)
 
       # Merge included definitions into current tree
       merge_included_grammar(grammar_tree, included_grammar, override)
@@ -230,6 +194,16 @@ module Rng
       end
     end
 
+    def resolve_included_grammar(file_path, base_dir, visited_files)
+      abs_path = track_visited_file(file_path, base_dir, visited_files)
+      raise "Include file not found: #{abs_path}" unless File.exist?(abs_path)
+
+      tree = parse_content_to_tree(File.read(abs_path))
+      process_raw_nodes!(tree)
+      process_includes(tree, File.dirname(abs_path), visited_files)
+      build_grammar_tree(tree)
+    end
+
     # Process raw_grammar/raw_override/raw_patterns nodes in-place
     #
     # This is needed when included files contain grammar blocks or

data/lib/rng/rnc_builder.rb

@@ -426,21 +426,10 @@ module Rng
       if attr.value
         value_str = attr.value.is_a?(Value) ? attr.value.value : attr.value.to_s
         result += "\"#{value_str}\""
-      # Check for choice of values
+      # Check for choice (values, text, refs, etc.) — delegate to the
+      # canonical choice builder so every member type is preserved
       elsif attr.choice
-        # Check if choice contains Value objects
-        choice_obj = attr.choice
-        if choice_obj.value.is_a?(Array)
-          # Choice with array of Value objects
-          values = choice_obj.value.map do |v|
-            v_str = v.is_a?(Value) ? v.value : v.to_s
-            "\"#{escape_rnc_string(v_str)}\""
-          end
-          result += values.join(' | ')
-        else
-          # Other choice patterns
-          result += build_pattern(attr.choice)
-        end
+        result += build_choice(attr.choice)
       # Check for datatype
       elsif attr.data
         result += if attr.data.type

data/lib/rng/rnc_parser.rb

@@ -261,6 +261,23 @@ module Rng
       (whitespace >> annotation).repeat(1)
     end
 
+    # Match a `[ ... ]` annotation WITHOUT capturing anything (produces a raw
+    # slice that is discarded in a sequence). Strings and nested brackets are
+    # matched as raw characters so no `.as` capture turns the surrounding
+    # content item into an array. Used to drop a content-position annotation.
+    rule(:annotation_skip) do
+      str('[') >> annotation_skip_inner.repeat >> str(']')
+    end
+
+    rule(:annotation_skip_inner) do
+      (str('"""') >> (str('"""').absent? >> any).repeat >> str('"""')) |
+        (str("'''") >> (str("'''").absent? >> any).repeat >> str("'''")) |
+        (str('"') >> ((str('\\') >> any) | (str('"').absent? >> match('[\u0000-\u001F\u007F]').absent? >> any)).repeat >> str('"')) |
+        (str("'") >> ((str('\\') >> any) | (str("'").absent? >> match('[\u0000-\u001F\u007F]').absent? >> any)).repeat >> str("'")) |
+        (str('[') >> annotation_skip_inner.repeat >> str(']')) |
+        (str('[').absent? >> str(']').absent? >> str('"').absent? >> str("'").absent? >> any)
+    end
+
     # Notation/annotation: [ key = "value" ] or just [ ... ]
     # Notations are only valid when attached to patterns using >>, not as standalone preamble items
     rule(:notation) do
@@ -312,14 +329,18 @@ module Rng
           value_literal >>
           (whitespace >> str('|') >> whitespace >> value_literal).repeat(1) >>
           whitespace >> str(')')).as(:datatype_except) |
-        # Non-parenthesized choice of value literals: "a" | "b" | "c"
-        (value_literal >> (whitespace >> str('|') >> whitespace >> value_literal).repeat(1).as(:value_choice)) |
+        # Non-parenthesized choice of items: "a" | "b", text | "a", "a" | ref, text | ref
+        (attribute_choice_item >> (whitespace >> str('|') >> whitespace >> attribute_choice_item).repeat(1).as(:attribute_choice)) |
         value_literal |
         datatype_ref |
         keyword('text').as(:text_type) |
         identifier.as(:ref)
     end
 
+    rule(:attribute_choice_item) do
+      keyword('text').as(:text_type) | value_literal | identifier.as(:ref)
+    end
+
     rule(:datatype_ref) do
       identifier.as(:prefix) >> str(':') >> identifier.as(:type) >>
         (whitespace >> str('{') >> whitespace >>
@@ -381,7 +402,7 @@ module Rng
     # Named pattern definition (e.g., "myPattern = element foo { text }")
     rule(:named_pattern) do
       (doc_comments >> whitespace).maybe.as(:docs) >>
-        annotations.maybe >>
+        (annotation_skip >> whitespace).repeat >>
         identifier.as(:name) >> whitespace >>
         (str('|=') | str('&=') | str('=')).as(:operator) >> whitespace >>
         pattern_list.as(:pattern)
@@ -402,14 +423,18 @@ module Rng
         (
           (whitespace >> str('&') >> whitespace >> content_item).repeat(1).as(:interleave_items) |
           (whitespace >> str('|') >> whitespace >> content_item).repeat(1).as(:choice_items) |
-          (comma? >> content_item).repeat(1).as(:sequence_items)
+          # A comma is required between top-level sequence items; a comma-less
+          # continuation would greedily swallow the following definition.
+          (whitespace >> comma >> whitespace >> content_item).repeat(1).as(:sequence_items)
         ).maybe
     end
 
     # Choice is handled at content level, not as separate pattern
     rule(:content_item) do
       (doc_comments >> whitespace).maybe >>
-        annotations.maybe >>
+        # Drop any content-position bracket annotation(s) before the item so the
+        # following pattern still matches (annotations are not modelled here).
+        (annotation_skip >> whitespace).repeat >>
         (element_def | attribute_def |
           # Datatype subtraction: identifier - ( value|identifier|choice|annotated )
           (identifier.as(:datatype_name) >>
@@ -668,18 +693,13 @@ module Rng
       result
     end
 
+    # Parse RNC content into a Grammar.
+    #
+    # Delegates to {IncludeProcessor#parse_content} so that this entry point
+    # and {Rng.parse_rnc} share a single canonical pipeline. With no location,
+    # include directives are not resolved.
     def self.parse(input)
-      parser = new
-      preprocessed = preprocess_hex_escapes(input.strip)
-      tree = parser.parse(preprocessed)
-
-      # Normalize parse tree
-      processor = ParseTreeProcessor.new(tree)
-      normalized = processor.normalize
-
-      # Convert to RNG XML and Grammar object
-      rng_xml = convert_to_rng(normalized.grammar_tree)
-      Grammar.from_xml(rng_xml)
+      IncludeProcessor.new.parse_content(input)
     end
 
     # Convert RNG schema to RNC