expressir 2.4.0 → 2.4.2

data/lib/expressir/express/streaming_builder.rb

@@ -112,15 +112,13 @@ module Expressir
       # @param key [String] The hash key name
       def on_hash_key(key)
         # Convert string key to symbol to match existing parser AST format
-        puts "DEBUG on_hash_key: #{key.inspect} (#{key.class})" if ENV["DEBUG_STREAMING"]
         @pending_key = key.to_sym
       end
 
       # Called when a hash value is about to be parsed
       #
       # @param key [String] The hash key name
-      def on_hash_value(key)
-        puts "DEBUG on_hash_value: #{key.inspect} (#{key.class})" if ENV["DEBUG_STREAMING"]
+      def on_hash_value(_key)
         @pending_key = nil
       end
 
@@ -241,191 +239,165 @@ module Expressir
       def convert_ast_format(obj)
         case obj
         when Hash
-          return obj if obj.empty?
-
-          # Process each key-value pair
-          result = {}
-          obj.each do |key, value|
-            # Check if this is a str or spaces key with an array value
-            result[key] = if %i[str spaces].include?(key) && value.is_a?(Array)
-                            # Concatenate character arrays
-                            value.compact.join
-                          else
-                            # Recursively convert the value
-                            convert_ast_format(value)
-                          end
-          end
-          result
+          convert_hash_node(obj)
         when Array
-          return obj if obj.empty?
-
-          # First, convert each element
-          converted_items = obj.map { |item| convert_ast_format(item) }
-
-          # Flatten nested arrays that come from grammar repetition (.repeat)
-          # These appear as arrays containing arrays (which may contain hashes)
-          # This handles the case where (op_comma >> item).repeat produces [[{...}, {...}], [{...}, {...}]]
-          # After inner conversion, this becomes [{multi-key hash}, {multi-key hash}]
-          # which should be flattened into the parent array
-          flattened_items = []
-          converted_items.each do |item|
-            if item.is_a?(Array)
-              # Flatten this array into the parent
-              flattened_items.concat(item)
-            else
-              flattened_items << item
-            end
+          convert_array_node(obj)
+        else
+          obj
+        end
+      end
+
+      def convert_hash_node(hash)
+        return hash if hash.empty?
+
+        result = {}
+        hash.each do |key, value|
+          result[key] = if %i[str spaces].include?(key) && value.is_a?(Array)
+                          value.compact.join
+                        else
+                          convert_ast_format(value)
+                        end
+        end
+        result
+      end
+
+      def convert_array_node(array)
+        return array if array.empty?
+
+        converted_items = array.map { |item| convert_ast_format(item) }
+
+        # Flatten nested arrays from grammar repetition (.repeat)
+        flattened = flatten_nested_arrays(converted_items)
+
+        non_empty = flattened.reject { |item| item.nil? || (item.is_a?(Array) && item.empty?) }
+
+        if non_empty.empty?
+          {}
+        elsif non_empty.all?(Hash)
+          merge_or_preserve_sequence(non_empty, flattened)
+        else
+          flattened
+        end
+      end
+
+      def flatten_nested_arrays(items)
+        result = []
+        items.each do |item|
+          if item.is_a?(Array)
+            result.concat(item)
+          else
+            result << item
           end
-          converted_items = flattened_items
+        end
+        result
+      end
+
+      def merge_or_preserve_sequence(non_empty, converted)
+        # Check for repetition patterns that should stay as arrays
+        return converted if same_key_repetition?(non_empty)
+        return converted if duplicate_key_repetition?(non_empty)
+
+        # Check for grouped repetition that needs unzipping
+        unzipped = try_unzip_repetition(non_empty)
+        return unzipped if unzipped
+
+        # Default: merge all single-key hashes into one hash
+        merge_sequence_hashes(non_empty)
+      end
 
-          # Check if this is an array of hashes (Parsanol sequence format)
-          # that should be merged into a single hash.
-          # Empty arrays and nil values are treated as nil/absent optional elements
-          non_empty_items = converted_items.reject do |item|
-            item.nil? || (item.is_a?(Array) && item.empty?)
+      def same_key_repetition?(items)
+        first_keys = items.first.keys
+        first_keys.length == 1 &&
+          items.length > 1 &&
+          items.all? { |item| item.keys.length == 1 && item.keys.first == first_keys.first }
+      end
+
+      def duplicate_key_repetition?(items)
+        all_keys = items.flat_map(&:keys)
+        key_counts = all_keys.tally
+        key_counts.any? { |_, count| count > 1 }
+      end
+
+      def try_unzip_repetition(items)
+        return nil unless items.length >= 2
+
+        first_item = items[0]
+        later_items = items[1..]
+        return nil unless first_item.keys.length == 1 && !first_item.values.first.is_a?(Array)
+
+        first_key = first_item.keys.first
+        repetition_item = later_items.find do |item|
+          item.key?(first_key) &&
+            item[first_key].is_a?(Array) &&
+            item[first_key].length > 1 &&
+            item.keys.length >= 2
+        end
+        return nil unless repetition_item
+
+        unzip_grouped_repetition(first_item, later_items)
+      end
+
+      def unzip_grouped_repetition(first_item, later_items)
+        result = [first_item]
+
+        later_items.each do |item|
+          array_keys = item.select { |_, v| v.is_a?(Array) && v.length > 1 }.keys
+
+          if array_keys.empty?
+            result << item
+          elsif array_keys.length == 1
+            expand_single_array_key(item, array_keys.first, result)
+          else
+            expand_multiple_array_keys(item, array_keys, result)
           end
+        end
 
-          if non_empty_items.empty?
-            # All items were nil or empty - return empty hash (sequence with all optionals absent)
-            {}
-          elsif non_empty_items.all?(Hash)
-            # Check if this is a repetition pattern where all hashes have the SAME single key
-            first_keys = non_empty_items.first.keys
-
-            if first_keys.length == 1 &&
-                non_empty_items.length > 1 &&
-                non_empty_items.all? do |item|
-                  item.keys.length == 1 && item.keys.first == first_keys.first
-                end
-              # Multiple items with the same single key - this is a repetition pattern
-              # Keep as array (each element is already a single-key hash)
-              return converted_items
-            end
+        result
+      end
 
-            # Check if any key appears in MULTIPLE hashes (repetition with separators pattern)
-            # Example: [{formalParameter: {...}}, {op_delim: {...}}, {formalParameter: {...}}]
-            # In this case, formalParameter appears in two different hashes
-            all_keys = non_empty_items.flat_map(&:keys)
-            key_counts = all_keys.tally
-            if key_counts.any? { |_, count| count > 1 }
-              # At least one key appears multiple times - this is a repetition pattern
-              # Keep as array
-              return converted_items
-            end
+      def expand_single_array_key(item, key, result)
+        values = item[key]
+        other_keys = item.keys - [key]
 
-            # Check if we have a repetition pattern that needs unzipping
-            # Pattern: [{item: first}, {separator: [...], item: [rest...]}]
-            # This happens when Parsanol groups repeated elements
-
-            # More specific detection:
-            # 1. First item has a single key with a non-array value
-            # 2. A later item has the SAME key with an array value (the grouped repetitions)
-            # 3. The later item also has a "separator" key (like op_comma)
-
-            if non_empty_items.length >= 2
-              first_item = non_empty_items[0]
-              later_items = non_empty_items[1..]
-
-              # Check if first item is simple (single key, non-array value)
-              if first_item.keys.length == 1 && !first_item.values.first.is_a?(Array)
-                first_key = first_item.keys.first
-
-                # Look for a later item that:
-                # 1. Has the same key as the first item
-                # 2. That key has an array value with length > 1
-                # 3. Also has at least one other key (the separator)
-                repetition_item = later_items.find do |item|
-                  item.key?(first_key) &&
-                    item[first_key].is_a?(Array) &&
-                    item[first_key].length > 1 &&
-                    item.keys.length >= 2
-                end
-
-                if repetition_item
-                  # This is a repetition pattern - unzip it
-                  # Build interleaved result
-                  result = [first_item]
-
-                  later_items.each do |item|
-                    # Find keys with array values
-                    array_keys = item.select do |_, v|
-                      v.is_a?(Array) && v.length > 1
-                    end.keys
-
-                    if array_keys.empty?
-                      # No array values - add as-is
-                      result << item
-                    elsif array_keys.length == 1
-                      # Single array key - expand it
-                      key = array_keys.first
-                      values = item[key]
-                      other_keys = item.keys - [key]
-
-                      values.each_with_index do |val, _idx|
-                        new_item = { key => val }
-                        other_keys.each do |ok|
-                          new_item[ok] = item[ok]
-                        end
-                        result << new_item
-                      end
-                    else
-                      # Multiple array keys - interleave them
-                      # All arrays should have the same length
-                      len = array_keys.map { |k| item[k].length }.max
-                      other_keys = item.keys - array_keys
-
-                      len.times do |idx|
-                        new_item = {}
-                        array_keys.each do |k|
-                          values = item[k]
-                          new_item[k] = values[idx] if idx < values.length
-                        end
-                        other_keys.each do |ok|
-                          new_item[ok] = item[ok]
-                        end
-                        result << new_item
-                      end
-                    end
-                  end
+        values.each do |val|
+          new_item = { key => val }
+          other_keys.each { |ok| new_item[ok] = item[ok] }
+          result << new_item
+        end
+      end
 
-                  return result
-                end
-              end
-            end
+      def expand_multiple_array_keys(item, array_keys, result)
+        len = array_keys.map { |k| item[k].length }.max
+        other_keys = item.keys - array_keys
 
-            # No repetition pattern detected - merge all hashes into one hash
-            # This handles sequences like: tENUMERATION >> tOF >> enumerationItems
-            # which produce: [{tENUMERATION: ...}, {tOF: ..., enumerationItems: ...}]
-            merged = {}
-            non_empty_items.each do |item|
-              item.each do |key, value|
-                # If key already exists, convert to array or append
-                if merged.key?(key)
-                  existing = merged[key]
-                  if existing.is_a?(Array)
-                    if value.is_a?(Array)
-                      existing.concat(value)
-                    else
-                      existing << value
-                    end
-                  else
-                    merged[key] = [existing, value]
-                  end
-                else
-                  merged[key] = value
-                end
+        len.times do |idx|
+          new_item = {}
+          array_keys.each do |k|
+            values = item[k]
+            new_item[k] = values[idx] if idx < values.length
+          end
+          other_keys.each { |ok| new_item[ok] = item[ok] }
+          result << new_item
+        end
+      end
+
+      def merge_sequence_hashes(items)
+        merged = {}
+        items.each do |item|
+          item.each do |key, value|
+            if merged.key?(key)
+              existing = merged[key]
+              if existing.is_a?(Array)
+                existing.is_a?(Array) && value.is_a?(Array) ? existing.concat(value) : existing << value
+              else
+                merged[key] = [existing, value]
               end
+            else
+              merged[key] = value
             end
-
-            # Process the merged hash for :str/:spaces concatenation
-            process_str_and_spaces(merged)
-          else
-            # Regular array - return the converted items
-            converted_items
           end
-        else
-          obj
         end
+        process_str_and_spaces(merged)
       end
 
       # Add a value to the current container on top of the stack
@@ -445,7 +417,6 @@ module Expressir
         else
           # This handles unexpected container types (debugging)
           if ENV["DEBUG_STREAMING"]
-            puts "DEBUG: Unknown container type: #{container.class}"
             puts "  Container value: #{container.inspect[0..200]}"
             puts "  Pending key: #{@pending_key.inspect}"
             puts "  Stack depth: #{@stack.length}"

data/lib/expressir/express.rb

@@ -2,30 +2,34 @@
 
 module Expressir
   module Express
+    autoload :AstKeyConverter, "#{__dir__}/express/ast_key_converter"
     autoload :Builder, "#{__dir__}/express/builder"
+    autoload :BuilderContext, "#{__dir__}/express/builder_context"
+    autoload :BuilderRegistry, "#{__dir__}/express/builder_registry"
     autoload :Builders, "#{__dir__}/express/builders"
-    # autoload :BuilderRegistry, "#{__dir__}/express/builder_registry"
 
     # Core classes (autoloaded for lazy loading)
     autoload :Cache, "#{__dir__}/express/cache"
     autoload :Error, "#{__dir__}/express/error"
     autoload :Formatter, "#{__dir__}/express/formatter"
     autoload :Formatters, "#{__dir__}/express/formatters"
+    autoload :Grammar, "#{__dir__}/express/grammar"
     autoload :HyperlinkFormatter, "#{__dir__}/express/hyperlink_formatter"
     autoload :LineMap, "#{__dir__}/express/line_map"
     autoload :ModelVisitor, "#{__dir__}/express/model_visitor"
+    autoload :NodePositionIndex, "#{__dir__}/express/node_position_index"
+    autoload :ParallelFiles, "#{__dir__}/express/parallel_files"
     autoload :Parser, "#{__dir__}/express/parser"
     autoload :PrettyFormatter, "#{__dir__}/express/pretty_formatter"
     autoload :RemarkAttacher, "#{__dir__}/express/remark_attacher"
     autoload :RemarkScanner, "#{__dir__}/express/remark_scanner"
     autoload :ResolveReferencesModelVisitor,
              "#{__dir__}/express/resolve_references_model_visitor"
+    autoload :SchemaBlockScanner, "#{__dir__}/express/schema_block_scanner"
     autoload :SchemaHeadFormatter, "#{__dir__}/express/schema_head_formatter"
+    autoload :SchemaSourceFormatter, "#{__dir__}/express/schema_source_formatter"
+    autoload :ScopeResolver, "#{__dir__}/express/scope_resolver"
+    autoload :SourceFormatter, "#{__dir__}/express/source_formatter"
     autoload :StreamingBuilder, "#{__dir__}/express/streaming_builder"
-
-    autoload :Transformer, "#{__dir__}/express/transformer"
   end
 end
-
-# Eagerly load builders (they register themselves at load time)
-require_relative "express/builder_registry"

data/lib/expressir/model/concerns.rb

@@ -21,5 +21,8 @@ module Expressir
 
     # Marker for types supporting where rules
     module HasWhereRules; end
+
+    # Marker for executable statements (ISO 10303-11 section 13)
+    module Statement; end
   end
 end

data/lib/expressir/model/data_types/generic_entity.rb

@@ -11,13 +11,13 @@ module Expressir
         key_value do
           map "_class", to: :_class, render_default: true
         end
-      end
 
-      # @return [Array<Declaration>]
-      def children
-        [
-          *remark_items,
-        ]
+        # @return [Array<Declaration>]
+        def children
+          [
+            *remark_items,
+          ]
+        end
       end
     end
   end

data/lib/expressir/model/declarations/entity.rb

@@ -9,6 +9,11 @@ module Expressir
         include HasInformalPropositions
         include HasWhereRules
 
+        collection_attributes :attributes, :derived_attributes,
+                              :inverse_attributes, :unique_rules,
+                              :where_rules, :informal_propositions,
+                              :remark_items
+
         attribute :abstract, :boolean
         attribute :supertype_expression, ModelElement
         attribute :subtype_of, ModelElement, collection: true

data/lib/expressir/model/declarations/function.rb

@@ -7,6 +7,11 @@ module Expressir
         include Identifier
         include ScopeContainer
 
+        collection_attributes :parameters, :types, :entities,
+                              :subtype_constraints, :functions, :procedures,
+                              :constants, :variables, :statements,
+                              :remark_items
+
         attribute :parameters, Parameter, collection: true
         attribute :return_type, ModelElement
         attribute :types, ModelElement, collection: true

data/lib/expressir/model/declarations/procedure.rb

@@ -7,6 +7,11 @@ module Expressir
         include Identifier
         include ScopeContainer
 
+        collection_attributes :parameters, :types, :entities,
+                              :subtype_constraints, :functions, :procedures,
+                              :constants, :variables, :statements,
+                              :remark_items
+
         attribute :parameters, Parameter, collection: true
         attribute :types, ModelElement, collection: true
         attribute :entities, Entity, collection: true

data/lib/expressir/model/declarations/rule.rb

@@ -9,6 +9,12 @@ module Expressir
         include HasInformalPropositions
         include HasWhereRules
 
+        collection_attributes :applies_to, :types, :entities,
+                              :subtype_constraints, :functions, :procedures,
+                              :constants, :variables, :statements,
+                              :where_rules, :informal_propositions,
+                              :remark_items
+
         attribute :applies_to, ModelElement, collection: true
         attribute :types, ModelElement, collection: true
         attribute :entities, Entity, collection: true

data/lib/expressir/model/declarations/schema.rb

@@ -7,6 +7,22 @@ module Expressir
         include Identifier
         include ScopeContainer
 
+        # Tree-walker collection attributes (TODO.bugs/12 — single source
+        # of truth; NodePositionIndex derives COLLECTION_REGISTRY from this).
+        collection_attributes :constants, :types, :entities,
+                              :subtype_constraints, :functions, :rules,
+                              :procedures, :remark_items
+
+        # Subset of this schema's collection attributes that hold named
+        # scope declarations (Function, Procedure, Rule, Entity, Type).
+        # Used by ScopeResolver to enumerate scope-capable children without
+        # duplicating the list. Single source of truth: this is the subset
+        # of NodePositionIndex::COLLECTION_REGISTRY[Schema] that holds
+        # scopes rather than remark_items / constants / subtype_constraints.
+        SCOPE_DECL_COLLECTIONS = %i[
+          functions procedures rules entities types
+        ].freeze
+
         attribute :file, :string
         attribute :version, SchemaVersion
         attribute :interfaces, Interface, collection: true
@@ -17,6 +33,7 @@ module Expressir
         attribute :functions, Function, collection: true
         attribute :rules, Rule, collection: true
         attribute :procedures, Procedure, collection: true
+        attribute :header, :string
         attribute :_class, :string, default: -> { self.class.name }
         attribute :selected, :boolean, default: false
         attribute :file_basename, :string
@@ -25,6 +42,7 @@ module Expressir
           map "_class", to: :_class, render_default: true
           map "id", to: :id
           map "file", to: :file
+          map "header", to: :header
           map "remarks", to: :remarks
           map "remark_items", to: :remark_items
           map "version", to: :version
@@ -39,6 +57,7 @@ module Expressir
         end
 
         liquid do
+          map :header, to: :header
           map :formatted, to: :formatted
           map :source, to: :source
           map :full_source, to: :full_source
@@ -74,17 +93,11 @@ module Expressir
         end
 
         def formatted
-          @formatted ||= to_s(no_remarks: true)
+          @formatted ||= format(no_remarks: false)
         end
 
         def source
-          @source ||= begin
-            formatter = Class.new(Expressir::Express::Formatter) do
-              include Expressir::Express::SchemaHeadFormatter
-              include Expressir::Express::HyperlinkFormatter
-            end
-            formatter.format(self)
-          end
+          @source ||= Expressir::Express::SchemaSourceFormatter.format(self)
         end
 
         private

data/lib/expressir/model/declarations/type.rb

@@ -9,6 +9,9 @@ module Expressir
         include HasInformalPropositions
         include HasWhereRules
 
+        collection_attributes :where_rules, :informal_propositions,
+                              :remark_items
+
         attribute :underlying_type, ModelElement
         attribute :where_rules, WhereRule, collection: true
         attribute :informal_propositions, InformalPropositionRule,

data/lib/expressir/model/exp_file.rb

@@ -14,6 +14,8 @@ module Expressir
       include HasId
       include ScopeContainer
 
+      collection_attributes :schemas
+
       attribute :path, :string
       attribute :schemas, Declarations::Schema, collection: true,
                                                 initialize_empty: true

data/lib/expressir/model/expressions/aggregate_initializer.rb

@@ -4,6 +4,7 @@ module Expressir
       # Specified in ISO 10303-11:2004
       # - section 12.9 Aggregate initializer
       class AggregateInitializer < ModelElement
+        child_attributes :items
         attribute :items, ModelElement, collection: true
         attribute :_class, :string, default: -> { self.class.name }
 

data/lib/expressir/model/expressions/aggregate_initializer_item.rb

@@ -4,6 +4,7 @@ module Expressir
       # Specified in ISO 10303-11:2004
       # - section 12.9 Aggregate initializer
       class AggregateInitializerItem < ModelElement
+        child_attributes :expression, :repetition
         attribute :expression, ModelElement
         attribute :repetition, ModelElement
         attribute :_class, :string, default: -> { self.class.name }

data/lib/expressir/model/expressions/binary_expression.rb

@@ -10,6 +10,7 @@ module Expressir
       # - section 12.6 Aggregate operators
       # - section 12.10 Complex entity instance construction operator
       class BinaryExpression < ModelElement
+        child_attributes :operand1, :operand2
         ADDITION = "ADDITION".freeze
         AND = "AND".freeze
         COMBINE = "COMBINE".freeze

data/lib/expressir/model/expressions/entity_constructor.rb

@@ -4,6 +4,7 @@ module Expressir
       # Specified in ISO 10303-11:2004
       # - section 9.2.6 Implicit declarations
       class EntityConstructor < ModelElement
+        child_attributes :parameters
         attribute :entity, ModelElement
         attribute :parameters, ModelElement, collection: true
         attribute :_class, :string, default: -> { self.class.name }

data/lib/expressir/model/expressions/function_call.rb

@@ -4,6 +4,7 @@ module Expressir
       # Specified in ISO 10303-11:2004
       # - section 12.8 Function call
       class FunctionCall < ModelElement
+        child_attributes :parameters
         attribute :function, ModelElement
         attribute :parameters, ModelElement, collection: true
         attribute :_class, :string, default: -> { self.class.name }

data/lib/expressir/model/expressions/interval.rb

@@ -4,6 +4,7 @@ module Expressir
       # Specified in ISO 10303-11:2004
       # - section 12.2.4 Interval expressions
       class Interval < ModelElement
+        child_attributes :low, :item, :high
         LESS_THAN = "LESS_THAN".freeze
         LESS_THAN_OR_EQUAL = "LESS_THAN_OR_EQUAL".freeze
 

data/lib/expressir/model/expressions/query_expression.rb

@@ -6,6 +6,8 @@ module Expressir
       class QueryExpression < ModelElement
         include Identifier
 
+        child_attributes :expression, :aggregate_source
+
         attribute :aggregate_source, ModelElement
         attribute :expression, ModelElement
         attribute :_class, :string, default: -> { self.class.name }

data/lib/expressir/model/expressions/unary_expression.rb

@@ -5,6 +5,7 @@ module Expressir
       # - section 12.1 Arithmetic operators
       # - section 12.4.1 NOT operator
       class UnaryExpression < ModelElement
+        child_attributes :operand
         MINUS = "MINUS".freeze
         NOT = "NOT".freeze
         PLUS = "PLUS".freeze