expressir 2.3.0 → 2.3.1

data/lib/expressir/express/builders/expression_builder.rb

@@ -45,9 +45,6 @@ module Expressir
 
           return term if rhs.nil? || (rhs.respond_to?(:empty?) && rhs.empty?)
 
-          # Handle both formats:
-          # - Ruby parser: rhs is Array of {:item => {...}} hashes
-          # - Native parser: rhs might be a Hash (merged) or Array
           rhs_array = rhs.is_a?(Array) ? rhs : [rhs]
 
           operands = [term]
@@ -79,9 +76,6 @@ module Expressir
 
           return factor if rhs.nil? || (rhs.respond_to?(:empty?) && rhs.empty?)
 
-          # Handle both formats:
-          # - Ruby parser: rhs is Array of {:item => {...}} hashes
-          # - Native parser: rhs might be a Hash (merged) or Array
           rhs_array = rhs.is_a?(Array) ? rhs : [rhs]
 
           operands = [factor]

data/lib/expressir/express/builders/function_decl_builder.rb

@@ -25,15 +25,15 @@ module Expressir
           functions = declarations.grep(Expressir::Model::Declarations::Function)
           procedures = declarations.grep(Expressir::Model::Declarations::Procedure)
           constants = if algorithm_head.is_a?(Hash) && algorithm_head[:constant_decl]
-                       build_constant_decl(algorithm_head[:constant_decl])
-                     else
-                       []
-                     end
+                        build_constant_decl(algorithm_head[:constant_decl])
+                      else
+                        []
+                      end
           variables = if algorithm_head.is_a?(Hash) && algorithm_head[:local_decl]
-                       build_local_decl(algorithm_head[:local_decl])
-                     else
-                       []
-                     end
+                        build_local_decl(algorithm_head[:local_decl])
+                      else
+                        []
+                      end
           statements = Builder.build_children(stmts)
 
           Expressir::Model::Declarations::Function.new(

data/lib/expressir/express/builders/procedure_decl_builder.rb

@@ -24,15 +24,15 @@ module Expressir
           functions = declarations.grep(Expressir::Model::Declarations::Function)
           procedures = declarations.grep(Expressir::Model::Declarations::Procedure)
           constants = if algorithm_head.is_a?(Hash) && algorithm_head[:constant_decl]
-                       build_constant_decl(algorithm_head[:constant_decl])
-                     else
-                       []
-                     end
+                        build_constant_decl(algorithm_head[:constant_decl])
+                      else
+                        []
+                      end
           variables = if algorithm_head.is_a?(Hash) && algorithm_head[:local_decl]
-                       build_local_decl(algorithm_head[:local_decl])
-                     else
-                       []
-                     end
+                        build_local_decl(algorithm_head[:local_decl])
+                      else
+                        []
+                      end
           statements = Builder.build_children(stmts)
 
           Expressir::Model::Declarations::Procedure.new(

data/lib/expressir/express/builders/rule_decl_builder.rb

@@ -25,15 +25,15 @@ module Expressir
           functions = declarations.grep(Expressir::Model::Declarations::Function)
           procedures = declarations.grep(Expressir::Model::Declarations::Procedure)
           constants = if algorithm_head.is_a?(Hash) && algorithm_head[:constant_decl]
-                       build_constant_decl(algorithm_head[:constant_decl])
-                     else
-                       []
-                     end
+                        build_constant_decl(algorithm_head[:constant_decl])
+                      else
+                        []
+                      end
           variables = if algorithm_head.is_a?(Hash) && algorithm_head[:local_decl]
-                       build_local_decl(algorithm_head[:local_decl])
-                     else
-                       []
-                     end
+                        build_local_decl(algorithm_head[:local_decl])
+                      else
+                        []
+                      end
           statements = Builder.build_children(stmts)
           where_rules = where_clause ? Builder.build({ where_clause: where_clause }) : []
 

data/lib/expressir/express/builders/syntax_builder.rb

@@ -7,14 +7,9 @@ module Expressir
       # Each parsed EXPRESS file is represented as an ExpFile containing schemas.
       class SyntaxBuilder
         # Build ExpFile from AST data
-        # @param ast_data [Hash, Array] The parsed AST with syntax node
-        #   - Ruby parser: Hash with :schema_decl key
-        #   - Native parser: Hash with :syntax wrapper containing :schemaDecl
+        # @param ast_data [Hash] The parsed AST with syntax node
         # @return [Model::ExpFile] ExpFile containing the parsed schemas
         def call(ast_data)
-          # Handle both formats:
-          # - Ruby parser: ast_data is a merged Hash {:schema_decl => [...], :spaces => {...}}
-          # - Native parser: ast_data is {:syntax => {:spaces, :schemaDecl => [...], :trailer}}
           schema_decl_data = extract_schema_decls(ast_data)
 
           schemas = Builder.build_children(schema_decl_data)
@@ -24,47 +19,10 @@ module Expressir
         private
 
         # Extract schema declarations from AST data.
-        # Handles both Ruby (snake_case) and native (camelCase) formats.
         def extract_schema_decls(ast_data)
           if ast_data.is_a?(Hash)
-            # Check for :syntax wrapper (native parser format)
-            if ast_data.key?(:syntax)
-              inner = ast_data[:syntax]
-              return inner[:schemaDecl] || inner[:schema_decl] if inner.is_a?(Hash)
-            end
-
-            # Ruby parser format: snake_case
-            return ast_data[:schema_decl] if ast_data.key?(:schema_decl)
-
-            # Native parser format: camelCase at top level (legacy)
-            return ast_data[:schemaDecl] if ast_data.key?(:schemaDecl)
-          elsif ast_data.is_a?(Array)
-            # Legacy native parser format: Array of Hashes
-            merged = {}
-            ast_data.each do |item|
-              if item.is_a?(Hash)
-                item.each do |_key, value|
-                  if value.is_a?(Array)
-                    value.each do |sub|
-                      if sub.is_a?(Hash)
-                        sub.each do |k, v|
-                          snake_key = k.to_s
-                            .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
-                            .gsub(/([a-z\d])([A-Z])/, '\1_\2')
-                            .downcase
-                            .to_sym
-                          merged[snake_key] ||= []
-                          merged[snake_key] = v if v.is_a?(Array)
-                        end
-                      end
-                    end
-                  end
-                end
-              end
-            end
-            return merged[:schema_decl]
+            ast_data[:schema_decl]
           end
-          nil
         end
       end
     end

data/lib/expressir/express/formatters/remark_formatter.rb

@@ -237,9 +237,7 @@ module Expressir
                 !node.untagged_remarks.nil? &&
                 skip_untagged_types.any? { |type| node.is_a?(type) }
 
-              remarks.concat(node.untagged_remarks.compact.select do |remark|
-                remark.is_a?(Model::RemarkInfo)
-              end.map do |remark|
+              remarks.concat(node.untagged_remarks.compact.grep(Model::RemarkInfo).map do |remark|
                 format_untagged_remark(remark)
               end)
             end

data/lib/expressir/express/parser.rb

@@ -14,6 +14,13 @@ module Expressir
         @@cached_parser = nil
         @@parser_mutex = Mutex.new
 
+        # Cache for schemaDecl grammar JSON (used for streaming parse)
+        @@cached_schema_grammar_json = nil
+
+        # Threshold for using memory-bounded fresh parse (bytes)
+        # Files above this use parse_fresh which has no packrat cache
+        LARGE_FILE_THRESHOLD = 1024 * 1024 # 1 MB
+
         # Get cached parser instance (thread-safe)
         # Reusing the parser avoids the overhead of reinitializing all rule definitions
         def self.cached_parser
@@ -59,6 +66,15 @@ module Expressir
           @@cached_grammar_json
         end
 
+        # Get cached grammar JSON for schemaDecl (used for streaming parse)
+        def self.cached_schema_grammar_json
+          return @@cached_schema_grammar_json if @@cached_schema_grammar_json
+
+          schema_atom = cached_parser.schemaDecl
+          @@cached_schema_grammar_json = Parsanol::Native.serialize_grammar(schema_atom)
+          @@cached_schema_grammar_json
+        end
+
         # Parse using native engine with Rust-side transformation (fastest)
         #
         # This method provides ~17x speedup over pure Ruby parsing.
@@ -74,7 +90,12 @@ module Expressir
           end
 
           grammar_atom = cached_parser.syntax
-          Parsanol::Native.parse(grammar_atom, source)
+          # Use fresh-parse (no cache) for large files to bound memory
+          if source.bytesize > LARGE_FILE_THRESHOLD
+            Parsanol::Native.parse_fresh(grammar_atom, source)
+          else
+            Parsanol::Native.parse(grammar_atom, source)
+          end
         end
 
         def cts(atom)
@@ -670,24 +691,37 @@ module Expressir
       # @param [String] file Express file path
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
+      # @param [Boolean] use_native use native parser (default: true when available)
       # @return [Model::ExpFile] ExpFile containing parsed schemas
       # @raise [SchemaParseFailure] if the schema file fails to parse
-      def self.from_file(file, skip_references: nil, include_source: nil, root_path: nil) # rubocop:disable Metrics/AbcSize
+      def self.from_file(file, skip_references: nil, include_source: nil, root_path: nil, use_native: nil) # rubocop:disable Metrics/AbcSize
         Expressir::Benchmark.measure_file(file) do
           source = File.read file
 
           # remove root path from file path
           schema_file = root_path ? Pathname.new(file.to_s).relative_path_from(root_path).to_s : file.to_s
 
+          use_native = Parser.native_available? if use_native.nil?
+
           begin
-            ast = Parser.cached_parser.parse source
+            ast = if use_native && Parser.native_available?
+                    begin
+                      Parser.parse_native(source)
+                    rescue StandardError
+                      # Native parser may fail on non-ASCII or edge cases;
+                      # fall back to Ruby parser
+                      Parser.cached_parser.parse source
+                    end
+                  else
+                    Parser.cached_parser.parse source
+                  end
           rescue Parsanol::ParseFailed => e
             # Instead of just printing, raise a proper error with file context
             raise Error::SchemaParseFailure.new(schema_file, e)
           end
 
           @exp_file = ::Expressir::Express::Builder.build_with_remarks(ast, source: source,
-                                                        include_source: include_source)
+                                                                            include_source: include_source)
 
           # Set file path on the ExpFile and propagate to schemas
           @exp_file.path = schema_file
@@ -712,16 +746,19 @@ module Expressir
       # @param [Array<String>] files Express file paths
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
+      # @param [Boolean] use_native use native parser (default: true when available)
       # @yield [filename, schemas, error] Optional block called for each file processed
       # @yieldparam filename [String] Name of the file being processed
       # @yieldparam schemas [Array, nil] Array of parsed schemas (nil if parsing failed)
       # @yieldparam error [Exception, nil] Error that occurred (nil if parsing succeeded)
       # @return [Model::Repository] Repository containing all parsed ExpFiles
-      def self.from_files(files, skip_references: nil, include_source: nil, root_path: nil)
+      def self.from_files(files, skip_references: nil, include_source: nil,
+root_path: nil, use_native: nil)
         all_exp_files = []
 
         files.each do |file|
-          exp_file = from_file(file, skip_references: true, root_path: root_path)
+          exp_file = from_file(file, skip_references: true,
+                                     root_path: root_path, use_native: use_native)
           all_exp_files << exp_file
 
           # Call the progress block if provided
@@ -751,18 +788,20 @@ module Expressir
       # @param [String] content Express content as string
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
-      # @param [Boolean] use_native use native parser if available (default: false - AST format differs slightly)
+      # @param [Boolean] use_native use native parser (default: true when available)
       # @param [Boolean] use_streaming use streaming builder for maximum performance (default: false)
       # @return [Model::ExpFile] Parsed ExpFile
       # @raise [SchemaParseFailure] if the content fails to parse
       def self.from_exp(content, skip_references: nil, include_source: nil,
-                         use_native: false, use_streaming: false)
+                         use_native: nil, use_streaming: false)
         # Streaming builder mode - uses Parsanol streaming callbacks
         if use_streaming && Parser.native_available? && defined?(Parsanol::Native.parse_with_builder)
           return from_exp_streaming(content, skip_references: skip_references,
                                              include_source: include_source)
         end
 
+        use_native = Parser.native_available? if use_native.nil?
+
         begin
           # Use cached parser instance for performance (avoids ~7ms Parser.new overhead)
           ast = if use_native && Parser.native_available?
@@ -774,8 +813,9 @@ module Expressir
           raise Error::SchemaParseFailure.new("(from string)", e)
         end
 
-        exp_file = ::Expressir::Express::Builder.build_with_remarks(ast, source: content,
-                                                   include_source: include_source)
+        exp_file = ::Expressir::Express::Builder.build_with_remarks(ast,
+                                                                    source: content,
+                                                                    include_source: include_source)
 
         exp_file.schemas.each do |schema|
           schema.file = nil
@@ -793,20 +833,21 @@ module Expressir
         exp_file
       end
 
-      # Parse using streaming builder (maximum performance)
+      # Parse using streaming builder (construct-by-construct)
       # @param [String] content Express content as string
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
       # @return [Model::ExpFile] Parsed ExpFile
       # @raise [SchemaParseFailure] if the content fails to parse
-      def self.from_exp_streaming(content, skip_references: nil, include_source: nil)
+      def self.from_exp_streaming_builder(content, skip_references: nil,
+include_source: nil)
         grammar_json = Parser.cached_grammar_json
         builder = ::Expressir::Express::StreamingBuilder.new(source: content,
-                                       include_source: include_source)
+                                                             include_source: include_source)
 
         begin
           exp_file = Parsanol::Native.parse_with_builder(grammar_json,
-                                                           content, builder)
+                                                         content, builder)
         rescue StandardError => e
           raise Error::SchemaParseFailure.new("(streaming)", e)
         end
@@ -826,6 +867,185 @@ module Expressir
 
         exp_file
       end
+
+      # Parse each schema separately with fresh arena (memory-bounded)
+      #
+      # This splits the source into schema blocks and parses each independently.
+      # Memory is bounded by the largest schema, not the entire file.
+      #
+      # @param content [String] EXPRESS source code
+      # @param skip_references [Boolean] Whether to skip reference resolution
+      # @param include_source [Boolean] Whether to include source in model
+      # @return [Expressir::Model::ExpFile] Parsed EXPRESS file
+      def self.from_exp_streaming(content, skip_references: nil,
+include_source: nil)
+        grammar_json = Parser.cached_schema_grammar_json
+
+        # Extract schema blocks from source
+        schema_blocks = extract_schema_blocks(content)
+
+        # Parse each schema with fresh arena
+        schemas = schema_blocks.map do |block|
+          ast = Parsanol::Native.parse_fresh(grammar_json, block[:source])
+          schema_model = Builder.build(ast)
+          schema_model.source = block[:source]
+          schema_model
+        rescue StandardError => e
+          raise Error::SchemaParseFailure.new(
+            "(schema #{block[:name] || 'unknown'})", e
+          )
+        end
+
+        # Build the file model
+        exp_file = Expressir::Model::ExpFile.new
+        exp_file.schemas = schemas
+
+        exp_file.schemas.each do |schema|
+          schema.file = nil
+          schema.file_basename = nil
+          schema.formatted = schema.to_s(no_remarks: true)
+        end
+
+        unless skip_references
+          Expressir::Benchmark.measure_references do
+            resolve_references_model_visitor = ResolveReferencesModelVisitor.new
+            resolve_references_model_visitor.visit(exp_file)
+          end
+        end
+
+        exp_file
+      end
+
+      # Extract individual schema blocks from EXPRESS source
+      #
+      # This uses a state machine to properly handle nested comments and strings.
+      #
+      # @param source [String] EXPRESS source
+      # @return [Array<Hash>] Array of {name: String, source: String} for each schema
+      def self.extract_schema_blocks(source)
+        blocks = []
+        pos = 0
+        len = source.length
+
+        while pos < len
+          # Skip whitespace and find SCHEMA keyword
+          skip_ws_and_comments(source, pos)
+          pos = skip_ws_and_comments(source, pos)
+
+          break if pos >= len
+
+          # Check for SCHEMA
+          if source[pos..].start_with?("SCHEMA")
+            result = parse_schema_block(source, pos)
+            if result
+              blocks << result
+              pos = result[:end_pos]
+              next
+            end
+          end
+
+          pos += 1
+        end
+
+        blocks
+      end
+
+      private_class_method :extract_schema_blocks
+
+      def self.parse_schema_block(source, start_pos)
+        # Must be at SCHEMA keyword
+        return nil unless source[start_pos..].start_with?("SCHEMA")
+
+        pos = start_pos + "SCHEMA".length
+        pos = skip_spaces(source, pos)
+
+        # Extract schema name (identifier)
+        name_start = pos
+        while pos < source.length && (source[pos] =~ /[a-zA-Z0-9_]/ || source[pos] == "_")
+          pos += 1
+        end
+        schema_name = source[name_start...pos]
+
+        return nil if schema_name.empty?
+
+        # Skip to END_SCHEMA
+        depth = 1
+        search_pos = pos
+        end_pos = nil
+
+        while search_pos < source.length
+          if source[search_pos] == '"'
+            # String literal - skip to end
+            search_pos += 1
+            while search_pos < source.length && source[search_pos] != '"'
+              search_pos += 1
+            end
+            search_pos += 1
+          elsif source[search_pos] == "(" && source[search_pos + 1] == "*"
+            # Comment - skip to end
+            search_pos += 2
+            while search_pos < source.length && !(source[search_pos] == "*" && source[search_pos + 1] == ")")
+              search_pos += 1
+            end
+            search_pos += 2
+          elsif source[search_pos..].start_with?("END_SCHEMA")
+            depth -= 1
+            if depth.zero?
+              end_pos = search_pos + "END_SCHEMA".length
+              # Skip trailing whitespace and semicolon
+              pos = end_pos
+              pos = skip_spaces(source, pos)
+              pos += 1 if source[pos] == ";" # semicolon
+              break
+            end
+            search_pos += "END_SCHEMA".length
+          else
+            search_pos += 1
+          end
+        end
+
+        return nil unless end_pos
+
+        schema_source = source[start_pos...end_pos]
+        {
+          name: schema_name,
+          source: schema_source,
+          start_pos: start_pos,
+          end_pos: pos,
+        }
+      end
+      private_class_method :parse_schema_block
+
+      def self.skip_spaces(source, pos)
+        while pos < source.length && [" ", "\t", "\n",
+                                      "\r"].include?(source[pos])
+          pos += 1
+        end
+        pos
+      end
+      private_class_method :skip_spaces
+
+      def self.skip_ws_and_comments(source, pos)
+        len = source.length
+        while pos < len
+          c = source[pos]
+          if [" ", "\t", "\n", "\r"].include?(c)
+            pos += 1
+          elsif c == "(" && source[pos + 1] == "*"
+            # Comment - skip to end
+            pos += 2
+            while pos < len - 1 && !(source[pos] == "*" && source[pos + 1] == ")")
+              pos += 1
+            end
+            pos += 2
+          else
+            break
+          end
+        end
+        pos
+      end
+      private_class_method :skip_ws_and_comments
+
       private_class_method :from_exp_streaming
     end
   end

data/lib/expressir/express/remark_attacher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-
 module Expressir
   module Express
     # Handles attaching remarks (comments) to model elements after parsing.
@@ -27,6 +25,15 @@ module Expressir
         remarks = extract_all_remarks
         attach_tagged_remarks(model, remarks)
         attach_untagged_remarks(model, remarks)
+
+        # Free expensive data structures after attachment is complete.
+        # These are only needed during the attach process.
+        @source = nil
+        @source_lines = nil
+        @scope_map = nil
+        @line_cache = nil
+        @remarks_cache = nil
+
         model
       end
 
@@ -242,7 +249,8 @@ module Expressir
               end
               # Fall back to schema prefix
               if target.nil?
-                target = create_implicit_remark_item(model, tag, get_schema_ids(model))
+                target = create_implicit_remark_item(model, tag,
+                                                     get_schema_ids(model))
               end
             end
 
@@ -280,6 +288,7 @@ module Expressir
 
         nil
       end
+
       # Done once per RemarkAttacher instance (O(file_lines)).
       # Each find_containing_scope call then becomes O(1).
       def build_scope_map
@@ -1021,21 +1030,41 @@ module Expressir
           # The parser always provides this via Slice#offset
           if node.source_offset
             pos = node.source_offset
-            line = get_line_number(pos)
-            source_end_line = get_line_number(pos + node.source.length)
-
-            # For container nodes, use the maximum end_line from children
-            # This is needed because source.length only covers the declaration, not the body
-            children_end_line = calculate_children_end_line(node)
-            end_line = [source_end_line,
-                        children_end_line].compact.max || source_end_line
-
-            result << {
-              node: node,
-              position: pos,
-              line: line,
-              end_line: end_line,
-            }
+            # Validate offset: native parser returns 0 for leaf nodes (WhereRule)
+            # where it can't determine the actual position. These have short
+            # expression-like source ("TRUE;") that doesn't appear at file start.
+            # Container nodes (Schema, Entity, Type) have declaration-like source
+            # that either starts at position 0 legitimately or is clearly valid.
+            valid = pos.positive?
+            if !valid && pos.zero? && node.source
+              src = node.source.to_s
+              # Accept position=0 if source is a declaration keyword line
+              valid = src.start_with?("SCHEMA", "ENTITY", "TYPE", "FUNCTION",
+                                     "PROCEDURE", "RULE", "CONSTANT", "VARIABLE",
+                                     "USE", "REFERENCE", "END_SCHEMA", "END_ENTITY",
+                                     "END_TYPE", "END_FUNCTION", "END_PROCEDURE",
+                                     "END_RULE", "END_CONSTANT", "END_VARIABLE")
+            end
+            if valid
+              line = get_line_number(pos)
+              source_end_line = get_line_number(pos + node.source.length)
+
+              # For container nodes, use the maximum end_line from children
+              # This is needed because source.length only covers the declaration, not the body
+              children_end_line = calculate_children_end_line(node)
+              end_line = [source_end_line,
+                          children_end_line].compact.max || source_end_line
+
+              result << {
+                node: node,
+                position: pos,
+                line: line,
+                end_line: end_line,
+              }
+            else
+              # Invalid offset — treat as unknown position
+              result << { node: node, position: nil, line: nil, end_line: nil }
+            end
           else
             # No source_offset available - should not happen if parser provides Slice
             result << { node: node, position: nil, line: nil, end_line: nil }

data/lib/expressir/express/transformer/remark_handling.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "parsanol"
-require "set"
 
 module Expressir
   module Express

data/lib/expressir/model/exp_file.rb

@@ -14,7 +14,8 @@ module Expressir
       include ScopeContainer
 
       attribute :path, :string
-      attribute :schemas, Declarations::Schema, collection: true, initialize_empty: true
+      attribute :schemas, Declarations::Schema, collection: true,
+                                                initialize_empty: true
       attribute :_class, :string, default: -> { self.class.name }
 
       key_value do

data/lib/expressir/model/model_element.rb

@@ -208,7 +208,7 @@ module Expressir
       def remark_infos
         return [] if untagged_remarks.nil?
 
-        untagged_remarks.select { |r| r.is_a?(RemarkInfo) }
+        untagged_remarks.grep(RemarkInfo)
       end
 
       private