expressir 2.2.1 → 2.3.1

data/lib/expressir/express/parser.rb

@@ -1,9 +1,4 @@
 require "parsanol"
-require_relative "error"
-require_relative "builder"
-require_relative "builders"
-require_relative "remark_attacher"
-require_relative "streaming_builder"
 
 module Expressir
   module Express
@@ -19,6 +14,13 @@ module Expressir
         @@cached_parser = nil
         @@parser_mutex = Mutex.new
 
+        # Cache for schemaDecl grammar JSON (used for streaming parse)
+        @@cached_schema_grammar_json = nil
+
+        # Threshold for using memory-bounded fresh parse (bytes)
+        # Files above this use parse_fresh which has no packrat cache
+        LARGE_FILE_THRESHOLD = 1024 * 1024 # 1 MB
+
         # Get cached parser instance (thread-safe)
         # Reusing the parser avoids the overhead of reinitializing all rule definitions
         def self.cached_parser
@@ -64,14 +66,36 @@ module Expressir
           @@cached_grammar_json
         end
 
-        # Parse using native engine (with caching)
+        # Get cached grammar JSON for schemaDecl (used for streaming parse)
+        def self.cached_schema_grammar_json
+          return @@cached_schema_grammar_json if @@cached_schema_grammar_json
+
+          schema_atom = cached_parser.schemaDecl
+          @@cached_schema_grammar_json = Parsanol::Native.serialize_grammar(schema_atom)
+          @@cached_schema_grammar_json
+        end
+
+        # Parse using native engine with Rust-side transformation (fastest)
+        #
+        # This method provides ~17x speedup over pure Ruby parsing.
+        # The transformation happens in Rust using to_parslet_compatible,
+        # producing Parslet-compatible output that Builder.build can consume directly.
+        #
+        # @param source [String] EXPRESS source code to parse
+        # @return [Hash, Array] Transformed AST in Parslet-compatible format
+        # @raise [LoadError] If native parser is not available
         def self.parse_native(source)
           unless native_available?
             raise LoadError, "Native parser not available"
           end
 
-          # Use Parsanol 2.0 API - parse returns Slice objects with position info
-          new.parse(source, mode: :native)
+          grammar_atom = cached_parser.syntax
+          # Use fresh-parse (no cache) for large files to bound memory
+          if source.bytesize > LARGE_FILE_THRESHOLD
+            Parsanol::Native.parse_fresh(grammar_atom, source)
+          else
+            Parsanol::Native.parse(grammar_atom, source)
+          end
         end
 
         def cts(atom)
@@ -218,7 +242,7 @@ module Expressir
           (attributeDecl >> op_colon >> parameterType >> op_decl >> expression >> op_delim).as(:derivedAttr)
         end
         rule(:digit) { match["0-9"] }
-        rule(:digits) { (digit >> digit.repeat) }
+        rule(:digits) { digit >> digit.repeat }
         rule(:domainRule) do
           ((ruleLabelId >> op_colon).maybe >> expression).as(:domainRule)
         end
@@ -228,7 +252,7 @@ module Expressir
         rule(:embeddedRemark) do
           (str("(*") >> (str("*)").absent? >> (embeddedRemark | any)).repeat >> str("*)")).as(:embeddedRemark)
         end
-        rule(:encodedCharacter) { (octet >> octet >> octet >> octet) }
+        rule(:encodedCharacter) { octet >> octet >> octet >> octet }
         rule(:encodedStringLiteral) do
           cts((str('"') >> encodedCharacter.repeat(1) >> str('"')).as(:str)).as(:encodedStringLiteral)
         end
@@ -667,26 +691,41 @@ module Expressir
       # @param [String] file Express file path
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
-      # @return [Model::Repository]
+      # @param [Boolean] use_native use native parser (default: true when available)
+      # @return [Model::ExpFile] ExpFile containing parsed schemas
       # @raise [SchemaParseFailure] if the schema file fails to parse
-      def self.from_file(file, skip_references: nil, include_source: nil, root_path: nil) # rubocop:disable Metrics/AbcSize
+      def self.from_file(file, skip_references: nil, include_source: nil, root_path: nil, use_native: nil) # rubocop:disable Metrics/AbcSize
         Expressir::Benchmark.measure_file(file) do
           source = File.read file
 
           # remove root path from file path
           schema_file = root_path ? Pathname.new(file.to_s).relative_path_from(root_path).to_s : file.to_s
 
+          use_native = Parser.native_available? if use_native.nil?
+
           begin
-            ast = Parser.cached_parser.parse source
+            ast = if use_native && Parser.native_available?
+                    begin
+                      Parser.parse_native(source)
+                    rescue StandardError
+                      # Native parser may fail on non-ASCII or edge cases;
+                      # fall back to Ruby parser
+                      Parser.cached_parser.parse source
+                    end
+                  else
+                    Parser.cached_parser.parse source
+                  end
           rescue Parsanol::ParseFailed => e
             # Instead of just printing, raise a proper error with file context
             raise Error::SchemaParseFailure.new(schema_file, e)
           end
 
-          @repository = Builder.build_with_remarks(ast, source: source,
-                                                        include_source: include_source)
+          @exp_file = ::Expressir::Express::Builder.build_with_remarks(ast, source: source,
+                                                                            include_source: include_source)
 
-          @repository.schemas.each do |schema|
+          # Set file path on the ExpFile and propagate to schemas
+          @exp_file.path = schema_file
+          @exp_file.schemas.each do |schema|
             schema.file = schema_file
             schema.file_basename = File.basename(schema_file, ".exp")
             schema.formatted = schema.to_s(no_remarks: true)
@@ -695,11 +734,11 @@ module Expressir
           unless skip_references
             Expressir::Benchmark.measure_references do
               @resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-              @resolve_references_model_visitor.visit(@repository)
+              @resolve_references_model_visitor.visit(@exp_file)
             end
           end
 
-          @repository
+          @exp_file
         end
       end
 
@@ -707,23 +746,23 @@ module Expressir
       # @param [Array<String>] files Express file paths
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
+      # @param [Boolean] use_native use native parser (default: true when available)
       # @yield [filename, schemas, error] Optional block called for each file processed
       # @yieldparam filename [String] Name of the file being processed
       # @yieldparam schemas [Array, nil] Array of parsed schemas (nil if parsing failed)
       # @yieldparam error [Exception, nil] Error that occurred (nil if parsing succeeded)
-      # @return [Model::Repository]
+      # @return [Model::Repository] Repository containing all parsed ExpFiles
       def self.from_files(files, skip_references: nil, include_source: nil,
-root_path: nil)
-        all_schemas = []
+root_path: nil, use_native: nil)
+        all_exp_files = []
 
         files.each do |file|
-          repository = from_file(file, skip_references: true,
-                                       root_path: root_path)
-          file_schemas = repository.schemas
-          all_schemas.concat(file_schemas)
+          exp_file = from_file(file, skip_references: true,
+                                     root_path: root_path, use_native: use_native)
+          all_exp_files << exp_file
 
           # Call the progress block if provided
-          yield(file, file_schemas, nil) if block_given?
+          yield(file, exp_file&.schemas, nil) if block_given?
         rescue StandardError => e
           # Call the progress block with the error if provided
           yield(file, nil, e) if block_given?
@@ -733,9 +772,7 @@ root_path: nil)
           raise unless e.is_a?(Error::SchemaParseFailure)
         end
 
-        @repository = Model::Repository.new(
-          schemas: all_schemas,
-        )
+        @repository = Model::Repository.new(files: all_exp_files)
 
         unless skip_references
           Expressir::Benchmark.measure_references do
@@ -751,18 +788,20 @@ root_path: nil)
       # @param [String] content Express content as string
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
-      # @param [Boolean] use_native use native parser if available (default: false - AST format differs slightly)
+      # @param [Boolean] use_native use native parser (default: true when available)
       # @param [Boolean] use_streaming use streaming builder for maximum performance (default: false)
-      # @return [Model::Repository] Parsed repository
+      # @return [Model::ExpFile] Parsed ExpFile
       # @raise [SchemaParseFailure] if the content fails to parse
       def self.from_exp(content, skip_references: nil, include_source: nil,
-                         use_native: false, use_streaming: false)
+                         use_native: nil, use_streaming: false)
         # Streaming builder mode - uses Parsanol streaming callbacks
         if use_streaming && Parser.native_available? && defined?(Parsanol::Native.parse_with_builder)
           return from_exp_streaming(content, skip_references: skip_references,
                                              include_source: include_source)
         end
 
+        use_native = Parser.native_available? if use_native.nil?
+
         begin
           # Use cached parser instance for performance (avoids ~7ms Parser.new overhead)
           ast = if use_native && Parser.native_available?
@@ -774,10 +813,11 @@ root_path: nil)
           raise Error::SchemaParseFailure.new("(from string)", e)
         end
 
-        repository = Builder.build_with_remarks(ast, source: content,
-                                                     include_source: include_source)
+        exp_file = ::Expressir::Express::Builder.build_with_remarks(ast,
+                                                                    source: content,
+                                                                    include_source: include_source)
 
-        repository.schemas.each do |schema|
+        exp_file.schemas.each do |schema|
           schema.file = nil
           schema.file_basename = nil
           schema.formatted = schema.to_s(no_remarks: true)
@@ -786,33 +826,33 @@ root_path: nil)
         unless skip_references
           Expressir::Benchmark.measure_references do
             resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-            resolve_references_model_visitor.visit(repository)
+            resolve_references_model_visitor.visit(exp_file)
           end
         end
 
-        repository
+        exp_file
       end
 
-      # Parse using streaming builder (maximum performance)
+      # Parse using streaming builder (construct-by-construct)
       # @param [String] content Express content as string
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
-      # @return [Model::Repository] Parsed repository
+      # @return [Model::ExpFile] Parsed ExpFile
       # @raise [SchemaParseFailure] if the content fails to parse
-      def self.from_exp_streaming(content, skip_references: nil,
+      def self.from_exp_streaming_builder(content, skip_references: nil,
 include_source: nil)
         grammar_json = Parser.cached_grammar_json
-        builder = StreamingBuilder.new(source: content,
-                                       include_source: include_source)
+        builder = ::Expressir::Express::StreamingBuilder.new(source: content,
+                                                             include_source: include_source)
 
         begin
-          repository = Parsanol::Native.parse_with_builder(grammar_json,
-                                                           content, builder)
+          exp_file = Parsanol::Native.parse_with_builder(grammar_json,
+                                                         content, builder)
         rescue StandardError => e
           raise Error::SchemaParseFailure.new("(streaming)", e)
         end
 
-        repository.schemas.each do |schema|
+        exp_file.schemas.each do |schema|
           schema.file = nil
           schema.file_basename = nil
           schema.formatted = schema.to_s(no_remarks: true)
@@ -821,12 +861,191 @@ include_source: nil)
         unless skip_references
           Expressir::Benchmark.measure_references do
             resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-            resolve_references_model_visitor.visit(repository)
+            resolve_references_model_visitor.visit(exp_file)
           end
         end
 
-        repository
+        exp_file
       end
+
+      # Parse each schema separately with fresh arena (memory-bounded)
+      #
+      # This splits the source into schema blocks and parses each independently.
+      # Memory is bounded by the largest schema, not the entire file.
+      #
+      # @param content [String] EXPRESS source code
+      # @param skip_references [Boolean] Whether to skip reference resolution
+      # @param include_source [Boolean] Whether to include source in model
+      # @return [Expressir::Model::ExpFile] Parsed EXPRESS file
+      def self.from_exp_streaming(content, skip_references: nil,
+include_source: nil)
+        grammar_json = Parser.cached_schema_grammar_json
+
+        # Extract schema blocks from source
+        schema_blocks = extract_schema_blocks(content)
+
+        # Parse each schema with fresh arena
+        schemas = schema_blocks.map do |block|
+          ast = Parsanol::Native.parse_fresh(grammar_json, block[:source])
+          schema_model = Builder.build(ast)
+          schema_model.source = block[:source]
+          schema_model
+        rescue StandardError => e
+          raise Error::SchemaParseFailure.new(
+            "(schema #{block[:name] || 'unknown'})", e
+          )
+        end
+
+        # Build the file model
+        exp_file = Expressir::Model::ExpFile.new
+        exp_file.schemas = schemas
+
+        exp_file.schemas.each do |schema|
+          schema.file = nil
+          schema.file_basename = nil
+          schema.formatted = schema.to_s(no_remarks: true)
+        end
+
+        unless skip_references
+          Expressir::Benchmark.measure_references do
+            resolve_references_model_visitor = ResolveReferencesModelVisitor.new
+            resolve_references_model_visitor.visit(exp_file)
+          end
+        end
+
+        exp_file
+      end
+
+      # Extract individual schema blocks from EXPRESS source
+      #
+      # This uses a state machine to properly handle nested comments and strings.
+      #
+      # @param source [String] EXPRESS source
+      # @return [Array<Hash>] Array of {name: String, source: String} for each schema
+      def self.extract_schema_blocks(source)
+        blocks = []
+        pos = 0
+        len = source.length
+
+        while pos < len
+          # Skip whitespace and find SCHEMA keyword
+          skip_ws_and_comments(source, pos)
+          pos = skip_ws_and_comments(source, pos)
+
+          break if pos >= len
+
+          # Check for SCHEMA
+          if source[pos..].start_with?("SCHEMA")
+            result = parse_schema_block(source, pos)
+            if result
+              blocks << result
+              pos = result[:end_pos]
+              next
+            end
+          end
+
+          pos += 1
+        end
+
+        blocks
+      end
+
+      private_class_method :extract_schema_blocks
+
+      def self.parse_schema_block(source, start_pos)
+        # Must be at SCHEMA keyword
+        return nil unless source[start_pos..].start_with?("SCHEMA")
+
+        pos = start_pos + "SCHEMA".length
+        pos = skip_spaces(source, pos)
+
+        # Extract schema name (identifier)
+        name_start = pos
+        while pos < source.length && (source[pos] =~ /[a-zA-Z0-9_]/ || source[pos] == "_")
+          pos += 1
+        end
+        schema_name = source[name_start...pos]
+
+        return nil if schema_name.empty?
+
+        # Skip to END_SCHEMA
+        depth = 1
+        search_pos = pos
+        end_pos = nil
+
+        while search_pos < source.length
+          if source[search_pos] == '"'
+            # String literal - skip to end
+            search_pos += 1
+            while search_pos < source.length && source[search_pos] != '"'
+              search_pos += 1
+            end
+            search_pos += 1
+          elsif source[search_pos] == "(" && source[search_pos + 1] == "*"
+            # Comment - skip to end
+            search_pos += 2
+            while search_pos < source.length && !(source[search_pos] == "*" && source[search_pos + 1] == ")")
+              search_pos += 1
+            end
+            search_pos += 2
+          elsif source[search_pos..].start_with?("END_SCHEMA")
+            depth -= 1
+            if depth.zero?
+              end_pos = search_pos + "END_SCHEMA".length
+              # Skip trailing whitespace and semicolon
+              pos = end_pos
+              pos = skip_spaces(source, pos)
+              pos += 1 if source[pos] == ";" # semicolon
+              break
+            end
+            search_pos += "END_SCHEMA".length
+          else
+            search_pos += 1
+          end
+        end
+
+        return nil unless end_pos
+
+        schema_source = source[start_pos...end_pos]
+        {
+          name: schema_name,
+          source: schema_source,
+          start_pos: start_pos,
+          end_pos: pos,
+        }
+      end
+      private_class_method :parse_schema_block
+
+      def self.skip_spaces(source, pos)
+        while pos < source.length && [" ", "\t", "\n",
+                                      "\r"].include?(source[pos])
+          pos += 1
+        end
+        pos
+      end
+      private_class_method :skip_spaces
+
+      def self.skip_ws_and_comments(source, pos)
+        len = source.length
+        while pos < len
+          c = source[pos]
+          if [" ", "\t", "\n", "\r"].include?(c)
+            pos += 1
+          elsif c == "(" && source[pos + 1] == "*"
+            # Comment - skip to end
+            pos += 2
+            while pos < len - 1 && !(source[pos] == "*" && source[pos + 1] == ")")
+              pos += 1
+            end
+            pos += 2
+          else
+            break
+          end
+        end
+        pos
+      end
+      private_class_method :skip_ws_and_comments
+
       private_class_method :from_exp_streaming
     end
   end

data/lib/expressir/express/pretty_formatter.rb

@@ -1,5 +1,3 @@
-require_relative "formatter"
-
 module Expressir
   module Express
     # Pretty formatter for EXPRESS schemas following ELF specification
@@ -100,22 +98,16 @@ module Expressir
         remark = node.untagged_remarks.first
         return "" if remark.nil?
 
-        # Handle both RemarkInfo and String
-        if remark.is_a?(Model::RemarkInfo)
-          text = remark.text
-          return "" if text.nil? || text.empty?
+        return "" unless remark.is_a?(Model::RemarkInfo)
 
-          # Include tag if present
-          formatted_text = remark.tagged? ? "\"#{remark.tag}\" #{text}" : text
+        text = remark.text
+        return "" if text.nil? || text.empty?
 
-          # Use format from RemarkInfo
-          remark.tail? ? " -- #{formatted_text}" : " (* #{formatted_text} *)"
-        else
-          # Legacy string - default to tail format
-          return "" if remark.empty?
+        # Include tag if present
+        formatted_text = remark.tagged? ? "\"#{remark.tag}\" #{text}" : text
 
-          " -- #{remark}"
-        end
+        # Use format from RemarkInfo
+        remark.tail? ? " -- #{formatted_text}" : " (* #{formatted_text} *)"
       end
 
       # Override to preserve tail remarks on attributes
@@ -254,33 +246,27 @@ module Expressir
       end
 
       # Format a single preamble remark
-      # @param remark [String, RemarkInfo] Remark text or RemarkInfo object
+      # @param remark [RemarkInfo] RemarkInfo object
       # @param indent_str [String] Indentation to use (optional)
       # @return [String] Formatted remark
       def format_preamble_remark(remark, _indent_str = "")
-        # Handle both String (legacy) and RemarkInfo
-        if remark.is_a?(Model::RemarkInfo)
-          text = remark.text
-
-          # Include tag if present
-          text = "\"#{remark.tag}\" #{text}" if remark.tagged?
-
-          # Use format from RemarkInfo
-          if remark.tail?
-            "-- #{text}"
-          elsif text.include?("\n")
-            # Embedded remark - always use embedded format for preamble
-            ["(*", text,
-             "*)"].join("\n")
-          else
-            "(* #{text} *)"
-          end
-        elsif remark.include?("\n")
-          # Legacy string handling
-          ["(*", remark,
+        return "" unless remark.is_a?(Model::RemarkInfo)
+
+        text = remark.text
+        return "" if text.nil? || text.empty?
+
+        # Include tag if present
+        text = "\"#{remark.tag}\" #{text}" if remark.tagged?
+
+        # Use format from RemarkInfo
+        if remark.tail?
+          "-- #{text}"
+        elsif text.include?("\n")
+          # Embedded remark - always use embedded format for preamble
+          ["(*", text,
            "*)"].join("\n")
         else
-          "(* #{remark} *)"
+          "(* #{text} *)"
         end
       end
 
@@ -307,18 +293,53 @@ module Expressir
       def format_repository(node)
         result = []
 
-        # Add preamble if source has remarks
-        # Use begin/rescue for duck typing (RBS-safe)
-        source_remarks = begin
-          node.source_remarks if node.is_a?(Model::Repository)
-        rescue NoMethodError
-          nil
+        # Format files if present
+        node.files&.each do |exp_file|
+          file_output = format_exp_file(exp_file)
+          result << file_output if file_output && !file_output.empty?
+        end
+
+        # Handle schemas directly added to repository (not via files)
+        direct_schemas = node.schemas.select do |s|
+          # Schema is direct if it's not in any file
+          node.files.nil? || node.files.none? { |f| f.schemas&.include?(s) }
         end
-        if source_remarks
-          result.concat(format_preamble(source_remarks))
+
+        if direct_schemas.any?
+          # Add preamble if repository has untagged remarks
+          if node.untagged_remarks && !node.untagged_remarks.empty?
+            result.concat(format_preamble(node.untagged_remarks))
+          end
+
+          # Add provenance
+          provenance = format_provenance
+          result << provenance if provenance
+          result << "" if provenance
+
+          # Add schemas
+          schemas_output = direct_schemas.map { |x| format(x) }.join("\n\n")
+          result << schemas_output if schemas_output
+        elsif result.empty?
+          # Empty repository - just add provenance
+          provenance = format_provenance
+          result << provenance if provenance
+        end
+
+        result.any? ? "#{result.join("\n")}\n" : ""
+      end
+
+      # Format ExpFile with file-level preamble
+      # @param node [Model::ExpFile] ExpFile node
+      # @return [String] Formatted file
+      def format_exp_file(node)
+        result = []
+
+        # Add file-level preamble if present
+        if node.untagged_remarks && !node.untagged_remarks.empty?
+          result.concat(format_preamble(node.untagged_remarks))
         end
 
-        # Add provenance
+        # Add provenance (only for first file or single file)
         provenance = format_provenance
         result << provenance if provenance
         result << "" if provenance
@@ -327,7 +348,7 @@ module Expressir
         schemas = node.schemas&.map { |x| format(x) }&.join("\n\n")
         result << schemas if schemas
 
-        "#{result.join("\n")}\n"
+        result.empty? ? "" : "#{result.join("\n")}\n"
       end
 
       # Format a block of constants with aligned colons and assignments