expressir 2.2.0 → 2.3.0

data/lib/expressir/express/parser.rb

@@ -1,9 +1,4 @@
 require "parsanol"
-require_relative "error"
-require_relative "builder"
-require_relative "builders"
-require_relative "remark_attacher"
-require_relative "streaming_builder"
 
 module Expressir
   module Express
@@ -64,14 +59,22 @@ module Expressir
           @@cached_grammar_json
         end
 
-        # Parse using native engine (with caching)
+        # Parse using native engine with Rust-side transformation (fastest)
+        #
+        # This method provides ~17x speedup over pure Ruby parsing.
+        # The transformation happens in Rust using to_parslet_compatible,
+        # producing Parslet-compatible output that Builder.build can consume directly.
+        #
+        # @param source [String] EXPRESS source code to parse
+        # @return [Hash, Array] Transformed AST in Parslet-compatible format
+        # @raise [LoadError] If native parser is not available
         def self.parse_native(source)
           unless native_available?
             raise LoadError, "Native parser not available"
           end
 
-          # Use Parsanol 2.0 API - parse returns Slice objects with position info
-          new.parse(source, mode: :native)
+          grammar_atom = cached_parser.syntax
+          Parsanol::Native.parse(grammar_atom, source)
         end
 
         def cts(atom)
@@ -218,7 +221,7 @@ module Expressir
           (attributeDecl >> op_colon >> parameterType >> op_decl >> expression >> op_delim).as(:derivedAttr)
         end
         rule(:digit) { match["0-9"] }
-        rule(:digits) { (digit >> digit.repeat) }
+        rule(:digits) { digit >> digit.repeat }
         rule(:domainRule) do
           ((ruleLabelId >> op_colon).maybe >> expression).as(:domainRule)
         end
@@ -228,7 +231,7 @@ module Expressir
         rule(:embeddedRemark) do
           (str("(*") >> (str("*)").absent? >> (embeddedRemark | any)).repeat >> str("*)")).as(:embeddedRemark)
         end
-        rule(:encodedCharacter) { (octet >> octet >> octet >> octet) }
+        rule(:encodedCharacter) { octet >> octet >> octet >> octet }
         rule(:encodedStringLiteral) do
           cts((str('"') >> encodedCharacter.repeat(1) >> str('"')).as(:str)).as(:encodedStringLiteral)
         end
@@ -667,7 +670,7 @@ module Expressir
       # @param [String] file Express file path
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
-      # @return [Model::Repository]
+      # @return [Model::ExpFile] ExpFile containing parsed schemas
       # @raise [SchemaParseFailure] if the schema file fails to parse
       def self.from_file(file, skip_references: nil, include_source: nil, root_path: nil) # rubocop:disable Metrics/AbcSize
         Expressir::Benchmark.measure_file(file) do
@@ -683,10 +686,12 @@ module Expressir
             raise Error::SchemaParseFailure.new(schema_file, e)
           end
 
-          @repository = Builder.build_with_remarks(ast, source: source,
+          @exp_file = ::Expressir::Express::Builder.build_with_remarks(ast, source: source,
                                                         include_source: include_source)
 
-          @repository.schemas.each do |schema|
+          # Set file path on the ExpFile and propagate to schemas
+          @exp_file.path = schema_file
+          @exp_file.schemas.each do |schema|
             schema.file = schema_file
             schema.file_basename = File.basename(schema_file, ".exp")
             schema.formatted = schema.to_s(no_remarks: true)
@@ -695,11 +700,11 @@ module Expressir
           unless skip_references
             Expressir::Benchmark.measure_references do
               @resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-              @resolve_references_model_visitor.visit(@repository)
+              @resolve_references_model_visitor.visit(@exp_file)
             end
           end
 
-          @repository
+          @exp_file
         end
       end
 
@@ -711,19 +716,16 @@ module Expressir
       # @yieldparam filename [String] Name of the file being processed
       # @yieldparam schemas [Array, nil] Array of parsed schemas (nil if parsing failed)
       # @yieldparam error [Exception, nil] Error that occurred (nil if parsing succeeded)
-      # @return [Model::Repository]
-      def self.from_files(files, skip_references: nil, include_source: nil,
-root_path: nil)
-        all_schemas = []
+      # @return [Model::Repository] Repository containing all parsed ExpFiles
+      def self.from_files(files, skip_references: nil, include_source: nil, root_path: nil)
+        all_exp_files = []
 
         files.each do |file|
-          repository = from_file(file, skip_references: true,
-                                       root_path: root_path)
-          file_schemas = repository.schemas
-          all_schemas.concat(file_schemas)
+          exp_file = from_file(file, skip_references: true, root_path: root_path)
+          all_exp_files << exp_file
 
           # Call the progress block if provided
-          yield(file, file_schemas, nil) if block_given?
+          yield(file, exp_file&.schemas, nil) if block_given?
         rescue StandardError => e
           # Call the progress block with the error if provided
           yield(file, nil, e) if block_given?
@@ -733,9 +735,7 @@ root_path: nil)
           raise unless e.is_a?(Error::SchemaParseFailure)
         end
 
-        @repository = Model::Repository.new(
-          schemas: all_schemas,
-        )
+        @repository = Model::Repository.new(files: all_exp_files)
 
         unless skip_references
           Expressir::Benchmark.measure_references do
@@ -753,7 +753,7 @@ root_path: nil)
       # @param [Boolean] include_source attach original source code to model elements
       # @param [Boolean] use_native use native parser if available (default: false - AST format differs slightly)
       # @param [Boolean] use_streaming use streaming builder for maximum performance (default: false)
-      # @return [Model::Repository] Parsed repository
+      # @return [Model::ExpFile] Parsed ExpFile
       # @raise [SchemaParseFailure] if the content fails to parse
       def self.from_exp(content, skip_references: nil, include_source: nil,
                          use_native: false, use_streaming: false)
@@ -774,10 +774,10 @@ root_path: nil)
           raise Error::SchemaParseFailure.new("(from string)", e)
         end
 
-        repository = Builder.build_with_remarks(ast, source: content,
-                                                     include_source: include_source)
+        exp_file = ::Expressir::Express::Builder.build_with_remarks(ast, source: content,
+                                                   include_source: include_source)
 
-        repository.schemas.each do |schema|
+        exp_file.schemas.each do |schema|
           schema.file = nil
           schema.file_basename = nil
           schema.formatted = schema.to_s(no_remarks: true)
@@ -786,33 +786,32 @@ root_path: nil)
         unless skip_references
           Expressir::Benchmark.measure_references do
             resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-            resolve_references_model_visitor.visit(repository)
+            resolve_references_model_visitor.visit(exp_file)
           end
         end
 
-        repository
+        exp_file
       end
 
       # Parse using streaming builder (maximum performance)
       # @param [String] content Express content as string
       # @param [Boolean] skip_references skip resolving references
       # @param [Boolean] include_source attach original source code to model elements
-      # @return [Model::Repository] Parsed repository
+      # @return [Model::ExpFile] Parsed ExpFile
       # @raise [SchemaParseFailure] if the content fails to parse
-      def self.from_exp_streaming(content, skip_references: nil,
-include_source: nil)
+      def self.from_exp_streaming(content, skip_references: nil, include_source: nil)
         grammar_json = Parser.cached_grammar_json
-        builder = StreamingBuilder.new(source: content,
+        builder = ::Expressir::Express::StreamingBuilder.new(source: content,
                                        include_source: include_source)
 
         begin
-          repository = Parsanol::Native.parse_with_builder(grammar_json,
+          exp_file = Parsanol::Native.parse_with_builder(grammar_json,
                                                            content, builder)
         rescue StandardError => e
           raise Error::SchemaParseFailure.new("(streaming)", e)
         end
 
-        repository.schemas.each do |schema|
+        exp_file.schemas.each do |schema|
           schema.file = nil
           schema.file_basename = nil
           schema.formatted = schema.to_s(no_remarks: true)
@@ -821,11 +820,11 @@ include_source: nil)
         unless skip_references
           Expressir::Benchmark.measure_references do
             resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-            resolve_references_model_visitor.visit(repository)
+            resolve_references_model_visitor.visit(exp_file)
           end
         end
 
-        repository
+        exp_file
       end
       private_class_method :from_exp_streaming
     end

data/lib/expressir/express/pretty_formatter.rb

@@ -1,5 +1,3 @@
-require_relative "formatter"
-
 module Expressir
   module Express
     # Pretty formatter for EXPRESS schemas following ELF specification
@@ -100,22 +98,16 @@ module Expressir
         remark = node.untagged_remarks.first
         return "" if remark.nil?
 
-        # Handle both RemarkInfo and String
-        if remark.is_a?(Model::RemarkInfo)
-          text = remark.text
-          return "" if text.nil? || text.empty?
+        return "" unless remark.is_a?(Model::RemarkInfo)
 
-          # Include tag if present
-          formatted_text = remark.tagged? ? "\"#{remark.tag}\" #{text}" : text
+        text = remark.text
+        return "" if text.nil? || text.empty?
 
-          # Use format from RemarkInfo
-          remark.tail? ? " -- #{formatted_text}" : " (* #{formatted_text} *)"
-        else
-          # Legacy string - default to tail format
-          return "" if remark.empty?
+        # Include tag if present
+        formatted_text = remark.tagged? ? "\"#{remark.tag}\" #{text}" : text
 
-          " -- #{remark}"
-        end
+        # Use format from RemarkInfo
+        remark.tail? ? " -- #{formatted_text}" : " (* #{formatted_text} *)"
       end
 
       # Override to preserve tail remarks on attributes
@@ -254,33 +246,27 @@ module Expressir
       end
 
       # Format a single preamble remark
-      # @param remark [String, RemarkInfo] Remark text or RemarkInfo object
+      # @param remark [RemarkInfo] RemarkInfo object
       # @param indent_str [String] Indentation to use (optional)
       # @return [String] Formatted remark
       def format_preamble_remark(remark, _indent_str = "")
-        # Handle both String (legacy) and RemarkInfo
-        if remark.is_a?(Model::RemarkInfo)
-          text = remark.text
-
-          # Include tag if present
-          text = "\"#{remark.tag}\" #{text}" if remark.tagged?
-
-          # Use format from RemarkInfo
-          if remark.tail?
-            "-- #{text}"
-          elsif text.include?("\n")
-            # Embedded remark - always use embedded format for preamble
-            ["(*", text,
-             "*)"].join("\n")
-          else
-            "(* #{text} *)"
-          end
-        elsif remark.include?("\n")
-          # Legacy string handling
-          ["(*", remark,
+        return "" unless remark.is_a?(Model::RemarkInfo)
+
+        text = remark.text
+        return "" if text.nil? || text.empty?
+
+        # Include tag if present
+        text = "\"#{remark.tag}\" #{text}" if remark.tagged?
+
+        # Use format from RemarkInfo
+        if remark.tail?
+          "-- #{text}"
+        elsif text.include?("\n")
+          # Embedded remark - always use embedded format for preamble
+          ["(*", text,
            "*)"].join("\n")
         else
-          "(* #{remark} *)"
+          "(* #{text} *)"
         end
       end
 
@@ -307,18 +293,53 @@ module Expressir
       def format_repository(node)
         result = []
 
-        # Add preamble if source has remarks
-        # Use begin/rescue for duck typing (RBS-safe)
-        source_remarks = begin
-          node.source_remarks if node.is_a?(Model::Repository)
-        rescue NoMethodError
-          nil
+        # Format files if present
+        node.files&.each do |exp_file|
+          file_output = format_exp_file(exp_file)
+          result << file_output if file_output && !file_output.empty?
+        end
+
+        # Handle schemas directly added to repository (not via files)
+        direct_schemas = node.schemas.select do |s|
+          # Schema is direct if it's not in any file
+          node.files.nil? || node.files.none? { |f| f.schemas&.include?(s) }
         end
-        if source_remarks
-          result.concat(format_preamble(source_remarks))
+
+        if direct_schemas.any?
+          # Add preamble if repository has untagged remarks
+          if node.untagged_remarks && !node.untagged_remarks.empty?
+            result.concat(format_preamble(node.untagged_remarks))
+          end
+
+          # Add provenance
+          provenance = format_provenance
+          result << provenance if provenance
+          result << "" if provenance
+
+          # Add schemas
+          schemas_output = direct_schemas.map { |x| format(x) }.join("\n\n")
+          result << schemas_output if schemas_output
+        elsif result.empty?
+          # Empty repository - just add provenance
+          provenance = format_provenance
+          result << provenance if provenance
+        end
+
+        result.any? ? "#{result.join("\n")}\n" : ""
+      end
+
+      # Format ExpFile with file-level preamble
+      # @param node [Model::ExpFile] ExpFile node
+      # @return [String] Formatted file
+      def format_exp_file(node)
+        result = []
+
+        # Add file-level preamble if present
+        if node.untagged_remarks && !node.untagged_remarks.empty?
+          result.concat(format_preamble(node.untagged_remarks))
         end
 
-        # Add provenance
+        # Add provenance (only for first file or single file)
         provenance = format_provenance
         result << provenance if provenance
         result << "" if provenance
@@ -327,7 +348,7 @@ module Expressir
         schemas = node.schemas&.map { |x| format(x) }&.join("\n\n")
         result << schemas if schemas
 
-        "#{result.join("\n")}\n"
+        result.empty? ? "" : "#{result.join("\n")}\n"
       end
 
       # Format a block of constants with aligned colons and assignments