expressir 2.4.1 → 2.4.2

data/lib/expressir/express/ast_key_converter.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Converts native-AST CamelCase keys to the snake_case keys the
+    # builders consume.
+    #
+    # build() descends into child nodes whose subtrees the parent already
+    # converted, so unchanged containers are marked with an invisible
+    # instance variable and skipped on re-visits — without it, every
+    # subtree is re-scanned once per ancestor level (~60 convert calls per
+    # model node on real schemas). The marker is invisible to equality,
+    # hashing, and inspection.
+    class AstKeyConverter
+      SNAKED_MARKER = :@_expressir_keys_snaked
+      UPPERCASE_PATTERN = /[A-Z]/
+
+      class << self
+        # Thread-local snake_case conversion cache. Thread-local avoids the
+        # mutable-constant anti-pattern while remaining thread-safe. The
+        # cache is bounded by the number of unique AST node-type names.
+        def snake_case(name)
+          cache[name] ||= begin
+            str = name.to_s
+            if /^[a-z_]+$/.match?(str)
+              str.to_sym
+            else
+              str
+                .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+                .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+                .downcase
+                .to_sym
+            end
+          end
+        end
+
+        # Returns the original object when no conversion is needed; the
+        # common no-conversion case allocates nothing and the conversion
+        # case walks the keys once.
+        def convert(obj)
+          case obj
+          when Hash
+            return obj if obj.empty?
+            return obj if obj.instance_variable_defined?(SNAKED_MARKER)
+
+            keys = obj.keys
+            converted_values = nil
+            new_keys = nil
+
+            keys.each_with_index do |k, i|
+              val = obj[k]
+              case val
+              when Hash, Array
+                unless val.empty?
+                  converted_val = convert(val)
+                  (converted_values ||= {})[k] = converted_val unless converted_val.equal?(val)
+                end
+              end
+
+              if k.match?(UPPERCASE_PATTERN)
+                (new_keys ||= keys.dup)[i] = snake_case(k)
+              end
+            end
+
+            return mark_snaked(obj) unless new_keys || converted_values
+
+            result = {}
+            keys.each_with_index do |k, i|
+              key = new_keys&.[](i) || k
+              result[key] = converted_values&.key?(k) ? converted_values[k] : obj[k]
+            end
+            mark_snaked(result)
+          when Array
+            return obj if obj.empty?
+            return obj if obj.instance_variable_defined?(SNAKED_MARKER)
+
+            needs_conversion = false
+            result = []
+
+            obj.each do |item|
+              case item
+              when Hash, Array
+                next if item.empty?
+
+                converted = convert(item)
+                result << converted
+                needs_conversion = true unless converted.equal?(item)
+              else
+                result << item
+              end
+            end
+
+            needs_conversion ? mark_snaked(result) : mark_snaked(obj)
+          else
+            obj
+          end
+        end
+
+        private
+
+        def cache
+          Thread.current[:expressir_snake_case_cache] ||= {}
+        end
+
+        def mark_snaked(obj)
+          obj.instance_variable_set(SNAKED_MARKER, true)
+          obj
+        rescue FrozenError
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/expressir/express/builder.rb

@@ -28,14 +28,6 @@ module Expressir
           current_context&.include_source
         end
 
-        # Thread-local snake_case conversion cache. Thread-local avoids the
-        # mutable-constant anti-pattern while remaining thread-safe.
-        # Each thread gets its own cache; the cache grows with the number of
-        # unique AST node-type names encountered (bounded by grammar size).
-        def snake_case_cache
-          Thread.current[:expressir_snake_case_cache] ||= {}
-        end
-
         # Register a builder for a node type.
         # @param node_type [Symbol] The AST node type
         # @param builder [#call] Optional callable that takes (ast_data)
@@ -57,8 +49,8 @@ module Expressir
             node_type = ast.keys.first
             node_data = ast[node_type]
 
-            handler_key = cached_snake_case(node_type)
-            snake_data = fast_convert_keys(node_data)
+            handler_key = AstKeyConverter.snake_case(node_type)
+            snake_data = AstKeyConverter.convert(node_data)
 
             builder = @register[handler_key]
             if builder
@@ -80,12 +72,12 @@ module Expressir
               ast.each_key do |key|
                 next if key == node_type
 
-                h_key = cached_snake_case(key)
+                h_key = AstKeyConverter.snake_case(key)
                 h_builder = @register[h_key]
                 next unless h_builder
 
                 n_data = ast[key]
-                s_data = fast_convert_keys(n_data)
+                s_data = AstKeyConverter.convert(n_data)
                 result = h_builder.call(s_data)
 
                 unless result.nil?
@@ -219,113 +211,10 @@ module Expressir
           builder.call(data)
         end
 
-        private
-
-        # Cached snake_case conversion
-        def cached_snake_case(name)
-          snake_case_cache[name] ||= begin
-            str = name.to_s
-            # Check if already snake_case
-            if /^[a-z_]+$/.match?(str)
-              str.to_sym
-            else
-              str
-                .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
-                .gsub(/([a-z\d])([A-Z])/, '\1_\2')
-                .downcase
-                .to_sym
-            end
-          end
-        end
-
-        # Optimized key conversion - returns original object when no conversion needed
-        # This avoids unnecessary allocations for AST nodes that don't need key conversion
-        def fast_convert_keys(obj)
-          case obj
-          when Hash
-            return obj if obj.empty?
-
-            # First pass: check if any conversion is needed
-            keys = obj.keys
-            needs_conversion = false
-            converted_values = nil
-
-            keys.each do |k|
-              key_str = k.to_s
-              # Check if key needs conversion (has uppercase)
-              if key_str.match?(/[A-Z]/)
-                needs_conversion = true
-              end
-
-              # Check if value needs conversion
-              val = obj[k]
-              case val
-              when Hash
-                next if val.empty?
-
-                converted_val = fast_convert_keys(val)
-                if !converted_val.equal?(val) # Identity check - same object?
-                  needs_conversion = true
-                  converted_values ||= {}
-                  converted_values[k] = converted_val
-                end
-              when Array
-                next if val.empty?
-
-                converted_val = fast_convert_keys(val)
-                if !converted_val.equal?(val)
-                  needs_conversion = true
-                  converted_values ||= {}
-                  converted_values[k] = converted_val
-                end
-              end
-            end
-
-            # Return original if no conversion needed (zero allocation!)
-            return obj unless needs_conversion
-
-            # Build result only when necessary
-            result = {}
-            keys.each do |k|
-              key_str = k.to_s
-              new_key = key_str.match?(/[A-Z]/) ? cached_snake_case(k) : k
-              new_val = converted_values&.key?(k) ? converted_values[k] : obj[k]
-              result[new_key] = new_val
-            end
-            result
-          when Array
-            return obj if obj.empty?
-
-            # Check if any element needs conversion
-            needs_conversion = false
-            result = []
-
-            obj.each do |item|
-              case item
-              when Hash
-                next if item.empty?
-
-                converted = fast_convert_keys(item)
-                result << converted
-                needs_conversion = true unless converted.equal?(item)
-              when Array
-                next if item.empty?
-
-                converted = fast_convert_keys(item)
-                result << converted
-                needs_conversion = true unless converted.equal?(item)
-              else
-                result << item
-              end
-            end
-
-            # Return original if no conversion needed
-            needs_conversion ? result : obj
-          else
-            obj
-          end
-        end
-
+        # Keys containing uppercase need snake-casing; testing the key
+        # directly avoids allocating `to_s` strings per key per pass.
+        # Key conversion lives in AstKeyConverter (MECE: converting AST
+        # keys is not building models).
         def extract_source_info(data)
           return nil unless data
 

data/lib/expressir/express/error.rb

@@ -45,6 +45,23 @@ module Expressir
         end
       end
 
+      # Error raised when parallel file parsing loses a worker or its data
+      class ParallelParseError < ExpressError
+        def initialize(message = "Parallel parsing failed")
+          super
+        end
+      end
+
+      # Error raised when streaming parsing is requested but unavailable.
+      # The streaming paths require a parsanol release whose
+      # parse_with_builder is stable; parse_fresh cannot parse EXPRESS
+      # without packrat memoization (see parsanol-ruby#52).
+      class StreamingUnsupportedError < ExpressError
+        def initialize(message = "Streaming parsing is not supported by the installed parsanol release")
+          super
+        end
+      end
+
       # Base class for visitor-related errors
       class VisitorError < ExpressError; end
 

data/lib/expressir/express/parallel_files.rb

@@ -0,0 +1,229 @@
+require "etc"
+
+module Expressir
+  module Express
+    # Fork-based worker pool for parsing many EXPRESS files in parallel.
+    # The native parser holds the GVL for the whole parse, so process-level
+    # parallelism is the only way to use multiple cores. Files are
+    # independent until reference resolution, which stays in the parent.
+    #
+    # Unlike sequential parsing, the progress block fires in file order
+    # only after all files have been parsed.
+    class ParallelFiles
+      DEFAULT_MAX_PROCESSES = 4
+      FRAME_HEADER_BYTES = 4
+
+      FORK_SUPPORTED = Process.respond_to?(:fork).freeze
+
+      # Forking is never the default: a library must not spawn processes on
+      # behalf of its host (forked children inherit broken thread and lock
+      # state, and fork does not exist on all Rubies). Parallelism requires
+      # an explicit max_processes > 1 from the caller and a platform that
+      # supports fork (e.g. not Windows); otherwise the request degrades
+      # to sequential parsing.
+      def self.sequential?(files, max_processes)
+        !FORK_SUPPORTED || max_processes.nil? || max_processes <= 1 ||
+          files.size < 3
+      end
+
+      # @param files [Array<String>] EXPRESS file paths
+      # @param max_processes [Integer, nil] worker cap; nil auto-selects
+      # @param parse [Proc] callback taking a file path, returning an ExpFile
+      # @param strict [Boolean] re-raise every error, including
+      #   Error::SchemaParseFailure, instead of skipping the file
+      # @yield [file, exp_file, error] called in original file order
+      # @return [Array<Expressir::Model::ExpFile, nil>] parsed files in order;
+      #   nil marks a file that failed with Error::SchemaParseFailure
+      def self.run(files, parse:, max_processes: nil, strict: false, &block)
+        new(files, max_processes, parse, block, strict).run
+      end
+
+      def initialize(files, max_processes, parse, block, strict)
+        @files = files
+        @parse = parse
+        @block = block
+        @strict = strict
+        @worker_count = [
+          files.size - 1,
+          max_processes || [Etc.nprocessors, DEFAULT_MAX_PROCESSES].min,
+        ].min
+      end
+
+      def run
+        job_pipes = Array.new(@worker_count) { IO.pipe }
+        result_pipes = Array.new(@worker_count) { IO.pipe }
+        pids = spawn_workers(job_pipes, result_pipes)
+
+        files_results = schedule_jobs(job_pipes, result_pipes)
+
+        ordered_pass(files_results)
+      ensure
+        cleanup(job_pipes, result_pipes, pids)
+      end
+
+      private
+
+      def spawn_workers(job_pipes, result_pipes)
+        job_pipes.each_index.map do |i|
+          job_r, job_w = job_pipes[i]
+          result_r, result_w = result_pipes[i]
+          fork do
+            job_w.close
+            result_r.close
+            other_pipes = (job_pipes + result_pipes).flatten -
+              [job_r, result_w]
+            other_pipes.each { |io| io.close unless io.closed? }
+            worker_loop(job_r, result_w)
+          end
+        end
+      end
+
+      # Assigns one job at a time to whichever worker reports a result, so
+      # each file is parsed exactly once and each worker holds at most one
+      # in-flight job (keeping result pipes free of interleaved frames).
+      def schedule_jobs(job_pipes, result_pipes)
+        writers = job_pipes.map(&:last)
+        readers = result_pipes.map(&:first)
+        files_results = Array.new(@files.size)
+        next_job = 0
+        busy = {}
+
+        writers.each_index do |wi|
+          break if next_job == @files.size
+
+          dispatch(writers[wi], next_job)
+          busy[wi] = true
+          next_job += 1
+        end
+
+        until busy.empty?
+          ready, = IO.select(readers.values_at(*busy.keys))
+          ready.each do |io|
+            wi = readers.index(io)
+            payload = Marshal.load(read_frame(io)) # rubocop:disable Security/MarshalLoad
+            files_results[payload[:index]] = payload
+
+            if next_job < @files.size
+              dispatch(writers[wi], next_job)
+              next_job += 1
+            else
+              writers[wi].close unless writers[wi].closed?
+              busy.delete(wi)
+            end
+          end
+        end
+
+        files_results
+      end
+
+      def dispatch(writer, job_index)
+        write_frame(writer, pack_frame(Marshal.dump([job_index, @files[job_index]])))
+      end
+
+      def worker_loop(job_r, result_w)
+        until job_r.eof?
+          data = read_frame(job_r)
+          index, file = Marshal.load(data) # rubocop:disable Security/MarshalLoad
+          begin
+            payload = { index: index, exp_file: @parse.call(file) }
+          rescue StandardError => e
+            payload = { index: index, error: transferable_error(e) }
+          end
+          write_frame(result_w, pack_frame(Marshal.dump(payload)))
+        end
+      rescue Errno::EPIPE
+        # parent went away; nothing to report to
+        exit!(0)
+      end
+
+      # SIGTERM cannot interrupt a worker blocked in the native parser (the
+      # GVL is held), so escalate to SIGKILL after a grace period instead of
+      # blocking in waitpid forever.
+      def cleanup(job_pipes, result_pipes, pids)
+        (job_pipes.to_a + result_pipes.to_a).each do |r, w|
+          r.close unless r.closed?
+          w.close unless w.closed?
+        end
+        pids.to_a.each { |pid| Process.kill("TERM", pid) if alive?(pid) }
+
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 2
+        pids.to_a.each do |pid|
+          while alive?(pid) && Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
+            Process.waitpid(pid, Process::WNOHANG)
+            sleep 0.05
+          end
+          Process.kill("KILL", pid) if alive?(pid)
+          reap(pid)
+        end
+      end
+
+      def ordered_pass(files_results)
+        files_results.each_with_index do |payload, index|
+          error = payload[:error]
+          @block&.call(@files[index], payload[:exp_file], error)
+          raise error if error && (@strict ||
+                                  !error.is_a?(Error::SchemaParseFailure))
+        end
+
+        files_results.map do |payload|
+          payload[:error] ? nil : payload[:exp_file]
+        end
+      end
+
+      # Errors carrying native-parser state (e.g. Parsanol::ParseFailed with
+      # its cause tree) cannot cross a fork boundary; rebuild them without
+      # the untransferable internals, preserving class and message.
+      def transferable_error(error)
+        Marshal.dump(error)
+        error
+      rescue StandardError
+        if error.is_a?(Error::SchemaParseFailure)
+          Error::SchemaParseFailure.new(error.filename,
+                                        StandardError.new(error.message))
+        else
+          StandardError.new(error.message)
+        end
+      end
+
+      def pack_frame(data)
+        [data.bytesize].pack("N") + data
+      end
+
+      def write_frame(io, frame)
+        io.write(frame)
+      end
+
+      def read_frame(io)
+        header = read_exactly(io, FRAME_HEADER_BYTES)
+        read_exactly(io, header.unpack1("N"))
+      end
+
+      def read_exactly(io, count)
+        data = +""
+        while data.bytesize < count
+          chunk = io.read(count - data.bytesize)
+          unless chunk
+            raise Error::ParallelParseError,
+                  "worker exited before sending its result"
+          end
+
+          data << chunk
+        end
+        data
+      end
+
+      def alive?(pid)
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH, Errno::EPERM
+        false
+      end
+
+      def reap(pid)
+        Process.waitpid(pid)
+      rescue Errno::ECHILD, Errno::EINVAL
+        nil
+      end
+    end
+  end
+end

data/lib/expressir/express/parser.rb

@@ -100,7 +100,32 @@ module Expressir
       # @yield [filename, schemas, error] Optional block called for each file
       # @return [Model::Repository] Repository containing all parsed ExpFiles
       def self.from_files(files, skip_references: nil, include_source: nil,
-root_path: nil, use_native: nil)
+root_path: nil, use_native: nil, max_processes: nil, &progress)
+        all_exp_files = if ParallelFiles.sequential?(files, max_processes)
+                          parse_files_sequentially(
+                            files, skip_references: skip_references, include_source: include_source,
+                                   root_path: root_path, use_native: use_native
+                          ) do |file, exp_file, error|
+                            progress&.call(file, exp_file&.schemas, error)
+                          end
+                        else
+                          ParallelFiles.run(
+                            files,
+                            max_processes: max_processes,
+                            parse: lambda do |file|
+                              from_file(file, skip_references: true, root_path: root_path,
+                                              use_native: use_native)
+                            end,
+                          ) do |file, exp_file, error|
+                            progress&.call(file, exp_file&.schemas, error)
+                          end
+                        end
+
+        build_repository(all_exp_files, skip_references: skip_references)
+      end
+
+      def self.parse_files_sequentially(files, skip_references: nil,
+include_source: nil, root_path: nil, use_native: nil, &block)
         all_exp_files = []
 
         files.each do |file|
@@ -108,12 +133,19 @@ root_path: nil, use_native: nil)
                                      root_path: root_path, use_native: use_native)
           all_exp_files << exp_file
 
-          yield(file, exp_file&.schemas, nil) if block_given?
+          yield(file, exp_file, nil) if block
         rescue StandardError => e
-          yield(file, nil, e) if block_given?
+          # Nil-pad so results align with files by index, exactly like the
+          # parallel path does.
+          all_exp_files << nil if e.is_a?(Error::SchemaParseFailure)
+          yield(file, nil, e) if block
           raise unless e.is_a?(Error::SchemaParseFailure)
         end
 
+        all_exp_files
+      end
+
+      def self.build_repository(all_exp_files, skip_references: nil)
         repository = Model::Repository.new(files: all_exp_files)
 
         unless skip_references
@@ -130,15 +162,18 @@ root_path: nil, use_native: nil)
       # @param skip_references [Boolean] skip resolving references
       # @param include_source [Boolean] attach original source code to model elements
       # @param use_native [Boolean] use native parser (default: true when available)
-      # @param use_streaming [Boolean] use streaming builder for maximum performance
+      # @param use_streaming [Boolean] unsupported on current parsanol;
+      #   passing true raises {Error::StreamingUnsupportedError}. The
+      #   streaming paths return when parsanol exposes a stable
+      #   parse_with_builder (see parsanol-ruby#52 and the TODO.max-perf/02
+      #   notes).
       # @return [Model::ExpFile] Parsed ExpFile
       # @raise [Error::SchemaParseFailure] if the content fails to parse
       def self.from_exp(content, skip_references: nil, include_source: nil,
                          use_native: nil, use_streaming: false)
         content = strip_bom(content)
-        if use_streaming && Grammar::Parser.native_available? && defined?(Parsanol::Native.parse_with_builder)
-          return from_exp_streaming(content, skip_references: skip_references,
-                                             include_source: include_source)
+        if use_streaming
+          raise Error::StreamingUnsupportedError
         end
 
         use_native = Grammar::Parser.native_available? if use_native.nil?
@@ -173,83 +208,6 @@ root_path: nil, use_native: nil)
         exp_file
       end
 
-      # Parse using streaming builder (construct-by-construct).
-      # @param content [String] EXPRESS source code
-      # @param skip_references [Boolean] skip resolving references
-      # @param include_source [Boolean] attach original source code to model elements
-      # @return [Model::ExpFile] Parsed ExpFile
-      # @raise [Error::SchemaParseFailure] if the content fails to parse
-      def self.from_exp_streaming_builder(content, skip_references: nil,
-include_source: nil)
-        grammar_json = Grammar::Parser.cached_grammar_json
-        builder = ::Expressir::Express::StreamingBuilder.new(source: content,
-                                                             include_source: include_source)
-
-        begin
-          exp_file = Parsanol::Native.parse_with_builder(grammar_json,
-                                                         content, builder)
-        rescue StandardError => e
-          raise Error::SchemaParseFailure.new("(streaming)", e)
-        end
-
-        exp_file.schemas.each do |schema|
-          schema.file = nil
-          schema.file_basename = nil
-        end
-
-        unless skip_references
-          Expressir::Benchmark.measure_references do
-            ResolveReferencesModelVisitor.new.visit(exp_file)
-          end
-        end
-
-        exp_file
-      end
-
-      # Parse each schema separately with fresh arena (memory-bounded).
-      #
-      # Splits source into schema blocks via {SchemaBlockScanner} and parses
-      # each independently. Memory is bounded by the largest schema, not the
-      # entire file.
-      #
-      # @param content [String] EXPRESS source code
-      # @param skip_references [Boolean] skip resolving references
-      # @param include_source [Boolean] attach original source code to model elements
-      # @return [Model::ExpFile] Parsed ExpFile
-      def self.from_exp_streaming(content, skip_references: nil,
-include_source: nil)
-        grammar_json = Grammar::Parser.cached_schema_grammar_json
-
-        schema_blocks = SchemaBlockScanner.extract_schema_blocks(content)
-
-        schemas = schema_blocks.map do |block|
-          ast = Parsanol::Native.parse_fresh(grammar_json, block[:source])
-          schema_model = Builder.build(ast)
-          schema_model.source = block[:source]
-          schema_model
-        rescue StandardError => e
-          raise Error::SchemaParseFailure.new(
-            "(schema #{block[:name] || 'unknown'})", e
-          )
-        end
-
-        exp_file = Expressir::Model::ExpFile.new
-        exp_file.schemas = schemas
-
-        exp_file.schemas.each do |schema|
-          schema.file = nil
-          schema.file_basename = nil
-        end
-
-        unless skip_references
-          Expressir::Benchmark.measure_references do
-            ResolveReferencesModelVisitor.new.visit(exp_file)
-          end
-        end
-
-        exp_file
-      end
-
       # Transfer file-level untagged remarks that appear before the first
       # SCHEMA keyword to the first schema's +header+ attribute so they are
       # accessible via +schema.header+ and through Liquid drops.
@@ -271,8 +229,8 @@ include_source: nil)
         exp_file.untagged_remarks -= header_remarks
       end
       private_class_method :transfer_header_to_schema
-
-      private_class_method :from_exp_streaming
+      private_class_method :parse_files_sequentially
+      private_class_method :build_repository
     end
   end
 end