expressir 1.2.10-aarch64-linux → 2.4.29-aarch64-linux

data/lib/expressir/express/node_position_index.rb

@@ -0,0 +1,342 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Builds and queries an index of model nodes by source position.
+    #
+    # One interface (`nearest_node_to`, `node_for_end_scope_at`, `each_node`),
+    # one implementation. Extracted from RemarkAttacher so the tree-walk +
+    # byte→line translation + nearest-node heuristics live in one place,
+    # tested independently of any remark logic.
+    #
+    # Reference: ISO 10303-11 — the parser yields source spans on ModelElement
+    # instances; this module turns those spans into a line-keyed index that
+    # remark attachment can query.
+    class NodePositionIndex
+      EMPTY = [].freeze
+      private_constant :EMPTY
+
+      # Single source of truth for "what collections does this node have"
+      # lives on the model — each class declares its own via the
+      # `collection_attributes` macro. Referenced by both RemarkAttacher
+      # and NodePositionIndex via this alias for backward compatibility.
+      COLLECTION_REGISTRY = Model::ModelElement.collection_registry
+
+      attr_reader :nodes
+
+      # Semantic entries starting on `line` (node order).
+      def starting_at(line)
+        semantic_by_line[line] || EMPTY
+      end
+
+      # Semantic entries ending on `line` (node order).
+      def ending_at(line)
+        semantic_by_end_line[line] || EMPTY
+      end
+
+      # Semantic entries whose span covers `line` (node order within the
+      # covering band).
+      def spanning(line)
+        band = line / BAND
+        bands = span_bands
+        return EMPTY unless bands.key?(band)
+
+        bands[band].select { |n| line.between?(n[:line], n[:end_line]) }
+      end
+
+      # Semantic entries with end_line < line, ascending by end_line.
+      def semantic_ending_before(line)
+        sorted = semantic_by_end_line_list
+        idx = sorted.bsearch_index { |n| n[:end_line] >= line } || sorted.size
+        sorted[0...idx]
+      end
+
+      # The first semantic entry (smallest line), for preamble remarks.
+      def first_semantic
+        first_by_line = semantic_by_line.keys.min
+        first_by_line && semantic_by_line[first_by_line]&.first
+      end
+
+      # Entries owned by `owner` within any of `collections`, in node order.
+      # Identity-keyed: ownership is object identity (the tree-walker's
+      # `.equal?` semantics), and value-equal model objects must not
+      # collide.
+      def children_in(owner, collections)
+        per_collection = children_index[owner]
+        return EMPTY unless per_collection
+
+        entries = per_collection.values_at(*collections).compact.flatten(1)
+        entries.sort_by { |n| node_order(n) }
+      end
+
+      # First node (in node order) that is_a?(type), memoized per type.
+      def first_node_of_type(type)
+        @first_of_type ||= {}
+        @first_of_type[type] ||= @nodes.find { |n| n[:node].is_a?(type) }&.dig(:node)
+      end
+
+      def semantic_entries
+        @semantic_entries ||=
+          @nodes.select { |n| n[:line] && semantic?(n[:node]) }
+      end
+
+      # Non-vivifying: missed lookups must not materialize keys, or
+      # `keys.min`-style queries see phantom lines.
+      def semantic_by_line
+        @semantic_by_line ||= begin
+          h = {}
+          semantic_entries.each { |n| (h[n[:line]] ||= []) << n }
+          h
+        end
+      end
+
+      def semantic_by_end_line
+        @semantic_by_end_line ||= begin
+          h = {}
+          semantic_entries.each { |n| (h[n[:end_line]] ||= []) << n if n[:end_line] }
+          h
+        end
+      end
+
+      def semantic_by_end_line_list
+        @semantic_by_end_line_list ||=
+          semantic_entries.select { |n| n[:end_line] }.sort_by { |n| n[:end_line] }
+      end
+
+      def children_index
+        @children_index ||= begin
+          index = {}.compare_by_identity
+          @nodes.each do |n|
+            next unless n[:owner] && n[:line]
+
+            per_owner = (index[n[:owner]] ||= {})
+            (per_owner[n[:collection]] ||= []) << n
+          end
+          index
+        end
+      end
+
+      def span_bands
+        @span_bands ||= begin
+          bands = Hash.new { |hash, key| hash[key] = [] }
+          semantic_entries.each do |n|
+            next unless n[:end_line]
+
+            ((n[:line] / BAND)..(n[:end_line] / BAND)).each do |band|
+              bands[band] << n
+            end
+          end
+          bands
+        end
+      end
+
+      BAND = 1024
+      private_constant :BAND
+
+      def initialize(model, line_map)
+        @model = model
+        @line_map = line_map
+        @nodes = build_sorted_nodes
+        @node_order = nil
+      end
+
+      # Returns the most-specific node whose span contains `remark_line`,
+      # preferring same-line starts/ends, then smallest containing span.
+      # Skips nodes that cannot own such a remark; see {#remark_scope?}.
+      def nearest_node_to(remark_line)
+        same_start = starting_at(remark_line).select { |n| remark_scope?(n[:node]) }
+        return same_start.last[:node] if same_start.any?
+
+        same_end = ending_at(remark_line).select { |n| remark_scope?(n[:node]) }
+        return same_end.last[:node] if same_end.any?
+
+        containing = spanning(remark_line).select { |n| remark_scope?(n[:node]) }
+
+        if containing.any?
+          exp_file_node = containing.find { |n| n[:node].is_a?(Model::ExpFile) }
+          if exp_file_node
+            first_schema_offset = exp_file_node[:node].schemas&.first&.source_offset
+            if first_schema_offset && remark_line < @line_map.line_number(first_schema_offset)
+              return exp_file_node[:node]
+            end
+          end
+          # Prefer non-ExpFile nodes (Schema, Entity, etc.) over ExpFile
+          # when both span the same range, since ExpFile is a file-level
+          # container and remarks inside a schema belong to the schema.
+          candidates = containing.reject { |n| n[:node].is_a?(Model::ExpFile) }
+          candidates = containing if candidates.empty?
+          candidates.min_by { |n| n[:end_line] - n[:line] }[:node]
+        else
+          before = semantic_ending_before(remark_line)
+          if before.any?
+            before.max_by { |n| n[:end_line] }[:node]
+          else
+            # Remark is before all nodes (e.g., preamble comment before SCHEMA).
+            # Attach to the first semantic node.
+            first_semantic&.dig(:node)
+          end
+        end
+      end
+
+      # Returns the node whose END_XXX declaration is on `remark_line`, when
+      # the line text matches one of the END_XXX keywords. Used to attach
+      # remarks that appear immediately after a scope closes.
+      def node_for_end_scope_at(remark_line, line_content)
+        node_type = case line_content
+                    when /END_SCHEMA/i then Model::Declarations::Schema
+                    when /END_ENTITY/i then Model::Declarations::Entity
+                    when /END_TYPE/i then Model::Declarations::Type
+                    when /END_FUNCTION/i then Model::Declarations::Function
+                    when /END_PROCEDURE/i then Model::Declarations::Procedure
+                    when /END_RULE/i then Model::Declarations::Rule
+                    end
+        return nil unless node_type
+
+        candidates = (0..2).flat_map do |back|
+          ending_at(remark_line - back)
+        end.select do |n|
+          n[:node].is_a?(node_type) &&
+            n[:end_line] <= remark_line && n[:end_line] >= remark_line - 2
+        end
+
+        candidates.min_by { |n| node_order(n) }&.dig(:node) ||
+          first_node_of_type(node_type)
+      end
+
+      private
+
+      def node_order(entry)
+        @node_order ||= begin
+          map = {}.compare_by_identity
+          @nodes.each_with_index { |n, i| map[n] = i }
+          map
+        end
+        @node_order[entry]
+      end
+
+      def semantic?(node)
+        !node.is_a?(Model::Repository) && !node.is_a?(Model::Cache)
+      end
+
+      # Whether a node can own a remark sitting on a line of its own.
+      # Distinct from Model::TakesInlineRemark, which asks whether a node can
+      # own a remark written AFTER it on the same line: a statement answers
+      # yes to both; an interface clause only to that one.
+      #
+      # An interface clause is indexed (so a remark trailing it can find it
+      # via the same-line lookup), but it encloses nothing: an own-line
+      # remark below `REFERENCE FROM x;` introduces whatever comes next, and
+      # giving it to the clause would both misplace it and lose it, since
+      # nothing renders remarks there.
+      def remark_scope?(node)
+        semantic?(node) && !node.is_a?(Model::Declarations::Interface)
+      end
+
+      def build_sorted_nodes
+        result = []
+        collect_nodes(@model, result, Set.new)
+        result.sort_by!.with_index { |n, i| [n[:position] || Float::INFINITY, i] }
+        result
+      end
+
+      def collect_nodes(node, result, visited, owner: nil, collection: nil)
+        return unless node
+        return if visited.include?(node.object_id)
+
+        visited.add(node.object_id)
+
+        if node.is_a?(Model::ModelElement) && node.source && node.source_offset
+          record_node_position(node, result, owner, collection)
+        else
+          result << { node: node, position: nil, line: nil, end_line: nil,
+                      owner: owner, collection: collection }
+        end
+
+        collect_children(node, result, visited)
+      end
+
+      def record_node_position(node, result, owner, collection)
+        pos = node.source_offset
+        valid = position_valid?(pos, node)
+        unless valid
+          result << { node: node, position: nil, line: nil, end_line: nil,
+                      owner: owner, collection: collection }
+          return
+        end
+
+        line = @line_map.line_number(pos)
+        source_end_line = @line_map.line_number(pos + node.source.length)
+        children_end_line = children_end_line_for(node)
+        end_line = [source_end_line, children_end_line].compact.max || source_end_line
+
+        result << { node: node, position: pos, line: line, end_line: end_line,
+                    owner: owner, collection: collection }
+      end
+
+      # The parser returns source_offset=0 for leaf nodes (WhereRule) where
+      # it cannot determine the actual position. Accept position=0 only when
+      # the source is a declaration keyword line — those legitimately start
+      # at the beginning of the file.
+      def position_valid?(pos, node)
+        return true if pos.positive?
+        return false unless pos.zero? && node.source
+
+        node.source.to_s.start_with?(
+          "SCHEMA", "ENTITY", "TYPE", "FUNCTION",
+          "PROCEDURE", "RULE", "CONSTANT", "VARIABLE",
+          "USE", "REFERENCE", "END_SCHEMA", "END_ENTITY",
+          "END_TYPE", "END_FUNCTION", "END_PROCEDURE",
+          "END_RULE", "END_CONSTANT", "END_VARIABLE"
+        )
+      end
+
+      def children_end_line_for(node)
+        end_lines = []
+
+        if node.is_a?(Model::Declarations::Schema)
+          Array(node.children).each do |child|
+            next unless child.is_a?(Model::ModelElement) && child.source_offset && child.source
+
+            end_lines << @line_map.line_number(child.source_offset + child.source.length)
+          end
+        end
+
+        each_collection_on(node) do |item|
+          next unless item.is_a?(Model::ModelElement) && item.source_offset && item.source
+
+          end_lines << @line_map.line_number(item.source_offset + item.source.length)
+        end
+
+        end_lines.max
+      end
+
+      def collect_children(node, result, visited)
+        if node.is_a?(Model::Declarations::Schema)
+          Array(node.children).each do |c|
+            collect_nodes(c, result, visited, owner: node, collection: :children)
+          end
+        end
+
+        each_collection_on(node) do |item, attr|
+          collect_nodes(item, result, visited, owner: node, collection: attr)
+        end
+      end
+
+      # Yields each child in any declared collection on the node (with the
+      # collection attribute name), based on the type-driven
+      # COLLECTION_REGISTRY. Returns nothing for nodes whose class is not
+      # registered.
+      def each_collection_on(node)
+        attrs = COLLECTION_REGISTRY[node.class]
+        return unless attrs
+
+        attrs.each do |attr|
+          collection = node.public_send(attr)
+          next unless collection.is_a?(Array)
+
+          collection.each { |item| yield(item, attr) }
+        end
+      end
+    end
+  end
+end

data/lib/expressir/express/parallel_files.rb

@@ -0,0 +1,229 @@
+require "etc"
+
+module Expressir
+  module Express
+    # Fork-based worker pool for parsing many EXPRESS files in parallel.
+    # The native parser holds the GVL for the whole parse, so process-level
+    # parallelism is the only way to use multiple cores. Files are
+    # independent until reference resolution, which stays in the parent.
+    #
+    # Unlike sequential parsing, the progress block fires in file order
+    # only after all files have been parsed.
+    class ParallelFiles
+      DEFAULT_MAX_PROCESSES = 4
+      FRAME_HEADER_BYTES = 4
+
+      FORK_SUPPORTED = Process.respond_to?(:fork).freeze
+
+      # Forking is never the default: a library must not spawn processes on
+      # behalf of its host (forked children inherit broken thread and lock
+      # state, and fork does not exist on all Rubies). Parallelism requires
+      # an explicit max_processes > 1 from the caller and a platform that
+      # supports fork (e.g. not Windows); otherwise the request degrades
+      # to sequential parsing.
+      def self.sequential?(files, max_processes)
+        !FORK_SUPPORTED || max_processes.nil? || max_processes <= 1 ||
+          files.size < 3
+      end
+
+      # @param files [Array<String>] EXPRESS file paths
+      # @param max_processes [Integer, nil] worker cap; nil auto-selects
+      # @param parse [Proc] callback taking a file path, returning an ExpFile
+      # @param strict [Boolean] re-raise every error, including
+      #   Error::SchemaParseFailure, instead of skipping the file
+      # @yield [file, exp_file, error] called in original file order
+      # @return [Array<Expressir::Model::ExpFile, nil>] parsed files in order;
+      #   nil marks a file that failed with Error::SchemaParseFailure
+      def self.run(files, parse:, max_processes: nil, strict: false, &block)
+        new(files, max_processes, parse, block, strict).run
+      end
+
+      def initialize(files, max_processes, parse, block, strict)
+        @files = files
+        @parse = parse
+        @block = block
+        @strict = strict
+        @worker_count = [
+          files.size - 1,
+          max_processes || [Etc.nprocessors, DEFAULT_MAX_PROCESSES].min,
+        ].min
+      end
+
+      def run
+        job_pipes = Array.new(@worker_count) { IO.pipe }
+        result_pipes = Array.new(@worker_count) { IO.pipe }
+        pids = spawn_workers(job_pipes, result_pipes)
+
+        files_results = schedule_jobs(job_pipes, result_pipes)
+
+        ordered_pass(files_results)
+      ensure
+        cleanup(job_pipes, result_pipes, pids)
+      end
+
+      private
+
+      def spawn_workers(job_pipes, result_pipes)
+        job_pipes.each_index.map do |i|
+          job_r, job_w = job_pipes[i]
+          result_r, result_w = result_pipes[i]
+          fork do
+            job_w.close
+            result_r.close
+            other_pipes = (job_pipes + result_pipes).flatten -
+              [job_r, result_w]
+            other_pipes.each { |io| io.close unless io.closed? }
+            worker_loop(job_r, result_w)
+          end
+        end
+      end
+
+      # Assigns one job at a time to whichever worker reports a result, so
+      # each file is parsed exactly once and each worker holds at most one
+      # in-flight job (keeping result pipes free of interleaved frames).
+      def schedule_jobs(job_pipes, result_pipes)
+        writers = job_pipes.map(&:last)
+        readers = result_pipes.map(&:first)
+        files_results = Array.new(@files.size)
+        next_job = 0
+        busy = {}
+
+        writers.each_index do |wi|
+          break if next_job == @files.size
+
+          dispatch(writers[wi], next_job)
+          busy[wi] = true
+          next_job += 1
+        end
+
+        until busy.empty?
+          ready, = IO.select(readers.values_at(*busy.keys))
+          ready.each do |io|
+            wi = readers.index(io)
+            payload = Marshal.load(read_frame(io)) # rubocop:disable Security/MarshalLoad
+            files_results[payload[:index]] = payload
+
+            if next_job < @files.size
+              dispatch(writers[wi], next_job)
+              next_job += 1
+            else
+              writers[wi].close unless writers[wi].closed?
+              busy.delete(wi)
+            end
+          end
+        end
+
+        files_results
+      end
+
+      def dispatch(writer, job_index)
+        write_frame(writer, pack_frame(Marshal.dump([job_index, @files[job_index]])))
+      end
+
+      def worker_loop(job_r, result_w)
+        until job_r.eof?
+          data = read_frame(job_r)
+          index, file = Marshal.load(data) # rubocop:disable Security/MarshalLoad
+          begin
+            payload = { index: index, exp_file: @parse.call(file) }
+          rescue StandardError => e
+            payload = { index: index, error: transferable_error(e) }
+          end
+          write_frame(result_w, pack_frame(Marshal.dump(payload)))
+        end
+      rescue Errno::EPIPE
+        # parent went away; nothing to report to
+        exit!(0)
+      end
+
+      # SIGTERM cannot interrupt a worker blocked in the native parser (the
+      # GVL is held), so escalate to SIGKILL after a grace period instead of
+      # blocking in waitpid forever.
+      def cleanup(job_pipes, result_pipes, pids)
+        (job_pipes.to_a + result_pipes.to_a).each do |r, w|
+          r.close unless r.closed?
+          w.close unless w.closed?
+        end
+        pids.to_a.each { |pid| Process.kill("TERM", pid) if alive?(pid) }
+
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 2
+        pids.to_a.each do |pid|
+          while alive?(pid) && Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
+            Process.waitpid(pid, Process::WNOHANG)
+            sleep 0.05
+          end
+          Process.kill("KILL", pid) if alive?(pid)
+          reap(pid)
+        end
+      end
+
+      def ordered_pass(files_results)
+        files_results.each_with_index do |payload, index|
+          error = payload[:error]
+          @block&.call(@files[index], payload[:exp_file], error)
+          raise error if error && (@strict ||
+                                  !error.is_a?(Error::SchemaParseFailure))
+        end
+
+        files_results.map do |payload|
+          payload[:error] ? nil : payload[:exp_file]
+        end
+      end
+
+      # Errors carrying native-parser state (e.g. Parsanol::ParseFailed with
+      # its cause tree) cannot cross a fork boundary; rebuild them without
+      # the untransferable internals, preserving class and message.
+      def transferable_error(error)
+        Marshal.dump(error)
+        error
+      rescue StandardError
+        if error.is_a?(Error::SchemaParseFailure)
+          Error::SchemaParseFailure.new(error.filename,
+                                        StandardError.new(error.message))
+        else
+          StandardError.new(error.message)
+        end
+      end
+
+      def pack_frame(data)
+        [data.bytesize].pack("N") + data
+      end
+
+      def write_frame(io, frame)
+        io.write(frame)
+      end
+
+      def read_frame(io)
+        header = read_exactly(io, FRAME_HEADER_BYTES)
+        read_exactly(io, header.unpack1("N"))
+      end
+
+      def read_exactly(io, count)
+        data = +""
+        while data.bytesize < count
+          chunk = io.read(count - data.bytesize)
+          unless chunk
+            raise Error::ParallelParseError,
+                  "worker exited before sending its result"
+          end
+
+          data << chunk
+        end
+        data
+      end
+
+      def alive?(pid)
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH, Errno::EPERM
+        false
+      end
+
+      def reap(pid)
+        Process.waitpid(pid)
+      rescue Errno::ECHILD, Errno::EINVAL
+        nil
+      end
+    end
+  end
+end