rigortype 0.4.0 → 0.4.2

data/lib/rigor/analysis/dependency_recorder.rb

@@ -22,12 +22,52 @@ module Rigor
       # It is a STACK because captures nest (a memoised callee body evaluating another memo-miss callee): an
       # outer capture must also see the inner window's reads, or its read-set would be non-transitive and a
       # later replay would drop the nested callee's edges.
+      #
+      # Every event reaches every capture open at the time, so an inner capture's sets are always a subset of
+      # each outer one's: the inner capture was pushed after the outer, and everything added since reached both.
+      # {withhold} swaps in a fresh stack and restores this one untouched, so it keeps the property too.
+      # {tee_read} and {tee_missing} rely on it to stop at the first capture that already holds an event.
       CAPTURE_KEY = :__rigor_dependency_capture_stack__
       private_constant :CAPTURE_KEY
 
+      # The read sets an {Accumulator} or {Capture} already holds in full, by identity: replayed into it, or
+      # captured by a window it saw every event of ({capture}). Both only ever grow and a {ReadSet} is frozen,
+      # so replaying such a set into the same target again adds nothing. Recording replays the same stored
+      # read set many times over: on Rigor's own `lib`, 71% of the replays into a consumer repeated one
+      # already made into it, carrying 65% of the replayed events.
+      module ReplayMemory
+        def holds?(read_set)
+          !@held.nil? && @held.key?(read_set)
+        end
+
+        # Called only once every event of `read_set` is in, so a replay that raises part-way leaves the
+        # target unmarked and the next replay adds the rest.
+        def hold!(read_set)
+          (@held ||= {}.compare_by_identity)[read_set] = true
+        end
+      end
+
+      # #1590 — the resolution-chain filings ({file_chain_once}) an {Accumulator} or {Capture} already holds in
+      # full: per chain (by identity; the table holds the chain, so its identity cannot be reused while the target
+      # lives), the starts filed from it and the discovery index each was filed against. A filing is a pure
+      # function of those three, so filing it again into the same target adds nothing.
+      module ChainMemory
+        def chain_filed?(chain, start, discovery)
+          table = @chains&.[](chain)
+          !table.nil? && table.key?(start) && table[start].equal?(discovery)
+        end
+
+        def chain_filed!(chain, start, discovery)
+          ((@chains ||= {}.compare_by_identity)[chain] ||= {})[start] = discovery
+        end
+      end
+
       # Mutable per-consumer accumulator. Frozen into a {Record} snapshot when `record_for` returns.
       class Accumulator
-        attr_reader :consumer, :sources, :missing, :symbol_sources, :ancestry_sources
+        include ReplayMemory
+        include ChainMemory
+
+        attr_reader :consumer, :sources, :missing, :symbol_sources, :ancestry_sources, :suspensions
 
         def initialize(consumer)
           @consumer = consumer
@@ -39,6 +79,15 @@ module Rigor
           # file-granularity by nature (a superclass edge touches the whole class).
           @symbol_sources = Hash.new { |h, k| h[k] = Set.new }
           @ancestry_sources = Set.new
+          @held = nil
+          @chains = nil
+          @suspensions = 0
+        end
+
+        # Counts the {record_for} windows of another consumer opened while this one was recording, which is
+        # when a capture window stops seeing only this consumer's events.
+        def suspend!
+          @suspensions += 1
         end
 
         def snapshot
@@ -65,16 +114,28 @@ module Rigor
       # self-read filter is applied at {replay} time, mirroring {read_site}.
       ReadSet = Data.define(:reads, :missing)
 
+      # What every window that saw nothing closes into: most windows {RbsDispatch}'s recording-mode memos
+      # open see nothing, and they need neither a copy of two empty Sets nor a place in a target's
+      # {ReplayMemory}.
+      EMPTY_READ_SET = Ractor.make_shareable(ReadSet.new(reads: Set.new, missing: Set.new))
+
       # Mutable capture accumulator; snapshot into a frozen {ReadSet} when the capture window closes.
       class Capture
+        include ReplayMemory
+        include ChainMemory
+
         attr_reader :reads, :missing
 
         def initialize
           @reads = Set.new
           @missing = Set.new
+          @held = nil
+          @chains = nil
         end
 
         def snapshot
+          return EMPTY_READ_SET if reads.empty? && missing.empty?
+
           ReadSet.new(reads: reads.dup.freeze, missing: missing.dup.freeze)
         end
       end
@@ -94,6 +155,7 @@ module Rigor
       # the previous recorder on exit.
       def record_for(consumer)
         previous = Thread.current[KEY]
+        previous&.suspend!
         accumulator = Accumulator.new(consumer.to_s)
         Thread.current[KEY] = accumulator
         @mutex.synchronize { @active_count += 1 }
@@ -109,6 +171,13 @@ module Rigor
         @active_count.positive?
       end
 
+      # Whether THIS thread's reads reach an accumulator: inside {record_for}, including a {withhold}
+      # window. {active?} answers for the process and is the cheap gate; this is the exact one, for a caller
+      # that must know whether an empty {capture} means "nothing to record" or "nothing was watching".
+      def recording?
+        !Thread.current[KEY].nil?
+      end
+
       # ADR-84 WD2 — runs the block under an observe-and-forward capture window and returns
       # `[block_result, read_set]`. Reads recorded during the window reach the current consumer's accumulator
       # unchanged (forwarding) and are additionally collected into the returned frozen {ReadSet} (observing),
@@ -116,6 +185,8 @@ module Rigor
       # {active?}; opening a capture with no accumulator active would collect reads no run is recording.
       def capture
         captures = (Thread.current[CAPTURE_KEY] ||= [])
+        accumulator = Thread.current[KEY]
+        suspensions = accumulator&.suspensions
         capture = Capture.new
         captures.push(capture)
         begin
@@ -123,9 +194,28 @@ module Rigor
         ensure
           captures.pop
         end
-        [result, capture.snapshot]
+        read_set = capture.snapshot
+        note_captured(read_set, captures, accumulator, suspensions)
+        [result, read_set]
       end
 
+      # Every event a window saw also reached the window outside it, and was filed for the consumer that was
+      # recording at the time, except the consumer's own self-reads, which a replay into that consumer drops
+      # anyway. So the enclosing window holds the closed window's read set in full, and so does the consumer's
+      # accumulator when no other consumer recorded in between. Noting that lets a later replay of the set
+      # there skip a pass that could add nothing: on Rigor's own `lib`, two thirds of the first replays into a
+      # consumer were of a set captured while that consumer was recording.
+      def note_captured(read_set, captures, accumulator, suspensions)
+        return if read_set.equal?(EMPTY_READ_SET)
+
+        captures.last&.hold!(read_set)
+        return unless accumulator && Thread.current[KEY].equal?(accumulator)
+        return unless accumulator.suspensions == suspensions
+
+        accumulator.hold!(read_set)
+      end
+      private_class_method :note_captured
+
       # Issue #992 — runs the block with its reads DETACHED from the current consumer (and from any enclosing
       # capture) and returns `[block_result, read_set]`, so a caller that learns only afterwards whether its
       # answer depended on those reads can {replay} them or drop them. `call.wrong-arity` reads a class's whole
@@ -154,22 +244,75 @@ module Rigor
       # capture so an enclosing capture window stays transitive when a memo hit substitutes for a body walk.
       def replay(read_set)
         accumulator = Thread.current[KEY]
-        return if accumulator.nil? || read_set.nil?
+        return if accumulator.nil? || read_set.nil? || read_set.equal?(EMPTY_READ_SET)
 
         captures = Thread.current[CAPTURE_KEY]
+        tee_read_set(captures, read_set) if captures && !captures.empty?
+        # Each set keeps its own insertion order whichever target is filled first, and a skipped replay is one
+        # that could only have re-added what the target already holds.
+        return if accumulator.holds?(read_set)
+
+        consumer = accumulator.consumer
         read_set.reads.each do |pair|
-          captures&.each { |c| c.reads.add(pair) }
           path, symbol = pair
-          next if path == accumulator.consumer
-
-          accumulate_read(accumulator, path, symbol)
+          accumulate_read(accumulator, path, symbol) unless path == consumer
         end
-        read_set.missing.each do |entry|
-          captures&.each { |c| c.missing.add(entry) }
-          accumulator.missing << entry
+        missing = accumulator.missing
+        read_set.missing.each { |entry| missing << entry }
+        accumulator.hold!(read_set)
+      end
+
+      # #1590 — runs the block, which files a resolution chain's edges from entry `start` (nil: the root and the
+      # whole chain) against `discovery`, unless the current consumer's accumulator AND the innermost open capture
+      # already hold that filing; the block files the same edges every time, so skipping it there changes nothing.
+      # Every event reaches every capture open at the time (see CAPTURE_KEY), so the innermost capture holding the
+      # filing means every open capture does. A filing the accumulator holds but a capture opened since does not is
+      # filed again, so that capture's read set, replayed elsewhere, still carries it; and {withhold} starts a fresh
+      # accumulator and capture, which hold nothing. The marks are made only once the block has run, as {hold!}'s are.
+      def file_chain_once(chain, start, discovery)
+        accumulator = Thread.current[KEY]
+        return if accumulator.nil?
+
+        captures = Thread.current[CAPTURE_KEY]
+        open = captures && !captures.empty?
+        if accumulator.chain_filed?(chain, start, discovery) &&
+           (!open || captures.last.chain_filed?(chain, start, discovery))
+          return
         end
+
+        yield
+        accumulator.chain_filed!(chain, start, discovery)
+        captures.each { |capture| capture.chain_filed!(chain, start, discovery) } if open
       end
 
+      # {replay}'s share for the open captures. The innermost capture having seen `read_set` already means every
+      # open capture holds all of it (see CAPTURE_KEY).
+      def tee_read_set(captures, read_set)
+        innermost = captures.last
+        return if innermost.holds?(read_set)
+
+        read_set.reads.each { |pair| tee_read(captures, pair) }
+        read_set.missing.each { |entry| tee_missing(captures, entry) }
+        innermost.hold!(read_set)
+      end
+      private_class_method :tee_read_set
+
+      # Adds a read pair to the open captures, innermost first, and stops at the first capture that already holds
+      # it: every capture outside that one holds it too (see CAPTURE_KEY). The capture sets end up exactly as
+      # adding it to every capture would leave them, in the same insertion order.
+      def tee_read(captures, pair)
+        index = captures.size - 1
+        index -= 1 while index >= 0 && captures[index].reads.add?(pair)
+      end
+      private_class_method :tee_read
+
+      # {tee_read} for a name-keyed `missing` entry.
+      def tee_missing(captures, entry)
+        index = captures.size - 1
+        index -= 1 while index >= 0 && captures[index].missing.add?(entry)
+      end
+      private_class_method :tee_missing
+
       # Records that the current consumer read a declaration / body whose definition site is `path_line` (a
       # `"path:line"` String, or nil). When `symbol` is given (a `"ClassName#method"` String), the read is a
       # method-call edge and is recorded at symbol granularity in `symbol_sources` in addition to the coarse
@@ -181,16 +324,26 @@ module Rigor
         accumulator = Thread.current[KEY]
         return if accumulator.nil? || path_line.nil?
 
-        path = path_line.split(":", 2).first
+        path = site_path(path_line)
         return unless path
 
         captures = Thread.current[CAPTURE_KEY]
-        captures&.each { |c| c.reads.add([path, symbol].freeze) }
+        tee_read(captures, [path, symbol].freeze) if captures && !captures.empty?
         return if path == accumulator.consumer
 
         accumulate_read(accumulator, path, symbol)
       end
 
+      # The path part of a `"path:line"` site: what `path_line.split(":", 2).first` answers, without the Array
+      # and the discarded line String.
+      def site_path(path_line)
+        colon = path_line.byteindex(":")
+        return path_line.byteslice(0, colon) if colon
+
+        path_line.empty? ? nil : path_line.dup
+      end
+      private_class_method :site_path
+
       # Records a NAME-KEYED cross-file dependency: this consumer's answer depends on what the project makes
       # of `name` under `kind`, so a recheck must re-analyze it when that changes. The recorded key is
       # inverted into `negative_dependents` and matched against the producer for its kind.
@@ -206,10 +359,53 @@ module Rigor
         return if accumulator.nil?
 
         entry = "#{kind}:#{name}"
-        Thread.current[CAPTURE_KEY]&.each { |c| c.missing.add(entry) }
+        captures = Thread.current[CAPTURE_KEY]
+        tee_missing(captures, entry) if captures && !captures.empty?
         accumulator.missing.add(entry)
       end
 
+      # {read_name} keyed on the LAST SEGMENT of a constant path, `name.delete_prefix("::").split("::").last`,
+      # which is the key the `class:` and `constant:` name edges use. The segment is found by scanning for the
+      # last separator rather than by building every segment: on Rigor's own `lib` that split cost about seven
+      # objects a call over half a million calls. A name the scan could read differently from `split` (one
+      # that is not ASCII, or carries a run of colons or a trailing one) takes `split`'s answer.
+      def read_last_segment(kind, name)
+        return if Thread.current[KEY].nil?
+
+        segment = last_segment(name)
+        read_name(kind, segment) if segment
+      end
+
+      def last_segment(name)
+        return name.delete_prefix("::").split("::").last unless plain_constant_path?(name)
+
+        start = name.start_with?("::") ? 2 : 0
+        separator = name.rindex("::")
+        cut = separator && separator >= start ? separator + 2 : start
+        cut == name.bytesize ? nil : name.byteslice(cut, name.bytesize - cut)
+      end
+      private_class_method :last_segment
+
+      def plain_constant_path?(name)
+        name.ascii_only? && !name.end_with?(":") && !name.include?(":::")
+      end
+      private_class_method :plain_constant_path?
+
+      # {read_name} for prebuilt, frozen `"kind:name"` keys: a caller that files the same keys on every read
+      # (a resolution chain's hook edges, #1641) builds them once and adds them here without a String apiece.
+      def read_keys(keys)
+        accumulator = Thread.current[KEY]
+        return if accumulator.nil?
+
+        captures = Thread.current[CAPTURE_KEY]
+        open = captures && !captures.empty?
+        missing = accumulator.missing
+        keys.each do |entry|
+          tee_missing(captures, entry) if open
+          missing.add(entry)
+        end
+      end
+
       # The miss-only spelling of {read_name}: a cross-file lookup of `name` (kind `:method` / `:class` /
       # `:toplevel` / …) that resolved to nothing. Kept as the name the internal spec's negative-edge
       # contract uses.
@@ -217,13 +413,14 @@ module Rigor
         read_name(kind, name)
       end
 
-      # The positive-edge aggregation shared by {read_site} and {replay}.
+      # The positive-edge aggregation shared by {read_site} and {replay}. `sources` is the union of the other
+      # two tables' paths, and this is the only writer of all three, so a read the finer table already holds is
+      # already in `sources` and costs one Set probe instead of two.
       def accumulate_read(accumulator, path, symbol)
-        accumulator.sources << path
         if symbol
-          accumulator.symbol_sources[path] << symbol
-        else
-          accumulator.ancestry_sources << path
+          accumulator.sources << path if accumulator.symbol_sources[path].add?(symbol)
+        elsif accumulator.ancestry_sources.add?(path)
+          accumulator.sources << path
         end
       end
       private_class_method :accumulate_read

data/lib/rigor/analysis/dependency_source_inference/builder.rb

@@ -42,6 +42,9 @@ module Rigor
             @class_to_gem = {}
             @budget_exceeded = []
             @gem_modes = {}
+            @refinements = {}
+            @bound_constants = Set.new
+            @bound_segments = Set.new
           end
 
           def absorb(outcome, budget, builder)
@@ -58,6 +61,9 @@ module Rigor
             @gem_modes[resolved.gem_name] = resolved.mode
             walked = builder.walker_outcome_for(resolved, budget)
             @catalog.merge!(walked.catalog)
+            @refinements = Inference::ScopeIndexer.merge_refinement_tables(@refinements, walked.refinements)
+            @bound_constants.merge(walked.bound)
+            @bound_segments.merge(walked.bound_segments)
             builder.record_class_to_gem(walked.catalog, resolved.gem_name, @class_to_gem)
             @budget_exceeded << resolved.gem_name if walked.truncated?
           end
@@ -68,7 +74,8 @@ module Rigor
               method_catalog: @catalog, budget_exceeded: @budget_exceeded,
               class_to_gem: @class_to_gem,
               budget_overrun_strategy: dependencies.budget_overrun_strategy,
-              gem_modes: @gem_modes
+              gem_modes: @gem_modes, refinements: @refinements, bound_constants: @bound_constants,
+              bound_segments: @bound_segments
             )
           end
         end

data/lib/rigor/analysis/dependency_source_inference/gem_resolver.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../../cache/descriptor"
+
 module Rigor
   module Analysis
     module DependencySourceInference
@@ -19,6 +21,11 @@ module Rigor
           def descriptor_key
             [gem_name, version, mode].freeze
           end
+
+          # ADR-10 slice 3 — this gem's row of a cache descriptor: its name, resolved version and mode.
+          def dependency_entry
+            Cache::Descriptor::DependencyEntry.new(gem_name: gem_name, gem_version: version, mode: mode)
+          end
         end
 
         # Unresolvable reasons. `:not_in_bundle` covers both the "RubyGems doesn't know this gem" case and the
@@ -43,6 +50,20 @@ module Rigor
           )
         end
 
+        # Issue #1814 — the {Resolved#dependency_entry} of each non-disabled `dependencies.source_inference` entry that
+        # resolves, in configuration order, without walking any gem. The run-result cache key and the incremental
+        # snapshot's fingerprint are built before analysis from inputs that are cheap to read, and a gem's walked
+        # source (its method catalogue, its refinement table, the constants it binds) is an input of the run that no
+        # analysed file records; a `Gemfile.lock` digest covers a `bundle update` and nothing else, so a project with no
+        # lockfile, or a gem installed beside the bundle, would replay the diagnostics of the gem's previous version.
+        # Unresolvable entries have no version to key on and add nothing: the gem appearing later adds its row.
+        def dependency_entries(dependencies)
+          dependencies.source_inference.reject(&:disabled?).filter_map do |entry|
+            resolved = resolve(entry)
+            resolved.dependency_entry if resolved.is_a?(Resolved)
+          end
+        end
+
         # Locator. `Gem.loaded_specs` reflects the bundle (cheap lookup, no filesystem walk); `find_by_name` is
         # the broader fallback for gems present on the gem path but not yet `require`'d. `Gem::MissingSpecError`
         # is a `LoadError` subclass, so the rescue covers both missing-spec and load-error signals.

data/lib/rigor/analysis/dependency_source_inference/index.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../../cache/descriptor"
+require_relative "../../inference/scope_indexer"
 
 module Rigor
   module Analysis
@@ -11,7 +12,8 @@ module Rigor
       # entries.
       class Index
         attr_reader :resolved_gems, :unresolvable, :method_catalog, :budget_exceeded,
-                    :class_to_gem, :budget_overrun_strategy, :gem_modes
+                    :class_to_gem, :budget_overrun_strategy, :gem_modes, :refinements, :bound_constants,
+                    :bound_segments
 
         # @param method_catalog — => Symbol}] the flat
         #   `(class_name, method_name) → :instance | :singleton` table produced by {Walker.walk}, aggregated
@@ -30,10 +32,21 @@ module Rigor
         #   gem-source and RBS both contribute under `mode: :full`. The map is keyed on `gem_name` (not
         #   class) because re-opened classes belong to the first gem they appeared in per `class_to_gem`;
         #   `mode_for(class_name)` chains the two lookups.
-        def initialize(
+        # @param refinements — issue #1672: the opt-in gems' `refine X do … end` bodies, `{refined class =>
+        #   {method => [refining modules]}}` ({Walker::Outcome#refinements}). The runner merges it into the
+        #   project's `discovered_refinements` seed, so `using M` for a gem's `M` reaches the check rules.
+        # @param bound_constants — issue #1814: every name the opt-in gems' files bind by a constant write without
+        #   declaring a class or module by it ({Walker::Outcome#bound}: `Shared::Target` for `module Shared; Target =
+        #   String; end`), unioned across the walked gems. A refine key one of them binds may name any value
+        #   (`refine Shared::Target` in the project or in the next gem), which the reader decides at read time
+        #   ({Inference::InEffectRefinements.class_wildcard_key?}) through {#bound_constant?}.
+        # @param bound_segments — the last segment of each such write whose owner the walk cannot spell
+        #   ({Walker::Outcome#bound_segments}: a `const_set`), which matches that segment of any key.
+        def initialize( # rubocop:disable Metrics/ParameterLists
           resolved_gems: [], unresolvable: [], method_catalog: {},
           budget_exceeded: [], class_to_gem: {},
-          budget_overrun_strategy: :walker_cap, gem_modes: {}
+          budget_overrun_strategy: :walker_cap, gem_modes: {}, refinements: {}, bound_constants: Set.new,
+          bound_segments: Set.new
         )
           @resolved_gems = resolved_gems.freeze
           @unresolvable = unresolvable.freeze
@@ -42,6 +55,9 @@ module Rigor
           @class_to_gem = class_to_gem.freeze
           @budget_overrun_strategy = budget_overrun_strategy
           @gem_modes = gem_modes.freeze
+          @refinements = Inference::ScopeIndexer.freeze_refinements(refinements)
+          @bound_constants = bound_constants.to_set.freeze
+          @bound_segments = bound_segments.to_set.freeze
           freeze
         end
 
@@ -62,6 +78,22 @@ module Rigor
           @class_to_gem[class_name]
         end
 
+        # Issue #1814 — does a walked gem bind `name` (a refine key, or a prefix of one: `Shared::Target`, `Pag::Enc`)
+        # by a constant write that declares no class or module, or bind a constant of `name`'s last segment under an
+        # owner the walk could not spell? Such a key may name any value (`module Shared; Target = String; end` then
+        # `refine(Shared::Target)`), which the walker cannot resolve and the reader decides
+        # ({Inference::InEffectRefinements.class_wildcard_key?}). A gem constant that merely shares a core class's name
+        # (`Names::String`) binds nothing a sibling spelling of `refine String` (`Ext::String`) could denote.
+        def bound_constant?(name)
+          @bound_constants.include?(name) || @bound_segments.include?(name.split("::").last)
+        end
+
+        # Does any walked gem bind a constant at all? False for a project that opts in no gem, so a reader asks nothing
+        # per key.
+        def binds_constants?
+          !(@bound_constants.empty? && @bound_segments.empty?)
+        end
+
         # ADR-10 slice 5c — per-class mode lookup. Chains `class_to_gem` + `gem_modes`; returns `nil` when
         # the class isn't owned by any opt-in gem in this run.
         def mode_for(class_name)
@@ -99,14 +131,7 @@ module Rigor
         # surfaces them as `dynamic.dependency-source.gem-not-found` diagnostics. Resolved-but-disabled
         # entries are also absent: the {Builder} skips them before resolution, so they never reach the index.
         def cache_descriptor
-          dependencies = @resolved_gems.map do |resolved|
-            Cache::Descriptor::DependencyEntry.new(
-              gem_name: resolved.gem_name,
-              gem_version: resolved.version,
-              mode: resolved.mode
-            )
-          end
-          Cache::Descriptor.new(dependencies: dependencies)
+          Cache::Descriptor.new(dependencies: @resolved_gems.map(&:dependency_entry))
         end
       end