audition 0.2.4 → 0.4.0

data/lib/audition/rewriters.rb

@@ -81,7 +81,10 @@ module Audition
         values = collector.values
         kinds = values.map { |v| classifier.classify(v) }
         flagged = kinds.reject do |k|
-          %i[shareable unknown].include?(k)
+          # A magic comment cannot fix opaque values, so they
+          # must not veto one either.
+          %i[shareable unknown instance_new opaque_call
+            shallow_opaque].include?(k)
         end
         scv_ok = values.any? &&
           values.all? { |value| deep_literal?(value, classifier) }
@@ -190,9 +193,9 @@ module Audition
     #   @x ||= expr
     #   return @x if defined?(@x); @x = expr
     #
-    # Preferred strategy is freeze-on-memoize, the pattern Rails
-    # core applies to its own code: the memoization stays exactly
-    # as written and only the memoized value becomes shareable
+    # Preferred strategy is freeze-on-memoize: the memoization
+    # stays exactly as written and only the memoized value
+    # becomes shareable
     # (`.freeze` appended; Ractor.make_shareable for containers).
     # Non-main Ractors may then read the ivar once it has been
     # computed; the first write must still happen on the main
@@ -325,7 +328,7 @@ module Audition
       end
 
       # Config setters (`def self.backend=(value); @backend =
-      # value; end`) get the Rails try_make_shareable recipe in
+      # value; end`) get a try-make-shareable recipe in
       # plain Ruby: shareable values are deeply frozen so reads
       # from any Ractor become legal, unshareable values keep
       # today's behavior through the rescue. Only bare local reads

data/lib/audition/static/analyzer.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "etc"
+require_relative "work_split"
 
 module Audition
   module Static
@@ -27,6 +27,10 @@ module Audition
 
       PARALLEL_THRESHOLD = 16
 
+      # Workers report in batches: a port message per file would
+      # cost more than the redraw it feeds.
+      TICK_STRIDE = 25
+
       # Scans files across Ractors when there are enough of them to
       # be worth the spawn cost. Checks are plain shareable classes
       # with deeply frozen catalogs, findings copy back through
@@ -34,47 +38,112 @@ module Audition
       # serial path.
       #
       # @param paths [Array<String>] files to analyze
-      # @param workers [Integer] Ractor count (defaults to
-      #   processor count minus one)
+      # @param workers [Integer, nil] Ractor count (defaults to
+      #   {#default_workers})
       # @param threshold [Integer] minimum file count before
       #   Ractors are used at all
+      # @param progress [Progress] ticked per file
       # @return [Array<Finding>]
-      def analyze_paths(paths, workers: default_workers,
-        threshold: PARALLEL_THRESHOLD)
+      def analyze_paths(paths, workers: nil,
+        threshold: PARALLEL_THRESHOLD, progress: Progress::SILENT)
+        workers ||= default_workers
         if paths.size < threshold || workers <= 1
-          return paths.flat_map { |path| analyze_path(path) }
+          return serial_analyze(paths, progress)
         end
 
-        parallel_analyze(paths, workers)
+        parallel_analyze(paths, workers, progress)
       rescue Ractor::Error => e
         # The silent fallback would otherwise mask a check that is
         # itself Ractor-hostile; surface it under -w.
         if $VERBOSE
-          warn "audition: parallel scan fell back to serial: " \
+          warn "Audition: parallel scan fell back to serial: " \
                "#{e.class}: #{e.message}"
         end
-        paths.flat_map { |path| analyze_path(path) }
+        progress.ractors = nil
+        serial_analyze(paths, progress)
       end
 
       private
 
-      def parallel_analyze(paths, workers)
+      def serial_analyze(paths, progress)
+        paths.flat_map do |path|
+          findings = analyze_path(path)
+          progress.tick
+          findings
+        end
+      end
+
+      # A port carries counts out of the workers while they run:
+      # the alternative is a status line frozen for the whole
+      # parallel phase, which is most of a large scan.
+      def parallel_analyze(paths, workers, progress)
         experimental = Warning[:experimental]
         Warning[:experimental] = false
-        slice = (paths.size / workers.to_f).ceil
         checks = @checks
-        paths.each_slice(slice).map do |chunk|
-          Ractor.new(chunk, checks) do |files, active_checks|
-            analyzer = Analyzer.new(checks: active_checks)
-            files.flat_map { |file| analyzer.analyze_path(file) }
+        port = progress.enabled? ? Ractor::Port.new : nil
+        chunks = balanced_chunks(paths, workers)
+        progress.ractors = chunks.size
+        ractors = chunks.map do |chunk|
+          # The sentinel goes out through `ensure` so a worker that
+          # raises still releases the drain loop; the exception
+          # itself still surfaces from `Ractor#value`.
+          Ractor.new(chunk, checks, port) do |files, active, tap|
+            analyzer = Analyzer.new(checks: active)
+            pending = 0
+            begin
+              files.flat_map do |file|
+                findings = analyzer.analyze_path(file)
+                pending += 1
+                if tap && pending >= TICK_STRIDE
+                  tap.send(pending)
+                  pending = 0
+                end
+                findings
+              end
+            ensure
+              tap&.send(pending)
+              tap&.send(:done)
+            end
           end
-        end.flat_map(&:value)
+        end
+        drain(port, ractors.size, progress) if port
+        ractors.flat_map(&:value)
       ensure
         Warning[:experimental] = experimental
       end
 
+      def drain(port, workers, progress)
+        done = 0
+        while done < workers
+          message = port.receive
+          if message == :done
+            done += 1
+          else
+            progress.tick(message)
+          end
+        end
+      rescue
+        # Narration is cosmetic; a closed port ends it quietly and
+        # `Ractor#value` still reports what went wrong.
+        nil
+      end
+
       def default_workers
-        [Etc.nprocessors - 1, 1].max
+        WorkSplit.workers
+      end
+
+      # Byte size stands in for parse cost, and the stat it costs
+      # is nothing beside the parse it schedules.
+      def balanced_chunks(paths, workers)
+        WorkSplit.chunks(
+          paths.map { |path| [path, file_size(path)] }, workers
+        )
+      end
+
+      def file_size(path)
+        File.size(path)
+      rescue SystemCallError
+        0
       end
 
       def analyze_file(file)

data/lib/audition/static/checks/dependency_class_state.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module Audition
+  module Static
+    module Checks
+      # Class-level state owned by a dependency. A gem's own source
+      # is outside the scanned tree, so the graph audit never sees
+      # the ivar and the call site is the only place left to flag.
+      # Generated type stubs in the target's tree record it: an
+      # attribute reader on a singleton class is a class-level ivar
+      # read by definition, and the generator marks those methods
+      # apart from ones that compute their answer.
+      class DependencyClassState < Base
+        explain :read,
+          severity: :warning,
+          message: "%{owner}.%{name} reads class-level state " \
+                   "owned by a dependency",
+          why: "The stub for %{owner} declares %{name} as an " \
+               "attribute on its singleton class, so the call " \
+               "returns a class-level instance variable. A " \
+               "non-main Ractor raises Ractor::IsolationError " \
+               "(\"can not get unshareable values from instance " \
+               "variables of classes/modules\") unless whatever " \
+               "the value happens to be is shareable.",
+          fix: "Read it once on the main Ractor and pass the " \
+               "value in, or make the dependency's assignment " \
+               "deeply shareable before any Ractor starts."
+
+        explain :write,
+          severity: :error,
+          message: "%{owner}.%{name}= writes class-level state " \
+                   "owned by a dependency",
+          why: "Assigning an attribute on %{owner}'s singleton " \
+               "class writes a class-level instance variable, " \
+               "which a non-main Ractor cannot do at all: it " \
+               "raises Ractor::IsolationError regardless of what " \
+               "the value is.",
+          fix: "Configure the dependency at boot on the main " \
+               "Ractor and leave it alone afterwards."
+
+        # The generator documents a reader it generated from an
+        # attribute; a hand-written method of the same shape gets
+        # its own prose and stays quiet.
+        STUB_READER = /Returns the value of attribute (\w+)/
+        STUB_SINGLETON = /\A\s*class\s+<<\s+self\b/
+        STUB_SCOPE = /\A\s*(?:class|module)\s+([A-Za-z0-9_:]+)/
+        STUB_DEF = /\A\s*def ([a-z_][A-Za-z0-9_]*)[(;]/
+        STUB_COMMENT = /\A\s*#/
+
+        class << self
+          attr_reader :attributes
+
+          # Owner name => set of attribute names, kept shareable
+          # for parallel scanning.
+          def attributes=(pairs)
+            grouped = pairs.group_by(&:first)
+              .transform_values { |v| v.map(&:last).uniq }
+            @attributes = Ractor.make_shareable( # audition:disable
+              grouped
+            )
+          end
+
+          # @param paths [Array<String>] scanned files; only
+          #   generated type stubs carry this evidence
+          # @return [void]
+          def learn(paths)
+            pairs = []
+            paths.each do |path|
+              next unless path.end_with?(".rbi")
+
+              read_stub(path, pairs)
+            end
+            self.attributes = pairs
+          end
+
+          private
+
+          # Stub indentation tracks nesting exactly, so the scope
+          # stack follows it rather than parsing the file.
+          def read_stub(path, pairs)
+            scope = []
+            singleton = []
+            indents = []
+            documented = nil
+            File.foreach(path) do |raw|
+              line = raw.rstrip
+              next if line.empty?
+
+              indent = line[/\A */].size
+              while indents.any? && indent <= indents.last
+                indents.pop
+                scope.pop
+                singleton.pop
+              end
+              documented =
+                step_stub(line, scope, singleton, indents,
+                  documented, pairs)
+            end
+          rescue SystemCallError
+            nil
+          end
+
+          def step_stub(line, scope, singleton, indents,
+            documented, pairs)
+            case line
+            when STUB_COMMENT
+              return STUB_READER.match(line)&.[](1) || documented
+            when STUB_SINGLETON
+              scope.push(scope.last)
+              singleton.push(true)
+              indents.push(line[/\A */].size)
+            when STUB_SCOPE
+              name = Regexp.last_match(1)
+              scope.push(scope.empty? ? name : "#{scope.last}::#{name}")
+              singleton.push(false)
+              indents.push(line[/\A */].size)
+            when STUB_DEF
+              name = Regexp.last_match(1)
+              if singleton.last && scope.last && documented == name
+                pairs << [scope.last, name]
+              end
+            end
+            nil
+          end
+        end
+
+        self.attributes = []
+
+        on :call_node do |node|
+          examine(node)
+        end
+
+        private
+
+        def examine(node)
+          owner = constant_name(node.receiver)
+          return unless owner
+
+          name = node.name.to_s
+          writer = name.end_with?("=")
+          attribute = writer ? name.chomp("=") : name
+          names = self.class.attributes[owner]
+          return unless names&.include?(attribute)
+
+          flag(node, writer ? :write : :read,
+            owner: owner, name: attribute)
+        end
+
+        def constant_name(node)
+          case node
+          when Prism::ConstantReadNode
+            node.name.to_s
+          when Prism::ConstantPathNode
+            node.location.slice.delete_prefix("::")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/audition/static/checks/mutable_constants.rb

@@ -30,21 +30,23 @@ module Audition
                "literals only; a method call returns a fresh " \
                "unfrozen object, so a non-main Ractor reading " \
                "this constant raises Ractor::IsolationError. " \
-               "Rails hit this with `.tr` and `Regexp.new` " \
-               "during its ractorization.",
-          fix: "Append `.freeze` to the call; a frozen String " \
-               "or Regexp is deeply shareable."
+               "Typical shapes: `.tr`, `Regexp.new`, and " \
+               "`Object.new` sentinels.",
+          fix: "Append `.freeze` to the call; a frozen String, " \
+               "Regexp, or bare Object is deeply shareable. " \
+               "BasicObject has no #freeze: use " \
+               "Object.new.freeze for such a sentinel."
 
         explain :mutable_container,
           severity: :error,
           message: "constant %{name} holds a mutable %{type} " \
                    "literal",
           why: CONSTANT_WHY,
-          fix: "Make it deeply shareable: " \
-               "`# shareable_constant_value: literal`, or " \
-               "wrap in Ractor.make_shareable(...). A bare " \
-               "`.freeze` is not enough when elements are " \
-               "themselves mutable."
+          fix: "Append `.freeze` when every element is itself " \
+               "shareable; otherwise make it deeply shareable with " \
+               "`# shareable_constant_value: literal` or " \
+               "Ractor.make_shareable(...), since a bare " \
+               "`.freeze` is shallow."
 
         explain :shallow_freeze,
           severity: :error,
@@ -62,17 +64,19 @@ module Audition
         explain :sync_primitive,
           severity: :error,
           message: "constant %{name} holds a %{klass}; sync " \
-                   "primitives are deliberately unshareable",
+                   "primitives and concurrent collections are " \
+                   "deliberately unshareable",
           why: "Mutex/Queue/ConditionVariable coordinate " \
                "threads inside one Ractor and can never be " \
                "shared across Ractors; any non-main Ractor " \
                "touching this constant raises " \
-               "Ractor::IsolationError.",
+               "Ractor::IsolationError. A concurrent-ruby Map " \
+               "defines no #freeze at all, so make_shareable " \
+               "raises NoMethodError on it.",
           fix: "Use Ractor::Port for cross-Ractor " \
                "coordination; keep a per-Ractor primitive " \
                "via Ractor.store_if_absent when the state it " \
-               "guards is per-Ractor too (Rails moved its " \
-               "template digest mutex this way); or use " \
+               "guards is per-Ractor too; or use " \
                "Ractor-safe structures (ractor_safe, ratomic " \
                "gems)."
 
@@ -94,12 +98,49 @@ module Audition
                "Hash, and freezing the Hash does not make " \
                "the block shareable; a non-main Ractor " \
                "reading this constant raises " \
-               "Ractor::IsolationError. Rails removed this " \
-               "pattern twice during its ractorization.",
+               "Ractor::IsolationError.",
           fix: "Use a plain frozen Hash with explicit keys, " \
                "or drop the default proc and fetch with a " \
                "literal default: hash.fetch(key, [])."
 
+        explain :unshareable_instance,
+          severity: :warning,
+          message: "constant %{name} holds an unfrozen %{klass} " \
+                   "instance",
+          why: "A fresh instance is unfrozen, so non-main " \
+               "Ractors raise Ractor::IsolationError reading " \
+               "it. A warning, not an error: .new can be " \
+               "overridden to return a shareable value.",
+          fix: "Freeze it when deeply immutable, wrap in " \
+               "Ractor.make_shareable, or keep a per-Ractor " \
+               "copy via Ractor.store_if_absent."
+
+        explain :shallow_opaque,
+          severity: :warning,
+          message: "constant %{name} is frozen at the top level " \
+                   "only; what it holds is unproven",
+          why: "Freezing is shallow: elements and instance " \
+               "variables stay as their calls returned them, and " \
+               "unless those are deeply frozen a non-main Ractor " \
+               "reading the constant raises " \
+               "Ractor::IsolationError. The dynamic probe settles " \
+               "it when the target boots.",
+          fix: "Build the value with Ractor.make_shareable for a " \
+               "deep freeze, or silence a known-shareable value " \
+               "with a disable comment."
+
+        explain :opaque_constant,
+          severity: :warning,
+          message: "constant %{name} holds the result of " \
+                   "%{method}; shareability unproven",
+          why: "Unless the call returns a deeply frozen value, " \
+               "non-main Ractors raise Ractor::IsolationError " \
+               "reading the constant. The dynamic probe settles " \
+               "it when the target boots.",
+          fix: "Freeze the result at definition time, or " \
+               "silence a known-shareable value with a disable " \
+               "comment."
+
         explain :constant_mutation,
           severity: :warning,
           message: "in-place %{method} on constant %{name}",
@@ -111,8 +152,8 @@ module Audition
           fix: "Build the complete value at load time and " \
                "freeze it (each_with_object then .freeze), " \
                "or move the registry behind a writer that " \
-               "rebuilds and refreezes on each change, the " \
-               "copy-on-write style Rails registries use. A " \
+               "rebuilds and refreezes on each change " \
+               "(copy-on-write). A " \
                "registry that plugins extend during boot is " \
                "frozen in the last boot hook (after_initialize) " \
                "rather than at definition, and writes after the " \
@@ -144,21 +185,34 @@ module Audition
 
           # A constant this file itself mutates in place is a
           # deliberate accumulator; freezing it would raise at
-          # the mutation site (sinatra's PARAMS_CONFIG). The
+          # the mutation site (sinatra's PARAMS_CONFIG). One it
+          # gives singleton methods raises the same way. The
           # finding stays, the autofix goes.
-          fix_ok = !mutated?(name)
-          case classifier.classify(value)
+          fix_ok = !mutated?(name) && !customized?(name)
+          kind = classifier.classify(value)
+          # A Sorbet cast returns its argument and a begin
+          # block its last statement: fixes, type names and
+          # depth checks target the value inside.
+          value = classifier.unwrap(value)
+          # Build-then-freeze: a bare `NAME.freeze` later in the
+          # same body makes the literal as good as frozen, so
+          # only provably mutable elements remain to report.
+          if kind == :mutable_container && frozen_later?(name)
+            kind = classifier.frozen_kind(value)
+            fix_ok = false
+          end
+          case kind
           when :mutable_string
             flag(node, :mutable_string, name: name,
               autofix: fix_ok ? append_freeze(value) : nil)
           when :mutable_call
             flag(node, :mutable_call, name: name,
               type: call_type(value), method: call_display(value),
-              autofix: fix_ok ? append_freeze(value) : nil)
+              autofix: fix_ok ? freeze_call(value) : nil)
           when :mutable_container
             flag(node, :mutable_container, name: name,
               type: container_type(value),
-              autofix: fix_ok ? wrap_make_shareable(value) : nil)
+              autofix: fix_ok ? freeze_container(value) : nil)
           when :shallow_freeze
             flag(node, :shallow_freeze, name: name,
               autofix:
@@ -183,13 +237,35 @@ module Audition
               autofix: wrappable ? wrap_make_shareable(value) : nil)
           when :default_proc
             flag(node, :hash_default_proc, name: name)
+          when :shallow_opaque
+            flag(node, :shallow_opaque, name: name)
+          # Build-then-freeze leaves depth unknown: stay silent.
+          when :instance_new
+            unless frozen_later?(name)
+              flag(node, :unshareable_instance, name: name,
+                klass:
+                  classifier.const_name(value.receiver) || "new")
+            end
+          when :opaque_call
+            unless frozen_later?(name)
+              flag(node, :opaque_constant, name: name,
+                method: opaque_display(value))
+            end
           end
         end
 
-        def mutated?(name)
+        def mutated?(name) = named_in?(file.mutated_constants, name)
+
+        def customized?(name)
+          named_in?(file.customized_constants, name)
+        end
+
+        def frozen_later?(name) = named_in?(file.frozen_constants, name)
+
+        def named_in?(names, name)
           bare = name.split("::").last
-          file.mutated_constants.any? do |mutated|
-            mutated == name || mutated.split("::").last == bare
+          names.any? do |other|
+            other == name || other.split("::").last == bare
           end
         end
 
@@ -262,6 +338,8 @@ module Audition
           when Prism::HashNode, Prism::KeywordHashNode then "Hash"
           when Prism::ArrayNode then "Array"
           when Prism::CallNode
+            return "Set" if value.name == :to_set
+
             classifier.const_name(value.receiver) || "container"
           else
             "container"
@@ -277,8 +355,11 @@ module Audition
         # Ternaries classify as strings when both branches are;
         # `.freeze` binds tighter than `?:`, so they get parens.
         def call_type(call)
-          owner = classifier.const_name(call.receiver)
-          (owner == "Regexp") ? "Regexp" : "String"
+          case classifier.const_name(call.receiver)
+          when "Regexp" then "Regexp"
+          when "Object", "BasicObject" then "Object"
+          else "String"
+          end
         end
 
         def call_display(call)
@@ -288,14 +369,32 @@ module Audition
           owner ? "#{owner}.#{call.name}" : "String##{call.name}"
         end
 
+        # The receiver is arbitrary, often a chain, so the
+        # fallback names only the method.
+        def opaque_display(call)
+          owner = classifier.const_name(call.receiver)
+          method =
+            if owner
+              "#{owner}.#{call.name}"
+            elsif call.receiver
+              ".#{call.name}"
+            else
+              call.name.to_s
+            end
+          "a #{method} call"
+        end
+
         # `.freeze` binds tighter than an operator: `"a" + "b".freeze`
         # freezes only "b", so operator calls get parentheses while
         # literals and parenthesized or argument-free calls take
         # the bare suffix.
         def bare_freezable?(value)
           case value
-          when Prism::StringNode, Prism::InterpolatedStringNode
+          when Prism::StringNode, Prism::InterpolatedStringNode,
+               Prism::HashNode
             true
+          when Prism::ArrayNode
+            !value.opening_loc.nil?
           when Prism::CallNode
             !value.opening_loc.nil? ||
               (!value.receiver.nil? && value.arguments.nil?)
@@ -305,6 +404,16 @@ module Audition
         end
 
         def append_freeze(value)
+          # `X = :a, :b` has no brackets; it gains them so the
+          # suffix freezes the whole array.
+          if value.is_a?(Prism::ArrayNode) && value.opening_loc.nil?
+            return Autofix.new(
+              start_offset: value.location.start_offset,
+              end_offset: value.location.end_offset,
+              replacement: "[#{value.location.slice}].freeze"
+            )
+          end
+
           if bare_freezable?(value)
             offset = value.location.end_offset
             Autofix.new(
@@ -322,6 +431,31 @@ module Audition
           end
         end
 
+        # A frozen bare Object is shareable; BasicObject has no
+        # #freeze, so its sentinel becomes a frozen Object.
+        def freeze_call(value)
+          return append_freeze(value) unless
+            classifier.const_name(value.receiver) == "BasicObject"
+
+          Autofix.new(
+            start_offset: value.location.start_offset,
+            end_offset: value.location.end_offset,
+            replacement: "Object.new.freeze",
+            safety: :unsafe
+          )
+        end
+
+        # Plain `.freeze` where every element is provably
+        # shareable, the plain-Ruby shape; the deep wrap only
+        # where a shallow freeze would not be enough.
+        def freeze_container(value)
+          if classifier.frozen_kind(value) == :shareable
+            append_freeze(value)
+          else
+            wrap_make_shareable(value)
+          end
+        end
+
         # `X = :a, :b` is an array literal without brackets; the
         # slice must gain them or the wrap becomes a multi-arg
         # call (mail's ATTRIBUTES).

data/lib/audition/static/checks/runtime_require.rb

@@ -22,7 +22,10 @@ module Audition
                "load-time side effects run at an arbitrary " \
                "point.",
           fix: "Require eagerly at boot, before Ractors are " \
-               "spawned."
+               "spawned. For an optional dependency, the " \
+               "class-level macro that enables the feature is " \
+               "a boot-time scope; the request-time method is " \
+               "not."
 
         explain :autoload,
           severity: :warning,