audition 0.4.0 → 0.4.1

data/lib/audition/dynamic/prober.rb

@@ -12,15 +12,15 @@ module Audition
     #   @return [Symbol] `:script`, `:require`, `:rack`, `:rails`,
     #     or `:capabilities`
     # @!attribute [r] raw
-    #   @return [Hash] the harness's parsed JSON, verbatim
+    #   @return [Hash] the harness's document, verbatim
     # @!attribute [r] findings
     #   @return [Array<Finding>] findings derived from `raw`
     # @!attribute [r] passed
     #   @return [Boolean] whether the target's own surface passed
     Result = Data.define(:mode, :raw, :findings, :passed)
 
-    # Spawns the harness subprocess per probe mode, parses its JSON,
-    # and converts observations into findings.
+    # Spawns the harness subprocess per probe mode, loads the
+    # document it prints, and converts observations into findings.
     class Prober
       HARNESS = File.expand_path("harness.rb", __dir__).freeze
 
@@ -152,10 +152,99 @@ module Audition
             findings: findings, passed: false)
         end
 
+        findings.concat(ractorize_findings(raw["ractorize"], entry))
         Result.new(mode: :rails, raw: raw, findings: findings,
           passed: own_clean?(findings))
       end
 
+      # What happened once the booted application was frozen: no
+      # entry point on this Rails, the first object it could not
+      # share, a request that broke on the frozen graph, or one
+      # that only broke inside a worker.
+      def ractorize_findings(info, entry)
+        return [] unless info.is_a?(Hash)
+
+        environment = entry[:environment]
+        prefix = own_prefix(entry[:root])
+        unless info["available"]
+          return [Finding.new(
+            check: "dynamic-rails",
+            severity: :info,
+            message: "Rails #{info["rails"]} has no ractorize!; " \
+                     "the application graph was not frozen",
+            why: "Rails 8.2 adds Rails::Application#ractorize!, " \
+                 "which deep-freezes the application and " \
+                 "everything it reaches; a lazy memoization on " \
+                 "any object in that graph raises FrozenError " \
+                 "afterwards. Without it the probe can only " \
+                 "sweep constants and class-level state.",
+            fix: "Upgrade to Rails 8.2 so the probe can freeze " \
+                 "the application and serve a request through it.",
+            path: environment,
+            line: nil
+          )]
+        end
+
+        unless info["ok"]
+          site = failure_site(info, prefix: prefix)
+          return [Finding.new(
+            check: "dynamic-rails",
+            severity: :error,
+            message: "ractorize! failed: #{describe(info)}",
+            why: "Rails::Application#ractorize! deep-freezes the " \
+                 "application and everything it reaches (routes, " \
+                 "middleware, configuration); the object the " \
+                 "error names is the first one that cannot be " \
+                 "shared. #{RUNTIME_WHY}",
+            fix: "Make the named object shareable: freeze it, " \
+                 "drop the Proc, Mutex, or IO it holds, or keep " \
+                 "it per-Ractor; then re-run.",
+            path: site&.first || environment,
+            line: site&.last
+          )]
+        end
+
+        main = info["main_request"] || {}
+        unless main["ok"]
+          site = failure_site(main, prefix: prefix)
+          return [Finding.new(
+            check: "dynamic-rails",
+            severity: :error,
+            message: "GET / after ractorize! failed on the main " \
+                     "Ractor: #{describe(main)}",
+            why: "The application is frozen, so a lazy " \
+                 "memoization on any object the request touches " \
+                 "writes an instance variable on a frozen object " \
+                 "and raises FrozenError. #{RUNTIME_WHY}",
+            fix: "Compute the value eagerly in initialize, warm " \
+                 "it in a freeze override that calls the reader " \
+                 "before super, or drop the memo and recompute.",
+            path: site&.first || environment,
+            line: site&.last
+          )]
+        end
+
+        ractor = info["ractor_request"] || {}
+        return [] if ractor["ok"]
+
+        site = failure_site(ractor, prefix: prefix)
+        [Finding.new(
+          check: "dynamic-rails",
+          severity: :error,
+          message: "GET / inside a Ractor after ractorize! " \
+                   "failed: #{describe(ractor)}",
+          why: "Serving on the main Ractor after ractorize! " \
+               "worked; inside a worker the request touched " \
+               "state a non-main Ractor cannot reach. " \
+               "#{RUNTIME_WHY}",
+          fix: "Remove global and class-level state touched " \
+               "during request handling; keep per-Ractor state " \
+               "in Ractor.store_if_absent.",
+          path: site&.first || environment,
+          line: site&.last
+        )]
+      end
+
       # A probe passes when the target's own surface is clean;
       # dependency errors surface in the findings and drive the
       # blocked verdict instead.
@@ -220,6 +309,24 @@ module Audition
         raw.fetch("class_state", []).each do |entry|
           findings << class_state_finding(entry, label)
         end
+        raw.fetch("unshareable_procs", []).each do |entry|
+          findings << runtime_finding(
+            entry, label,
+            check: "runtime-unshareable-proc",
+            severity: :warning,
+            message: "Rails could not make a callback block " \
+                     "Ractor-shareable: #{entry["proc"]}",
+            why: "With unshareable_proc_action set to :warn, " \
+                 "Rails ran Ractor.shareable_proc on this block " \
+                 "and it raised Ractor::IsolationError, so the " \
+                 "callback keeps an unshareable Proc; a non-main " \
+                 "Ractor running it raises. #{RUNTIME_WHY}",
+            fix: "Capture only shareable values: freeze the " \
+                 "local, inline it, or hoist a shareable leaf " \
+                 "such as a Symbol into a fresh local assigned " \
+                 "once before the block."
+          )
+        end
         raw.fetch("class_variables", []).each do |entry|
           findings << runtime_finding(
             entry, label,
@@ -462,12 +569,12 @@ module Audition
 
       # -- subprocess plumbing -------------------------------------
 
-      # Harness output can carry arbitrary target bytes; force
-      # valid UTF-8 before any string work or a binary exception
-      # message crashes the whole run.
+      # The harness prints one Marshal document; anything else on
+      # its stdout (a crash before the document) is reported with
+      # the tail of its stderr. Stderr can carry arbitrary target
+      # bytes, so it is forced to valid UTF-8 first.
       def run(mode, payload = {}, root: nil)
         out, err, timed_out = execute(mode, payload, root: root)
-        out = sanitize(out)
         err = sanitize(err)
         if timed_out
           return {"error" => {
@@ -475,8 +582,11 @@ module Audition
             "message" => "harness exceeded #{@timeout}s"
           }}
         end
-        JSON.parse(out)
-      rescue JSON::ParserError
+        document = Marshal.load(out)
+        raise TypeError, "not a document" unless document.is_a?(Hash)
+
+        document
+      rescue TypeError, ArgumentError, EOFError
         {"error" => {
           "class" => "HarnessFailure",
           "message" => err.split("\n").last(5).join("; ")
@@ -514,7 +624,8 @@ module Audition
           end
         end
         Open3.popen3(env, *cmd, **opts) do |stdin, stdout, stderr, wait|
-          stdin.write(JSON.generate(payload))
+          stdin.binmode
+          stdin.write(Marshal.dump(payload))
           stdin.close
           out_reader = reader(stdout)
           err_reader = reader(stderr)

data/lib/audition/finding.rb

@@ -66,12 +66,16 @@ module Audition
   #   @return [Boolean] see {#dependency?}
   # @!attribute [r] test
   #   @return [Boolean] see {#test?}
+  # @!attribute [r] subject
+  #   @return [String, nil] the object a finding is about, in a
+  #     form the dynamic probe can match ("Owner/@ivar"), so a
+  #     runtime proof of shareability can retire the static guess
   Finding = Data.define(
     :check, :severity, :message, :why, :fix,
-    :path, :line, :source, :autofix, :dependency, :test
+    :path, :line, :source, :autofix, :dependency, :test, :subject
   ) do
     def initialize(source: nil, autofix: nil, dependency: false,
-      test: false, **rest)
+      test: false, subject: nil, **rest)
       super
     end
 

data/lib/audition/reconciliation.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Audition
+  # What the dynamic probe proved shareable retires the static
+  # pass's guesses about the same objects. A constant the probe
+  # read as shareable needs no static warning at all; class-level
+  # state whose every value was shareable after boot keeps its
+  # finding as an info note, because reads are legal from any
+  # Ractor and only a later write would raise.
+  module Reconciliation
+    PROVEN_WHY =
+      "The dynamic probe found every value this variable held " \
+      "after boot shareable, so reads from any Ractor are legal; " \
+      "only a later write would raise Ractor::IsolationError, " \
+      "and writes must stay on the main Ractor."
+    PROVEN_FIX =
+      "Keep every write at boot on the main Ractor, or warm the " \
+      "memo before spawning Ractors."
+
+    # @param findings [Array<Finding>] static findings
+    # @param results [Array<Dynamic::Result>] the probes that ran
+    # @return [Array<Finding>] findings with the disproven ones
+    #   dropped or downgraded
+    def self.apply(findings, results)
+      constants, ivars = proven(results)
+      return findings if constants.empty? && ivars.empty?
+
+      findings.filter_map do |finding|
+        case finding.check
+        when "mutable-constants"
+          site = [realpath(finding.path), finding.line]
+          constants.include?(site) ? nil : finding
+        when "class-level-state"
+          if finding.severity != :info && finding.subject &&
+              ivars.include?(finding.subject)
+            finding.with(
+              severity: :info,
+              message: "#{finding.message}, shareable in the " \
+                       "dynamic probe",
+              why: PROVEN_WHY,
+              fix: PROVEN_FIX
+            )
+          else
+            finding
+          end
+        else
+          finding
+        end
+      end
+    end
+
+    # @return [Array(Set, Set)] constant sites ([path, line]) and
+    #   class-level ivars ("Owner/@name") observed shareable
+    def self.proven(results)
+      constants = Set.new
+      ivars = Set.new
+      results.each do |result|
+        raw = result.raw
+        next unless raw.is_a?(Hash)
+
+        Array(raw["proven_constants"]).each do |path, line|
+          constants << [realpath(path), line]
+        end
+        Array(raw["class_state"]).each do |entry|
+          unshareable = Array(entry["unshareable"])
+          owner = entry["const"].to_s
+          Array(entry["ivars"]).each do |ivar|
+            next if unshareable.include?(ivar)
+
+            ivars << "#{owner}/#{ivar}"
+            # A write through a singleton attribute only knows
+            # its owner's last segment.
+            ivars << "#{owner.split("::").last}/#{ivar}"
+          end
+        end
+      end
+      [constants, ivars]
+    end
+
+    def self.realpath(path)
+      File.realpath(path)
+    rescue SystemCallError, TypeError
+      path
+    end
+  end
+end

data/lib/audition/rewriters.rb

@@ -227,6 +227,9 @@ module Audition
           kinds = ops.map { |op| op[:kind] }
           next if kinds.include?(:other)
           next if ops.any? { |op| op[:body] }
+          # A memo written inside an on_main block already runs
+          # on the main Ractor; the audit rates the shape itself.
+          next if ops.any? { |op| op[:proxied] }
 
           memos = memo_sites(ops)
           if memos.empty?
@@ -620,9 +623,29 @@ module Audition
           @sclass_depth = 0
           @def_stack = []
           @defined_depth = 0
+          @proxy_depth = 0
           super
         end
 
+        # A block handed to `on_main` runs on the main Ractor by
+        # construction (the read-then-proxy escape hatch), so the
+        # ivar writes inside it are recorded as proxied.
+        def visit_call_node(node)
+          block = node.block
+          unless node.name == :on_main && block.is_a?(Prism::BlockNode)
+            return super
+          end
+
+          visit(node.receiver) if node.receiver
+          visit(node.arguments) if node.arguments
+          @proxy_depth += 1
+          begin
+            visit(block)
+          ensure
+            @proxy_depth -= 1
+          end
+        end
+
         def visit_class_node(node)
           scoped(node.constant_path.location.slice, :class) do
             super(node)
@@ -766,7 +789,8 @@ module Audition
             def_id: current_def && current_def[:id],
             def_name: current_def && current_def[:name],
             def_node: current_def && current_def[:node],
-            class_owner: @namespace.last[:kind] == :class
+            class_owner: @namespace.last[:kind] == :class,
+            proxied: @proxy_depth.positive?
           }
         end
       end

data/lib/audition/static/checks/instance_memoization.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module Audition
+  module Static
+    module Checks
+      # Lazy memoization on an instance is harmless until the
+      # instance is frozen: Ractor.make_shareable freezes every
+      # object it reaches, and the next `@x ||=` raises FrozenError.
+      # Two shapes make the freeze provable from the class alone: an
+      # initialize that ends by freezing self (a value object), and a
+      # `freeze` override (the class expects to be frozen). In the
+      # first the memo can never run; in the second it must be
+      # warmed inside the override, before super, which is the
+      # compute-on-freeze pattern. Class-level memos belong to the
+      # graph audit.
+      class InstanceMemoization < Base
+        explain :memo_after_self_freeze,
+          severity: :error,
+          message: "instance memoization %{ivar} in #%{method} on " \
+                   "a class that freezes itself in initialize",
+          why: "The instance is frozen before any other method " \
+               "runs, so the first call writes an instance " \
+               "variable on a frozen object and raises " \
+               "FrozenError.",
+          fix: "Compute the value in initialize, before the " \
+               "freeze, and expose it with attr_reader; or drop " \
+               "the memo and recompute on each call."
+
+        explain :memo_not_warmed,
+          severity: :warning,
+          message: "freeze override leaves %{ivar} cold; " \
+                   "#%{method} memoizes it lazily",
+          why: "Ractor.make_shareable calls freeze, so an " \
+               "instance frozen through this override raises " \
+               "FrozenError the first time #%{method} runs " \
+               "afterwards.",
+          fix: "Warm it in the override: call #%{method} (or " \
+               "assign %{ivar}) before super, the " \
+               "compute-on-freeze pattern; or compute it in " \
+               "initialize."
+
+        def initialize(file)
+          super
+          @contexts = []
+          @sclass_depth = 0
+        end
+
+        def visit_class_node(node) = scoped { super }
+
+        def visit_module_node(node) = scoped { super }
+
+        def visit_singleton_class_node(node)
+          @sclass_depth += 1
+          super
+        ensure
+          @sclass_depth -= 1
+        end
+
+        # Only plain instance methods count; a def is not entered,
+        # so nothing inside a method body opens a context.
+        def visit_def_node(node)
+          context = @contexts.last
+          return unless context && node.receiver.nil? &&
+            @sclass_depth.zero?
+
+          record_method(context, node)
+        end
+
+        private
+
+        def scoped
+          @contexts.push(
+            {memos: {}, methods: {}, self_freeze: false, warm: nil}
+          )
+          saved = @sclass_depth
+          @sclass_depth = 0
+          yield
+        ensure
+          @sclass_depth = saved
+          report(@contexts.pop)
+        end
+
+        def record_method(context, node)
+          case node.name
+          when :initialize
+            context[:self_freeze] = self_freezing?(node)
+          when :freeze
+            context[:warm] = touched_by(node)
+          else
+            context[:methods][node.name] ||= touched_by(node)
+            memo_sites(node).each do |ivar, write|
+              context[:memos][ivar] ||= {method: node.name, node: write}
+            end
+          end
+        end
+
+        def report(context)
+          memos = context[:memos]
+          return if memos.empty?
+
+          if context[:self_freeze]
+            memos.each do |ivar, memo|
+              flag(memo[:node], :memo_after_self_freeze,
+                ivar: ivar, method: memo[:method])
+            end
+          elsif context[:warm]
+            reached, warmed = warmed_closure(context)
+            memos.each do |ivar, memo|
+              next if reached.include?(memo[:method]) ||
+                warmed.include?(ivar)
+
+              flag(memo[:node], :memo_not_warmed,
+                ivar: ivar, method: memo[:method])
+            end
+          end
+        end
+
+        # Everything the override reaches through the class's own
+        # instance methods: a memo is warm when its method runs on
+        # the way, or when any method on the way assigns its ivar.
+        def warmed_closure(context)
+          methods = context[:methods]
+          reached = []
+          warmed = context[:warm][:ivars].dup
+          queue = context[:warm][:calls].dup
+          until queue.empty?
+            name = queue.shift
+            next if reached.include?(name)
+
+            reached << name
+            touched = methods[name] or next
+
+            warmed.concat(touched[:ivars])
+            queue.concat(touched[:calls])
+          end
+          [reached, warmed]
+        end
+
+        # initialize ends with `freeze`, `self.freeze`, or
+        # `Ractor.make_shareable(self)`.
+        def self_freezing?(node)
+          last = statements_of(node.body)&.last
+          return false unless last.is_a?(Prism::CallNode)
+
+          receiver = last.receiver
+          if last.name == :freeze
+            (receiver.nil? || receiver.is_a?(Prism::SelfNode)) &&
+              last.arguments.nil?
+          elsif last.name == :make_shareable
+            last.arguments&.arguments&.first.is_a?(Prism::SelfNode)
+          else
+            false
+          end
+        end
+
+        def statements_of(body)
+          case body
+          when Prism::StatementsNode then body.body
+          when Prism::BeginNode then body.statements&.body
+          end
+        end
+
+        # The methods a body calls on self and the ivars it assigns.
+        def touched_by(node)
+          calls = []
+          ivars = []
+          each_descendant(node.body) do |child|
+            case child
+            when Prism::CallNode
+              receiver = child.receiver
+              if receiver.nil? || receiver.is_a?(Prism::SelfNode)
+                calls << child.name
+              end
+            when Prism::InstanceVariableWriteNode,
+                 Prism::InstanceVariableOrWriteNode
+              ivars << child.name.to_s
+            end
+          end
+          {calls: calls, ivars: ivars}
+        end
+
+        # `@x ||= v`, and `@x = v` guarded by `defined?(@x)` in the
+        # same method.
+        def memo_sites(node)
+          or_writes = []
+          writes = {}
+          guarded = []
+          each_descendant(node.body) do |child|
+            case child
+            when Prism::InstanceVariableOrWriteNode
+              or_writes << [child.name.to_s, child]
+            when Prism::InstanceVariableWriteNode
+              writes[child.name.to_s] ||= child
+            when Prism::DefinedNode
+              value = child.value
+              if value.is_a?(Prism::InstanceVariableReadNode)
+                guarded << value.name.to_s
+              end
+            end
+          end
+          guarded.each do |ivar|
+            or_writes << [ivar, writes[ivar]] if writes[ivar]
+          end
+          or_writes
+        end
+
+        # Blocks and lambdas inside a method still write self's
+        # ivars; a nested def or class does not.
+        def each_descendant(node)
+          queue = [node].compact
+          until queue.empty?
+            current = queue.shift
+            yield current
+            current.compact_child_nodes.each do |child|
+              next if child.is_a?(Prism::DefNode) ||
+                child.is_a?(Prism::ClassNode) ||
+                child.is_a?(Prism::ModuleNode)
+
+              queue << child
+            end
+          end
+        end
+      end
+    end
+  end
+end