audition 0.3.0 → 0.4.0

data/lib/audition/static/literal_classifier.rb

@@ -16,6 +16,10 @@ module Audition
     #   :proc              lambda or proc
     #   :default_proc      Hash.new with a block; the block
     #                      survives .freeze and stays unshareable
+    #   :instance_new      unfrozen instance of an arbitrary class
+    #   :opaque_call       method call with an unprovable return
+    #   :shallow_opaque    frozen at the top, but what it holds
+    #                      is an unprovable call result
     #   :unknown           cannot tell statically
     class LiteralClassifier
       SYNC_PRIMITIVES = %w[
@@ -51,6 +55,18 @@ module Audition
       ].freeze
       FORMATTERS = %i[format sprintf].freeze
       REGEXP_FACTORIES = %i[new union compile].freeze
+      # File methods returning a fresh path String.
+      FILE_PATH_METHODS = %i[
+        expand_path join dirname basename absolute_path realpath
+      ].freeze
+
+      # Calls returning a shareable primitive on any receiver.
+      SHAREABLE_RETURNS = %i[
+        to_i to_int to_f to_r to_c to_sym size length count
+        bytesize ord hash
+      ].freeze
+      # Sorbet's inline casts, which return their value argument.
+      SORBET_CASTS = %i[let cast must].freeze
 
       # @param frozen_string_literal [Boolean] whether the file has
       #   the frozen_string_literal magic comment
@@ -61,6 +77,7 @@ module Audition
       # @param node [Prism::Node] an expression node
       # @return [Symbol] classification, see class docs
       def classify(node)
+        node = begin_value(node)
         case node
         when Prism::IntegerNode, Prism::FloatNode,
              Prism::RationalNode, Prism::ImaginaryNode,
@@ -94,6 +111,29 @@ module Audition
         end
       end
 
+      # The value an expression hands back once begin blocks
+      # and inline casts are peeled off.
+      def unwrap(node)
+        loop do
+          inner = begin_value(node)
+          inner = cast_value(inner) || inner
+          break node if inner.equal?(node)
+
+          node = inner
+        end
+      end
+
+      # A begin block's value is its last statement. A rescue,
+      # else, or ensure clause can supply a different one, so
+      # only the plain form resolves.
+      def begin_value(node)
+        return node unless node.is_a?(Prism::BeginNode) &&
+          node.rescue_clause.nil? && node.else_clause.nil? &&
+          node.ensure_clause.nil?
+
+        node.statements&.body&.last || node
+      end
+
       # `::Mutex` and `Mutex` are the same constant for matching
       # purposes; the leading colons are stripped.
       def const_name(node)
@@ -105,6 +145,17 @@ module Audition
         end
       end
 
+      # The value inside a Sorbet inline cast, or nil. The cast
+      # returns its argument, so fixes and type names belong on
+      # the value, not the cast.
+      def cast_value(node)
+        return nil unless node.is_a?(Prism::CallNode)
+        return nil unless const_name(node.receiver) == "T" &&
+          SORBET_CASTS.include?(node.name)
+
+        node.arguments&.arguments&.first
+      end
+
       # What `value.freeze` would classify as, for the check to
       # choose a plain `.freeze` over a deep wrap: :shareable
       # when every element is provably shareable, :shallow_freeze
@@ -165,9 +216,11 @@ module Audition
             return :mutable_container
           end
 
-          :unknown
-        when :[], :to_set
+          name ? :instance_new : :opaque_call
+        when :to_set
           set_kind(node)
+        when :[]
+          index_kind(node)
         when :define
           (const_name(receiver) == "Data") ? :shareable : :unknown
         when :make_shareable
@@ -175,14 +228,46 @@ module Audition
         when :lambda, :proc
           (receiver.nil? && node.block) ? :proc : :unknown
         else
-          :unknown
+          opaque_kind(node)
         end
       end
 
+      # Catch-all for unrecognized calls: shareable returns pass,
+      # predicates stay silent, everything else is opaque.
+      def opaque_kind(node)
+        return :shareable if SHAREABLE_RETURNS.include?(node.name)
+        return :unknown if node.name.end_with?("?")
+
+        if (value = cast_value(node))
+          return classify(value)
+        end
+
+        :opaque_call
+      end
+
+      # Indexing into a constant, another call's result, or a
+      # fresh instance returns a value of unprovable shareability.
+      # Sorbet type constructors and ENV, whose values are frozen
+      # strings, stay silent.
+      def index_kind(node)
+        owner = const_name(node.receiver)
+        return set_kind(node) if owner == "Set"
+        if owner.nil?
+          receiver_kind = classify(node.receiver)
+          return :opaque_call if %i[opaque_call instance_new]
+            .include?(receiver_kind)
+          return :unknown
+        end
+        return :unknown if owner == "ENV" ||
+          owner == "T" || owner.start_with?("T::")
+
+        :opaque_call
+      end
+
       def classify_freeze(node, receiver)
         return :unknown unless node.arguments.nil? && receiver
 
-        case receiver
+        case (receiver = begin_value(receiver))
         when Prism::StringNode
           :shareable
         when Prism::ArrayNode, Prism::HashNode
@@ -194,6 +279,7 @@ module Audition
           when :default_proc then :default_proc
           when :mutable_call then :shareable
           when :mutable_container then frozen_kind(receiver)
+          when :instance_new, :opaque_call then :shallow_opaque
           else :unknown
           end
         else
@@ -214,6 +300,7 @@ module Audition
           owner = const_name(receiver)
           (name == :new && owner == "String") ||
             (owner == "Kernel" && FORMATTERS.include?(name)) ||
+            (owner == "File" && FILE_PATH_METHODS.include?(name)) ||
             STRING_ONLY_METHODS.include?(name)
         else
           false
@@ -243,12 +330,12 @@ module Audition
       end
 
       # A Set built from literals is a container of them:
-      # `Set.new([...])`, `Set[...]`, `%w[...].to_set`. Anything
-      # else, such as `Set.new(compute)` or a mapping block,
-      # stays unknown.
+      # `Set.new([...])`, `Set[...]`, `%w[...].to_set`. A Set
+      # built from an opaque source or a mapping block is still
+      # a fresh unfrozen Set, mutable no matter its contents.
       def set_kind(node)
         elements = set_elements(node)
-        elements ? elements_kind(elements) : :unknown
+        elements ? elements_kind(elements) : :mutable_container
       end
 
       def set_elements(node)
@@ -308,20 +395,33 @@ module Audition
         body && body.size == 1 && body[0]
       end
 
-      # Fold element classifications: everything provably shareable
-      # gives :shareable; anything provably mutable gives
-      # :shallow_freeze; a sync primitive poisons the whole
-      # container; anything unknowable gives :unknown (stay silent
-      # rather than guess).
+      # Fold element classifications by the strongest evidence:
+      # a sync primitive poisons the whole container; a provably
+      # mutable element makes it :shallow_freeze; an opaque call
+      # result makes it :shallow_opaque. A bare constant read is
+      # commonly a class or another frozen constant, so on its
+      # own it keeps the container silent (:unknown), but it
+      # cannot excuse a bad element elsewhere.
+      VERDICT_RANK = {
+        shareable: 0, unknown: 1, shallow_opaque: 2,
+        shallow_freeze: 3
+      }.freeze
+
       def deep_classify(elements)
         verdict = :shareable
         elements.each do |element|
           element_children(element).each do |child|
-            case classify(child)
-            when :shareable then nil
-            when :sync_primitive then return :sync_primitive
-            when :unknown then return :unknown
-            else verdict = :shallow_freeze
+            kind =
+              case classify(child)
+              when :shareable then :shareable
+              when :sync_primitive then return :sync_primitive
+              when :unknown then :unknown
+              when :instance_new, :opaque_call, :shallow_opaque
+                :shallow_opaque
+              else :shallow_freeze
+              end
+            if VERDICT_RANK[kind] > VERDICT_RANK[verdict]
+              verdict = kind
             end
           end
         end

data/lib/audition/static/native_extensions.rb

@@ -22,9 +22,9 @@ module Audition
       INIT = /\bInit_\w+\s*\(/
       SOURCES = "*.{c,cc,cpp,cxx,m,mm,h,hpp,rs,zig}"
       BUILD_FILES = %w[Cargo.toml build.zig extconf.rb].freeze
-      # Harness crates and test trees under ext/ are not compiled
-      # into the extension; a fuzz target that links the VM itself
-      # may call rb_ext_ractor_safe without the extension doing so.
+      # Harness crates and test trees are not compiled into the
+      # extension; a fuzz target that links the VM itself may call
+      # rb_ext_ractor_safe without the extension doing so.
       HARNESS_DIRS = %w[fuzz benches tests examples test spec].freeze
 
       SILENT_WHY =
@@ -51,7 +51,7 @@ module Audition
       DECLARED_WHY =
         "rb_ext_ractor_safe(true) is the maintainer's assertion " \
         "that the extension keeps no process-global mutable state; " \
-        "Ruby does not verify it and neither can audition. Its " \
+        "Ruby does not verify it and neither can Audition. Its " \
         "methods run from any Ractor, in parallel, on the strength " \
         "of that assertion alone."
       DECLARED_FIX =
@@ -91,12 +91,16 @@ module Audition
         nil
       end
 
-      # One finding per extension directory (ext/<name>), anchored
+      # One finding per extension directory, anchored
       # at the declaration when there is one, else at Init_* or the
       # build file, where the declaration belongs.
       def source_finding(dir, root)
         sources = sources_under(dir)
-        label = dir.delete_prefix("#{root}/")
+        label = if dir == root
+          File.basename(root)
+        else
+          dir.delete_prefix("#{root}/")
+        end
         path, line = locate(sources, DECLARATION)
         if path
           return finding(:info, path, line,
@@ -111,18 +115,26 @@ module Audition
           "safety", SILENT_WHY + SOURCE_TAIL, SILENT_FIX)
       end
 
-      # ext/<name> directories holding native sources; ext/ itself
-      # when the sources sit directly in it.
+      # Directories holding native sources, found by the build file
+      # that compiles them rather than by convention: a gem is free
+      # to put its extension anywhere. A manifest above another one
+      # is dropped, so a workspace does not report the sources of
+      # the crates under it a second time.
       def extension_dirs(root)
-        ext = File.join(root, "ext")
-        return [] unless File.directory?(ext)
-
-        dirs = Dir[File.join(ext, "*")].sort.select do |dir|
-          File.directory?(dir) && !skipped?(File.basename(dir)) &&
-            sources_under(dir).any?
+        dirs = build_dirs(root).select { |dir| sources_under(dir).any? }
+        dirs.reject do |dir|
+          dirs.any? { |other| other.start_with?("#{dir}/") }
         end
-        dirs << ext if Dir[File.join(ext, SOURCES)].any?
-        dirs
+      end
+
+      def build_dirs(root)
+        pattern = "{#{BUILD_FILES.join(",")}}"
+        Dir[File.join(root, "**", pattern)].filter_map do |path|
+          relative = path.delete_prefix("#{root}/").split("/")
+          next if relative[0..-2].any? { |part| skipped?(part) }
+
+          File.dirname(path)
+        end.uniq.sort
       end
 
       def sources_under(dir)

data/lib/audition/static/work_split.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module Audition
+  module Static
+    # How a scan is divided between Ractor workers.
+    module WorkSplit
+      # Ractors run on a fixed pool of native threads sized by
+      # RUBY_MAX_CPU, which only the environment can set. Spawning
+      # past it buys no parallelism and costs a Ractor each.
+      RACTOR_CPU_DEFAULT = 8
+
+      module_function
+
+      # One worker per core the Ractor pool can actually run. The
+      # main Ractor only waits while workers scan, so no core is
+      # held back for it.
+      #
+      # @return [Integer]
+      def workers
+        Etc.nprocessors.clamp(1, ractor_cpu_limit)
+      end
+
+      def ractor_cpu_limit
+        limit = ENV["RUBY_MAX_CPU"].to_i
+        limit.positive? ? limit : RACTOR_CPU_DEFAULT
+      end
+
+      # Longest-processing-time-first: deal the heaviest item onto
+      # the lightest worker. Consecutive files are neighbors in the
+      # tree and so alike in size, so contiguous slices come out
+      # lopsided: one worker can draw a slice weighing several times
+      # the mean and still be running once the rest have finished.
+      # Greedy is enough here: LPT finishes within 4/3 of an
+      # optimal split.
+      #
+      # @param weighted [Array<Array>] `[item, weight]` pairs
+      # @param count [Integer] worker count
+      # @return [Array<Array>] one chunk of items per busy worker
+      def chunks(weighted, count)
+        chunks = Array.new(count) { [] }
+        loads = Array.new(count, 0)
+        weighted.sort_by { |item, weight| [-weight, item] }
+          .each do |item, weight|
+            lightest = loads.index(loads.min)
+            chunks[lightest] << item
+            loads[lightest] += weight
+          end
+        chunks.reject(&:empty?)
+      end
+    end
+  end
+end

data/lib/audition/target.rb

@@ -12,6 +12,12 @@ module Audition
       vendor node_modules tmp log coverage pkg .git .bundle
     ].freeze
 
+    # Directories holding the target's tests rather than the code
+    # a production boot loads. The default; `test_dirs` in
+    # .audition.yml replaces it for a project that files them
+    # elsewhere.
+    TEST_DIRS = %w[test spec features].freeze
+
     # Cargo and Zig build output: full of .so/.dylib artifacts that
     # are not the extension `require` loads.
     BUILD_DIRS = %w[target zig-out].freeze
@@ -20,6 +26,11 @@ module Audition
     # "bundle" on macOS, "so" everywhere else, Windows included).
     COMPILED = "*.{bundle,so}"
 
+    # Generated type stubs. The extension is the convention; where
+    # the generator writes them is not, so they are found by
+    # extension anywhere in the tree.
+    STUBS = "*.rbi"
+
     # @return [Symbol] one of `:script`, `:gem`, `:rack`, `:rails`,
     #   `:directory`, `:bundle`
     attr_reader :type
@@ -34,6 +45,10 @@ module Audition
     #   shipped with the target; empty for scripts and file lists
     attr_reader :compiled_files
 
+    # @return [Array<String>] generated type stubs (.rbi) found
+    #   anywhere in the target's tree
+    attr_reader :stub_files
+
     # @return [Hash, nil] dynamic probe entry (`:mode` plus
     #   mode-specific keys), nil for static-only targets
     attr_reader :entry
@@ -75,7 +90,8 @@ module Audition
         type: :files,
         root: Dir.pwd,
         ruby_files: paths,
-        entry: nil
+        entry: nil,
+        stub_files: stubs(Dir.pwd)
       )
     end
 
@@ -96,10 +112,26 @@ module Audition
         type: :script,
         root: File.dirname(path),
         ruby_files: [path],
-        entry: {mode: :script, path: path}
+        # A file inside a Rails root cannot run standalone; the
+        # script probe would report a misleading boot failure, so
+        # such files stay static-only.
+        entry: rails_member?(path) ? nil : {mode: :script, path: path},
+        stub_files: stubs(File.dirname(path))
       )
     end
 
+    def self.rails_member?(path)
+      dir = File.dirname(File.expand_path(path))
+      until dir == (parent = File.dirname(dir))
+        return true if File.file?(
+          File.join(dir, "config", "application.rb")
+        )
+
+        dir = parent
+      end
+      false
+    end
+
     def self.from_directory(dir)
       application_rb = File.join(dir, "config", "application.rb")
       config_ru = File.join(dir, "config.ru")
@@ -117,7 +149,8 @@ module Audition
           root: dir,
           ruby_files: glob(dir),
           entry: nil,
-          compiled_files: compiled(dir)
+          compiled_files: compiled(dir),
+          stub_files: stubs(dir)
         )
       end
     end
@@ -131,16 +164,21 @@ module Audition
           glob(File.join(spec.full_gem_path, rp))
         end,
         entry: {mode: :require, feature: name,
+                load_paths: spec.full_require_paths,
                 root: spec.full_gem_path},
-        compiled_files: compiled_for(spec)
+        compiled_files: compiled_for(spec),
+        stub_files: stubs(spec.full_gem_path)
       )
     rescue Gem::MissingSpecError
       raise Error,
         "#{name} is not a file, directory, or installed gem"
     end
 
+    # An app's own Ruby is not confined to app/lib/config: local
+    # gems, engines and tests boot into the same process, so the
+    # whole root is scanned.
     def self.rails_target(dir)
-      files = %w[app lib config].flat_map { |d| glob(File.join(dir, d)) }
+      files = glob(dir)
       files << File.join(dir, "config.ru")
       new(
         type: :rails,
@@ -151,7 +189,8 @@ module Audition
           environment: File.join(dir, "config", "environment.rb"),
           root: dir
         },
-        compiled_files: compiled(dir)
+        compiled_files: compiled(dir),
+        stub_files: stubs(dir)
       )
     end
 
@@ -160,24 +199,31 @@ module Audition
         type: :rack,
         root: dir,
         ruby_files: [config_ru] + glob(dir),
-        entry: {mode: :rack, config_ru: config_ru},
-        compiled_files: compiled(dir)
+        entry: {mode: :rack, config_ru: config_ru, root: dir},
+        compiled_files: compiled(dir),
+        stub_files: stubs(dir)
       )
     end
 
+    # Reading require_paths off the gemspec would mean evaluating
+    # it, which the static pass never does; `lib` is the packaging
+    # default, and a gem that keeps its code elsewhere falls back to
+    # the whole checkout rather than scanning nothing.
     def self.gem_dir_target(dir, gemspec)
       lib = File.join(dir, "lib")
+      paths = File.directory?(lib) ? [lib] : [dir]
       new(
         type: :gem,
         root: dir,
-        ruby_files: glob(lib),
+        ruby_files: paths.flat_map { |path| glob(path) },
         entry: {
           mode: :require,
           feature: File.basename(gemspec, ".gemspec"),
-          load_paths: [lib],
+          load_paths: paths,
           root: dir
         },
-        compiled_files: compiled(dir)
+        compiled_files: compiled(dir),
+        stub_files: stubs(dir)
       )
     end
 
@@ -199,6 +245,14 @@ module Audition
       end.sort
     end
 
+    # Generated type stubs anywhere under a directory.
+    #
+    # @param dir [String]
+    # @return [Array<String>]
+    def self.stubs(dir)
+      glob(dir, STUBS)
+    end
+
     # macOS debug-symbol bundles (x.bundle.dSYM/...) carry a file
     # with the extension's name that nothing ever loads.
     def self.compiled(dir)
@@ -222,15 +276,30 @@ module Audition
 
     private_class_method :from_file, :from_directory, :from_gem_name,
       :rails_target, :rack_target, :gem_dir_target,
-      :glob, :compiled, :normalize
+      :rails_member?, :glob, :compiled, :normalize
 
     def initialize(type:, root:, ruby_files:, entry:,
-      compiled_files: [])
+      compiled_files: [], stub_files: [])
       @type = type
       @root = root
       @ruby_files = ruby_files
       @entry = entry
       @compiled_files = compiled_files
+      @stub_files = stub_files
+    end
+
+    # Whether a path is test/spec code rather than code the
+    # production boot loads.
+    #
+    # @param path [String] absolute or root-relative
+    # @param dirs [Array<String>] directory names that hold tests
+    # @return [Boolean]
+    def test_file?(path, dirs: TEST_DIRS)
+      relative = File.expand_path(path, root)
+        .delete_prefix("#{root}/")
+      parts = relative.split("/")
+      parts[0..-2].any? { |p| dirs.include?(p) } ||
+        parts.last.to_s.end_with?("_test.rb", "_spec.rb")
     end
   end
 end

data/lib/audition/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Audition
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/audition.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
 require_relative "audition/version"
+require_relative "audition/progress"
 require_relative "audition/finding"
 require_relative "audition/target"
 require_relative "audition/static/source_file"
 require_relative "audition/static/literal_classifier"
 require_relative "audition/static/checks"
 require_relative "audition/static/graph_audit"
+require_relative "audition/static/gem_calls"
 require_relative "audition/static/native_extensions"
 require_relative "audition/static/analyzer"
 require_relative "audition/dynamic/prober"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: audition
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Yaroslav Markin
@@ -43,14 +43,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: table_tennis
   requirement: !ruby/object:Gem::Requirement
@@ -102,24 +102,30 @@ files:
 - lib/audition/dynamic/prober.rb
 - lib/audition/finding.rb
 - lib/audition/fixer.rb
+- lib/audition/progress.rb
 - lib/audition/report.rb
 - lib/audition/report/github.rb
 - lib/audition/report/json.rb
 - lib/audition/report/style.rb
+- lib/audition/report/sweep.rb
 - lib/audition/report/text.rb
 - lib/audition/rewriters.rb
 - lib/audition/static/analyzer.rb
 - lib/audition/static/checks.rb
 - lib/audition/static/checks/base.rb
+- lib/audition/static/checks/dependency_class_state.rb
 - lib/audition/static/checks/global_variables.rb
 - lib/audition/static/checks/mutable_constants.rb
 - lib/audition/static/checks/ractor_isolation.rb
 - lib/audition/static/checks/runtime_require.rb
 - lib/audition/static/checks/unsafe_calls.rb
+- lib/audition/static/checks/unshareable_reads.rb
+- lib/audition/static/gem_calls.rb
 - lib/audition/static/graph_audit.rb
 - lib/audition/static/literal_classifier.rb
 - lib/audition/static/native_extensions.rb
 - lib/audition/static/source_file.rb
+- lib/audition/static/work_split.rb
 - lib/audition/target.rb
 - lib/audition/version.rb
 homepage: https://github.com/yaroslav/audition
@@ -142,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Probe Ruby code for Ractor-readiness
 test_files: []