fast_cov 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d5eb20089cfe981168592cd5672e1919746a5186c61f151aff87eea7319eb1c
-  data.tar.gz: af45acb129fd680beaa1a906248ad1aedf03e10a5d40d02fbeaa0971e9112b44
+  metadata.gz: 1b276767c3b77601bb2cb06febd1476fcab8c5f94fc5b714842d0ddcce6a0fb9
+  data.tar.gz: 44bd1a0b268802dc8c7b6e5687bf52fcb969849191fdb0d5471cab87350e43cc
 SHA512:
-  metadata.gz: 97cdc30f83fb1970d97d6cddc04758463043a85ee81ccd5a22011e0b017a04d8b9b82be90931b42bdf5565ef163ac84cc1544ec43ecae2cc1727041770adfc52
-  data.tar.gz: baa28e4ae365439250404b285eb0577a167de1cb907a3c37b33f09d3a99bf53bf4e6ca0ac6fd2ff84360a048f2795de7ca709ffd11c1236b0846ae66a4f4a75a
+  metadata.gz: 178d1061ae15c475719b2da16297df134be25798a6bc22ec81aece28a180d7db2e5914296fc54596d3ea858b57538be177dc96f203d2625004c2f369078b6d44
+  data.tar.gz: abdf03b462db505b675d29b332ddc933a6af415cef35a474608d9d8330993b52f294b84923a00dd7c9371bcd7311b3e1518e1ab9772088fbb529740ff74fbfbc

data/README.md

@@ -6,7 +6,7 @@ FastCov hooks directly into the Ruby VM's event system, avoiding the overhead of
 
 ## Requirements
 
-- Ruby >= 3.4.0 (MRI only)
+- Ruby >= 3.2.0 (MRI only)
 - macOS or Linux
 
 ## Installation
@@ -32,7 +32,7 @@ coverage = FastCov::CoverageMap.new
 coverage.root = File.expand_path("app")
 coverage.use(FastCov::FileTracker)
 
-result = coverage.start do
+result = coverage.build do
   # ... run a test ...
 end
 
@@ -53,6 +53,8 @@ coverage.ignored_paths = Rails.root.join("vendor")
 
 coverage.use(FastCov::FileTracker)
 coverage.use(FastCov::FactoryBotTracker)
+coverage.use(FastCov::ConstGetTracker)
+coverage.use(FastCov::FixtureKitTracker)
 ```
 
 ### Options
@@ -66,9 +68,13 @@ coverage.use(FastCov::FactoryBotTracker)
 ### Lifecycle
 
 ```ruby
-coverage.start        # starts tracking and returns the CoverageMap
-coverage.stop         # stops tracking and returns a Set
-coverage.start { ... } # block form: start, yield, stop
+coverage.start         # starts tracking, returns self
+result = coverage.stop # stops tracking, returns a Set
+
+# Block form: start, yield, stop
+result = coverage.build do
+  # ...
+end
 ```
 
 Native line coverage is always enabled. Extra trackers registered with `use` are additive.
@@ -77,7 +83,11 @@ Native line coverage is always enabled. Extra trackers registered with `use` are
 
 ### FileTracker
 
-Tracks files read from disk during coverage, including JSON, YAML, ERB templates, and any file accessed via `File.read` or read-mode `File.open`.
+Tracks files read from disk during coverage, including YAML, JSON, ERB templates, and any file accessed via `File.read`, read-mode `File.open`, `YAML.load_file`, `YAML.safe_load_file`, or `YAML.unsafe_load_file`.
+
+The YAML methods are patched directly to handle Bootsnap's compile cache, which bypasses `File.open` for YAML files.
+
+When a file is read indirectly (e.g., `YAML.load_file` calling through Psych), the tracker walks the caller stack to find the first in-root frame and creates a connected dependency.
 
 ```ruby
 coverage.use(FastCov::FileTracker)
@@ -107,62 +117,112 @@ This catches patterns such as:
 
 It does not catch direct constant references such as `Foo::Bar` in source code.
 
-## Low-level native coverage
+### FixtureKitTracker
 
-`FastCov::Coverage` is still available as a low-level primitive:
+Tracks [fixture_kit](https://github.com/Gusto/fixture_kit) fixture definition files when fixtures are used. Requires fixture_kit >= 0.14.0.
+
+Fixture definitions run once during cache generation (`before(:context)`), then every test replays cached SQL without executing Ruby. This tracker uses fixture_kit's callback hooks to:
+
+1. Track files touched during fixture generation and create connected dependencies
+2. Record fixture definition files (including parent chain) when tests mount fixtures
 
 ```ruby
-cov = FastCov::Coverage.new(
-  root: "/repo/app",
-  ignored_paths: ["/repo/app/vendor"],
-  threads: true
-)
+coverage.use(FastCov::FixtureKitTracker)
 ```
 
-This API is mainly useful for internal use and low-level tests. `CoverageMap` is the intended public orchestration API.
-
 ## StaticMap
 
-`FastCov::StaticMap` is a build-time API for static dependency mapping. It parses Ruby files with Prism, resolves literal constant references, and builds a direct dependency graph. Transitive closures are computed lazily on demand.
+`FastCov::StaticMap` is a build-time API for static dependency mapping. It parses Ruby files with Prism, resolves literal constant references, and builds a dependency graph. Transitive closures are computed lazily on demand.
 
 ```ruby
 static_map = FastCov::StaticMap.new(root: Rails.root)
 static_map.build("spec/**/*_spec.rb")
 
 # Direct dependencies for a single file
-static_map.dependencies("/app/spec/models/user_spec.rb")
-# => ["/app/app/models/user.rb"]
+static_map.direct_dependencies("spec/models/user_spec.rb")
+# => ["app/models/user.rb"]
 
-# Transitive closure (computed and cached on first call)
-static_map.transitive_dependencies("/app/spec/models/user_spec.rb")
-# => ["/app/app/models/account.rb", "/app/app/models/user.rb"]
-
-# Raw direct graph
-static_map.direct_graph
-# => { "/app/spec/models/user_spec.rb" => ["/app/app/models/user.rb"], ... }
+# Transitive dependencies (computed and cached on first call)
+static_map.dependencies("spec/models/user_spec.rb")
+# => ["app/models/user.rb", "app/models/account.rb"]
 ```
 
 The instance caches constant resolution results, so reusing the same instance across multiple `build` calls is efficient.
 
-#### Options
+### Options
 
 | Option | Type | Default | Description |
 |---|---|---|---|
 | `root` | String or Pathname | required | Absolute project root. Only resolved files under this path are included. |
-| `ignored_paths` | String or Array<String> | `[]` | Files or directories to exclude from the graph and recursive traversal. |
-| `*patterns` (on `build`) | String(s) or Array | required | File paths or globs to traverse. Relative paths are expanded against `root`. |
+| `ignored_paths` | String or Array | `[]` | Files or directories to exclude from the graph and recursive traversal. |
+| `concurrency` | Integer | `Etc.nprocessors` | Number of threads for parallel file parsing. |
 
-#### How it works
+### How it works
 
-- `build` traverses reachable files and stores a direct dependency graph
-- `dependencies` returns direct dependencies for a file
-- `transitive_dependencies` computes and caches the transitive closure lazily
+- `build(*patterns)` traverses reachable files and stores a direct dependency graph
+- `direct_dependencies(file)` returns direct dependencies for a file
+- `dependencies(file)` computes and caches the transitive closure lazily
 - Constant resolution results are cached and reused across `build` calls
 - Resolves each reference from most-specific lexical candidate to least-specific
 - Uses `const_defined?` and `const_source_location` to resolve literal constant references to source files
 
 This is intended for a booted application process. It requires constants to be eager-loaded. It will not see dynamic constant lookups that are not expressed as literal constants in the source.
 
+## TestMap
+
+`FastCov::TestMap` handles test mapping serialization and aggregation. It accumulates mappings from test runs, writes gzipped fragment files, and merges fragments from multiple CI nodes.
+
+### Accumulating mappings
+
+```ruby
+test_map = FastCov::TestMap.new
+
+# Record which files each test depends on
+test_map.add("spec/models/" => coverage_map.stop)
+
+# Query: which tests cover this file?
+test_map.dependencies("app/models/user.rb")
+# => ["spec/models/"]
+
+# Write gzipped fragment for later aggregation
+test_map.dump("tmp/test_mapping.node_0.gz")
+```
+
+### Aggregating fragments
+
+Merge fragments from multiple CI nodes via k-way merge:
+
+```ruby
+aggregator = FastCov::TestMap.aggregate(Dir["tmp/test_mapping.*.gz"])
+
+# Hook into progress events
+aggregator.on(:sort) { |fragments, batches| puts "#{fragments} fragments -> #{batches} batches" }
+aggregator.on(:sorted) { |elapsed| puts "Sorted in #{elapsed.round(2)}s" }
+aggregator.on(:merge) { |processed, total| print "#{processed}/#{total}\r" }
+aggregator.on(:merged) { |files, elapsed| puts "Merged #{files} files in #{elapsed.round(2)}s" }
+
+# Iterate in batches — yields Hash of { file => [deps] }
+aggregator.each(10_000) do |batch|
+  database.bulk_write(batch)
+end
+```
+
+### Options
+
+| Option | Type | Default | Description |
+|---|---|---|---|
+| `readers:` | Integer | `min(100, ulimit/2)` | Max concurrent readers for k-way merge. Auto-detected from OS file descriptor limit. |
+
+### Fragment format
+
+Tab-delimited, gzipped. One line per source file, first column is the file, remaining columns are dependencies:
+
+```
+source_file\tdep1\tdep2\tdep3
+```
+
+Aggregation owns sorting — fragments are unsorted, intermediates are sorted during the merge process using pure Ruby (no shell commands).
+
 ## Writing custom trackers
 
 There are two approaches: a minimal custom tracker, or inheriting from `AbstractTracker`.
@@ -200,6 +260,7 @@ end
 - thread-aware recording
 - lifecycle management
 - class-level `record` dispatch for patched hooks
+- caller stack traversal via `Utils.resolve_caller` for indirect calls
 
 ```ruby
 class MyTracker < FastCov::AbstractTracker
@@ -209,7 +270,8 @@ class MyTracker < FastCov::AbstractTracker
 
   module MyPatch
     def some_method(...)
-      MyTracker.record { some_file_path }
+      # record(path) auto-resolves the caller via stack traversal
+      MyTracker.record(some_file_path)
       super
     end
   end

data/lib/fast_cov/test_map/aggregator.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "zlib"
+
+module FastCov
+  class TestMap
+    # Handles k-way merge of sorted fragment files.
+    # Created by TestMap.aggregate, not instantiated directly.
+    #
+    # Usage:
+    #   aggregator = FastCov::TestMap.aggregate(Dir["tmp/test_mapping.*.gz"])
+    #   aggregator.on(:sorted) { |elapsed| puts "Sorted in #{elapsed.round(2)}s" }
+    #   aggregator.on(:merged) { |files, elapsed| puts "Merged #{files} files in #{elapsed.round(2)}s" }
+    #   aggregator.each(10_000) { |batch| database.bulk_write(batch) }
+    class Aggregator
+      def initialize(fragment_paths, max_readers)
+        @fragment_paths = fragment_paths
+        @max_readers = max_readers
+        @hooks = {}
+      end
+
+      # Register a callback for an aggregation event.
+      #
+      # Events:
+      #   :sort    — before sorting. Yields (fragment_count, batch_count)
+      #   :sorted  — after sorting. Yields (elapsed)
+      #   :merge   — during merge. Yields (processed_lines, total_lines)
+      #   :merged  — after merging. Yields (file_count, elapsed)
+      def on(event, &block)
+        @hooks[event] = block
+        self
+      end
+
+      # Iterate over merged results.
+      # Yields a Hash of { file => dependencies } per batch.
+      # Default batch_size is 1.
+      def each(batch_size = 1, &block)
+        raise ArgumentError, "each requires a block" unless block
+        return if @fragment_paths.empty?
+
+        Dir.mktmpdir("fastcov") do |tmpdir|
+          intermediates, total_lines = create_intermediates(tmpdir)
+          readers = intermediates.map { |f| Reader.new(f) }
+          kway_merge(readers, batch_size, total_lines, &block)
+        ensure
+          readers&.each(&:close)
+        end
+      end
+
+      private
+
+      def emit(event, *args)
+        @hooks[event]&.call(*args)
+      end
+
+      def measure
+        t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        result = yield
+        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0
+        [result, elapsed]
+      end
+
+      def create_intermediates(intermediates_dir)
+        batch_size = (@fragment_paths.size.to_f / @max_readers).ceil
+        batches = @fragment_paths.each_slice(batch_size).to_a
+
+        emit(:sort, @fragment_paths.size, batches.size)
+
+        total_lines = 0
+        intermediates, elapsed = measure do
+          batches.each_with_index.map do |batch, i|
+            intermediate = File.join(intermediates_dir, "intermediate_#{i}.txt")
+            lines = batch.flat_map { |f| Zlib::GzipReader.open(f) { |gz| gz.readlines } }
+            total_lines += lines.size
+            lines.sort!
+            File.write(intermediate, lines.join)
+            intermediate
+          end
+        end
+
+        emit(:sorted, elapsed)
+        [intermediates, total_lines]
+      end
+
+      def kway_merge(readers, batch_size, total_lines, &block)
+        unique_files = 0
+        processed_lines = 0
+        batch = {}
+
+        _, elapsed = measure do
+          loop do
+            active = readers.reject(&:exhausted?)
+            break if active.empty?
+
+            min_path = active.map(&:file_path).min
+
+            merged = Set.new
+            active.each do |reader|
+              if reader.file_path == min_path
+                processed_lines += 1
+                merged.merge(reader.dependencies)
+                reader.advance
+              end
+            end
+
+            emit(:merge, processed_lines, total_lines)
+            unique_files += 1
+
+            batch[min_path] = merged.to_a.sort
+            if batch.size >= batch_size
+              block.call(batch)
+              batch = {}
+            end
+          end
+        end
+
+        block.call(batch) unless batch.empty?
+
+        emit(:merged, unique_files, elapsed)
+      end
+    end
+  end
+end

data/lib/fast_cov/test_map/reader.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module FastCov
+  class TestMap
+    # Reads a single sorted fragment file (gzipped or plain text).
+    # Merges consecutive entries with the same file path, which occur when
+    # multiple fragments are concatenated and sorted into an intermediate.
+    class Reader
+      attr_reader :file_path, :dependencies
+
+      def initialize(path)
+        @io = path.to_s.end_with?(".gz") ? Zlib::GzipReader.open(path) : File.open(path)
+        @exhausted = false
+        @next_file_path = nil
+        @next_dependencies = nil
+        read_line
+        advance
+      end
+
+      def exhausted?
+        @exhausted
+      end
+
+      def advance
+        if @next_file_path.nil?
+          @exhausted = true
+          @file_path = nil
+          @dependencies = []
+          return
+        end
+
+        @file_path = @next_file_path
+        @dependencies = @next_dependencies
+        read_line
+
+        # Merge consecutive lines with the same file path
+        while @next_file_path == @file_path
+          @dependencies.concat(@next_dependencies)
+          read_line
+        end
+      end
+
+      def close
+        @io.close
+      end
+
+      private
+
+      def read_line
+        line = @io.gets
+        if line.nil?
+          @next_file_path = nil
+          @next_dependencies = nil
+          return
+        end
+
+        parts = line.chomp.split("\t")
+        @next_file_path = parts[0]
+        @next_dependencies = parts[1..] || []
+      end
+    end
+  end
+end

data/lib/fast_cov/test_map.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "zlib"
+
+module FastCov
+  # In-memory test mapping that records which files each test depends on.
+  # Can be dumped to a gzipped TSV fragment file for later aggregation.
+  #
+  # Usage:
+  #   # Accumulate mappings (e.g., in an RSpec formatter)
+  #   test_map = FastCov::TestMap.new
+  #   test_map.add("spec/models/user_spec.rb" => coverage_result)
+  #   test_map.dump("tmp/test_mapping.node_0.gz")
+  #
+  #   # Query mappings
+  #   test_map.dependencies("app/models/user.rb")
+  #   # => ["spec/models/user_spec.rb"]
+  #
+  #   # Aggregate fragments from multiple nodes
+  #   aggregator = FastCov::TestMap.aggregate(Dir["tmp/test_mapping.*.gz"])
+  #   aggregator.on(:sorted) { |elapsed| puts "Sorted in #{elapsed.round(2)}s" }
+  #   aggregator.on(:merged) { |files, elapsed| puts "Merged #{files} files in #{elapsed.round(2)}s" }
+  #   aggregator.each(10_000) { |batch| database.bulk_write(batch) }
+  class TestMap
+    autoload :Aggregator, File.expand_path("test_map/aggregator", __dir__)
+    autoload :Reader, File.expand_path("test_map/reader", __dir__)
+
+    DEFAULT_MAX_READERS = [100, Process.getrlimit(Process::RLIMIT_NOFILE).first / 2].min
+
+    def initialize
+      @mapping = {}
+    end
+
+    # Record test -> dependency mappings.
+    # Accepts a Hash of { test_path => dependencies }.
+    def add(mappings)
+      mappings.each do |test_path, deps|
+        deps.each do |dep|
+          next if dep == test_path
+
+          (@mapping[dep] ||= Set.new) << test_path
+        end
+      end
+    end
+
+    # Returns the test paths that depend on the given file.
+    def dependencies(file)
+      @mapping[file]&.to_a
+    end
+
+    # Write the accumulated mappings as a gzipped TSV fragment.
+    def dump(path)
+      FileUtils.mkdir_p(File.dirname(path))
+
+      lines = @mapping.map { |file, deps| "#{file}\t#{deps.to_a.join("\t")}\n" }
+      Zlib::GzipWriter.open(path) { |gz| gz.write(lines.join) }
+    end
+
+    # Number of unique source files mapped.
+    def size
+      @mapping.size
+    end
+
+    # Create an Aggregator for merging fragment files.
+    # Accepts file paths or glob patterns.
+    def self.aggregate(*patterns, readers: DEFAULT_MAX_READERS)
+      fragment_paths = patterns.flatten.flat_map { |p| p.include?("*") ? Dir.glob(p).sort : p }
+      Aggregator.new(fragment_paths, readers)
+    end
+  end
+end

data/lib/fast_cov/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FastCov
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

data/lib/fast_cov.rb

@@ -13,4 +13,5 @@ module FastCov
   autoload :ConstGetTracker, File.expand_path("fast_cov/trackers/const_get_tracker", __dir__)
   autoload :FixtureKitTracker, File.expand_path("fast_cov/trackers/fixture_kit_tracker", __dir__)
   autoload :StaticMap, File.expand_path("fast_cov/static_map", __dir__)
+  autoload :TestMap, File.expand_path("fast_cov/test_map", __dir__)
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fast_cov
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Ngan Pham
@@ -115,6 +115,9 @@ files:
 - lib/fast_cov/dev.rb
 - lib/fast_cov/static_map.rb
 - lib/fast_cov/static_map/reference_extractor.rb
+- lib/fast_cov/test_map.rb
+- lib/fast_cov/test_map/aggregator.rb
+- lib/fast_cov/test_map/reader.rb
 - lib/fast_cov/trackers/abstract_tracker.rb
 - lib/fast_cov/trackers/const_get_tracker.rb
 - lib/fast_cov/trackers/factory_bot_tracker.rb