rspec-tracer 1.2.3 → 2.0.0.pre.2

data/lib/rspec_tracer/remote_cache/archive.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'rubygems/package'
+require 'zlib'
+
+module RSpecTracer
+  # Internal RemoteCache — see {RSpecTracer} for the user-facing surface.
+  # @api private
+  module RemoteCache
+    # tar+gzip pack/extract for the remote-cache S3 payload. Pure
+    # Ruby stdlib (`rubygems/package` + `zlib`) - no shell-out, no
+    # new gem deps, works on every supported interpreter (MRI 3.1+
+    # and JRuby 9.4 both bundle both modules).
+    #
+    # Wire format: single `.tar.gz` containing `last_run.json` at the
+    # archive root plus a `<run_id>/` directory with the 15 JSON
+    # files. Replaces the 1.x per-file layout on S3 (N+1 objects per
+    # cache -> 1 object), shrinks payload ~4-6x via gzip on the highly-
+    # redundant JSON (shared example IDs across files), and cuts the
+    # per-download/upload request count from 15+ to exactly 2
+    # (cp for download, cp for upload).
+    #
+    # Local cache_path layout is UNCHANGED - the archive is pack/unpack
+    # boundary for transit only. User-facing filenames in
+    # `USER_FACING_SURFACE.md` section 6 stay as documented; external tooling
+    # that walks `rspec_tracer_cache/` sees the same 15-file layout.
+    module Archive
+      # Internal constant.
+      # @api private
+      CACHE_FILENAME = 'cache.tar.gz'
+
+      # Pack the relevant contents of `cache_path` into `dest_path`.
+      # Required: `cache_path/last_run.json` and `cache_path/<run_id>/`.
+      # Raises ArgumentError on a malformed cache; any I/O error during
+      # pack propagates to the caller (S3Backend wraps in S3BackendError
+      # + rescues at the orchestrator boundary for graceful degradation).
+      def self.pack(cache_path:, run_id:, dest_path:)
+        validate_pack_args!(cache_path, run_id, dest_path)
+        last_run, run_dir = resolve_pack_sources!(cache_path, run_id)
+
+        File.open(dest_path, 'wb') do |file|
+          Zlib::GzipWriter.wrap(file) do |gz|
+            Gem::Package::TarWriter.new(gz) do |tar|
+              add_file(tar, last_run, 'last_run.json')
+              Dir[File.join(run_dir, '*.json')].each do |path|
+                add_file(tar, path, File.join(run_id, File.basename(path)))
+              end
+            end
+          end
+        end
+        dest_path
+      end
+
+      # Internal helper for the tracer pipeline.
+      # @api private
+      def self.validate_pack_args!(cache_path, run_id, dest_path)
+        raise ArgumentError, 'cache_path is required' if cache_path.nil? || cache_path.empty?
+        raise ArgumentError, 'run_id is required' if run_id.nil? || run_id.empty?
+        raise ArgumentError, 'dest_path is required' if dest_path.nil? || dest_path.empty?
+      end
+      private_class_method :validate_pack_args!
+
+      # Internal helper for the tracer pipeline.
+      # @api private
+      def self.resolve_pack_sources!(cache_path, run_id)
+        last_run = File.join(cache_path, 'last_run.json')
+        raise ArgumentError, "missing last_run.json at #{last_run}" unless File.file?(last_run)
+
+        run_dir = File.join(cache_path, run_id)
+        raise ArgumentError, "missing run dir at #{run_dir}" unless File.directory?(run_dir)
+
+        [last_run, run_dir]
+      end
+      private_class_method :resolve_pack_sources!
+
+      # Extract `archive_path` into `dest_dir`. Overwrites existing
+      # files (run-dir already present gets replaced). Raises on a
+      # malformed archive; caller rescues.
+      def self.extract(archive_path:, dest_dir:)
+        raise ArgumentError, 'archive_path is required' if archive_path.nil? || archive_path.empty?
+        raise ArgumentError, 'dest_dir is required' if dest_dir.nil? || dest_dir.empty?
+        raise ArgumentError, "missing archive at #{archive_path}" unless File.file?(archive_path)
+
+        FileUtils.mkdir_p(dest_dir)
+        File.open(archive_path, 'rb') do |file|
+          Zlib::GzipReader.wrap(file) do |gz|
+            Gem::Package::TarReader.new(gz) do |tar|
+              tar.each { |entry| write_entry(entry, dest_dir) }
+            end
+          end
+        end
+        dest_dir
+      end
+
+      # `add_file_simple` (not `add_file`) because Zlib::GzipWriter is
+      # non-seekable; the 2-arg `add_file` needs to back-patch the size
+      # into the tar header after writing content, which requires
+      # `io.pos=`. `add_file_simple` takes the size upfront, writes the
+      # header, then streams content - compatible with gzip.
+      def self.add_file(tar, local_path, archive_name)
+        stat = File.stat(local_path)
+        tar.add_file_simple(archive_name, stat.mode & 0o777, stat.size) do |io|
+          File.open(local_path, 'rb') { |src| IO.copy_stream(src, io) }
+        end
+      end
+      private_class_method :add_file
+
+      # Internal helper for the tracer pipeline.
+      # @api private
+      def self.write_entry(entry, dest_dir)
+        return unless entry.file?
+
+        safe_name = safe_entry_name(entry.full_name)
+        return if safe_name.nil?
+
+        dest = File.join(dest_dir, safe_name)
+        FileUtils.mkdir_p(File.dirname(dest))
+        File.open(dest, 'wb') { |out| IO.copy_stream(entry, out) }
+      end
+      private_class_method :write_entry
+
+      # Refuse absolute paths or `..` traversal. Both are illegal in a
+      # well-formed cache entry name; silently dropping them beats
+      # trusting a remote-sourced value to write anywhere on disk.
+      # Public because RedisBackend reuses the same guard on hash field
+      # names (the Redis equivalent of tar entry names).
+      def self.safe_entry_name(name)
+        return nil if name.nil? || name.empty?
+        return nil if name.start_with?('/')
+        return nil if name.split('/').include?('..')
+
+        name
+      end
+    end
+  end
+end

data/lib/rspec_tracer/remote_cache/backend.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module RSpecTracer
+  # Internal RemoteCache — see {RSpecTracer} for the user-facing surface.
+  # @api private
+  module RemoteCache
+    # Protocol every remote-cache backend must satisfy. S3Backend,
+    # LocalFsBackend, and RedisBackend all implement it. The shared-
+    # examples group in `spec/contracts/remote_cache_backend.rb` asserts
+    # the full contract on every implementation.
+    #
+    # Tier routing lives inside each backend, not in the protocol. A
+    # backend is constructed with branch + default_branch + test_suite_id
+    # state, and routes uploads/downloads to either the main tier
+    # (`main/<sha>/...`) or the pr tier (`pr/<branch>/<sha>/...`) based
+    # on whether the current branch is the default.
+    #
+    # Graceful-degradation contract (same as Storage::Backend):
+    #   - `download(ref)` returns true/false, never raises on wire or
+    #     validation failures. A failed download is "cold run" from the
+    #     orchestrator's perspective.
+    #   - `upload(ref)` raises on wire failure so the Rake task can
+    #     report a meaningful exit status, but the orchestrator wraps
+    #     the call in a rescue so test runs never propagate non-zero.
+    #   - `branch_refs(branch_name)` returns `{}` when no refs file
+    #     exists. Missing is not an error.
+    #   - `write_branch_refs(branch_name, refs)` is a no-op for main
+    #     tier writes (main branches do not use branch_refs; history
+    #     rewrites are not expected on the default branch).
+    #   - `prune!(...)` returns the count of refs removed and never
+    #     raises on a LIST / DELETE wire error (logs + returns what it
+    #     managed to delete). Scoped to the backend's own tier.
+    #   - `prune_all!(pr_branch_ttl_seconds:)` walks every PR branch
+    #     under the configured prefix, applies the ttl to each, and
+    #     deletes dead branches whole. Cross-tier (every pr branch),
+    #     unlike `prune!` which is own-tier only. Returns the count of
+    #     refs removed across all branches. Never raises. No-op when
+    #     ttl is nil / non-positive.
+    #
+    # This module is intentionally documentation-only - it does not
+    # define stubs that raise NotImplementedError, because mutant would
+    # flag every `raise` as an alive mutation with no way to kill it.
+    # The shared-examples contract is the real gate.
+    #
+    # @example Registering a custom remote-cache backend
+    #   RSpecTracer.configure do
+    #     remote_cache_backend MyCustomBackend, custom_opt: 'value'
+    #   end
+    module Backend
+      # Internal constant.
+      # @api private
+      REQUIRED_METHODS = %i[
+        download
+        upload
+        branch_refs
+        write_branch_refs
+        prune!
+        prune_all!
+      ].freeze
+
+      # Verifies a candidate object satisfies the backend protocol.
+      # Used by {RSpecTracer::Configuration#remote_cache_backend} to
+      # gate custom-backend registration.
+      #
+      # @param backend [Object] candidate backend instance
+      # @return [Boolean] true if every {REQUIRED_METHODS} entry is
+      #   responded to
+      def self.conforms?(backend)
+        REQUIRED_METHODS.all? { |m| backend.respond_to?(m) }
+      end
+    end
+  end
+end

data/lib/rspec_tracer/remote_cache/git_ancestry.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require 'English'
+require 'set'
+
+module RSpecTracer
+  # Internal RemoteCache — see {RSpecTracer} for the user-facing surface.
+  # @api private
+  module RemoteCache
+    # Git ancestry walker. Given the current branch + default branch,
+    # answers "which commit should I upload cache under?" and "which
+    # prior commits should I try as cache candidates?"
+    #
+    # Behavior preserved verbatim from 1.x `RemoteCache::Repo` -
+    # USER_FACING_SURFACE.md pins the 25-commit ancestry, branch_refs,
+    # history-rewrite resilience, merge-commit handling, and shallow-
+    # clone guidance as user contracts. Any change here affects every
+    # existing CI config that sets `fetch-depth: 25` or relies on the
+    # current "nearest ancestor" heuristic.
+    #
+    # Four scenarios this class handles:
+    #
+    #   1. Main branch build (GIT_BRANCH == GIT_DEFAULT_BRANCH):
+    #      merge_base_branch! is a no-op. branch_ref = HEAD (or HEAD^1
+    #      if HEAD is itself a merge commit, e.g. main just absorbed a
+    #      feature branch via --no-ff merge). Ancestry walks HEAD^'s
+    #      linear history up to 25 commits.
+    #
+    #   2. GitHub Actions PR (checkout of refs/pull/N/merge, detached
+    #      HEAD on a synthetic merge commit):
+    #      merge_base_branch! runs `git fetch origin <branch>:<branch>`
+    #      + `git checkout <branch>` + `git merge origin/<default>
+    #      --no-edit --no-ff`. After this HEAD is a (possibly new) merge
+    #      commit. branch_ref = HEAD^1 (the PR branch tip). Ancestry
+    #      walks HEAD^1's 25-commit history UNION `HEAD^1..origin/HEAD`
+    #      (default branch commits the PR hasn't absorbed yet).
+    #
+    #   3. Jenkins / CircleCI / Travis PR (checkout of raw PR branch):
+    #      merge_base_branch! materializes the merge commit that GHA
+    #      provides automatically. Result is identical to scenario 2.
+    #
+    #   4. PR branch behind main: same flow as 2/3. The explicit merge
+    #      ensures RSpec runs against the merged state and ancestry
+    #      picks up main's recent commits as candidates.
+    #
+    # The merge is a WORKING-TREE MUTATION. Callers should run
+    # `merge_base_branch!` exactly once at the start of each Rake task
+    # invocation (download + upload each instantiate a fresh orchestrator
+    # and re-run the merge; idempotent on a clean tree). See
+    # RSPEC_TRACER.md "Caching on CI" for the user-facing rationale.
+    class GitAncestry
+      # Internal GitAncestryError — see {RSpecTracer} for the user-facing surface.
+      # @api private
+      class GitAncestryError < StandardError; end
+
+      # Matches 1.x. 25 commits is a 14-year-stable tradeoff between
+      # cache hit rate on slow trunks and ancestry-walk cost.
+      ANCESTRY_DEPTH = 25
+
+      # Internal attribute.
+      # @api private
+      attr_reader :default_branch_name, :branch_name
+
+      # Internal method on the tracer pipeline.
+      # @api private
+      def initialize(default_branch:, branch:)
+        raise GitAncestryError, 'default_branch is required' if default_branch.nil? || default_branch.to_s.empty?
+        raise GitAncestryError, 'branch is required' if branch.nil? || branch.to_s.empty?
+
+        @default_branch_name = default_branch.to_s.chomp
+        @branch_name = branch.to_s.chomp
+      end
+
+      # True when this is a PR build (current branch != default branch).
+      def pr_build?
+        @branch_name != @default_branch_name
+      end
+
+      # True when HEAD has two parents (i.e., a merge commit). Cached.
+      def merge_commit?
+        return @merge_commit if defined?(@merge_commit)
+
+        @merge_commit = system('git', 'rev-parse', 'HEAD^2', out: File::NULL, err: File::NULL)
+      end
+
+      # Materialize the PR-merged state: fetch + checkout + merge default.
+      # No-op on main builds. Idempotent on a clean tree (a subsequent
+      # `git merge origin/<default>` when already merged produces "Already
+      # up to date", no new merge commit).
+      #
+      # Raises GitAncestryError on fetch/checkout/merge failure. The
+      # orchestrator catches and logs; the user's Rake task exits
+      # cleanly with a warning but proceeds to cold-run.
+      def merge_base_branch!
+        return if @default_branch_name == @branch_name
+
+        pull_remote_branch! if current_branch != @branch_name
+        merge_default_branch!
+        reset_memo!
+      end
+
+      # The SHA to upload cache under, and to seed ancestry walk from.
+      # When HEAD is a merge commit (normal PR case after merge_base_branch!
+      # runs), branch_ref = HEAD^1 (the real branch tip). Uploading under
+      # HEAD directly would key the cache under a synthetic merge SHA
+      # that no future build's ancestry walk can reach.
+      def branch_ref
+        return @branch_ref if defined?(@branch_ref)
+
+        head = head_ref
+        if merge_commit?
+          parents = merged_parents
+          @branch_ref = parents.first
+          @ignored_refs = [head]
+        else
+          @branch_ref = head
+          @ignored_refs = []
+        end
+        @branch_ref
+      end
+
+      # Hash{sha => committer_timestamp} of candidate ancestor commits.
+      # Newest-first ordering is applied by the orchestrator when merging
+      # with branch_refs; this method returns insertion order.
+      #
+      # Walk rule:
+      #   refs = Set[]
+      #   if merge_commit?: refs |= rev-list --max-count=25 branch_ref..origin/HEAD
+      #   refs |= rev-list --max-count=25 branch_ref
+      #   refs -= ignored_refs  # drop synthetic HEAD when merged
+      #
+      # Returns {} when the walk yields nothing (new repo, shallow
+      # clone shorter than 25, etc.). Not an error.
+      def ancestry_refs
+        return @ancestry_refs if defined?(@ancestry_refs)
+
+        branch_ref # materialize ignored_refs
+
+        ref_list = Set.new
+        ref_list |= rev_list("#{@branch_ref}..origin/HEAD") if merge_commit?
+        ref_list |= rev_list(@branch_ref.to_s)
+
+        @ancestry_refs = refs_committer_timestamp(ref_list - @ignored_refs)
+      end
+
+      private
+
+      # Internal method on the tracer pipeline.
+      # @api private
+      def reset_memo!
+        remove_instance_variable(:@merge_commit) if defined?(@merge_commit)
+        remove_instance_variable(:@branch_ref) if defined?(@branch_ref)
+        remove_instance_variable(:@ignored_refs) if defined?(@ignored_refs)
+        remove_instance_variable(:@ancestry_refs) if defined?(@ancestry_refs)
+      end
+
+      # Internal method on the tracer pipeline.
+      # @api private
+      def current_branch
+        branch = `git rev-parse --abbrev-ref HEAD`.chomp
+        raise GitAncestryError, 'Could not determine current branch' unless $CHILD_STATUS.success?
+
+        branch
+      end
+
+      # Internal method on the tracer pipeline.
+      # @api private
+      def pull_remote_branch!
+        fetched = system(
+          'git', 'fetch', 'origin', "#{@branch_name}:#{@branch_name}",
+          out: File::NULL, err: File::NULL
+        )
+        checked_out = fetched && system(
+          'git', 'checkout', @branch_name,
+          out: File::NULL, err: File::NULL
+        )
+        raise GitAncestryError, "Could not pull remote branch #{@branch_name}" unless checked_out
+      end
+
+      # Internal method on the tracer pipeline.
+      # @api private
+      def merge_default_branch!
+        merged = system(
+          'git', 'merge', "origin/#{@default_branch_name}",
+          '--no-edit', '--no-ff',
+          out: File::NULL, err: File::NULL
+        )
+        raise GitAncestryError, "Could not merge #{@default_branch_name} into #{@branch_name}" unless merged
+      end
+
+      # Internal method on the tracer pipeline.
+      # @api private
+      def head_ref
+        head = `git rev-parse HEAD`.chomp
+        raise GitAncestryError, 'Could not find HEAD commit sha' unless $CHILD_STATUS.success?
+
+        head
+      end
+
+      # Internal method on the tracer pipeline.
+      # @api private
+      def merged_parents
+        parents = []
+        first_parent = `git rev-parse HEAD^1`.chomp
+        parents << first_parent if $CHILD_STATUS.success?
+        second_parent = `git rev-parse HEAD^2`.chomp
+        parents << second_parent if $CHILD_STATUS.success?
+        raise GitAncestryError, 'Could not find merge commit parents' if parents.length != 2
+
+        parents
+      end
+
+      # Internal method on the tracer pipeline.
+      # @api private
+      def rev_list(spec)
+        output = `git rev-list --max-count=#{ANCESTRY_DEPTH} #{spec}`.chomp
+        raise GitAncestryError, "Could not list revs for #{spec}" unless $CHILD_STATUS.success?
+
+        output.split
+      end
+
+      # Internal method on the tracer pipeline.
+      # @api private
+      def refs_committer_timestamp(ref_list)
+        return {} if ref_list.empty?
+
+        command = <<-COMMAND.strip.gsub(/\s+/, ' ')
+          git show
+            --no-patch
+            --format="%H %ct"
+            #{ref_list.to_a.join(' ')}
+        COMMAND
+
+        output = `#{command}`.chomp
+        raise GitAncestryError, 'Could not fetch committer timestamps' unless $CHILD_STATUS.success?
+
+        output.split("\n").to_h(&:split).transform_values(&:to_i)
+      end
+    end
+  end
+end