saboteur 0.8.0 → 0.9.0

data/lib/saboteur/engine/coverage_inputs_snapshot.rb

@@ -0,0 +1,146 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "securerandom"
+require_relative "../text"
+
+module Saboteur
+  class Engine
+    # The inputs a coverage map was built from, saved beside it as
+    # inputs-<map key>.json: relative path => content digest, plus the
+    # settings digest, the runtime identity (RunFingerprint.runtime_identity
+    # and the map format) and the project root.
+    #
+    # Diagnostic only (#46). A restored cache is untrusted data, so a
+    # snapshot is accepted only when everything checks: size and entry caps,
+    # an exact key set, every field's type and shape, the map key in the file
+    # name, the project root, and the map file next to it with the digest
+    # recorded when the pair was written. Anything else reads as no snapshot.
+    # It still must not be the authority for reusing part of a map: it
+    # describes the inputs, not what a test read from them.
+    #
+    # The pair is written map first, snapshot last, through a temp file and a
+    # rename, so a snapshot never names a map that is not on disk.
+    module CoverageInputsSnapshot
+      FORMAT = 2
+      PREFIX = "inputs-"
+      MAX_BYTES = 8 * 1024 * 1024
+      MAX_FILES = 100_000
+      MAX_PATH = 4096
+      MAX_CANDIDATES = 20
+      HEX = /\A[0-9a-f]{64}\z/
+      MAX_SEED = (2**32) - 1
+      KEYS = %w[files format map_digest map_key root runtime seed settings].freeze
+
+      module_function
+
+      def path(cache_dir, map_key)
+        File.join(cache_dir, "#{PREFIX}#{map_key}.json")
+      end
+
+      def map_path(cache_dir, map_key)
+        File.join(cache_dir, "coverage-#{map_key}.json")
+      end
+
+      # The snapshot of the inputs now; map_digest is filled in by #write. The
+      # seed is the one the map's spec run used: a partial rerun must reuse it.
+      def build(root:, files:, settings:, map_key:, seed:)
+        { "format" => FORMAT, "map_key" => map_key, "map_digest" => nil, "root" => digest_of(root),
+          "runtime" => runtime_digest, "seed" => seed, "settings" => digest_of(settings.to_s),
+          "files" => files.to_h { |file| [relative(root, file), file_digest(file)] } }
+      end
+
+      def runtime_digest
+        digest_of(JSON.generate(RunFingerprint.runtime_identity.merge(map_format: Selection::CoverageMap::FORMAT)))
+      end
+
+      # Best effort: a failed write leaves no snapshot and never stops a run.
+      # The temp file is removed however the write ends, an interrupt included.
+      def write(cache_dir, data)
+        target = path(cache_dir, data["map_key"])
+        map_digest = file_digest(map_path(cache_dir, data["map_key"]))
+        return unless map_digest
+
+        temp = "#{target}.#{SecureRandom.hex(8)}.tmp"
+        begin
+          File.open(temp, "wx", 0o600) { |out| out.write(JSON.generate(data.merge("map_digest" => map_digest))) }
+          File.rename(temp, target)
+        ensure
+          FileUtils.rm_f(temp)
+        end
+      rescue SystemCallError, EncodingError, JSON::JSONError
+        nil
+      end
+
+      # The newest snapshot that passes every check, or nil.
+      def latest(cache_dir, root)
+        candidates = Dir.glob(File.join(cache_dir, "#{PREFIX}*.json")).sort_by { |file| -mtime(file) }
+        candidates.first(MAX_CANDIDATES).each do |file|
+          data = read(file, cache_dir, root)
+          return data if data
+        end
+        nil
+      end
+
+      # Only a regular file is read, never a symlink, FIFO or device (their
+      # File.size is 0 and a read would not end), and never more than the cap.
+      def read(file, cache_dir, root)
+        return nil unless File.lstat(file).file?
+
+        text = File.open(file, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) { |io| io.read(MAX_BYTES + 1) }
+        return nil if text.nil? || text.bytesize > MAX_BYTES
+
+        data = JSON.parse(Text.utf8_strict(text))
+        data if valid?(data, file, cache_dir, root)
+      rescue SystemCallError, JSON::ParserError, EncodingError
+        nil
+      end
+
+      def valid?(data, file, cache_dir, root)
+        return false unless data.is_a?(Hash) && data.keys.sort == KEYS && data["format"] == FORMAT
+        return false unless %w[runtime settings map_digest].all? { |key| hex?(data[key]) }
+        return false unless data["seed"].is_a?(Integer) && data["seed"].between?(0, MAX_SEED)
+        return false unless data["root"] == digest_of(root)
+        # The key becomes a path only after this: a name with a separator can never equal a basename.
+        return false unless File.basename(file) == "#{PREFIX}#{data['map_key']}.json"
+
+        files_valid?(data["files"]) && file_digest(map_path(cache_dir, data["map_key"])) == data["map_digest"]
+      end
+
+      def files_valid?(files)
+        files.is_a?(Hash) && files.size <= MAX_FILES && files.all? do |name, digest|
+          name.valid_encoding? && name.bytesize <= MAX_PATH && (digest == false || hex?(digest))
+        end
+      end
+
+      def hex?(value)
+        value.is_a?(String) && value.match?(HEX)
+      end
+
+      def mtime(file)
+        File.mtime(file).to_f
+      rescue SystemCallError
+        0.0
+      end
+
+      def relative(root, file)
+        file.delete_prefix("#{root}/")
+      end
+
+      # Only a regular file is read: a symlink, FIFO or device is false, the
+      # same as a missing file, so it never reads outside the tree or blocks.
+      def file_digest(file)
+        File.lstat(file).file? && Digest::SHA256.file(file).hexdigest
+      rescue SystemCallError
+        false
+      end
+
+      def digest_of(text)
+        Digest::SHA256.hexdigest(text)
+      end
+    end
+  end
+end

data/lib/saboteur/engine/coverage_miss_note.rb

@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+
+module Saboteur
+  class Engine
+    # Says why the coverage cache missed (#46). The map is keyed on the
+    # content of every source, spec and support file, so a pull request that
+    # edits one never hits a map built on the base branch; without a note the
+    # run just sits through a whole-suite pass. The note names how many input
+    # files changed since the last map, and points at the spec scoping flags.
+    #
+    # The comparison needs the earlier map's inputs: see
+    # CoverageInputsSnapshot, which validates what is read back from the
+    # cache (diagnostic only, never the authority for reusing a map). Every
+    # path shown goes through the caller's line cleaner, since a file name, or
+    # a key in a restored snapshot, is untrusted text that a CI log would
+    # otherwise read as a workflow command.
+    class CoverageMissNote
+      NAMED = 3
+      HINT = "; --test narrows the specs the build runs"
+
+      # settings: whatever else is part of the cache key (config inputs, version).
+      # map_key: the key the map about to be built is saved under.
+      # seed: the seed that build runs the specs with.
+      # clean: turns one path into a line-safe string.
+      def initialize(cache_dir:, root:, files:, settings:, map_key:, seed:, clean:)
+        @cache_dir = cache_dir
+        @root = root
+        @clean = clean
+        @current = CoverageInputsSnapshot.build(root: root, files: files, settings: settings, map_key: map_key,
+                                                seed: seed)
+      end
+
+      # The inputs snapshot of the run about to build its map.
+      attr_reader :current
+
+      def message(scoped:)
+        head = "coverage cache miss: #{reason}; the map build runs "
+        scoped ? "#{head}the selected specs once" : "#{head}the whole suite once#{HINT}"
+      end
+
+      # Call after the map is saved: the snapshot is written last and names it.
+      def save
+        CoverageInputsSnapshot.write(@cache_dir, @current)
+      end
+
+      private
+
+      def reason
+        earlier = CoverageInputsSnapshot.latest(@cache_dir, @root)
+        return "no earlier map to compare with" unless earlier
+
+        now = @current["files"]
+        before = earlier["files"]
+        parts = []
+        parts << files_note(now, before) unless now == before
+        parts << "settings, Ruby or saboteur version changed" if settings_changed?(earlier)
+        parts.empty? ? "inputs match the last snapshot but its map file is missing or unusable" : parts.join("; ")
+      end
+
+      # "2 edited, 1 added, 1 removed", then the first few paths. Edited:
+      # present on both sides with another digest.
+      def files_note(now, before)
+        edited = (now.keys & before.keys).reject { |name| now[name] == before[name] }
+        added = now.keys - before.keys
+        removed = before.keys - now.keys
+        counts = { "edited" => edited, "added" => added, "removed" => removed }
+                 .reject { |_, names| names.empty? }.map { |word, names| "#{names.size} #{word}" }
+        "input files changed since the last map: #{counts.join(', ')} (#{named(edited + added + removed)})"
+      end
+
+      def named(names)
+        shown = names.sort.first(NAMED).map { |name| @clean.call(name) }.join(", ")
+        names.size > NAMED ? "#{shown}, +#{names.size - NAMED} more" : shown
+      end
+
+      def settings_changed?(earlier)
+        earlier["settings"] != @current["settings"] || earlier["runtime"] != @current["runtime"]
+      end
+    end
+  end
+end

data/lib/saboteur/engine/coverage_partial.rb

@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # CoverageStore's opt-in partial rebuild (#46): after a cache miss, start
+    # from the last map and rerun only the spec files PartialInvalidation
+    # calls stale. Anything short of a clean answer (no earlier map, a change
+    # that needs a full rebuild, a failed rerun, an empty result) returns nil
+    # and the caller builds the full map, so this can only save time.
+    # Uses the store's @config, @root, @integration and @diagnostics and the
+    # shadow helpers it shares (shadow_invalidation, shadow_rerun,
+    # shadow_clean).
+    module CoveragePartial
+      private
+
+      # Same preconditions as the shadow run: a scoped or parted map is not
+      # what a partial rebuild starts from.
+      def partial_use?
+        @config["partial_coverage"] == "use" && @config["cache"] && @config["tests"].empty? && @parts == 1
+      end
+
+      def partial_map(files, note, preloaded)
+        cache = @config.path("cache_dir")
+        base = CoverageInputsSnapshot.latest(cache, @root)
+        base_map = base && Selection::CoverageMap.load(CoverageInputsSnapshot.map_path(cache, base["map_key"]))
+        return nil unless base_map
+
+        stale = shadow_invalidation(base_map, files).stale_specs(base, note.current)
+        return nil if stale == PartialInvalidation::FULL
+
+        merged = stale.empty? ? base_map : PartialMerge.call(base_map, shadow_rerun(files, stale, base["seed"]), stale)
+        merged = merged.with_preloaded(preloaded)
+        return nil if merged.tests.empty?
+
+        refuse_empty_map(merged, files, preloaded)
+        @diagnostics.inform("reusing the earlier coverage map, rerunning #{stale.size} of " \
+                            "#{@integration.test_files.size} spec files")
+        merged
+      rescue BudgetPhaseTimeout
+        raise
+      rescue StandardError => e
+        @diagnostics.warn("partial coverage failed, so the full map is built\n  #{shadow_clean(e.message)}")
+        nil
+      end
+    end
+  end
+end

data/lib/saboteur/engine/coverage_parts.rb

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Saboteur
+  class Engine
+    # CoverageStore's saved parts (#427): the map of each finished part of a
+    # parallel build is kept beside the whole-map cache, so a rerun after a
+    # failed or budget-stopped build only rebuilds the parts that did not
+    # finish. Uses the store's @config and cache_key.
+    module CoverageParts
+      private
+
+      # One finished part's map, beside the cache of the whole map and under
+      # the same key, so it is valid for exactly the inputs that cache is.
+      def part_path(files, index, count)
+        File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}-part-#{index + 1}-of-#{count}.json")
+      end
+
+      # Best effort, like the verdict checkpoint: a full disk or a read-only
+      # cache dir must not hide the failure this build is about to report.
+      # The saved maps of earlier runs' finished parts that still hold exactly
+      # their part's examples; nil for a part that has to be built.
+      def saved_parts(files, count, counts)
+        Array.new(count) do |index|
+          next unless @config["cache"]
+
+          map = Selection::CoverageMap.load(part_path(files, index, count))
+          map if map && map.tests.size == counts[index]
+        end
+      end
+
+      # The saved parts are spent once the whole map exists.
+      def forget_parts(files, count)
+        count.times do |index|
+          FileUtils.rm_f(part_path(files, index, count))
+        rescue SystemCallError
+          nil
+        end
+      end
+
+      def save_part(map, files, index, count)
+        map.save(part_path(files, index, count)) if @config["cache"] && !map.tests.empty?
+      rescue SystemCallError
+        nil
+      end
+    end
+  end
+end

data/lib/saboteur/engine/coverage_shadow.rb

@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Saboteur
+  class Engine
+    # CoverageStore's opt-in shadow step (#46): after a cache miss, rebuild
+    # the map partially from the last one and compare it with the full map
+    # the run just built. Uses the store's @config, @root, @integration,
+    # @supervisor and @diagnostics, and its record and map_from.
+    module CoverageShadow
+      private
+
+      # The opt-in shadow run (#46) needs the single whole-suite build: a
+      # scoped or parted map is not what a partial rebuild is compared with.
+      def shadow?
+        @config["partial_coverage"] == "shadow" && @config["cache"] && @config["tests"].empty? && @parts == 1
+      end
+
+      # Never stops a run: the full map is already built and is the one used.
+      # Writes shadow-<map key>.json beside the map, with the seconds the full
+      # build and the partial rerun took, and warns on a mismatch.
+      def shadow(files, full_map, note, build_seconds)
+        cache = @config.path("cache_dir")
+        base = CoverageInputsSnapshot.latest(cache, @root)
+        base_map = base && Selection::CoverageMap.load(CoverageInputsSnapshot.map_path(cache, base["map_key"]))
+        return unless base_map
+
+        rerun_seconds = nil
+        report = ShadowRun.call(base: base, base_map: base_map, current: note.current, full_map: full_map,
+                                invalidation: shadow_invalidation(base_map, files), fallback_files: files,
+                                rerun: lambda { |specs, seed|
+                                  from = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+                                  shadow_rerun(files, specs, seed).tap do
+                                    rerun_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - from
+                                  end
+                                }, clean: method(:shadow_clean))
+        report = report.merge("full_build_seconds" => build_seconds.round(2),
+                              "rerun_seconds" => rerun_seconds&.round(2))
+        save_shadow_report(cache, report, cache_key(files))
+      rescue StandardError => e
+        @diagnostics.warn("partial coverage shadow run failed, so the full map is used\n  #{shadow_clean(e.message)}")
+      end
+
+      def shadow_invalidation(base_map, files)
+        PartialInvalidation.new(map: base_map, root: @root, included: files.map(&method(:relative)),
+                                spec_files: @integration.test_files.map(&method(:relative)),
+                                status_file: @integration.status_file?(@root))
+      end
+
+      def relative(file)
+        file.delete_prefix("#{@root}/")
+      end
+
+      # The stale spec files, run once in a child with the seed the base map
+      # was built with, over exactly the runnable spec files named. It writes
+      # the store's usual map build log, which the full build is done with.
+      def shadow_rerun(files, specs, seed)
+        Execution::ForkReadiness.prepare_heap
+        admitted = build_timeout
+        result = @supervisor.run(timeout: admitted, label: "coverage child died during the shadow rerun") do |writer|
+          record(writer, files, only: specs.map { |file| File.join(@root, file) }, seed: seed)
+        end
+        map_from(result, admitted)
+      end
+
+      def save_shadow_report(cache, report, key)
+        target = File.join(cache, "shadow-#{key}.json")
+        temp = "#{target}.#{SecureRandom.hex(8)}.tmp"
+        begin
+          File.open(temp, "wx", 0o600) { |out| out.write(JSON.pretty_generate(report)) }
+          File.rename(temp, target)
+        ensure
+          FileUtils.rm_f(temp)
+        end
+        warn_mismatch(report, target) if report["result"] == "mismatch"
+      end
+
+      def warn_mismatch(report, target)
+        first = report["differences"].first
+        @diagnostics.warn("partial coverage shadow run: this run used the full map\n  " \
+                          "the partial map differs from the full map in #{report['total']} place(s), " \
+                          "first #{first['kind']} #{first['name']}\n  Report: #{shadow_clean(target)}")
+      end
+
+      def shadow_clean(value)
+        plain_line(Text.utf8(value.to_s))
+      end
+    end
+  end
+end