hecks 6.0.0 → 7.0.0

data/lib/hecks/hecks/adapters/git_rehearsal/impacted.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "spec_map"
+require_relative "impacted/spec_run"
+require_relative "impacted/skips"
+
+module Hecks
+  module Adapters
+    class GitRehearsal
+      # The stage that runs the specs a push's changes touch, in the rehearsal's copy of the commit.
+      #
+      # The changed files are those between the remote tracking ref and the commit; `SpecMap` names
+      # the specs they affect, each is run alone through the launcher, and one that is red turns the
+      # stage red unless it is red on the remote tracking ref too (run in a second detached copy,
+      # made only when something failed), which is reported as pre-existing and passes.
+      #
+      # A file that fails among the others is run again alone, once, when nothing else of the stage
+      # runs: one that passes then is flaky under parallel load, reported by name and counted in
+      # the rehearsal record, and does not turn the stage red; one that fails alone is red.
+      class Impacted
+        # The stage's name in the report.
+        NAME = "impacted specs"
+
+        # The most spec files a rehearsal runs.
+        DEFAULT_LIMIT = 60
+
+        # How many spec files run at once.
+        DEFAULT_WIDTH = 2
+
+        # What the stage works with.
+        #
+        # @!attribute [r] git
+        #   @return [Git] the adapter every git call goes through
+        # @!attribute [r] run
+        #   @return [SpecRun] runs one spec file in a directory
+        # @!attribute [r] skips
+        #   @return [Skips] decides which specs cannot run here
+        # @!attribute [r] bench
+        #   @return [#call] given a commit and a block, yields a second copy of it
+        # @!attribute [r] flakes
+        #   @return [#call, nil] given flaky files, counts them in the record, answers each count
+        Tools = Struct.new(:git, :run, :skips, :bench, :flakes)
+
+        # @param directory [String] the copy of the commit
+        # @param branch [String] the branch pushed; its remote tracking ref is `origin/<branch>`
+        # @param tools [Tools] what the stage works with
+        # @param limit [Integer] the most spec files to run
+        # @param width [Integer] how many spec files run at once
+        def initialize(directory, branch:, tools:, limit: DEFAULT_LIMIT, width: DEFAULT_WIDTH)
+          @directory = directory
+          @remote = "origin/#{branch}"
+          @git = tools.git
+          @run = tools.run
+          @skips = tools.skips
+          @bench = tools.bench
+          @flakes = tools.flakes
+          @limit = limit
+          @width = [width.to_i, 1].max
+        end
+
+        # The stage as a rehearsal runs it.
+        #
+        # @param directory [String] the copy of the commit
+        # @param branch [String] the branch being pushed
+        # @param capture [#call] given a directory, a command and `env:`, runs it there
+        # @param limit [Integer, nil] the most spec files to run; the default when nil
+        # @param width [Integer, nil] how many run at once; the default when nil
+        # @return [Impacted]
+        def self.build(directory, branch:, capture:, limit: nil, width: nil)
+          git = Git.new
+          scratch = -> { capture.call(directory, "pg_isready", env: {}).ok? }
+          bench = ->(commit, &block) { Bench.new(git, directory).rehearse(commit, &block) }
+          flakes = ->(names) { Record.new(git, directory).count_flakes(names) }
+          tools = Tools.new(git, SpecRun.new(capture), Skips.new(directory, scratch: scratch), bench, flakes)
+          new(directory, branch: branch, tools: tools, limit: limit || DEFAULT_LIMIT, width: width || DEFAULT_WIDTH)
+        end
+
+        # @return [Stages::Result] the stage's result
+        def result
+          base = commit(@remote)
+          return Stages::Result.new(NAME, true, " (no #{@remote} to compare with)") unless base
+
+          mapping = SpecMap.new(@directory).map(changed(base), limit: @limit, skip: @skips)
+          verdict(mapping, base)
+        end
+
+        private
+
+        def verdict(mapping, base)
+          outcomes = run_all(mapping.specs)
+          flaky, red = alone(outcomes.reject(&:passed))
+          known, fresh = red.partition { |outcome| red_on_base?(outcome, base) }
+          notes = notes(mapping, outcomes.size, known) + flaky_notes(flaky)
+          fresh.empty? ? Stages::Result.new(NAME, true, notes_in_brackets(notes)) : failed(fresh, notes)
+        end
+
+        def failed(fresh, notes) = Stages::Result.new(NAME, false, ([fresh.map(&:line).join("; ")] + notes).join("; "))
+
+        # Runs each failed file again alone, after the batch; answers the specs that then passed and
+        # the outcomes of those that failed again.
+        def alone(failed)
+          again = failed.map { |outcome| @run.call(@directory, outcome.spec, io: @skips.io?(outcome.spec)) }
+          flaky, red = again.partition(&:passed)
+          [flaky.map(&:spec), red]
+        end
+
+        def flaky_notes(flaky)
+          counts = counted(flaky)
+          flaky.map do |spec|
+            times = counts[spec].to_i
+            "flaky under parallel load: #{spec}#{" (flaky in #{times} rehearsals)" if times > 1}"
+          end
+        end
+
+        # How many rehearsals each flaky file has been flaky in; empty when the record cannot say.
+        def counted(flaky)
+          return {} if flaky.empty? || @flakes.nil?
+
+          @flakes.call(flaky)
+        rescue StandardError
+          {}
+        end
+
+        def notes(mapping, ran, known)
+          [("#{ran} spec file(s) run" if ran.positive?), mapping.cap_note, *skipped(mapping),
+           ("pre-existing red: #{known.map(&:spec).join(", ")}" unless known.empty?)].compact.reject(&:empty?)
+        end
+
+        def skipped(mapping)
+          mapping.skipped.group_by { |_spec, why| why }.map { |why, specs| "skipped: #{why} (#{specs.size})" }
+        end
+
+        def notes_in_brackets(notes) = notes.empty? ? " (nothing impacted)" : " (#{notes.join("; ")})"
+
+        def changed(base)
+          out = @git.capture("diff", "--name-only", "--diff-filter=d", base, "HEAD", chdir: @directory)
+          out.ok? ? paths(out.out) : []
+        end
+
+        def paths(text) = text.lines.map(&:strip).reject(&:empty?)
+
+        def commit(ref)
+          out = @git.capture("rev-parse", "--verify", "--quiet", "#{ref}^{commit}", chdir: @directory)
+          out.ok? ? out.out.strip : nil
+        end
+
+        def run_all(specs)
+          queue = Queue.new
+          specs.each_with_index { |spec, index| queue << [spec, index] }
+          found = Array.new(specs.size)
+          Array.new([@width, specs.size].min) { Thread.new { work(queue, found) } }.each(&:join)
+          found
+        end
+
+        def work(queue, found)
+          loop do
+            spec, index = queue.pop(true)
+            found[index] = @run.call(@directory, spec, io: @skips.io?(spec))
+          end
+        rescue ThreadError
+          nil
+        end
+
+        # Whether the spec is red on the remote tracking ref too: it exists there and fails there.
+        def red_on_base?(outcome, base)
+          @base_outcomes ||= {}
+          @base_outcomes.fetch(outcome.spec) do
+            @base_outcomes[outcome.spec] = on_base(outcome.spec, base)
+          end
+        end
+
+        def on_base(spec, base)
+          @bench.call(base) do |directory|
+            File.exist?(File.join(directory, spec)) && !@run.call(directory, spec, io: @skips.io?(spec)).passed
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/hecks/adapters/git_rehearsal/line.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "reading"
+
+module Hecks
+  module Adapters
+    class GitRehearsal
+      # The line rehearsals of one repository wait in, which is a line of one: at most one rehearsal
+      # runs per repository, and every land that arrives meanwhile is folded into a single pending
+      # mark, so the running rehearsal runs once more for the latest commit instead of N queued
+      # runs.
+      #
+      # The running rehearsal holds a non-blocking `flock` (released by the system if the process is
+      # killed) on a file in the repository's common git directory. The pending mark is a file
+      # beside it, replaced atomically; it holds the branch and the commit.
+      class Line
+        include Reading
+
+        # The files in the common git directory.
+        RUNNING = "hecks-push-rehearsal.running"
+        PENDING = "hecks-push-rehearsal.pending"
+
+        # @param git [Git] the adapter every git call goes through
+        # @param checkout [String] the repository's top level
+        # @param scope [String] each scope has its own line, so quick runs alongside deep
+        def initialize(git, checkout, scope: "quick")
+          @git = git
+          @checkout = checkout
+          @suffix = scope == "quick" ? ".quick" : ""
+        end
+
+        # Runs the block as the repository's one running rehearsal.
+        #
+        # @yield [] holding the lock
+        # @return [Object, Symbol] the block's value, or `:busy` (the block not run) when another
+        #   rehearsal is running
+        def running
+          File.open(path(RUNNING), File::RDWR | File::CREAT) do |lock|
+            return :busy unless lock.flock(File::LOCK_EX | File::LOCK_NB)
+
+            yield
+          end
+        end
+
+        # Asks the running rehearsal to run once more, for the latest commit of the branch.
+        #
+        # @param branch [String] the branch to rehearse
+        # @param commit [String] the commit that landed, named in the mark
+        # @return [String] the mark, `pending: rehearse <sha>`
+        def mark(branch, commit)
+          temp = "#{path(PENDING)}.#{Process.pid}.tmp"
+          File.write(temp, "#{branch} #{commit}\n")
+          File.rename(temp, path(PENDING))
+          "pending: rehearse #{short(commit)}"
+        end
+
+        # Takes the pending mark, if any, leaving none: a land arriving after this marks anew.
+        #
+        # @return [String, nil] the branch to rehearse again, nil when nothing is pending
+        def take
+          file = path(PENDING)
+          return unless File.file?(file)
+
+          branch = File.read(file).split.first
+          FileUtils.rm_f(file)
+          branch
+        end
+
+        private
+
+        def path(name) = File.join(read("rev-parse", "--path-format=absolute", "--git-common-dir"), "#{name}#{@suffix}")
+      end
+    end
+  end
+end

data/lib/hecks/hecks/adapters/git_rehearsal/reading.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Hecks
+  module Adapters
+    class GitRehearsal
+      # The git reads a rehearsal makes: each answers what git printed or
+      # refuses with what git said.
+      #
+      # An includer sets `@git` (the `Git` adapter) and `@checkout` (the repository's top level).
+      module Reading
+        private
+
+        def read(*args, dir: @checkout)
+          result = @git.capture(*args, chdir: dir)
+          raise ConsoleCapture::Failure, "git #{args.first} failed in #{dir}: #{result.err.strip}" unless result.ok?
+
+          result.out.strip
+        end
+
+        def short(commit) = commit[0, 9]
+      end
+    end
+  end
+end

data/lib/hecks/hecks/adapters/git_rehearsal/record.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require_relative "reading"
+
+module Hecks
+  module Adapters
+    class GitRehearsal
+      # The last rehearsal's result per scope, kept in files in the repository's common git
+      # directory
+      # (never committed, shared by every worktree) as `<commit> <green|red> <unix time> <failing
+      # stages>`. The `quick` scope's file is `hecks-push-rehearsal.quick`; the `deep`
+      # scope's is `hecks-push-rehearsal` (the only file before scopes existed).
+      class Record
+        include Reading
+
+        # The file in the common git directory.
+        FILE = "hecks-push-rehearsal"
+
+        # The file beside it counting, per spec file, the rehearsals it was flaky in: a JSON map.
+        FLAKES = "#{FILE}.flakes".freeze
+
+        # The `quick` scope's file.
+        QUICK = "#{FILE}.quick".freeze
+
+        # What the file holds.
+        #
+        # @!attribute [r] commit
+        #   @return [String] the commit rehearsed
+        # @!attribute [r] verdict
+        #   @return [String] `green` or `red`
+        # @!attribute [r] at
+        #   @return [Time] when it ended
+        # @!attribute [r] failed
+        #   @return [String] the stages that failed, comma separated; empty when green
+        Entry = Struct.new(:commit, :verdict, :at, :failed)
+
+        # @param git [Git] the adapter every git call goes through
+        # @param checkout [String] the repository's top level
+        def initialize(git, checkout)
+          @git = git
+          @checkout = checkout
+        end
+
+        # Records a result atomically (a temp file in the same directory, renamed over the record)
+        # unless the record holds a newer commit's result. Newer is decided by ancestry
+        # (`git merge-base --is-ancestor`), else by committer time. The check and the rename happen
+        # under a lock.
+        #
+        # @param commit [String] the commit rehearsed
+        # @param verdict [String] `green` or `red`
+        # @param results [Array<Stages::Result>] the stages
+        # @param scope [String] `quick` or `deep`: which scope's record this is
+        # @return [String] `recorded`, or the newer record kept and how that was decided
+        def write(commit, verdict, results, scope: "quick")
+          failed = results.reject(&:passed).map(&:name).join(",")
+          locked(scope) do
+            kept = newer_kept(commit, scope)
+            next kept if kept
+
+            replace("#{commit} #{verdict} #{Time.now.to_i} #{failed}\n", path(scope))
+            "recorded"
+          end
+        end
+
+        # Counts one more rehearsal in which each spec file was flaky (failed in a batch, passed
+        # alone). The read, the increment and the rename happen under the record's lock, so
+        # concurrent rehearsals lose no count.
+        #
+        # @param specs [Array<String>] the flaky spec files
+        # @return [Hash{String => Integer}] each of them with the rehearsals it has been flaky in
+        def count_flakes(specs)
+          locked do
+            counts = flake_counts
+            specs.each { |spec| counts[spec] = counts.fetch(spec, 0) + 1 }
+            replace(JSON.generate(counts), flakes_path) unless specs.empty?
+            counts.slice(*specs)
+          end
+        end
+
+        # @return [Hash{String => Integer}] the rehearsals each spec file has been flaky in
+        def flake_counts
+          JSON.parse(File.read(flakes_path))
+        rescue Errno::ENOENT, JSON::ParserError
+          {}
+        end
+
+        # @param scope [String] `quick` or `deep`
+        # @return [Entry, nil] the last rehearsal of that scope, nil when none has run
+        def last(scope = "quick")
+          return unless File.file?(path(scope))
+
+          commit, verdict, at, failed = File.read(path(scope)).split(" ", 4)
+          Entry.new(commit, verdict, Time.at(at.to_i), failed.to_s.strip)
+        end
+
+        private
+
+        def path(scope = "quick") = File.join(common_dir, scope == "quick" ? QUICK : FILE)
+
+        def flakes_path = File.join(common_dir, FLAKES)
+
+        def common_dir = read("rev-parse", "--path-format=absolute", "--git-common-dir")
+
+        def locked(scope = "quick", &)
+          File.open("#{path(scope)}.lock", File::RDWR | File::CREAT) do |lock|
+            lock.flock(File::LOCK_EX)
+            yield
+          end
+        end
+
+        # Writes the record through a temp file in its directory, so a reader sees the old record or
+        # the new one whole.
+        def replace(line, target = path)
+          temp = "#{target}.#{Process.pid}.tmp"
+          File.write(temp, line)
+          File.rename(temp, target)
+        ensure
+          FileUtils.rm_f(temp) if temp
+        end
+
+        # The recorded result that outranks this commit's, as a sentence; nil when this one wins.
+        def newer_kept(commit, scope)
+          held = last(scope)
+          return unless held && held.commit != commit
+          return if ancestor?(held.commit, commit)
+          return kept(held, commit, "#{short(commit)} is an ancestor of it") if ancestor?(commit, held.commit)
+          return unless committed_at(held.commit) > committed_at(commit)
+
+          kept(held, commit, "by commit time #{short(held.commit)} is later (neither descends from the other)")
+        end
+
+        def kept(held, commit, why) = "kept the record for #{short(held.commit)}, newer than #{short(commit)}: #{why}"
+
+        def ancestor?(older, newer) = @git.capture("merge-base", "--is-ancestor", older, newer, chdir: @checkout).ok?
+
+        # A commit's committer time; 0 for one this repository does not hold.
+        def committed_at(commit)
+          result = @git.capture("show", "-s", "--format=%ct", commit, chdir: @checkout)
+          result.ok? ? result.out.to_i : 0
+        end
+      end
+    end
+  end
+end

data/lib/hecks/hecks/adapters/git_rehearsal/spec_map/corpus.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Hecks
+  module Adapters
+    class GitRehearsal
+      class SpecMap
+        # The spec files of a tree and what each says, read once.
+        class Corpus
+          # @param root [String] the tree
+          def initialize(root)
+            @root = root
+            @texts = {}
+          end
+
+          # @return [Array<String>] every `*_spec.rb` under `spec/`, relative to the root, sorted
+          def files = @files ||= Dir.glob("spec/**/*_spec.rb", base: @root).sort
+
+          # @param spec [String] a spec path relative to the root
+          # @return [String] its text
+          def text(spec) = @texts[spec] ||= File.read(File.join(@root, spec)).scrub
+        end
+      end
+    end
+  end
+end

data/lib/hecks/hecks/adapters/git_rehearsal/spec_map/needles.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Hecks
+  module Adapters
+    class GitRehearsal
+      class SpecMap
+        # What a spec says when it is about a file: the file's path, how a require spells it, and
+        # the constant or domain it declares.
+        class Needles
+          DOMAIN = /\.(bluebook|hecksagon|world|behaviors)\z/
+          DOMAIN_NAME = /^\s*(?:Hecks\.(?:bluebook|hecksagon|world|behaviors)|domain_module|domain)\s+"([^"]+)"/
+          NESTING = /^(\s*)(?:class|module)\s+([A-Z][\w:]*)/
+          GENERIC = %w[Hecks Adapters Tools Language Framework].freeze
+
+          # @param file [String] the changed file, to read
+          # @param path [String] the same file relative to the root
+          def initialize(file, path)
+            @path = path
+            @text = File.exist?(file) ? File.read(file).scrub : ""
+          end
+
+          # @return [Boolean] whether there is nothing to look for
+          def none? = pattern.nil?
+
+          # @param text [String] a spec's text
+          # @return [Boolean] whether the spec is about the file
+          def match?(text) = pattern.match?(text)
+
+          private
+
+          def pattern
+            return @pattern if defined?(@pattern)
+
+            parts = paths.map { |needle| Regexp.escape(needle) } + names.map { |name| "\\b#{Regexp.escape(name)}\\b" }
+            @pattern = parts.empty? ? nil : Regexp.new(parts.join("|"))
+          end
+
+          def paths
+            bare = @path.delete_prefix("lib/").delete_suffix(".rb")
+            [@path, bare, bare.split("/").last(2).join("/")].uniq
+          end
+
+          def names = (DOMAIN.match?(@path) ? domain_names : [nesting]).compact.reject { |name| GENERIC.include?(name) }.uniq
+
+          def domain_names = @text.scan(DOMAIN_NAME).flatten
+
+          # The innermost class or module of the first chain of nested declarations.
+          def nesting
+            chain = []
+            @text.scan(NESTING) do |indent, name|
+              break unless chain.empty? || indent.size == chain.size * 2
+
+              chain << name
+            end
+            chain.last&.split("::")&.last
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/hecks/adapters/git_rehearsal/spec_map.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require_relative "spec_map/corpus"
+require_relative "spec_map/needles"
+
+module Hecks
+  module Adapters
+    class GitRehearsal
+      # Maps the files a push changes to the spec files that exercise them: conservatively and
+      # cheaply, by name and by text, with no index and no dependency beyond the standard library.
+      #
+      # A changed spec runs itself. A changed Ruby file runs the spec named for it and the specs
+      # that require it or mention its path or its constant. A changed bluebook, hecksagon, world or
+      # behaviors file runs the specs that mention its path or domain name (and, for the language's
+      # own bluebooks, the shape-ledger, conformance and corpus specs). A changed `rust/` file runs
+      # the Ruby specs that drive it, never cargo; a changed `docs/` file runs the docs specs.
+      class SpecMap
+        # What was mapped.
+        #
+        # @!attribute [r] specs
+        #   @return [Array<String>] the specs to run, closest first, at most the limit
+        # @!attribute [r] total
+        #   @return [Integer] how many specs the change maps to once skipped ones are left out
+        # @!attribute [r] skipped
+        #   @return [Hash{String => String}] each spec left out and why
+        Mapping = Struct.new(:specs, :total, :skipped) do
+          # @return [Boolean] whether the limit left specs out
+          def capped? = total > specs.size
+
+          # @return [String] how many ran and that the rest run in CI; empty when none were left out
+          def cap_note = capped? ? "capped at #{specs.size} of #{total}; the rest run in CI" : ""
+        end
+
+        # The spec families a changed `docs/` file and a changed `rust/` file run.
+        DOCS = %w[docs_ reference_ changelog_ adr_].freeze
+        RUST = %w[rust_ semantics_corpus_spec].freeze
+        LANGUAGE = %r{\Alib/hecks/language/}
+        LEDGER = /shape_ledger|conformance|corpus/
+        EXTENSIONS = /\..*\z/
+
+        # @param root [String] the tree whose `spec/` is searched
+        def initialize(root)
+          @root = root
+          @corpus = Corpus.new(root)
+        end
+
+        # @param changed [Array<String>] changed paths relative to the root
+        # @param limit [Integer] the most specs to return
+        # @param skip [#call] given a spec, nil to run it or the reason to leave it out
+        # @return [Mapping]
+        def map(changed, limit:, skip: ->(_spec) {})
+          ranked = tiers(changed).sort_by { |spec, tier| [tier, spec] }.map(&:first)
+          reasons = ranked.to_h { |spec| [spec, skip.call(spec)] }.compact
+          kept = ranked - reasons.keys
+          Mapping.new(kept.first(limit), kept.size, reasons)
+        end
+
+        private
+
+        # The closest tier each spec is reached at: 0 itself, 1 named for the file, 2 mentions it.
+        def tiers(changed)
+          changed.each_with_object({}) do |path, found|
+            reach(path).each { |spec, tier| found[spec] = [found[spec], tier].compact.min }
+          end
+        end
+
+        def reach(path)
+          return { path => 0 } if itself?(path)
+
+          found = named(path).to_h { |spec| [spec, 1] }
+          found.merge!(mentioning(path).to_h { |spec| [spec, 2] }) { |_spec, near, _far| near }
+          families(path).each { |spec| found[spec] ||= 3 }
+          found
+        end
+
+        def itself?(path) = path.end_with?("_spec.rb") && File.exist?(File.join(@root, path))
+
+        def named(path)
+          wanted = "#{File.basename(path).sub(EXTENSIONS, "")}_spec.rb"
+          @corpus.files.select { |spec| File.basename(spec) == wanted }
+        end
+
+        def mentioning(path)
+          return [] if path.start_with?("docs/", "rust/")
+
+          needles = Needles.new(File.join(@root, path), path)
+          return [] if needles.none?
+
+          @corpus.files.select { |spec| needles.match?(@corpus.text(spec)) }
+        end
+
+        def families(path)
+          return starting(DOCS) if path.start_with?("docs/")
+          return starting(RUST) if path.start_with?("rust/")
+
+          path.match?(LANGUAGE) && Needles::DOMAIN.match?(path) ? @corpus.files.grep(LEDGER) : []
+        end
+
+        def starting(prefixes)
+          @corpus.files.select { |spec| prefixes.any? { |prefix| File.basename(spec).start_with?(prefix) } }
+        end
+      end
+    end
+  end
+end