holdfast 0.1.0

data/lib/holdfast/inputs.rb

@@ -0,0 +1,105 @@
+module Holdfast
+  # The pin inputs of one target, in branchout's format plus three keys of
+  # holdfast's own: "change": true on an input the change is meant to alter,
+  # "raises" for an intended error, and a "generate" list whose entries hegel
+  # expands into concrete inputs. The expansion is derandomized with a fixed
+  # seed, so base and head (and every rerun) see the same values.
+  #
+  #   {"generate": [{"receiver": {"ruby": "create(:loan, due_on: Date.new(2026, 1, 1) - %{days})"},
+  #                  "args": [], "draws": {"days": {"integers": [0, 40]}},
+  #                  "cases": 20, "change_when": "days > 10"}]}
+  class Inputs
+    Entry = Data.define(:input, :change, :note)
+    OWN_KEYS = %w[change raises change_when draws cases].freeze
+    SEED = 20_261_107
+
+    def self.path(root, target) = File.join(Holdfast.dir(root), "evidence", "#{Holdfast.slug(target)}.inputs.json")
+
+    def self.read(root, target, boundaries: [])
+      file = path(root, target)
+      return nil unless File.file?(file)
+
+      data = JSON.parse(File.read(file))
+      new(file, data, boundaries)
+    rescue JSON::ParserError => e
+      raise Holdfast::Error, "#{file} is not JSON: #{e.message.lines.first.strip}"
+    end
+
+    attr_reader :file, :raw
+
+    def initialize(file, raw, boundaries)
+      @file, @raw, @boundaries = file, raw, boundaries
+    end
+
+    def entries
+      @entries ||= Array(raw["inputs"]).map { Entry.new(input: _1.except(*OWN_KEYS), change: _1["change"] == true, note: _1["note"]) } +
+                   Array(raw["generate"]).each_with_index.flat_map { |spec, i| generated(spec, i) }
+    end
+
+    def declared_errors = Array(raw["inputs"]).filter_map { _1["raises"] }
+    def generated_count = entries.count { _1.note.to_s.start_with?("drawn ") }
+
+    private
+
+    def generated(spec, index)
+      draws = spec.fetch("draws") { raise Holdfast::Error, "#{file} generate[#{index}] has no \"draws\"" }
+      values = drawn_values(draws, spec.fetch("cases", 20), SEED + index)
+      values.uniq.map do |names|
+        input = fill(spec.except(*OWN_KEYS), names)
+        change = spec["change_when"] ? condition_holds?(spec["change_when"], names) : spec["change"] == true
+        note = "drawn #{names.map { "#{_1}=#{_2.inspect}" }.join(", ")}"
+        Entry.new(input: input.merge("note" => note), change:, note:)
+      end
+    end
+
+    # hegel draws the values; integer ranges also get each boundary literal of
+    # the target, one below it and one above it, so every side of a
+    # comparison the method makes gets at least one input.
+    def drawn_values(draws, cases, seed)
+      finite = draws.values.map { _1["one_of"] || (_1["booleans"] && [true, false]) }
+      # A small finite space is listed whole; a random draw from it repeats.
+      if finite.all? && finite.map(&:size).inject(1, :*) <= [cases, 120].max
+        return finite.first.product(*finite.drop(1)).map { draws.keys.zip(_1).to_h }
+      end
+
+      require "hegel"
+      found = []
+      Hegel.test(test_cases: cases, derandomize: true, seed:, verbosity: :quiet) do |tc|
+        found << draws.to_h { |name, kind| [name, tc.draw(generator(kind))] }
+      end
+      edges = draws.select { _2.key?("integers") }.flat_map do |name, kind|
+        low, high = kind["integers"]
+        @boundaries.flat_map { [_1 - 1, _1, _1 + 1] }.select { (low..high).cover?(_1) }.map { |value| [name, value] }
+      end
+      found + edges.map { |name, value| (found.first || {}).merge(name => value) }
+    end
+
+    def generator(kind)
+      g = Hegel::Generators
+      case kind.keys.first
+      when "integers" then g.integers(min_value: kind["integers"][0], max_value: kind["integers"][1])
+      when "booleans" then g.booleans
+      when "one_of" then g.sampled_from(kind["one_of"])
+      else raise Holdfast::Error, "#{file}: unknown draw #{kind.keys.first}; use integers, booleans or one_of"
+      end
+    end
+
+    def fill(value, names)
+      case value
+      when Hash
+        return names.fetch(value["draw"]) if value.keys == ["draw"]
+
+        value.transform_values { fill(_1, names) }
+      when Array then value.map { fill(_1, names) }
+      when String then value.gsub(/%\{(\w+)\}/) { names.fetch(Regexp.last_match(1)).inspect }
+      else value
+      end
+    end
+
+    def condition_holds?(condition, names)
+      scope = Object.new
+      names.each { |name, v| scope.define_singleton_method(name) { v } }
+      scope.instance_eval(condition) ? true : false
+    end
+  end
+end

data/lib/holdfast/record.rb

@@ -0,0 +1,132 @@
+require "digest"
+
+module Holdfast
+  # What this project needs to run a test, learned once by running it and
+  # kept in .holdfast/record.json for every stage. Each learning step keeps
+  # its command and outcome, so a failed setup is one fact the verdicts cite
+  # instead of a load error on every row.
+  class Record
+    # jitcatch numbers base 1 and head 2; a pin runs in a checkout of its own.
+    SIDES = { "base" => "1", "head" => "2", "pin" => "3" }.freeze
+    COVERAGE = %w[coverage/jitcatch-coverage.json tmp/jitcatch-coverage.json].freeze
+
+    attr_reader :data
+
+    def self.path(root) = File.join(Holdfast.dir(root), "record.json")
+
+    def self.load_or_learn(root, relearn: false)
+      return new(JSON.parse(File.read(path(root)))) if File.file?(path(root)) && !relearn
+
+      learn(root).tap { _1.write(root) }
+    end
+
+    def self.learn(root)
+      facts = read_facts(root)
+      steps = []
+      if facts["rails"] && facts["database"]["server"]
+        SIDES.each do |side, number|
+          steps << run_step(root, "create the #{side} database (#{database_name(facts, number) || "TEST_ENV_NUMBER=#{number}"})", facts["side_setup"],
+                            side_env(facts, number))
+        end
+      end
+      if steps.all? { _1["ok"] } && (probe = smallest_test(root, facts["framework"]))
+        cmd = facts["framework"] == "rspec" ? "#{facts["bundle"]}rspec #{probe}" : "#{facts["bundle"]}ruby -Itest #{probe}"
+        steps << run_step(root, "boot one test", cmd, side_env(facts, SIDES["base"]))
+      end
+      failed = steps.find { !_1["ok"] }
+      new(facts.merge("learned_at" => Time.now.utc.iso8601, "steps" => steps,
+                      "status" => failed ? "setup_failed" : "ok",
+                      "failure" => failed && "#{failed["name"]}: `#{failed["command"]}` exited #{failed["exit"]}"))
+    end
+
+    def self.read_facts(root)
+      lock = File.read(File.join(root, "Gemfile.lock")) rescue ""
+      framework = lock.match?(/^    rspec-core /) && File.directory?(File.join(root, "spec")) ? "rspec" : "minitest"
+      db_yml = File.read(File.join(root, "config/database.yml")) rescue ""
+      test_section = db_yml[/^test:.*\z/m].to_s
+      adapter = test_section[/adapter:\s*(\w+)/, 1] || db_yml[/adapter:\s*(\w+)/, 1]
+      server = %w[postgresql mysql2 trilogy].include?(adapter)
+      {
+        "framework" => framework,
+        "bundle" => File.file?(File.join(root, "Gemfile")) ? "bundle exec " : "",
+        "rails" => File.file?(File.join(root, "config/environment.rb")),
+        "helper" => framework == "rspec" ? (File.file?(File.join(root, "spec/rails_helper.rb")) ? "rails_helper" : "spec_helper") : "test_helper",
+        "database" => { "adapter" => adapter, "server" => server, "numbered" => test_section.include?("TEST_ENV_NUMBER"),
+                        "url" => server ? database_url(root, adapter) : nil }.compact,
+        "env" => db_yml.scan(/ENV(?:\.fetch\(|\[)\s*["'](\w+)["']/).flatten.uniq - %w[TEST_ENV_NUMBER RAILS_ENV],
+        "side_setup" => server ? "bin/rails db:create db:schema:load" : nil,
+        "base_branch" => base_branch(root),
+        "coverage" => COVERAGE.find { File.file?(File.join(root, _1)) }
+      }
+    end
+
+    # Two apps whose database.yml both say app_test<N> would share one
+    # database on a shared server. Each app gets its own names, from its
+    # directory and a digest of its path, through DATABASE_URL, which Rails
+    # prefers over database.yml and jitcatch numbers per side.
+    def self.database_url(root, adapter)
+      yml = YAML.safe_load(ERB.new(File.read(File.join(root, "config/database.yml"))).result, aliases: true)
+      test = yml.fetch("test", {})
+      scheme = { "postgresql" => "postgres", "mysql2" => "mysql2", "trilogy" => "trilogy" }.fetch(adapter)
+      user = [test["username"], test["password"]].compact.join(":")
+      host = test["host"] || "localhost"
+      name = "hf_#{File.basename(root).downcase.gsub(/[^a-z0-9]+/, "_")}_#{Digest::SHA1.hexdigest(root)[0, 6]}"
+      "#{scheme}://#{"#{user}@" unless user.empty?}#{host}#{":#{test["port"]}" if test["port"]}/#{name}"
+    rescue StandardError
+      nil
+    end
+
+    def self.side_env(facts, number)
+      url = facts.dig("database", "url")
+      { "RAILS_ENV" => "test", "TEST_ENV_NUMBER" => number }.merge(url ? { "DATABASE_URL" => "#{url}_#{number}" } : {})
+    end
+
+    def self.database_name(facts, number) = facts.dig("database", "url")&.then { "#{_1[%r{[^/]+\z}]}_#{number}" }
+
+    def self.base_branch(root)
+      remote = Open3.capture2("git", "symbolic-ref", "--short", "refs/remotes/origin/HEAD", chdir: root, err: File::NULL).first.strip
+      return remote unless remote.empty?
+
+      %w[main master].find { system("git", "rev-parse", "--verify", "--quiet", _1, chdir: root, out: File::NULL) } || "HEAD"
+    end
+
+    def self.smallest_test(root, framework)
+      glob = framework == "rspec" ? "spec/**/*_spec.rb" : "test/**/*_test.rb"
+      Dir.glob(glob, base: root).min_by { File.size(File.join(root, _1)) }
+    end
+
+    def self.run_step(root, name, command, extra_env)
+      env = extra_env.merge("BUNDLE_GEMFILE" => File.join(root, "Gemfile"))
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      out, status = Open3.capture2e(env, "/bin/sh", "-c", command, chdir: root)
+      { "name" => name, "command" => command, "env" => extra_env, "ok" => status.success?, "exit" => status.exitstatus,
+        "seconds" => (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round(1),
+        "tail" => status.success? ? nil : failure_lines(out) }.compact
+    end
+
+    # Rake ends every failure with the same two lines; the cause is above them.
+    def self.failure_lines(out)
+      lines = out.lines.map(&:rstrip).reject { _1.strip.empty? || _1.start_with?("Tasks:", "(See full trace") }
+      (lines.grep(/error|refused|could not|failed|bad/i).first(4) + lines.last(2)).uniq
+    end
+
+    def initialize(data) = @data = data
+    def ok? = data["status"] == "ok"
+    def failure = data["failure"]
+    def framework = data["framework"]
+    def env_names = data["env"]
+    def side_setup = data["side_setup"]
+    def bundle = data["bundle"]
+    def helper = data["helper"]
+    def base_branch = data["base_branch"]
+    def coverage = data["coverage"]
+    def database_url = data.dig("database", "url")
+    def env_for(side) = self.class.side_env(data, SIDES.fetch(side))
+    def test_suffix = framework == "rspec" ? "_spec.rb" : "_test.rb"
+
+    def write(root)
+      FileUtils.mkdir_p(Holdfast.dir(root))
+      File.write(self.class.path(root), "#{JSON.pretty_generate(data)}\n")
+    end
+  end
+end