holdfast 0.1.0

data/lib/holdfast/runtime/observe.rb

@@ -0,0 +1,136 @@
+# Loaded by holdfast's pin tests inside the app's own suite. It calls one
+# method on one input and turns what it saw into plain JSON: the returned
+# value (a record by its attributes, not its class), the error it raised,
+# and the attributes of each record the call created or changed. A pin test
+# compares that JSON with the JSON the base side produced, and names only the
+# paths that differ.
+require "json"
+require "date"
+
+module Holdfast
+  module Observe
+    # Ids, foreign keys and timestamps differ between two runs of the same
+    # input, so they never decide a pin.
+    VOLATILE = /\A(id|created_at|updated_at)\z|_id\z/
+    RECENT = 86_400
+
+    module Track
+      def _create_record(*)
+        super.tap { (Thread.current[:holdfast_writes] ||= []) << [:created, self] if Thread.current[:holdfast_writes] }
+      end
+
+      def _update_record(*)
+        super.tap { (Thread.current[:holdfast_writes] ||= []) << [:updated, self] if Thread.current[:holdfast_writes] }
+      end
+    end
+    ActiveRecord::Base.prepend(Track) if defined?(ActiveRecord::Base)
+
+    module_function
+
+    def observe(context, input, target)
+      const_name, kind, name = target.match(/\A(.+?)([#.])(\w+[?!=]?)\z/).captures
+      const = Object.const_get(const_name)
+      within_savepoint do
+        run_ruby(context, input["before"]) if input["before"]
+        receiver = if input.key?("receiver") then value(context, input["receiver"])
+                   elsif kind == "." then const
+                   else const.new(*value(context, input.fetch("new", [])))
+                   end
+        args = value(context, Array(input["args"]))
+        kwargs = value(context, input.fetch("kwargs", {})).transform_keys(&:to_sym)
+        Thread.current[:holdfast_writes] = []
+        outcome = begin
+          { "returns" => project(receiver.__send__(name, *args, **kwargs)) }
+        rescue StandardError => e
+          { "raises" => e.class.name, "message" => e.message.gsub(/\d{4,}/, "N").lines.first.to_s.strip }
+        end
+        writes = Thread.current[:holdfast_writes]
+        outcome["writes"] = written(writes) unless writes.empty?
+        outcome
+      ensure
+        Thread.current[:holdfast_writes] = nil
+      end
+    end
+
+    # Paths whose values differ, as "path: base X, head Y" lines.
+    def differences(expected, actual, path = "")
+      return [] if expected == actual
+
+      if expected.is_a?(Hash) && actual.is_a?(Hash)
+        (expected.keys | actual.keys).flat_map { differences(expected[_1], actual[_1], path.empty? ? _1.to_s : "#{path}.#{_1}") }
+      elsif expected.is_a?(Array) && actual.is_a?(Array) && expected.size == actual.size
+        expected.each_index.flat_map { differences(expected[_1], actual[_1], "#{path}[#{_1}]") }
+      else
+        ["#{path.empty? ? "value" : path}: base #{short(expected)}, head #{short(actual)}"]
+      end
+    end
+
+    def short(v) = (v.nil? ? "(none)" : v.is_a?(String) ? v : JSON.generate(v)).then { _1.size > 90 ? "#{_1[0, 87]}..." : _1 }
+
+    def within_savepoint
+      return yield unless defined?(ActiveRecord::Base)
+
+      result = nil
+      ActiveRecord::Base.transaction(requires_new: true) do
+        result = yield
+        raise ActiveRecord::Rollback
+      end
+      result
+    end
+
+    def run_ruby(context, code) = context.instance_eval(code.is_a?(Hash) ? code.fetch("ruby") : code.to_s)
+
+    def value(context, v)
+      case v
+      when Hash then v.keys == ["ruby"] ? run_ruby(context, v) : v.to_h { [_1, value(context, _2)] }
+      when Array then v.map { value(context, _1) }
+      else v
+      end
+    end
+
+    def written(writes)
+      seen = {}
+      writes.each do |how, record|
+        key = [how, record.class.name, record.object_id]
+        if how == :created
+          seen[key] = { "created" => record.class.name, "attributes" => attributes(record) }
+        else
+          changed = record.saved_changes.keys.grep_v(VOLATILE)
+          next if changed.empty?
+
+          entry = (seen[key] ||= { "updated" => record.class.name, "changed" => {} })
+          changed.each { entry["changed"][_1] = project(record.public_send(_1)) }
+        end
+      end
+      seen.values.sort_by { JSON.generate(_1) }
+    end
+
+    def attributes(record) = record.attributes.reject { |k, _| k.match?(VOLATILE) }.sort.to_h.transform_values { project(_1) }
+
+    def project(v, depth = 0)
+      return v.class.name if depth > 3
+
+      case v
+      when nil, true, false, Integer, String then v
+      when Float then v.round(6)
+      when Symbol then ":#{v}"
+      when Date then v.respond_to?(:hour) ? time(v) : "Date #{v.iso8601}"
+      when Time then time(v)
+      when ->(x) { defined?(ActiveSupport::TimeWithZone) && x.is_a?(ActiveSupport::TimeWithZone) } then time(v)
+      when ->(x) { defined?(BigDecimal) && x.is_a?(BigDecimal) } then v.to_s("F")
+      when ->(x) { defined?(ActiveRecord::Base) && x.is_a?(ActiveRecord::Base) }
+        { "record" => v.class.name, "persisted" => v.persisted? }.merge(attributes(v))
+      when ->(x) { defined?(ActiveRecord::Relation) && x.is_a?(ActiveRecord::Relation) } then v.to_a.first(20).map { project(_1, depth + 1) }
+      when Array then v.first(50).map { project(_1, depth + 1) }
+      when Hash then v.to_h { [_1.is_a?(Symbol) ? ":#{_1}" : _1.to_s, project(_2, depth + 1)] }
+      when Data then { "data" => v.class.name }.merge(v.to_h.to_h { [_1.to_s, project(_2, depth + 1)] })
+      when Struct then { "struct" => v.class.name }.merge(v.to_h.to_h { [_1.to_s, project(_2, depth + 1)] })
+      else { "object" => v.class.name }
+      end
+    end
+
+    # A time near now comes from the clock, not the input, so it is pinned
+    # as recent only.
+    def time(t) = (Time.now.to_f - t.to_f).abs < RECENT ? "Time (now)" : "Time #{t.utc.iso8601}"
+  end
+end

data/lib/holdfast/runtime/reach.rb

@@ -0,0 +1,104 @@
+# Loaded through RUBYOPT into each test process jitcatch starts, before the
+# app: it records which lines of the app each test file ran, so a row can say
+# whether a test reached the changed lines or only named the method. It runs
+# with no holdfast code loaded and writes one JSON file per process.
+require "coverage"
+require "json"
+
+# eval: true also counts code compiled from a file name, which is how
+# ActionView runs a template. Branches tell whether the body of a
+# `return x if y` ran, which line counts cannot.
+Coverage.start(lines: true, branches: true, eval: true) unless Coverage.running?
+
+# Which of the values the change added or removed (HOLDFAST_VALUES, a JSON
+# list) an input drew: a row written with that value. Reads do not count: a
+# scope that selects by a constant puts every member in each query.
+HOLDFAST_WANTED = (JSON.parse(ENV.fetch("HOLDFAST_VALUES", "[]")) rescue []).map(&:to_s)
+HOLDFAST_DREW = []
+unless HOLDFAST_WANTED.empty?
+  attach = TracePoint.new(:end) do |tp|
+    next unless defined?(ActiveSupport::Notifications)
+
+    tp.disable
+    ActiveSupport::Notifications.subscribe("sql.active_record") do |*, payload|
+      sql = payload[:sql].to_s
+      next unless sql.match?(/\A\s*(INSERT|UPDATE)/i)
+
+      binds = payload[:type_casted_binds]
+      binds = binds.call if binds.respond_to?(:call)
+      seen = Array(binds).map(&:to_s) + sql.scan(/'((?:[^']|'')*)'/).flatten
+      HOLDFAST_DREW.concat(HOLDFAST_WANTED & seen)
+      HOLDFAST_DREW.uniq!
+    end
+  end
+  attach.enable
+end
+
+# What ran since the last take, as { file => lines } and { file => arms },
+# for the app's own files. Each take clears the counts, so the file's total
+# is the union of every take.
+HOLDFAST_ROOT = "#{Dir.pwd}/"
+HOLDFAST_TOTAL = { "hits" => Hash.new { |h, k| h[k] = [] }, "arms" => Hash.new { |h, k| h[k] = [] } }
+def holdfast_take
+  hits = {}
+  arms = {}
+  Coverage.result(stop: false, clear: true).each do |path, counts|
+    next unless path.start_with?(HOLDFAST_ROOT)
+
+    relative = path.delete_prefix(HOLDFAST_ROOT)
+    next unless relative.start_with?("app/", "lib/")
+
+    lines = counts.is_a?(Hash) ? counts[:lines] : counts
+    ran = Array(lines).each_with_index.filter_map { |count, i| i + 1 if count.to_i.positive? }
+    taken = (counts.is_a?(Hash) && counts[:branches] || {}).values.flat_map { |a| a.filter_map { |(_, _, line, column, last, last_column), n| "#{line}:#{column}-#{last}:#{last_column}" if n.positive? } }
+    hits[relative] = ran unless ran.empty?
+    arms[relative] = taken unless taken.empty?
+  end
+  hits.each { |k, v| HOLDFAST_TOTAL["hits"][k] |= v }
+  arms.each { |k, v| HOLDFAST_TOTAL["arms"][k] |= v }
+  { "hits" => hits, "arms" => arms }
+end
+
+# One test file holds examples that agree and examples the change added, so
+# a file's lines cannot say which example drew a new branch. Each example's
+# own lines are kept by the line it is declared on, the line jitcatch
+# reports for it.
+HOLDFAST_EXAMPLES = {}
+module HoldfastRSpecExample
+  def run(*)
+    holdfast_take
+    super
+  ensure
+    HOLDFAST_EXAMPLES[metadata[:line_number]] = holdfast_take
+  end
+end
+module HoldfastMinitestTest
+  def run(*)
+    holdfast_take
+    super
+  ensure
+    line = (method(name).source_location&.last rescue nil)
+    HOLDFAST_EXAMPLES[line] = holdfast_take if line
+  end
+end
+holdfast_hook = TracePoint.new(:end) do |tp|
+  if defined?(RSpec::Core::Example) && !RSpec::Core::Example.include?(HoldfastRSpecExample)
+    RSpec::Core::Example.prepend(HoldfastRSpecExample)
+    tp.disable
+  elsif defined?(Minitest::Test) && !Minitest::Test.include?(HoldfastMinitestTest)
+    Minitest::Test.prepend(HoldfastMinitestTest)
+    tp.disable
+  end
+end
+holdfast_hook.enable
+
+at_exit do
+  dir = ENV["HOLDFAST_REACH_DIR"]
+  test = ARGV.reverse.find { _1.include?(".jitcatch/") }
+  next unless dir && test
+
+  holdfast_take
+  record = { "side" => ENV["TEST_ENV_NUMBER"], "test" => test.sub(%r{\A.*?\.jitcatch/}, ""), "hits" => HOLDFAST_TOTAL["hits"],
+             "drew" => HOLDFAST_DREW, "arms" => HOLDFAST_TOTAL["arms"], "examples" => HOLDFAST_EXAMPLES.compact }
+  File.write(File.join(dir, "reach-#{ENV["TEST_ENV_NUMBER"]}-#{Process.pid}.json"), JSON.generate(record))
+end