recheck 0.0.1 → 0.5.0

data/lib/recheck/reporters.rb

@@ -0,0 +1,277 @@
+module Recheck
+  module Reporter
+    class Base
+      @subclasses = []
+
+      # Register subclasses for `recheck reporters`.
+      class << self
+        attr_reader :subclasses
+
+        def inherited(subclass)
+          super
+          @subclasses << subclass
+        end
+      end
+
+      def self.help
+      end
+
+      def initialize(arg:)
+      end
+
+      def fetch_record_id(record)
+        if Recheck.unloaded_is_a? record, "ActiveRecord::Base"
+          record.id.to_s
+          # or: record.to_global_id, if you want to override in
+          # your_app/recheck/reporter/base_reporter.rb
+        elsif Recheck.unloaded_is_a? record, "Sequel::Model"
+          record.pk.to_s # may be an array
+        else
+          record.to_s
+        end
+      end
+
+      # A recheck run flows like this, with indicated calls to each reporter.
+      #
+      # around_run -> for each Checker class:
+      #   around_checker ->
+      #     around_query ->
+      #       run each query() method
+      #       for each 'check_' method on the checker:
+      #         for each record queried:
+      #           around_check ->
+      #             check(record)
+
+      def around_run(checkers: [])
+        total_count = yield
+      end
+
+      def around_checker(checker:, queries: [], checks: [])
+        counts = yield
+      end
+
+      def around_query(checker:, query:, checks: [])
+        yield
+      end
+
+      def around_check(checker:, query:, check:, record:)
+        result = yield
+      end
+
+      def halt(checker:, query:, error:, check: nil)
+        # running the checker was halted, so there's no result available for yield
+      end
+    end # Base
+
+    class Cron < Base
+      def self.help
+        "Prints failures/exceptions but nothing on pass. For use in cron jobs, which use silence to incidate success."
+      end
+
+      def initialize(arg:)
+        raise ArgumentError, "does not take options" unless arg.nil?
+        @errors = []
+      end
+
+      def around_run(checkers: [])
+        total_counts = yield
+
+        if total_counts.any_errors?
+          puts "Total: #{total_counts.summary}"
+        end
+      end
+
+      def around_checker(checker:, queries:, checks:)
+        @errors = []
+
+        counts = yield
+
+        if counts.any_errors?
+          puts "#{checker.class}: #{counts.summary}"
+          print_errors
+        end
+      end
+
+      def around_check(checker:, query:, check:, record:)
+        result = yield
+        @errors << result if result.is_a? Error
+      end
+
+      def halt(checker:, query:, error:, check: nil)
+        @errors << error
+      end
+
+      def print_errors
+        failure_details = []
+        grouped_errors = @errors.group_by { |e| [e.checker, e.query, e.check, e.type] }
+
+        grouped_errors.each do |(checker, query, check), group_errors|
+          case group_errors.first.type
+          when :fail
+            ids = group_errors.map { |e| fetch_record_id(e.record) }.join(", ")
+            failure_details << "  #{checker}##{query} -> #{check} failed for records: #{ids}"
+          when :exception
+            error = group_errors.first
+            error_message = "  #{checker}##{query} -> #{check} exception #{error.exception.message} for #{group_errors.size} records"
+            failure_details << error_message
+            failure_details << error.record.full_message(highlight: false, order: :top) if error.record.respond_to?(:full_message)
+          when :blanket
+            failure_details << "  #{checker}: Skipping because the first 20 checks all failed. Either there's a lot of bad data or there's something wrong with the checks."
+          end
+        end
+        puts failure_details
+      end
+    end # Cron
+
+    class Default < Base
+      def self.help
+        "Used when no --reporter is named. Prints incremental progress to stdout. No options."
+      end
+
+      def initialize(arg:)
+        raise ArgumentError, "does not take options" unless arg.nil?
+        @current_counts = CountStats.new
+        @errors = []
+      end
+
+      def around_run(checkers: [])
+        total_counts = yield
+
+        puts "Total: #{total_counts.summary}"
+        puts "Queries found no records to check (this is OK when a checker queries for invalid data)" if total_counts.all_zero?
+
+        total_counts
+      end
+
+      def around_checker(checker:, queries:, checks:, check: [])
+        @errors = []
+
+        print "#{checker.class}: "
+        counts = yield
+
+        # don't double-print last progress indicator
+        print_progress unless @current_counts.total % 1000 == 0
+        print_check_summary(counts)
+        print_errors
+
+        counts
+      end
+
+      def around_check(checker:, query:, check:, record:)
+        result = yield
+
+        @current_counts.increment(result.type)
+        print_progress if @current_counts.total % 1000 == 0
+
+        @errors << result if result.is_a? Error
+      end
+
+      def halt(checker:, query:, error:, check: nil)
+        @errors << error
+      end
+
+      def print_check_summary(counts)
+        puts "  #{counts.summary}"
+      end
+
+      def print_errors
+        failure_details = []
+        grouped_errors = @errors.group_by { |e| [e.checker, e.query, e.check, e.type] }
+
+        grouped_errors.each do |(checker, query, check), group_errors|
+          case group_errors.first.type
+          when :fail
+            ids = group_errors.map { |e| fetch_record_id(e.record) }.join(", ")
+            failure_details << "  #{checker.class}##{query} -> #{check} failed for records: #{ids}"
+          when :exception
+            error = group_errors.first
+            error_message = "  #{checker.class}##{query} -> #{check} exception #{error.exception.message} for #{group_errors.size} records"
+            failure_details << error_message
+            failure_details << error.exception.full_message(highlight: false, order: :top) if error.exception.respond_to?(:full_message)
+          when :no_query_methods
+            failure_details << "  #{checker.class}: Did not define .query_methods"
+          when :no_queries
+            failure_details << "  #{checker.class} Defines .query_methods, but it didn't return any"
+          when :no_check_methods
+            failure_details << "  #{checker.class}: Did not define .check_methods"
+          when :no_checks
+            failure_details << "  #{checker.class} Defines .check_methods, but it didn't return any"
+          when :blanket
+            failure_details << "  #{checker.class}: Skipping because the first 20 checks all failed. Either there's a lot of bad data or there's something wrong with the checker."
+          else
+            failure_details << "  #{checker.class} unknown error"
+          end
+        end
+        puts failure_details
+      end
+
+      def print_progress
+        print @current_counts.all_pass? ? "." : "x"
+        @current_counts = CountStats.new
+      end
+    end # Default
+
+    class Json < Base
+      def self.help
+        "Outputs JSON-formatted results to a file or stdout. Arg is filename or blank for stdout."
+      end
+
+      def initialize(arg:)
+        @filename = arg
+        @results = {}
+      end
+
+      def around_checker(checker:, queries:, checks:, check: [])
+        @results[checker.class.to_s] = checks.to_h { |method|
+          [method, {
+            counts: CountStats.new,
+            fail: [],
+            exception: []
+          }]
+        }
+        yield
+      end
+
+      def around_check(checker:, query:, check:, record:)
+        result = yield
+
+        # puts "around_check(checker: #{checker}, query: #{query}, check: #{check.inspect}, record: #{record}"
+        check ||= query
+        @results[checker.class.to_s][check][:counts].increment(result.type)
+        case result.type
+        when :fail
+          @results[checker.class.to_s][check][:fail] << fetch_record_id(result.record)
+        when :exception
+          @results[checker.class.to_s][check][:exception] << {
+            id: fetch_record_id(result.record),
+            message: result.exception.message,
+            backtrace: result.exception.backtrace
+          }
+        end
+      end
+
+      def around_run(checkers)
+        yield
+        if @filename
+          File.write(@filename, @results.to_json)
+        else
+          puts @results.to_json
+        end
+      end
+
+      def halt(checker:, query:, error:, check: "meta")
+        @results[checker.class.to_s][check][:halt] = error.type
+      end
+    end # Json
+
+    class Silent < Base
+      def self.help
+        "Prints nothing. Useful for checks that can automatically fix issues."
+      end
+
+      def initialize(arg:)
+        raise ArgumentError, "does not take options" unless arg.nil?
+      end
+    end # Silent
+  end
+end

data/lib/recheck/results.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Recheck
+  ERROR_TYPES = [:fail, :exception, :blanket, :no_query_methods, :no_queries, :no_check_methods, :no_checks].freeze
+  RESULT_TYPES = ([:pass] + ERROR_TYPES).freeze
+
+  # This doesn't track all the fields because Recheck is about finding errors and failures.
+  # If you need more data, please tell me about your use case?
+  Pass = Data.define do
+    def type
+      :pass
+    end
+  end
+
+  Error = Data.define(:type, :checker, :query, :check, :record, :exception) do
+    def initialize(*args)
+      super
+      raise ArgumentError unless ERROR_TYPES.include? type
+    end
+  end
+end

data/lib/recheck/runner.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module Recheck
+  class HookDidNotYield < RuntimeError; end
+
+  class HookYieldedTwice < RuntimeError; end
+
+  class UnexpectedHookYield < RuntimeError; end
+
+  class UnexpectedReporterYield < RuntimeError; end
+
+  class Yields
+    def initialize
+      @executions = {}
+    end
+
+    def expect(hook:, reporter:)
+      @executions[hook] ||= {}
+      @executions[hook][reporter] = false
+      # puts "expect #{hook}, #{reporter.class.name}, id #{reporter.id}"
+    end
+
+    def ran(hook:, reporter:)
+      raise UnexpectedHookYield, "Ran an unexpected hook #{hook} (for reporter #{reporter})" unless @executions.include? hook
+      raise UnexpectedReporterYield, "Ran an expected hook #{hook} for an unexpected reporter #{reporter}" unless @executions[hook].include? reporter
+      raise HookYieldedTwice, "Ran a hook #{hook} twice for reporter #{reporter}" unless @executions[hook][reporter] == false
+
+      # puts "ran #{hook}, #{reporter}, #{reporter.id}"
+      @executions[hook][reporter] = true
+    end
+
+    def raise_unless_all_reporters_yielded(hook:)
+      didnt_yield = @executions[hook].filter { |reporter, ran| ran == false }
+      raise HookDidNotYield, "Reporter(s) [#{didnt_yield.keys.join(", ")}] did not yield in their #{hook} hook" if didnt_yield.any?
+    end
+  end
+
+  class Runner
+    PASSTHROUGH_EXCEPTIONS = [
+      # ours
+      HookDidNotYield, HookYieldedTwice, UnexpectedHookYield,
+      # Ruby's
+      NoMemoryError, SignalException, SystemExit
+    ]
+
+    def initialize(checkers: [], reporters: [])
+      # maintain order and we want to check/report in user-provided order; Set lacks .reverse
+      @checkers = checkers.uniq
+      @reporters = reporters.uniq
+      @yields = Yields.new
+    end
+
+    # compose reporter hooks so they each see the block fire once at 'yield'
+    def reduce(hook:, kwargs: {}, reporters: [], &blk)
+      reporters.reverse.reduce(blk) do |proc, reporter|
+        @yields.expect(hook:, reporter:)
+        -> {
+          result = nil
+          reporter.public_send(hook, **kwargs) {
+            @yields.ran(hook:, reporter:)
+            result = proc.call.freeze
+          }
+          result
+        }
+      end.call
+    end
+
+    # only for calling from inside run()
+    def cant_run reporters:, checker:, queries:, checks:, type:
+      checker_counts = CountStats.new
+      checker_counts.increment type
+      @total_counts << checker_counts
+
+      error = Error.new(checker:, query: nil, check: nil, record: nil, type:, exception: nil)
+      reduce(reporters:, hook: :around_checker, kwargs: {checker:, queries:, checks:}) do
+        reporters.each { it.halt(checker:, query: nil, check: nil, error:) }
+        checker_counts
+      end
+    end
+
+    # n queries * n check methods * n records = O(1) right?
+    def run
+      @total_counts = CountStats.new
+      # All happy families are alike; each unhappy family is unhappy in its own way.
+      pass = Pass.new
+
+      # for want of a monad...
+      reduce(reporters: @reporters, hook: :around_run, kwargs: {checkers: @checkers}) do
+        # for each checker...
+        @checkers.each do |checker|
+          checker_counts = CountStats.new
+          if !checker.class.respond_to?(:query_methods)
+            cant_run(reporters: @reporters, checker:, type: :no_query_methods, queries: nil, checks: nil)
+            next
+          end
+          if (queries = checker.class.query_methods).empty?
+            cant_run(reporters: @reporters, checker:, type: :no_queries, queries:, checks: nil)
+            next
+          end
+
+          if !checker.class.respond_to?(:check_methods)
+            cant_run(reporters: @reporters, checker:, type: :no_check_methods, queries:, checks: nil)
+            next
+          end
+          if (checks = checker.class.check_methods).empty?
+            cant_run(reporters: @reporters, checker:, type: :no_checks, queries:, checks:)
+            next
+          end
+
+          reduce(reporters: @reporters, hook: :around_checker, kwargs: {checker:, queries:, checks:}) do
+            # for each query_...
+            queries.each do |query|
+              reduce(reporters: @reporters, hook: :around_query, kwargs: {checker:, query:, checks:}) do
+                checker_counts.increment :queries
+                # for each record...
+                # TODO: must handle if the query method yields (find_each) OR returns (current)
+                (checker.public_send(query) || []).each do |record|
+                  # for each check_method...
+                  checks.each do |check|
+                    raw_result = nil
+                    reduce(reporters: @reporters, hook: :around_check, kwargs: {checker:, query:, check:, record:}) do
+                      raw_result = checker.public_send(check, record)
+                      result = raw_result ? pass : Error.new(checker:, query:, check:, record:, type: :fail, exception: nil)
+
+                      checker_counts.increment(result.type)
+                      break if checker_counts.reached_blanket_failure?
+
+                      result
+                    rescue *PASSTHROUGH_EXCEPTIONS
+                      raise
+                    rescue => e
+                      Error.new(checker:, query:, check:, record:, type: :exception, exception: e)
+                    end
+                  end
+                  @yields.raise_unless_all_reporters_yielded(hook: :around_check)
+
+                  # if the first 20 error out, halt the check method, it's probably buggy
+                  if checker_counts.reached_blanket_failure?
+                    checker_counts.increment :blanket
+
+                    error = Error.new(checker:, query:, check: nil, record: nil, type: :blanket, exception: nil)
+                    @reporters.each { it.halt(checker:, query:, check: nil, error:) }
+
+                    break
+                  end
+                end
+                nil # yield nothing around_query
+              end
+              @yields.raise_unless_all_reporters_yielded(hook: :around_query)
+            rescue *PASSTHROUGH_EXCEPTIONS
+              raise
+            rescue => e
+              # puts "outer rescue: #{e.inspect}"
+              @reporters.each do |check_reporter|
+                result = Error.new(checker:, query:, check: nil, record: nil, type: :exception, exception: e)
+                check_reporter.around_check(checker:, query: query, check: nil, record: nil) { result }
+              end
+            end
+            checker_counts
+          end
+          @yields.raise_unless_all_reporters_yielded(hook: :around_checker)
+          @total_counts << checker_counts
+        end
+        @total_counts
+      end
+      @yields.raise_unless_all_reporters_yielded(hook: :around_run)
+      @total_counts
+    end
+  end
+end

data/lib/recheck/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Recheck
+  VERSION = '0.5.0'
+end

data/lib/recheck.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Recheck
+  # Check if an obj.is_a? Foo without having to depend on or load the foo gem.
+  def self.unloaded_is_a? obj, class_name
+    raise ArgumentError, "unloaded_is_a? takes class_name as a String" unless class_name.is_a? String
+
+    Object.const_defined?(class_name) && obj.is_a?(Object.const_get(class_name))
+  end
+end
+
+require_relative "../vendor/optimist"
+require_relative "recheck/checkers"
+require_relative "recheck/cli"
+require_relative "recheck/commands"
+require_relative "recheck/results"
+require_relative "recheck/count_stats"
+require_relative "recheck/reporters"
+require_relative "recheck/runner"
+require_relative "recheck/version"

data/template/recheck_helper.rb

@@ -0,0 +1,18 @@
+# This file is automatically required before running any checks.
+# Customize it to load your application environment and provide utility methods.
+
+# For Rails applications:
+require File.expand_path("../config/environment", __dir__)
+
+# For non-Rails applications, you might want to do something like:
+# $LOAD_PATH.unshift File.expand_path('../../lib', __dir__)
+# require 'your_app'
+
+# Load helpers and reporters; not checkers because all loaded checkers are run
+Dir.glob([
+  "#{__dir__}/*_helper*.rb",
+  "#{__dir__}/reporter/**/*.rb"
+]).sort.each { |file| require_relative file }
+
+# Add any other setup here.
+# You could also share code by writing a YourAppChecker class (or classes) for your checkers to inherit from.