verity 0.1.0

data/lib/verity/reporters/documentation_reporter.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Verity
+  module Reporters
+    # Public: Verbose reporter that prints one indented line per test, nesting
+    # output under group headers. Uses ANSI colors (green/red/yellow/magenta)
+    # when outputting to a TTY, unless suppressed by NO_COLOR or forced via
+    # FORCE_COLOR / VERITY_FORCE_COLOR.
+    class DocumentationReporter
+      include Verity::Reporter
+
+      ESC = "\e["
+      RESET = "#{ESC}0m"
+      PASS_STYLE = "#{ESC}32m"
+      FAIL_STYLE = "#{ESC}31m"
+      SKIP_STYLE = "#{ESC}33m"
+      ERROR_STYLE = "#{ESC}35m"
+
+      # Public: Create a new DocumentationReporter.
+      #
+      # io    - IO object for output (default $stdout).
+      # color - Boolean to force color on/off, or nil for auto-detect.
+      def initialize(io = $stdout, color: nil)
+        @io = io
+        @color_override = color
+      end
+
+      # Public: Print a "Running N tests..." header.
+      def on_run_start(total:, worker_id:)
+        @last_group_path = nil
+        return if total.nil?
+
+        @io.puts "Running #{total} tests..."
+        @io.puts
+      end
+
+      # Public: Print a status-labeled line for the completed test, indented
+      # under its group headers.
+      def on_test_complete(result:, worker_id:)
+        path = Array(result.test.group_path)
+        emit_group_headers(path)
+        indent = "  " * (path.size + 1)
+        case result.status
+        when :pass
+          @io.puts "#{indent}#{paint("pass", PASS_STYLE)}  #{result.test.description}"
+        when :fail
+          @io.puts "#{indent}#{paint("FAIL", FAIL_STYLE)}  #{result.test.description}\n         #{result.error.message}"
+        when :error
+          msg = "#{result.error.class}: #{result.error.message}"
+          @io.puts "#{indent}#{paint("ERROR", ERROR_STYLE)} #{result.test.description}\n         #{msg}"
+        when :skip
+          @io.puts "#{indent}#{paint("skip", SKIP_STYLE)}  #{result.test.description}"
+        end
+      end
+
+      # Public: Print the final summary line with counts and optional color.
+      def on_run_finish(summary:, worker_id:)
+        t = summary[:total]
+        p = summary[:passed]
+        f = summary[:failed]
+        e = summary[:errored]
+        sk = summary[:skipped].to_i
+        if color?
+          parts = [
+            "\n#{t} tests:",
+            "#{paint("#{p} passed", PASS_STYLE)},",
+            "#{paint("#{f} failed", FAIL_STYLE)},",
+            "#{paint("#{e} errored", ERROR_STYLE)}"
+          ]
+          parts << ", #{paint("#{sk} skipped", SKIP_STYLE)}" if sk.positive?
+          line = "#{parts.join(" ")}#{RESET}"
+        else
+          line = "\n#{t} tests: #{p} passed, #{f} failed, #{e} errored"
+          line += ", #{sk} skipped" if sk.positive?
+        end
+        line += " (focus)" if summary[:focus]
+        @io.puts line
+      end
+
+      # Public: Delegate to ParallelSummaryReporter for the multi-worker summary.
+      def on_parallel_complete(counts:, problem_rows:)
+        ParallelSummaryReporter.new(@io).emit(counts:, problem_rows:)
+      end
+
+      private
+
+      def paint(text, sequence)
+        return text unless color?
+
+        "#{sequence}#{text}#{RESET}"
+      end
+
+      def color?
+        return @color_override unless @color_override.nil?
+
+        return false if ENV.key?("NO_COLOR")
+        return true if truthy_env?(ENV["FORCE_COLOR"]) || truthy_env?(ENV["VERITY_FORCE_COLOR"])
+
+        @io.respond_to?(:tty?) && @io.tty?
+      end
+
+      def truthy_env?(value)
+        %w[1 true yes].include?(value&.downcase)
+      end
+
+      def emit_group_headers(path)
+        last = @last_group_path || []
+        common = 0
+        n = [last.size, path.size].min
+        while common < n && last[common] == path[common]
+          common += 1
+        end
+
+        (common...path.size).each do |i|
+          @io.puts "#{"  " * i}#{path[i]}"
+        end
+        @last_group_path = path.dup
+      end
+    end
+  end
+end

data/lib/verity/reporters/dots_reporter.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Verity
+  module Reporters
+    # Public: Minimal reporter that prints a single character per test:
+    # "." for pass, "F" for failure, "E" for error. No color.
+    class DotsReporter
+      include Verity::Reporter
+
+      # Public: Create a new DotsReporter.
+      #
+      # io - IO object for output (default $stdout).
+      def initialize(io = $stdout)
+        @io = io
+      end
+
+      # Public: Print a dot, F, E, or S (skip) for the completed test.
+      def on_test_complete(result:, worker_id:)
+        char =
+          case result.status
+          when :pass then "."
+          when :fail then "F"
+          when :error then "E"
+          when :skip then "S"
+          end
+        @io.print char
+        @io.flush
+      end
+
+      # Public: Print the final summary line with counts.
+      def on_run_finish(summary:, worker_id:)
+        t = summary[:total]
+        p = summary[:passed]
+        f = summary[:failed]
+        e = summary[:errored]
+        line = "\n\n#{t} tests: #{p} passed, #{f} failed, #{e} errored"
+        line += ", #{summary[:skipped]} skipped" if summary[:skipped].to_i.positive?
+        line += " (focus)" if summary[:focus]
+        @io.puts line
+      end
+
+      # Public: Delegate to ParallelSummaryReporter for the multi-worker summary.
+      def on_parallel_complete(counts:, problem_rows:)
+        ParallelSummaryReporter.new(@io).emit(counts:, problem_rows:)
+      end
+    end
+  end
+end

data/lib/verity/reporters/null_reporter.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Verity
+  module Reporters
+    # Public: Silent reporter that discards all output. Used in forked worker
+    # processes and in tests where reporter output is unwanted.
+    class NullReporter
+      include Verity::Reporter
+    end
+  end
+end

data/lib/verity/reporters/parallel_summary_reporter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Verity
+  module Reporters
+    # Internal: Shared helper that emits the multi-worker summary block.
+    # Typically called from a reporter's on_parallel_complete to print
+    # aggregate counts and list any failures or errors from the manifest.
+    class ParallelSummaryReporter
+      def initialize(io = $stdout)
+        @io = io
+      end
+
+      # Public: Write the parallel-run summary to the IO stream.
+      #
+      # counts       - Hash with String status keys ("passed", "failed", etc.)
+      #                and Integer counts.
+      # problem_rows - Array of Hashes with :fingerprint, :description, :status,
+      #                and :failure from Manifest#failures_for_report.
+      #
+      # Returns nothing.
+      def emit(counts:, problem_rows:)
+        passed = counts.fetch("passed", 0)
+        failed = counts.fetch("failed", 0)
+        errored = counts.fetch("errored", 0)
+        pending = counts.fetch("pending", 0)
+        running = counts.fetch("running", 0)
+        skipped = counts.fetch("skipped", 0)
+        total = passed + failed + errored + pending + running
+
+        line = "Parallel run finished (#{total} tests in manifest: #{passed} passed, #{failed} failed, #{errored} errored, #{pending} pending, #{running} running"
+        line += ", #{skipped} skipped" if skipped > 0
+        line += ")"
+        @io.puts "\n#{line}"
+
+        return if problem_rows.empty?
+
+        @io.puts "\nFailures and errors:"
+        problem_rows.each do |row|
+          fp = row[:fingerprint]
+          desc = row[:description]
+          st = row[:status]
+          @io.puts "  #{st}  #{desc} (#{fp})"
+          next if row[:failure].nil? || row[:failure].empty?
+
+          msg = row[:failure]["message"] || row[:failure][:message]
+          @io.puts "         #{msg}" if msg
+        end
+      end
+    end
+  end
+end

data/lib/verity/reporters/test_reporter.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Verity
+  module Reporters
+    # Public: In-memory reporter that records every callback for later
+    # inspection. Produces no I/O — designed for use in Verity's own tests
+    # and tooling.
+    class TestReporter
+      include Verity::Reporter
+
+      def initialize
+        @run_starts = []
+        @test_completes = []
+        @run_finishes = []
+        @parallel_finishes = []
+      end
+
+      # Public: Array of Hashes recorded from on_run_start calls.
+      # Each Hash contains :total and :worker_id.
+      #
+      # Public: Array of Hashes recorded from on_test_complete calls.
+      # Each Hash contains :status, :error (exception or nil), and :worker_id.
+      #
+      # Public: Array of Hashes recorded from on_run_finish calls.
+      # Each Hash contains :summary and :worker_id.
+      #
+      # Public: Array of Hashes recorded from on_parallel_complete calls.
+      # Each Hash contains :counts and :problem_rows.
+      attr_reader :run_starts, :test_completes, :run_finishes, :parallel_finishes
+
+      def on_run_start(total:, worker_id:)
+        @run_starts << { total: total, worker_id: worker_id }
+      end
+
+      def on_test_complete(result:, worker_id:)
+        @test_completes << { status: result.status, error: result.error, worker_id: worker_id }
+      end
+
+      def on_run_finish(summary:, worker_id:)
+        @run_finishes << { summary: summary, worker_id: worker_id }
+      end
+
+      def on_parallel_complete(counts:, problem_rows:)
+        @parallel_finishes << { counts: counts, problem_rows: problem_rows }
+      end
+    end
+  end
+end

data/lib/verity/runner.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module Verity
+  # Public: Executes tests, fires reporter hooks, and records results back to
+  # the manifest. A single Runner instance services one worker process.
+  class Runner
+    # Public: Immutable outcome of running a single test.
+    #
+    # test   - The Verity::Test that was executed.
+    # status - Symbol :pass, :fail, :error, or :skip.
+    # error  - Exception instance or nil.
+    Result = Data.define(:test, :status, :error)
+
+    # Public: Create a new Runner.
+    #
+    # reporter - Object implementing Verity::Reporter (default: from configuration).
+    def initialize(reporter: nil)
+      @reporter = reporter || Verity.configuration.reporter
+    end
+
+    # Public: Run a list of tests in-process without a manifest. Primarily
+    # used for simple single-worker execution.
+    #
+    # tests - Array of Verity::Test (default: all registered tests).
+    #
+    # Returns true if every test passed.
+    def run(tests = Registry.all)
+      run_worker(tests, worker_id: 0)
+    end
+
+    CONFLICT_RETRY_INTERVAL = 0.05
+
+    # Public: Claim and execute tests from a shared manifest until none remain.
+    # Fires before_worker_start hooks, then loops claim_next until exhausted.
+    # When resource resolvers are registered, builds a conflict exclusion list
+    # before each claim and sleeps briefly when blocked by running tests.
+    #
+    # manifest  - A Verity::Manifest instance.
+    # worker_id - Integer identifying this worker.
+    #
+    # Returns true if every executed test passed.
+    def run_manifest(manifest, worker_id:)
+      Verity.hooks[:before_worker_start].each(&:call)
+
+      @reporter.on_run_start(total: manifest.example_count, worker_id: worker_id)
+
+      results = []
+      loop do
+        claimed = next_claim(manifest, worker_id)
+        if claimed == :blocked
+          sleep CONFLICT_RETRY_INTERVAL
+          next
+        end
+        break unless claimed
+
+        test = Registry.find(claimed.fingerprint)
+        unless test
+          err = RuntimeError.new(
+            "Test fingerprint not in Registry (load files before replace_tests): #{claimed.fingerprint}"
+          )
+          manifest.record_error(claimed.fingerprint, err)
+          result = Result.new(test: synthetic_test_from_claim(claimed), status: :error, error: err)
+          results << result
+          @reporter.on_test_complete(result: result, worker_id: worker_id)
+          next
+        end
+
+        result = run_with_hooks(test)
+        results << result
+        @reporter.on_test_complete(result: result, worker_id: worker_id)
+
+        case result.status
+        when :pass  then manifest.record_pass(test.fingerprint)
+        when :fail  then manifest.record_failure(test.fingerprint, result.error)
+        when :error then manifest.record_error(test.fingerprint, result.error)
+        end
+      end
+
+      Registry.all.select { Verity.skipped?(_1) }.each do |t|
+        @reporter.on_test_complete(
+          result: Result.new(test: t, status: :skip, error: nil),
+          worker_id: worker_id
+        )
+      end
+
+      without_skip = Registry.all.reject { Verity.skipped?(_1) }
+      skipped = Registry.all.count { Verity.skipped?(_1) }
+      focus = Verity.focus_filter_active?(without_skip)
+
+      @reporter.on_run_finish(
+        summary: build_summary(results, skipped: skipped, focus: focus),
+        worker_id: worker_id
+      )
+      results.all? { |r| r.status == :pass }
+    end
+
+    private
+
+    # Returns the next ClaimedRow, nil when the queue is empty, or the symbol
+    # :blocked when pending tests exist but all conflict with running tests.
+    def next_claim(manifest, worker_id)
+      return manifest.claim_next(worker_id) if Verity.resource_resolvers.empty?
+
+      running_res = manifest.running_resources
+      exclude = Verity.conflict_exclusion_list(running_res)
+      claimed = manifest.claim_next(worker_id, exclude: exclude)
+      return claimed if claimed
+      exclude.empty? ? nil : :blocked
+    end
+
+    def run_worker(tests, worker_id:)
+      without_skip = tests.reject { Verity.skipped?(_1) }
+      list =
+        if without_skip.any? { Verity.focus_tag?(_1) }
+          without_skip.select { Verity.focus_tag?(_1) }
+        else
+          without_skip
+        end
+
+      skipped = tests.count { Verity.skipped?(_1) }
+      focus = Verity.focus_filter_active?(without_skip)
+
+      @reporter.on_run_start(total: list.size, worker_id: worker_id)
+
+      results = []
+      list.each do |t|
+        r = run_with_hooks(t)
+        results << r
+        @reporter.on_test_complete(result: r, worker_id: worker_id)
+      end
+
+      tests.select { Verity.skipped?(_1) }.each do |t|
+        r = Result.new(test: t, status: :skip, error: nil)
+        @reporter.on_test_complete(result: r, worker_id: worker_id)
+      end
+
+      @reporter.on_run_finish(
+        summary: build_summary(results, skipped: skipped, focus: focus),
+        worker_id: worker_id
+      )
+      results.all? { |r| r.status == :pass }
+    end
+
+    def build_summary(results, skipped:, focus:)
+      {
+        total: results.size,
+        passed: results.count { |r| r.status == :pass },
+        failed: results.count { |r| r.status == :fail },
+        errored: results.count { |r| r.status == :error },
+        skipped: skipped,
+        focus: focus
+      }
+    end
+
+    def synthetic_test_from_claim(claimed)
+      Test.new(
+        fingerprint: claimed.fingerprint,
+        description: claimed.description,
+        tags: claimed.tags.map(&:to_sym),
+        timeout: claimed.timeout,
+        requires: claimed.requires.map(&:to_sym),
+        resources: claimed.resources.transform_keys(&:to_sym),
+        file: claimed.file,
+        line: claimed.line,
+        fn: -> {},
+        group_path: [],
+        inherited_group_tags: [],
+        group_scopes: []
+      )
+    end
+
+    def run_with_hooks(test)
+      result = nil
+      begin
+        Verity.hooks[:before_test].each(&:call)
+        result = execute(test)
+      rescue => e
+        result = Result.new(test:, status: :error, error: e)
+      ensure
+        Verity.hooks[:after_test].each(&:call)
+      end
+      result
+    end
+
+    def execute(test)
+      body = proc { test.fn.call }
+      if (sec = timeout_seconds_for(test))
+        Timeout.timeout(sec, TestTimeoutError, &body)
+      else
+        body.call
+      end
+      Result.new(test:, status: :pass, error: nil)
+    rescue AssertionError => e
+      Result.new(test:, status: :fail, error: e)
+    rescue TestTimeoutError => e
+      Result.new(test:, status: :error, error: e)
+    rescue => e
+      Result.new(test:, status: :error, error: e)
+    end
+
+    def timeout_seconds_for(test)
+      Verity.validate_test_timeout!(test.timeout)
+      return nil if test.timeout.nil?
+
+      test.timeout.to_f
+    end
+  end
+end

data/lib/verity/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Verity
+  VERSION = "0.1.0"
+end