checkset 0.1.0

data/lib/checkset/prep.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Checkset
+  class Prep
+    attr_reader :credentials, :target_url
+
+    def initialize(credentials:, target_url:)
+      @credentials = credentials
+      @target_url = target_url
+    end
+
+    # Override in subclass — perform the prep action.
+    # Receives the same Playwright page the check will use,
+    # so browser state (cookies, localStorage) persists.
+    def satisfy(page)
+      raise NotImplementedError, "#{self.class}#satisfy must be implemented"
+    end
+
+    # Override in subclass — optional optimization.
+    # Return true to skip satisfy (e.g., already signed in).
+    def satisfied?(page)
+      false
+    end
+
+    # Resolve a prep name to its class.
+    # :sign_in_as_admin → Preps::SignInAsAdmin
+    def self.resolve(name)
+      return name if name.is_a?(Class) && name < Checkset::Prep
+
+      const_name = name.to_s.split("_").map(&:capitalize).join
+      if defined?(Preps) && Preps.const_defined?(const_name)
+        Preps.const_get(const_name)
+      else
+        raise NameError, "Could not resolve prep :#{name} — expected class Preps::#{const_name}"
+      end
+    end
+  end
+end

data/lib/checkset/reporter/cli.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Checkset
+  module Reporter
+    class CLI
+      COLORS = {
+        green:  "\e[32m",
+        red:    "\e[31m",
+        yellow: "\e[33m",
+        bold:   "\e[1m",
+        dim:    "\e[2m",
+        reset:  "\e[0m"
+      }.freeze
+
+      def initialize(target_url:, suite_name: nil, io: $stdout)
+        @target_url = target_url
+        @suite_name = suite_name
+        @io = io
+        @color = io.respond_to?(:tty?) && io.tty?
+        @mutex = Mutex.new
+      end
+
+      def print_header
+        puts
+        if @suite_name
+          puts "#{bold}Checkset#{reset} — #{bold}#{@suite_name}#{reset} → #{@target_url}"
+        else
+          puts "#{bold}Checkset#{reset} — #{@target_url}"
+        end
+        puts line
+        puts
+      end
+
+      def report_result(result)
+        @mutex.synchronize do
+          status = format_status(result.status)
+          name = result.check_name.to_s.split("::").last || result.check_name
+          duration = result.duration ? format("%.1fs", result.duration) : "?"
+          dots = "." * [2, 50 - name.length].max
+
+          puts "  #{status}  #{name} #{dim}#{dots}#{reset} #{duration}"
+
+          if result.failed?
+            result.step_results.select(&:failed?).each do |sr|
+              puts "        #{red}✗#{reset} #{sr.type} #{dim}\"#{sr.name}\"#{reset} — #{sr.error}"
+            end
+
+            result.step_results.select(&:failed?).each do |sr|
+              if sr.screenshot_path
+                puts "        #{dim}screenshot: #{sr.screenshot_path}#{reset}"
+              end
+            end
+
+            if result.trace_path
+              puts "        #{dim}trace: #{result.trace_path}#{reset}"
+            end
+
+            if result.error && result.step_results.none?(&:failed?)
+              puts "        #{red}✗#{reset} #{result.error}"
+            end
+          end
+
+          if result.status == :skipped
+            puts "        #{yellow}⊘#{reset} #{result.error || "skipped"}"
+          end
+        end
+      end
+
+      def print_summary(results)
+        puts
+        puts line
+
+        passed = results.count(&:passed?)
+        failed = results.count(&:failed?)
+        skipped = results.count { |r| r.status == :skipped }
+        total_duration = results.sum { |r| r.duration || 0 }
+
+        parts = []
+        parts << "#{green}#{passed} passed#{reset}" if passed > 0
+        parts << "#{red}#{failed} failed#{reset}" if failed > 0
+        parts << "#{yellow}#{skipped} skipped#{reset}" if skipped > 0
+
+        puts "#{parts.join(", ")} #{dim}(#{format("%.1fs", total_duration)})#{reset}"
+      end
+
+      def print_results_path(path)
+        puts "#{dim}Results: #{path}#{reset}"
+        puts
+      end
+
+      private
+
+      def puts(text = "")
+        @io.puts(text)
+      end
+
+      def line
+        "━" * 50
+      end
+
+      def format_status(status)
+        case status
+        when :passed  then "#{green}#{bold} PASS #{reset}"
+        when :failed  then "#{red}#{bold} FAIL #{reset}"
+        when :skipped then "#{yellow}#{bold} SKIP #{reset}"
+        else               "  ??  "
+        end
+      end
+
+      def bold   = @color ? COLORS[:bold]   : ""
+      def dim    = @color ? COLORS[:dim]    : ""
+      def green  = @color ? COLORS[:green]  : ""
+      def red    = @color ? COLORS[:red]    : ""
+      def yellow = @color ? COLORS[:yellow] : ""
+      def reset  = @color ? COLORS[:reset]  : ""
+    end
+  end
+end

data/lib/checkset/reporter/json.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Checkset
+  module Reporter
+    class JSON
+      def initialize(configuration: Checkset.configuration)
+        @configuration = configuration
+      end
+
+      def write(results, target_url: @configuration.target_url, suite_name: nil)
+        dir = @configuration.results_dir
+        FileUtils.mkdir_p(dir)
+
+        timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
+        prefix = suite_name ? "results_#{suite_name}" : "results"
+        path = File.join(dir, "#{prefix}_#{timestamp}.json")
+
+        total_duration = results.sum { |r| r.duration || 0 }
+
+        data = {
+          run_at: Time.now.iso8601,
+          target_url: target_url,
+          duration: total_duration.round(2),
+          summary: {
+            total: results.size,
+            passed: results.count(&:passed?),
+            failed: results.count(&:failed?),
+            skipped: results.count { |r| r.status == :skipped }
+          },
+          checks: results.map(&:to_h)
+        }
+
+        File.write(path, ::JSON.pretty_generate(data))
+        path
+      end
+    end
+  end
+end

data/lib/checkset/result.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Checkset
+  class Result
+    attr_reader :check_name, :status, :started_at, :finished_at,
+                :step_results, :prep_results, :trace_path,
+                :error, :error_backtrace
+
+    def initialize(check_name:)
+      @check_name = check_name
+      @status = :pending
+      @started_at = nil
+      @finished_at = nil
+      @step_results = []
+      @prep_results = []
+      @trace_path = nil
+      @error = nil
+      @error_backtrace = nil
+    end
+
+    def start!
+      @status = :running
+      @started_at = Time.now
+    end
+
+    def pass!
+      @status = :passed
+      @finished_at = Time.now
+    end
+
+    def fail!(error: nil)
+      @status = :failed
+      @finished_at = Time.now
+      if error.is_a?(Exception)
+        @error = error.message
+        @error_backtrace = error.backtrace&.first(10)
+      elsif error
+        @error = error.to_s
+      end
+    end
+
+    def skip!(reason: nil)
+      @status = :skipped
+      @finished_at = Time.now
+      @error = reason
+    end
+
+    def add_step_result(step_result)
+      @step_results << step_result
+    end
+
+    def add_prep_result(prep_result)
+      @prep_results << prep_result
+    end
+
+    def trace_path=(path)
+      @trace_path = path
+    end
+
+    def duration
+      return nil unless @started_at && @finished_at
+      @finished_at - @started_at
+    end
+
+    def passed?
+      @status == :passed
+    end
+
+    def failed?
+      @status == :failed
+    end
+
+    def to_h
+      {
+        check: @check_name,
+        status: @status,
+        duration: duration,
+        preps: @prep_results.map(&:to_h),
+        steps: @step_results.map(&:to_h),
+        trace_path: @trace_path,
+        error: @error,
+        error_backtrace: @error_backtrace
+      }.compact
+    end
+  end
+end

data/lib/checkset/runner.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Checkset
+  class Runner
+    def initialize(configuration: Checkset.configuration, filter: nil, tag: nil, target_url: nil, suite_name: nil, browser_manager: nil, preloaded: false)
+      @configuration = configuration
+      @filter = filter
+      @tag = tag
+      @target_url = target_url || @configuration.target_url
+      @suite_name = suite_name
+      @browser_manager = browser_manager || BrowserManager.new(configuration: configuration)
+      @owns_browser = browser_manager.nil?
+      @preloaded = preloaded
+      @credentials = Credentials.new(provider: configuration.credentials_provider)
+    end
+
+    # Preload all check and prep files without running anything.
+    # Call this before spawning concurrent suite runners.
+    def self.preload(configuration: Checkset.configuration)
+      configuration.checks_paths.each do |path|
+        Dir.glob(File.join(path, "**", "*.rb")).sort.each { |f| require File.expand_path(f) }
+      end
+      configuration.preps_paths.each do |path|
+        Dir.glob(File.join(path, "**", "*.rb")).sort.each { |f| require File.expand_path(f) }
+      end
+    end
+
+    def run
+      if @preloaded
+        compute_check_files
+      else
+        load_check_files
+        load_prep_files
+      end
+      check_classes = discover_checks
+
+      if check_classes.empty?
+        Checkset.logger.warn("No checks found in #{@configuration.checks_paths.inspect}")
+        return []
+      end
+
+      @browser_manager.start! if @owns_browser
+
+      results = if @configuration.parallel > 1
+                  run_parallel(check_classes, pool_size: @configuration.parallel)
+                else
+                  check_classes.map { |klass| run_check(klass) }
+                end
+
+      results
+    ensure
+      @browser_manager.stop! if @owns_browser
+    end
+
+    private
+
+    def compute_check_files
+      @check_files = []
+
+      @configuration.checks_paths.each do |path|
+        if @suite_name
+          @check_files.concat(Dir.glob(File.join(path, "*.rb")).sort)
+          @check_files.concat(Dir.glob(File.join(path, @suite_name, "**", "*.rb")).sort)
+        else
+          @check_files.concat(Dir.glob(File.join(path, "**", "*.rb")).sort)
+        end
+      end
+
+      @check_files.map! { |f| File.expand_path(f) }
+    end
+
+    def load_check_files
+      compute_check_files
+      @check_files.each { |f| require f }
+    end
+
+    def load_prep_files
+      @configuration.preps_paths.each do |path|
+        Dir.glob(File.join(path, "**", "*.rb")).sort.each { |f| require File.expand_path(f) }
+      end
+    end
+
+    def discover_checks
+      classes = ObjectSpace.each_object(Class).select { |klass| klass < Checkset::Check }
+
+      if @check_files
+        classes.select! do |klass|
+          source = klass.instance_method(:call).source_location&.first
+          source && @check_files.include?(source)
+        end
+      end
+
+      if @filter
+        classes.select! { |klass| klass.name&.include?(@filter) }
+      end
+
+      if @tag
+        classes.select! { |klass| klass.tags.include?(@tag) }
+      end
+
+      # Sort top-level checks first, then suite-specific checks, alphabetically within each group
+      top_level_files = @suite_name ? @configuration.checks_paths.flat_map { |p| Dir.glob(File.join(p, "*.rb")).map { |f| File.expand_path(f) } } : []
+
+      classes.sort_by do |klass|
+        source = klass.instance_method(:call).source_location&.first
+        top_level = top_level_files.include?(source) ? 0 : 1
+        [top_level, klass.name || ""]
+      end
+    end
+
+    def run_check(klass)
+      max_attempts = @configuration.retries + 1
+
+      max_attempts.times do |attempt|
+        result = attempt_check(klass)
+
+        # Return immediately on pass, skip, or last attempt
+        if result.passed? || result.status == :skipped || attempt == max_attempts - 1
+          return result
+        end
+
+        Checkset.logger.info("Retrying #{klass.name} (attempt #{attempt + 2}/#{max_attempts})")
+      end
+    end
+
+    def attempt_check(klass)
+      result = Result.new(check_name: klass.name || klass.to_s)
+      result.start!
+
+      @browser_manager.with_context do |page, context|
+        # Run preps
+        prep_ok = run_preps(klass, page, result)
+        unless prep_ok
+          result.skip!(reason: "Prep failed")
+          save_trace(context, result)
+          next
+        end
+
+        # Run the check
+        check = klass.new(
+          page: page,
+          result: result,
+          credentials: @credentials,
+          target_url: @target_url,
+          configuration: @configuration
+        )
+
+        begin
+          check.call
+
+          if result.step_results.any?(&:failed?)
+            result.fail!
+            save_trace(context, result)
+          else
+            result.pass!
+            discard_trace(context)
+          end
+        rescue => e
+          # Emergency screenshot
+          emergency_screenshot(page, klass)
+          result.fail!(error: e)
+          save_trace(context, result)
+        end
+      end
+
+      result
+    end
+
+    def run_preps(klass, page, result)
+      klass.preps.each do |prep_name|
+        step_result = StepResult.new(name: prep_name.to_s, type: :prep)
+        step_result.start!
+
+        begin
+          prep_class = Prep.resolve(prep_name)
+          prep = prep_class.new(credentials: @credentials, target_url: @target_url)
+
+          if prep.satisfied?(page)
+            step_result.pass!
+          else
+            prep.satisfy(page)
+            step_result.pass!
+          end
+        rescue => e
+          step_result.fail!(error: e)
+          result.add_prep_result(step_result)
+          return false
+        end
+
+        result.add_prep_result(step_result)
+      end
+
+      true
+    end
+
+    def run_parallel(classes, pool_size:)
+      results = []
+      mutex = Mutex.new
+      queue = Queue.new
+      classes.each { |klass| queue << klass }
+
+      workers = pool_size.times.map do
+        Thread.new do
+          while (klass = begin; queue.pop(true); rescue ThreadError; nil; end)
+            r = run_check(klass)
+            mutex.synchronize { results << r }
+          end
+        end
+      end
+
+      workers.each(&:join)
+      results
+    end
+
+    def save_trace(context, result)
+      check_name = result.check_name.to_s.gsub("::", "_").downcase
+      timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
+      dir = @configuration.traces_dir
+      FileUtils.mkdir_p(dir)
+      path = File.join(dir, "#{check_name}_#{timestamp}.zip")
+      context.tracing.stop(path: path)
+      result.trace_path = path
+    rescue => e
+      Checkset.logger.warn("Failed to save trace: #{e.message}")
+    end
+
+    def discard_trace(context)
+      context.tracing.stop
+    rescue => e
+      Checkset.logger.warn("Failed to discard trace: #{e.message}")
+    end
+
+    def emergency_screenshot(page, klass)
+      check_name = (klass.name || "anonymous").gsub("::", "_").downcase
+      dir = File.join(@configuration.screenshots_dir, check_name)
+      FileUtils.mkdir_p(dir)
+      timestamp = Time.now.strftime("%Y%m%d_%H%M%S_%L")
+      page.screenshot(path: File.join(dir, "EMERGENCY_#{timestamp}.png"), fullPage: true)
+    rescue => e
+      Checkset.logger.warn("Emergency screenshot failed: #{e.message}")
+    end
+  end
+end

data/lib/checkset/step_result.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Checkset
+  class StepResult
+    attr_reader :name, :type, :status, :started_at, :finished_at,
+                :screenshot_path, :error, :error_backtrace
+
+    def initialize(name:, type:)
+      @name = name
+      @type = type # :verify, :step, :prep
+      @status = :pending
+      @started_at = nil
+      @finished_at = nil
+      @screenshot_path = nil
+      @error = nil
+      @error_backtrace = nil
+    end
+
+    def start!
+      @status = :running
+      @started_at = Time.now
+    end
+
+    def pass!(screenshot_path: nil)
+      @status = :passed
+      @finished_at = Time.now
+      @screenshot_path = screenshot_path
+    end
+
+    def fail!(error:, screenshot_path: nil)
+      @status = :failed
+      @finished_at = Time.now
+      @screenshot_path = screenshot_path
+      if error.is_a?(Exception)
+        @error = error.message
+        @error_backtrace = error.backtrace&.first(10)
+      else
+        @error = error.to_s
+      end
+    end
+
+    def skip!(reason: nil)
+      @status = :skipped
+      @finished_at = Time.now
+      @error = reason
+    end
+
+    def duration
+      return nil unless @started_at && @finished_at
+      @finished_at - @started_at
+    end
+
+    def passed?
+      @status == :passed
+    end
+
+    def failed?
+      @status == :failed
+    end
+
+    def to_h
+      {
+        name: @name,
+        type: @type,
+        status: @status,
+        duration: duration,
+        screenshot_path: @screenshot_path,
+        error: @error,
+        error_backtrace: @error_backtrace
+      }.compact
+    end
+  end
+end

data/lib/checkset/suite.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Checkset
+  class Suite
+    attr_reader :name, :target_url
+
+    def initialize(name:, target_url:)
+      @name = name.to_s
+      @target_url = target_url
+    end
+  end
+end

data/lib/checkset/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Checkset
+  VERSION = "0.1.0"
+end

data/lib/checkset.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "logger"
+
+require_relative "checkset/version"
+require_relative "checkset/configuration"
+require_relative "checkset/credentials"
+require_relative "checkset/step_result"
+require_relative "checkset/result"
+require_relative "checkset/browser_manager"
+require_relative "checkset/prep"
+require_relative "checkset/check"
+require_relative "checkset/suite"
+require_relative "checkset/config_file"
+require_relative "checkset/runner"
+require_relative "checkset/reporter/cli"
+require_relative "checkset/reporter/json"
+require_relative "checkset/init"
+require_relative "checkset/cli"
+
+module Checkset
+  class Error < StandardError; end
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def logger
+      @logger ||= Logger.new($stdout, level: Logger::INFO)
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+
+    def reset!
+      @configuration = Configuration.new
+      @logger = nil
+    end
+  end
+end