checkset 0.1.0

data/examples/checkset.yml

@@ -0,0 +1,23 @@
+# checkset.yml — Maps suite names to target URLs
+#
+# Checks are auto-discovered by folder:
+#   examples/checks/hey/*.rb      → belongs to "hey" suite
+#   examples/checks/basecamp/*.rb → belongs to "basecamp" suite
+#   examples/checks/*.rb          → top-level, runs in EVERY suite
+#
+# Usage:
+#   bundle exec checkset --config examples/checkset.yml              # runs all suites
+#   bundle exec checkset --config examples/checkset.yml --suite hey  # runs just "hey"
+#   bundle exec checkset --target https://foo.com                    # ignores yml, original behavior
+
+checks_dir: examples/checks
+
+suites:
+  hey:
+    target: https://hey.com
+  basecamp:
+    target: https://basecamp.com
+  once:
+    target: https://once.com
+  ruby:
+    target: https://www.ruby-lang.org

data/exe/checkset

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "checkset"
+Checkset::CLI.new(ARGV).run

data/lib/checkset/browser_manager.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Checkset
+  class BrowserManager
+    def initialize(configuration: Checkset.configuration)
+      @configuration = configuration
+      @execution = nil
+      @playwright = nil
+      @browser = nil
+    end
+
+    def start!
+      require "playwright"
+
+      if @configuration.playwright_server_url
+        start_remote!
+      else
+        start_local!
+      end
+    end
+
+    def stop!
+      @browser&.close
+      @execution&.stop
+      @browser = nil
+      @playwright = nil
+      @execution = nil
+    end
+
+    def with_context
+      viewport = @configuration.viewport_size
+      context = @browser.new_context(
+        viewport: { width: viewport[:width], height: viewport[:height] }
+      )
+      context.set_default_timeout(@configuration.default_timeout)
+      context.tracing.start(screenshots: true, snapshots: true)
+
+      page = context.new_page
+      yield page, context
+    ensure
+      context&.close
+    end
+
+    private
+
+    def start_local!
+      cli_path = @configuration.playwright_cli_path || find_playwright_cli
+      launch_options = { headless: @configuration.headless }
+      launch_options[:slowMo] = @configuration.slow_mo if @configuration.slow_mo
+
+      @execution = Playwright.create(playwright_cli_executable_path: cli_path)
+      @playwright = @execution.playwright
+      @browser = @playwright.send(@configuration.browser_type).launch(**launch_options)
+    end
+
+    def start_remote!
+      @playwright = Playwright.connect_to_playwright_server(@configuration.playwright_server_url)
+      launch_options = { headless: @configuration.headless }
+      launch_options[:slowMo] = @configuration.slow_mo if @configuration.slow_mo
+
+      @browser = @playwright.send(@configuration.browser_type).launch(**launch_options)
+    end
+
+    def find_playwright_cli
+      # Check common locations
+      candidates = [
+        File.join(Dir.pwd, "node_modules", ".bin", "playwright"),
+        `which playwright 2>/dev/null`.strip
+      ]
+
+      candidates.each do |path|
+        return path if !path.empty? && File.exist?(path)
+      end
+
+      raise <<~MSG
+        Could not find Playwright CLI. Either:
+        1. Install it: npm install playwright
+        2. Set PLAYWRIGHT_CLI_PATH environment variable
+        3. Configure: Checkset.configure { |c| c.playwright_cli_path = "/path/to/playwright" }
+      MSG
+    end
+  end
+end

data/lib/checkset/check.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "uri"
+
+module Checkset
+  class Check
+    attr_reader :page, :result, :credentials, :target_url
+
+    # --- Class-level DSL ---
+
+    def self.prep(*names)
+      @preps ||= []
+      @preps.concat(names)
+    end
+
+    def self.preps
+      @preps || []
+    end
+
+    def self.description(text = nil)
+      if text
+        @description = text
+      else
+        @description
+      end
+    end
+
+    def self.tags(*tag_list)
+      if tag_list.any?
+        @tags = tag_list
+      else
+        @tags || []
+      end
+    end
+
+    # --- Instance ---
+
+    def initialize(page:, result:, credentials:, target_url:, configuration: Checkset.configuration)
+      @page = page
+      @result = result
+      @credentials = credentials
+      @target_url = target_url
+      @configuration = configuration
+      @step_counter = 0
+    end
+
+    # Override in subclass
+    def call
+      raise NotImplementedError, "#{self.class}#call must be implemented"
+    end
+
+    private
+
+    def verify(name, &block)
+      step_result = StepResult.new(name: name, type: :verify)
+      step_result.start!
+
+      begin
+        # Screenshot before assertion (captures current state)
+        screenshot_path = take_screenshot(name, "verify")
+
+        passed = block.call
+        if passed
+          step_result.pass!(screenshot_path: screenshot_path)
+        else
+          # Take full-page failure screenshot
+          fail_screenshot = take_screenshot(name, "FAIL_verify", full_page: true)
+          step_result.fail!(error: "Verification '#{name}' returned falsy", screenshot_path: fail_screenshot)
+        end
+      rescue => e
+        fail_screenshot = take_screenshot(name, "FAIL_verify", full_page: true)
+        step_result.fail!(error: e, screenshot_path: fail_screenshot)
+      end
+
+      @result.add_step_result(step_result)
+      step_result
+      # verify does NOT re-raise — continues collecting failures
+    end
+
+    def step(name, &block)
+      step_result = StepResult.new(name: name, type: :step)
+      step_result.start!
+
+      begin
+        block.call
+
+        # Screenshot after action (captures result)
+        screenshot_path = take_screenshot(name, "step")
+        step_result.pass!(screenshot_path: screenshot_path)
+      rescue => e
+        fail_screenshot = take_screenshot(name, "FAIL_step", full_page: true)
+        step_result.fail!(error: e, screenshot_path: fail_screenshot)
+        @result.add_step_result(step_result)
+        raise # step re-raises to halt execution
+      end
+
+      @result.add_step_result(step_result)
+      step_result
+    end
+
+    def visit(path)
+      url = URI.join(@target_url, path).to_s
+      @page.goto(url)
+    end
+
+    def take_screenshot(name, type, full_page: false)
+      @step_counter += 1
+      check_name = sanitize_name(self.class.name || "anonymous")
+      step_name = sanitize_name(name)
+      timestamp = Time.now.strftime("%Y%m%d_%H%M%S_%L")
+      filename = "#{@step_counter}_#{type}_#{step_name}_#{timestamp}.png"
+
+      dir = File.join(@configuration.screenshots_dir, check_name)
+      FileUtils.mkdir_p(dir)
+      path = File.join(dir, filename)
+
+      @page.screenshot(path: path, fullPage: full_page)
+      path
+    rescue => e
+      Checkset.logger.warn("Screenshot failed: #{e.message}")
+      nil
+    end
+
+    def sanitize_name(name)
+      name.to_s.gsub("::", "_").gsub(/[^a-zA-Z0-9_-]/, "_").downcase
+    end
+  end
+end

data/lib/checkset/cli.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "fileutils"
+
+module Checkset
+  class CLI
+    def initialize(argv)
+      @argv = argv
+      @init = @argv.first == "init"
+      @filter = nil
+      @suite_name = nil
+      @config_path = "checkset.yml"
+      @domain = nil
+      @tag = nil
+      @clean = false
+      parse_options! unless @init
+    end
+
+    def run
+      if @init
+        Init.run
+        return
+      end
+
+      if @clean
+        clean!
+        return
+      end
+
+      config = Checkset.configuration
+
+      if config.target_url
+        run_single(config)
+      elsif (config_file = ConfigFile.load(@config_path, domain: @domain))
+        run_suites(config, config_file)
+      else
+        $stderr.puts "Error: --target URL is required (or provide a checkset.yml)"
+        $stderr.puts "Usage: bundle exec checkset --target https://staging.myapp.com"
+        exit 1
+      end
+    end
+
+    private
+
+    def clean!
+      config = Checkset.configuration
+      dirs = [config.screenshots_dir, config.traces_dir, config.results_dir]
+
+      dirs.each do |dir|
+        if File.directory?(dir)
+          FileUtils.rm_rf(dir)
+          puts "Removed #{dir}"
+        end
+      end
+
+      puts "Clean!"
+    end
+
+    def run_single(config)
+      cli_reporter = Reporter::CLI.new(target_url: config.target_url)
+      json_reporter = Reporter::JSON.new(configuration: config)
+
+      cli_reporter.print_header
+
+      runner = Runner.new(configuration: config, filter: @filter, tag: @tag)
+      results = runner.run
+
+      results.each { |r| cli_reporter.report_result(r) }
+      cli_reporter.print_summary(results)
+
+      json_path = json_reporter.write(results)
+      cli_reporter.print_results_path(json_path)
+
+      exit 1 if results.any?(&:failed?)
+    end
+
+    def run_suites(config, config_file)
+      config.checks_paths = [config_file.checks_dir] if config_file.checks_dir
+
+      suites = if @suite_name
+                 suite = config_file.find_suite(@suite_name)
+                 unless suite
+                   $stderr.puts "Error: Suite '#{@suite_name}' not found in #{@config_path}"
+                   $stderr.puts "Available suites: #{config_file.suites.map(&:name).join(', ')}"
+                   exit 1
+                 end
+                 [suite]
+               else
+                 config_file.suites
+               end
+
+      json_reporter = Reporter::JSON.new(configuration: config)
+      browser_manager = BrowserManager.new(configuration: config)
+      started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      # Preload all check/prep files before spawning threads
+      Runner.preload(configuration: config)
+
+      begin
+        browser_manager.start!
+
+        mutex = Mutex.new
+        suite_outputs = {}
+
+        threads = suites.map do |suite|
+          Thread.new do
+            runner = Runner.new(
+              configuration: config,
+              filter: @filter,
+              tag: @tag,
+              target_url: suite.target_url,
+              suite_name: suite.name,
+              browser_manager: browser_manager,
+              preloaded: true
+            )
+            results = runner.run
+            json_path = mutex.synchronize { json_reporter.write(results, target_url: suite.target_url, suite_name: suite.name) }
+
+            suite_outputs[suite.name] = { suite: suite, results: results, json_path: json_path }
+          end
+        end
+
+        threads.each(&:join)
+      ensure
+        browser_manager.stop!
+      end
+
+      # Print results in suite order
+      any_failed = false
+      suites.each do |suite|
+        output = suite_outputs[suite.name]
+        next unless output
+
+        cli_reporter = Reporter::CLI.new(target_url: suite.target_url, suite_name: suite.name)
+        cli_reporter.print_header
+        output[:results].each { |r| cli_reporter.report_result(r) }
+        cli_reporter.print_summary(output[:results])
+        cli_reporter.print_results_path(output[:json_path])
+
+        any_failed = true if output[:results].any?(&:failed?)
+      end
+
+      total_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+      all_results = suites.flat_map { |s| suite_outputs.dig(s.name, :results) || [] }
+      passed = all_results.count(&:passed?)
+      failed = all_results.count(&:failed?)
+      skipped = all_results.count { |r| r.status == :skipped }
+
+      puts
+      puts "━" * 50
+      puts "All suites: #{passed + failed + skipped} checks across #{suites.size} suites — #{passed} passed, #{failed} failed, #{skipped} skipped (#{format("%.1fs", total_duration)})"
+      puts
+
+      exit 1 if any_failed
+    end
+
+    def parse_options!
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: checkset [options] [CheckName]\n       checkset init"
+        opts.separator ""
+        opts.separator "Options:"
+
+        opts.on("--target URL", "Target URL to verify against") do |url|
+          Checkset.configuration.target_url = url
+        end
+
+        opts.on("--suite NAME", "Run only the named suite from checkset.yml") do |name|
+          @suite_name = name
+        end
+
+        opts.on("--config FILE", "Path to checkset.yml (default: checkset.yml)") do |path|
+          @config_path = path
+        end
+
+        opts.on("--domain DOMAIN", "Base domain for suite targets (overrides base_domain in yml)") do |domain|
+          @domain = domain
+        end
+
+        opts.on("--headed", "Run browser in headed mode") do
+          Checkset.configuration.headless = false
+        end
+
+        opts.on("--headless", "Run browser in headless mode (default)") do
+          Checkset.configuration.headless = true
+        end
+
+        opts.on("--browser TYPE", "Browser type: chromium, firefox, webkit (default: chromium)") do |type|
+          Checkset.configuration.browser_type = type.to_sym
+        end
+
+        opts.on("--screenshots-dir DIR", "Directory for screenshots") do |dir|
+          Checkset.configuration.screenshots_dir = dir
+        end
+
+        opts.on("--timeout MS", Integer, "Default timeout in milliseconds") do |ms|
+          Checkset.configuration.default_timeout = ms
+        end
+
+        opts.on("--playwright-server URL", "WebSocket URL for remote Playwright server") do |url|
+          Checkset.configuration.playwright_server_url = url
+        end
+
+        opts.on("--slow-mo MS", Integer, "Slow down actions by N milliseconds") do |ms|
+          Checkset.configuration.slow_mo = ms
+        end
+
+        opts.on("--parallel N", Integer, "Number of concurrent checks (default: 1)") do |n|
+          Checkset.configuration.parallel = n
+        end
+
+        opts.on("--checks-dir DIR", "Directory containing check files (default: checks)") do |dir|
+          Checkset.configuration.checks_paths = [dir]
+        end
+
+        opts.on("--preps-dir DIR", "Directory containing prep files (default: preps)") do |dir|
+          Checkset.configuration.preps_paths = [dir]
+        end
+
+        opts.on("--tag TAG", "Only run checks tagged with TAG") do |tag|
+          @tag = tag.to_sym
+        end
+
+        opts.on("--retries N", Integer, "Retry failed checks N times (default: 0)") do |n|
+          Checkset.configuration.retries = n
+        end
+
+        opts.on("--clean", "Remove all screenshots, traces, and results") do
+          @clean = true
+        end
+
+        opts.on("-v", "--version", "Print version") do
+          puts "checkset #{Checkset::VERSION}"
+          exit
+        end
+
+        opts.on("-h", "--help", "Print help") do
+          puts opts
+          exit
+        end
+      end
+
+      remaining = parser.parse(@argv)
+      @filter = remaining.first if remaining.any?
+    end
+  end
+end

data/lib/checkset/config_file.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Checkset
+  class ConfigFile
+    attr_reader :suites, :checks_dir, :base_domain
+
+    def self.load(path = "checkset.yml", domain: nil)
+      return nil unless File.exist?(path)
+
+      new(path, domain: domain)
+    end
+
+    def initialize(path, domain: nil)
+      data = YAML.safe_load_file(path, permitted_classes: [Symbol])
+      @checks_dir = data["checks_dir"]
+      @base_domain = domain || data["base_domain"]
+      @suites = parse_suites(data)
+    end
+
+    def find_suite(name)
+      @suites.find { |s| s.name == name.to_s }
+    end
+
+    private
+
+    def parse_suites(data)
+      suites_data = data.fetch("suites", {})
+
+      suites_data.map do |name, config|
+        Suite.new(
+          name: name,
+          target_url: interpolate(config.fetch("target"))
+        )
+      end
+    end
+
+    def interpolate(value)
+      # Replace %{domain} with base_domain
+      result = value.gsub("%{domain}") do
+        @base_domain || raise("No domain set — use --domain or set base_domain in checkset.yml")
+      end
+
+      # Replace ${ENV_VAR} with environment variables
+      result.gsub(/\$\{([^}]+)\}/) do
+        ENV.fetch($1) { raise "Environment variable #{$1} is not set" }
+      end
+    end
+  end
+end

data/lib/checkset/configuration.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Checkset
+  class Configuration
+    attr_accessor :target_url,
+                  :playwright_cli_path,
+                  :playwright_server_url,
+                  :headless,
+                  :screenshots_dir,
+                  :traces_dir,
+                  :results_dir,
+                  :default_timeout,
+                  :credentials_provider,
+                  :checks_paths,
+                  :preps_paths,
+                  :browser_type,
+                  :viewport_size,
+                  :slow_mo,
+                  :parallel,
+                  :retries
+
+    def initialize
+      @target_url           = ENV["CHECKSET_TARGET_URL"]
+      @playwright_cli_path  = ENV["PLAYWRIGHT_CLI_PATH"]
+      @playwright_server_url = ENV["PLAYWRIGHT_SERVER_URL"]
+      @headless             = true
+      @screenshots_dir      = "tmp/checkset/screenshots"
+      @traces_dir           = "tmp/checkset/traces"
+      @results_dir          = "tmp/checkset/results"
+      @default_timeout      = 10_000
+      @credentials_provider = :env
+      @checks_paths         = ["checks"]
+      @preps_paths          = ["preps"]
+      @browser_type         = :chromium
+      @viewport_size        = { width: 1280, height: 720 }
+      @slow_mo              = nil
+      @parallel             = 1
+      @retries              = 0
+    end
+  end
+end

data/lib/checkset/credentials.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Checkset
+  class Credentials
+    def initialize(provider: Checkset.configuration.credentials_provider)
+      @provider = provider
+    end
+
+    def fetch(key)
+      case @provider
+      when :env
+        ENV.fetch(key.to_s.upcase)
+      when :rails_credentials
+        fetch_rails_credential(key)
+      else
+        raise ArgumentError, "Unknown credentials provider: #{@provider}"
+      end
+    end
+
+    alias_method :[], :fetch
+
+    private
+
+    def fetch_rails_credential(key)
+      if defined?(Rails) && Rails.application.respond_to?(:credentials)
+        value = Rails.application.credentials.dig(:checkset, key)
+        return value if value
+      end
+
+      # Fall back to ENV
+      ENV.fetch(key.to_s.upcase)
+    end
+  end
+end

data/lib/checkset/init.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Checkset
+  class Init
+    CHECKSET_YML = <<~YAML
+      # Base domain for %{domain} interpolation in suite targets
+      # base_domain: example.com
+
+      # Directory containing check files (default: checks)
+      # checks_dir: checks
+
+      # Define suites to run checks against multiple targets:
+      # suites:
+      #   app:
+      #     target: https://app.%{domain}
+      #   admin:
+      #     target: https://admin.%{domain}
+    YAML
+
+    EXAMPLE_CHECK = <<~RUBY
+      # frozen_string_literal: true
+
+      class Checks::HomepageLoads < Checkset::Check
+        description "Verifies the homepage loads correctly"
+        tags :homepage
+
+        def call
+          visit "/"
+
+          verify "page has a title" do
+            page.title.is_a?(String) && !page.title.empty?
+          end
+
+          verify "page has visible content" do
+            page.text_content("body").strip.length > 0
+          end
+        end
+      end
+    RUBY
+
+    def self.run(base_dir: ".")
+      new(base_dir: base_dir).run
+    end
+
+    def initialize(base_dir: ".")
+      @base_dir = base_dir
+    end
+
+    def run
+      create_file("checkset.yml", CHECKSET_YML)
+      create_file(File.join("checks", "homepage_loads.rb"), EXAMPLE_CHECK)
+    end
+
+    private
+
+    def create_file(relative_path, content)
+      path = File.join(@base_dir, relative_path)
+
+      if File.exist?(path)
+        puts "  exists  #{relative_path}"
+      else
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, content)
+        puts "  create  #{relative_path}"
+      end
+    end
+  end
+end