flaky-friend 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e3410a7b5cc61c235701fe0eba23923ffdafe5d46ce95ab2065b8218f35e3ce2
+  data.tar.gz: e419f801b44b0bc80eb5af88b4da5bf2014b6731f1c3f2b0c550b59698ae7389
+SHA512:
+  metadata.gz: bee24490b01ced67fb5ba16addb18f390c0e03b08946fa1309b5ae64e33d961895741e3c59abdf7cd84c0991791762ede207ce94db895acbbf5f9b5d4ade6a9b
+  data.tar.gz: da12cc1017c510ca264e89b583fa59577c1823d2ab3c8c87b4362f906f38f3c99b48d20d3660fddf786ad97c2ce3b618e5e03326b4d9010e932cfbb052a0ebe5

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Flytedesk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,215 @@
+# Flaky
+
+Track, rank, and reproduce flaky CI test failures in Rails projects.
+
+Flaky fetches test results from your CI provider, stores failures in a local SQLite database, ranks tests by flakiness, and helps reproduce failures under simulated CI conditions.
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+# From GitHub
+gem 'flaky', github: 'Flytedesk/flaky', group: [:development, :test]
+
+# Or from a local path during development
+gem 'flaky', path: '../flaky', group: [:development, :test]
+```
+
+Then `bundle install`.
+
+## Configuration
+
+Create an initializer (e.g. `config/initializers/flaky.rb`):
+
+```ruby
+if defined?(Flaky)
+  Flaky.configure do |c|
+    c.provider = :semaphore        # or :github_actions
+    c.project  = "my-project"      # CI project name
+    c.branch   = "main"            # branch to track
+  end
+end
+```
+
+### Prerequisites by provider
+
+**Semaphore**: Install and authenticate the [`sem` CLI](https://docs.semaphoreci.com/reference/sem-command-line-tool/).
+
+**GitHub Actions**: Install and authenticate the [`gh` CLI](https://cli.github.com/).
+
+## Rake Tasks
+
+### `rake flaky:fetch[age]`
+
+Fetch recent CI results and store failures in the local database.
+
+```sh
+rake flaky:fetch              # last 24 hours (default)
+rake flaky:fetch[168h]        # last 7 days
+rake flaky:fetch[2160h]       # last 90 days
+```
+
+For each workflow on the configured branch, fetches all test job logs, parses RSpec output for failures and random seeds, and inserts new records into `tmp/flaky.db`.
+
+### `rake flaky:rank[since]`
+
+Rank flaky tests by failure frequency and suggest the next one to investigate.
+
+```sh
+rake flaky:rank               # last 30 days (default)
+rake flaky:rank[7]            # last 7 days
+```
+
+Output:
+
+```
+Flaky tests on main (last 30 days, 42 CI runs):
+
+Fails  Location                                           Last Failure
+------------------------------------------------------------------------------------------
+5      ...spec/system/inventory_search_modal_spec.rb:83    2026-04-12 09:15:22
+
+  > Next to investigate: packs/.../inventory_search_modal_spec.rb:83
+    Inventory search modal filters by enrollment
+    Seeds: 6432, 51203, 8891
+```
+
+### `rake flaky:history[spec_location]`
+
+Show the full failure timeline for a specific test, including every seed and CI job it failed in.
+
+```sh
+rake flaky:history[inventory_search_modal_spec.rb:83]
+rake flaky:history[inventory_search_modal_spec.rb]     # all failures in this file
+```
+
+### `rake flaky:stress[spec,iterations,seed,ci]`
+
+Run a test repeatedly to reproduce a flaky failure or prove a fix is stable.
+
+```sh
+# 20 iterations with random seeds
+rake flaky:stress[path/to/spec.rb:83]
+
+# 50 iterations with a specific seed and CI simulation
+rake flaky:stress[path/to/spec.rb:83,50,6432,true]
+```
+
+Arguments:
+- `spec` (required) -- spec file path, optionally with line number
+- `iterations` -- number of runs (default: 20)
+- `seed` -- RSpec random seed; omit for random each run
+- `ci` -- `true` to enable CI environment simulation (default: false)
+
+Results are recorded to the database and shown in `rake flaky:report`.
+
+### `rake flaky:report`
+
+Summary dashboard showing overall flaky test health.
+
+```sh
+rake flaky:report
+```
+
+Output:
+
+```
+=== Flaky Test Report (main) ===
+
+CI Runs tracked:     42
+Failed runs:         8 (19.0%)
+Total test failures: 14
+Unique flaky specs:  6
+Last fetch:          2026-04-14 20:39:57
+
+7-day trend:         3 failures (prior 7 days: 5)
+                     v Trending better
+
+Top 5 flaky tests:
+--------------------------------------------------------------------------------
+  1. packs/.../inventory_search_modal_spec.rb:83 (5x)
+     Inventory search modal filters by enrollment
+
+Recent stress runs:
+--------------------------------------------------------------------------------
+  packs/.../inventory_search_modal_spec.rb:83 -- 18/20 passed (10.0% failure rate) [CI sim]
+```
+
+## CI Simulation
+
+When `ci=true` is passed to `rake flaky:stress`, the gem simulates CI environment constraints:
+
+1. **Rack middleware latency** -- adds 30ms delay per HTTP request (approximates the difference between a Mac and an f1-standard-2 CI machine). Configurable via `FLAKY_LATENCY_MS` env var.
+
+2. **Reduced Puma threads** -- the host app should conditionally reduce Capybara's Puma threads when `FLAKY_CI_SIMULATE=1` is set:
+
+```ruby
+# spec/support/capybara_drivers.rb (or equivalent)
+max_threads = ENV["FLAKY_CI_SIMULATE"] ? 2 : 8
+Capybara.server = :puma, { Silent: true, Threads: "1:#{max_threads}" }
+```
+
+The middleware is auto-inserted by the Railtie in test environment when `FLAKY_CI_SIMULATE=1`.
+
+## Database
+
+Failures are stored in SQLite at `tmp/flaky.db` (auto-created on first use). The schema is managed internally and migrated automatically.
+
+Tables:
+- `ci_runs` -- one row per CI workflow on the tracked branch
+- `job_results` -- one row per test job (unit tests, system tests, etc.)
+- `test_failures` -- one row per individual test failure with spec file, line, description, and seed
+- `stress_runs` -- one row per stress test session
+
+The database is local and should be gitignored (typically already is via `tmp/`).
+
+## Custom Providers
+
+To add a CI provider, implement the three-method interface and register it:
+
+```ruby
+class Flaky::Providers::CircleCI < Flaky::Providers::Base
+  def fetch_workflows(age: "24h")
+    # Return [{ id:, pipeline_id:, branch:, created_at: }, ...]
+  end
+
+  def fetch_jobs(pipeline_id:)
+    # Return [{ id:, name:, block_name:, result: }, ...]
+  end
+
+  def fetch_log(job_id:)
+    # Return raw log string
+  end
+end
+
+Flaky.register_provider(:circleci, Flaky::Providers::CircleCI)
+```
+
+The log parser is CI-agnostic -- it extracts failures, seeds, and counts from standard RSpec output. Your provider just needs to return the raw log text.
+
+## Typical Workflow
+
+```sh
+# 1. Fetch recent CI data
+rake flaky:fetch[168h]
+
+# 2. See what's flaky
+rake flaky:rank
+
+# 3. Investigate the top offender
+rake flaky:history[the_flaky_spec.rb:42]
+
+# 4. Try to reproduce it locally with CI simulation
+rake flaky:stress[the_flaky_spec.rb:42,30,6432,true]
+
+# 5. Fix the test, then prove the fix holds
+rake flaky:stress[the_flaky_spec.rb:42,50,,true]
+
+# 6. Check overall health
+rake flaky:report
+```
+
+## License
+
+MIT

data/lib/flaky/commands/fetch.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative "../database"
+require_relative "../log_parser"
+
+module Flaky
+  module Commands
+    class Fetch
+      def initialize(age: "24h")
+        @age = age
+        @db = Database.new
+        @parser = LogParser.new
+      end
+
+      def execute
+        provider = Flaky.provider
+        branch = Flaky.configuration.branch
+        conn = @db.connection
+
+        workflows = provider.fetch_workflows(age: @age)
+        main_workflows = workflows.select { |w| w[:branch] == branch }
+
+        if main_workflows.empty?
+          puts "No #{branch} workflows found in the last #{@age}."
+          return
+        end
+
+        new_workflows = 0
+        new_failures = 0
+        total_jobs = 0
+
+        main_workflows.each do |wf|
+          # Skip if already fetched
+          existing = conn.get_first_value("SELECT 1 FROM ci_runs WHERE workflow_id = ?", wf[:id])
+          next if existing
+
+          # Determine pipeline result by fetching jobs
+          jobs = provider.fetch_jobs(pipeline_id: wf[:pipeline_id])
+          pipeline_result = jobs.any? { |j| j[:result] == "failed" } ? "failed" : "passed"
+
+          conn.execute(
+            "INSERT INTO ci_runs (workflow_id, pipeline_id, branch, result, created_at) VALUES (?, ?, ?, ?, ?)",
+            [wf[:id], wf[:pipeline_id], wf[:branch], pipeline_result, wf[:created_at]]
+          )
+          new_workflows += 1
+
+          jobs.each do |job|
+            total_jobs += 1
+            log = provider.fetch_log(job_id: job[:id])
+            parsed = @parser.parse(log)
+
+            conn.execute(
+              "INSERT OR IGNORE INTO job_results (job_id, workflow_id, job_name, block_name, result, example_count, failure_count, seed, duration_seconds) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+              [job[:id], wf[:id], job[:name], job[:block_name], parsed.failure_count.to_i > 0 ? "failed" : "passed",
+               parsed.example_count, parsed.failure_count, parsed.seed, parsed.duration_seconds]
+            )
+
+            parsed.failures.each do |failure|
+              conn.execute(
+                "INSERT OR IGNORE INTO test_failures (workflow_id, job_id, job_name, spec_file, line_number, description, seed, branch, failed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                [wf[:id], job[:id], job[:name], failure.spec_file, failure.line_number, failure.description,
+                 parsed.seed, wf[:branch], wf[:created_at]]
+              )
+              new_failures += 1
+            end
+          end
+        end
+
+        puts "Fetched #{new_workflows} new workflow(s), #{total_jobs} job(s) parsed, #{new_failures} failure(s) recorded."
+      ensure
+        @db.close
+      end
+    end
+  end
+end

data/lib/flaky/commands/history.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require_relative "../database"
+
+module Flaky
+  module Commands
+    class History
+      def initialize(spec_location:)
+        @spec_location = spec_location
+        @db = Database.new
+      end
+
+      def execute
+        conn = @db.connection
+
+        file, line = parse_location(@spec_location)
+
+        conditions = ["tf.spec_file LIKE ?"]
+        params = ["%#{file}%"]
+
+        if line
+          conditions << "tf.line_number = ?"
+          params << line
+        end
+
+        rows = conn.execute(<<~SQL, params)
+          SELECT
+            tf.spec_file,
+            tf.line_number,
+            tf.description,
+            tf.seed,
+            tf.job_name,
+            tf.branch,
+            tf.failed_at,
+            cr.workflow_id
+          FROM test_failures tf
+          JOIN ci_runs cr ON cr.workflow_id = tf.workflow_id
+          WHERE #{conditions.join(" AND ")}
+          ORDER BY tf.failed_at DESC
+        SQL
+
+        if rows.empty?
+          puts "No failures found matching '#{@spec_location}'."
+          return
+        end
+
+        first = rows.first
+        puts "Failure history for #{first['spec_file']}:#{first['line_number']}"
+        puts "  #{first['description']}\n\n"
+        puts format("%-20s %-8s %-30s %s", "Date", "Seed", "Job", "Workflow")
+        puts "-" * 100
+
+        rows.each do |row|
+          puts format("%-20s %-8d %-30s %s", row["failed_at"], row["seed"], row["job_name"], row["workflow_id"])
+        end
+
+        puts "\nTotal failures: #{rows.length}"
+
+        seeds = rows.map { |r| r["seed"] }.uniq
+        puts "Unique seeds: #{seeds.join(', ')}"
+        puts "\nTo reproduce: rake flaky:stress[#{first['spec_file']}:#{first['line_number']},10,#{seeds.first},true]"
+      ensure
+        @db.close
+      end
+
+      private
+
+      def parse_location(loc)
+        if loc.include?(":")
+          parts = loc.rpartition(":")
+          [parts[0], parts[2].to_i]
+        else
+          [loc, nil]
+        end
+      end
+    end
+  end
+end

data/lib/flaky/commands/rank.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative "../database"
+
+module Flaky
+  module Commands
+    class Rank
+      def initialize(since_days: 30, min_failures: 1)
+        @since_days = since_days
+        @min_failures = min_failures
+        @db = Database.new
+      end
+
+      def execute
+        conn = @db.connection
+        branch = Flaky.configuration.branch
+
+        rows = conn.execute(<<~SQL, [branch, "-#{@since_days} days", @min_failures])
+          SELECT
+            tf.spec_file,
+            tf.line_number,
+            tf.description,
+            COUNT(*) as failure_count,
+            MAX(tf.failed_at) as last_failure,
+            GROUP_CONCAT(DISTINCT tf.seed) as seeds
+          FROM test_failures tf
+          JOIN ci_runs cr ON cr.workflow_id = tf.workflow_id
+          WHERE cr.branch = ?
+            AND cr.created_at >= datetime('now', ?)
+          GROUP BY tf.spec_file, tf.line_number
+          HAVING COUNT(*) >= ?
+          ORDER BY failure_count DESC, last_failure DESC
+        SQL
+
+        if rows.empty?
+          puts "No flaky tests found in the last #{@since_days} days."
+          return
+        end
+
+        total_runs = conn.get_first_value(
+          "SELECT COUNT(DISTINCT workflow_id) FROM ci_runs WHERE branch = ? AND created_at >= datetime('now', ?)",
+          [branch, "-#{@since_days} days"]
+        ).to_i
+
+        puts "Flaky tests on #{branch} (last #{@since_days} days, #{total_runs} CI runs):\n\n"
+        puts format("%-6s %-50s %s", "Fails", "Location", "Last Failure")
+        puts "-" * 90
+
+        rows.each_with_index do |row, i|
+          location = "#{row['spec_file']}:#{row['line_number']}"
+          truncated = location.length > 48 ? "...#{location[-45..]}" : location
+          puts format("%-6d %-50s %s", row["failure_count"], truncated, row["last_failure"])
+
+          if i == 0
+            puts "\n  \e[33m▶ Next to investigate:\e[0m #{location}"
+            puts "    #{row['description']}"
+            puts "    Seeds: #{row['seeds']}"
+            puts ""
+          end
+        end
+      ensure
+        @db.close
+      end
+    end
+  end
+end

data/lib/flaky/commands/report.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative "../database"
+
+module Flaky
+  module Commands
+    class Report
+      def initialize
+        @db = Database.new
+      end
+
+      def execute
+        conn = @db.connection
+        branch = Flaky.configuration.branch
+
+        total_runs = conn.get_first_value("SELECT COUNT(*) FROM ci_runs WHERE branch = ?", branch).to_i
+        failed_runs = conn.get_first_value("SELECT COUNT(*) FROM ci_runs WHERE branch = ? AND result = 'failed'", branch).to_i
+        total_failures = conn.get_first_value("SELECT COUNT(*) FROM test_failures WHERE branch = ?", branch).to_i
+        unique_specs = conn.get_first_value("SELECT COUNT(DISTINCT spec_file || ':' || line_number) FROM test_failures WHERE branch = ?", branch).to_i
+        last_fetch = conn.get_first_value("SELECT MAX(fetched_at) FROM ci_runs")
+
+        puts "=== Flaky Test Report (#{branch}) ==="
+        puts ""
+        puts "CI Runs tracked:     #{total_runs}"
+        puts "Failed runs:         #{failed_runs} (#{total_runs > 0 ? (failed_runs.to_f / total_runs * 100).round(1) : 0}%)"
+        puts "Total test failures: #{total_failures}"
+        puts "Unique flaky specs:  #{unique_specs}"
+        puts "Last fetch:          #{last_fetch || 'never'}"
+
+        # Recent trend
+        recent = conn.get_first_value(
+          "SELECT COUNT(*) FROM test_failures WHERE branch = ? AND failed_at >= datetime('now', '-7 days')", branch
+        ).to_i
+        prior = conn.get_first_value(
+          "SELECT COUNT(*) FROM test_failures WHERE branch = ? AND failed_at >= datetime('now', '-14 days') AND failed_at < datetime('now', '-7 days')", branch
+        ).to_i
+
+        puts ""
+        puts "7-day trend:         #{recent} failures (prior 7 days: #{prior})"
+
+        if recent > prior
+          puts "                     \e[31m▲ Trending worse\e[0m"
+        elsif recent < prior
+          puts "                     \e[32m▼ Trending better\e[0m"
+        else
+          puts "                     → Stable"
+        end
+
+        # Top 5 flaky tests
+        top = conn.execute(<<~SQL, [branch])
+          SELECT
+            spec_file,
+            line_number,
+            description,
+            COUNT(*) as failure_count,
+            MAX(failed_at) as last_failure
+          FROM test_failures
+          WHERE branch = ?
+          GROUP BY spec_file, line_number
+          ORDER BY failure_count DESC
+          LIMIT 5
+        SQL
+
+        if top.any?
+          puts "\nTop 5 flaky tests:"
+          puts "-" * 80
+          top.each_with_index do |row, i|
+            puts "  #{i + 1}. #{row['spec_file']}:#{row['line_number']} (#{row['failure_count']}x)"
+            puts "     #{row['description']}"
+          end
+        end
+
+        # Recent stress runs
+        stress = conn.execute("SELECT * FROM stress_runs ORDER BY created_at DESC LIMIT 3")
+        if stress.any?
+          puts "\nRecent stress runs:"
+          puts "-" * 80
+          stress.each do |run|
+            total = run["passes"] + run["failures"]
+            rate = total > 0 ? (run["failures"].to_f / total * 100).round(1) : 0
+            ci_flag = run["ci_simulation"] == 1 ? " [CI sim]" : ""
+            puts "  #{run['spec_location']} — #{run['passes']}/#{total} passed (#{rate}% failure rate)#{ci_flag}"
+          end
+        end
+      ensure
+        @db.close
+      end
+    end
+  end
+end

data/lib/flaky/commands/stress.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require_relative "../database"
+
+module Flaky
+  module Commands
+    class Stress
+      def initialize(spec_location:, iterations: 20, seed: nil, ci_simulate: false, timeout: 600)
+        @spec_location = spec_location
+        @iterations = iterations
+        @seed = seed
+        @ci_simulate = ci_simulate
+        @timeout = timeout
+        @db = Database.new
+      end
+
+      def execute
+        env = {}
+        env["FLAKY_CI_SIMULATE"] = "1" if @ci_simulate
+
+        passes = 0
+        failures = 0
+        failed_seeds = []
+        start_time = Time.now
+
+        puts "Stress testing: #{@spec_location}"
+        puts "  Iterations: #{@iterations}, Seed: #{@seed || 'random'}, CI simulation: #{@ci_simulate}"
+        puts ""
+
+        @iterations.times do |i|
+          elapsed = Time.now - start_time
+          if elapsed > @timeout
+            puts "\n\nTimeout reached (#{@timeout}s) after #{i} iterations."
+            break
+          end
+
+          run_seed = @seed || rand(100_000)
+          cmd = "bundle exec rspec #{@spec_location} --seed #{run_seed} --format progress 2>&1"
+
+          output = nil
+          success = nil
+          IO.popen(env, cmd) do |io|
+            output = io.read
+            io.close
+            success = $?.success?
+          end
+
+          if success
+            passes += 1
+            print "\e[32m.\e[0m"
+          else
+            failures += 1
+            failed_seeds << run_seed
+            print "\e[31mF\e[0m"
+          end
+        end
+
+        total = passes + failures
+        rate = total > 0 ? (failures.to_f / total * 100).round(1) : 0
+
+        puts "\n\n#{total} runs: #{passes} passed, #{failures} failed"
+        puts "Failure rate: #{rate}%"
+
+        if failed_seeds.any?
+          puts "Failed seeds: #{failed_seeds.join(', ')}"
+          puts "\nTo reproduce a specific failure:"
+          puts "  FLAKY_CI_SIMULATE=#{@ci_simulate ? '1' : '0'} bundle exec rspec #{@spec_location} --seed #{failed_seeds.first}"
+        end
+
+        # Record to database
+        conn = @db.connection
+        conn.execute(
+          "INSERT INTO stress_runs (spec_location, seed, iterations, passes, failures, ci_simulation) VALUES (?, ?, ?, ?, ?, ?)",
+          [@spec_location, @seed, @iterations, passes, failures, @ci_simulate ? 1 : 0]
+        )
+
+        exit(failures > 0 ? 1 : 0)
+      ensure
+        @db.close
+      end
+    end
+  end
+end

data/lib/flaky/configuration.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Flaky
+  class Configuration
+    PROVIDERS = {}
+
+    attr_accessor :project, :branch, :db_path
+
+    def initialize
+      @provider_name = nil
+      @project = nil
+      @branch = "main"
+      @db_path = nil # resolved lazily
+    end
+
+    def provider=(name)
+      @provider_name = name.to_sym
+    end
+
+    def provider_instance
+      klass = PROVIDERS[@provider_name] || raise(Error, "Unknown provider: #{@provider_name}. Registered: #{PROVIDERS.keys.join(', ')}")
+      klass.new(self)
+    end
+
+    def resolved_db_path
+      @db_path || (defined?(Rails) ? Rails.root.join("tmp", "flaky.db").to_s : "tmp/flaky.db")
+    end
+
+    def self.register_provider(name, klass)
+      PROVIDERS[name.to_sym] = klass
+    end
+  end
+end

data/lib/flaky/database.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+
+module Flaky
+  class Database
+    SCHEMA_VERSION = 1
+
+    def initialize(path = nil)
+      @path = path || Flaky.configuration.resolved_db_path
+    end
+
+    def connection
+      @connection ||= begin
+        dir = File.dirname(@path)
+        FileUtils.mkdir_p(dir) unless File.directory?(dir)
+        db = SQLite3::Database.new(@path)
+        db.results_as_hash = true
+        db.execute("PRAGMA journal_mode=WAL")
+        db.execute("PRAGMA foreign_keys=ON")
+        migrate!(db)
+        db
+      end
+    end
+
+    def close
+      @connection&.close
+      @connection = nil
+    end
+
+    private
+
+    def migrate!(db)
+      version = db.get_first_value("PRAGMA user_version").to_i
+
+      if version < 1
+        db.execute_batch(<<~SQL)
+          CREATE TABLE IF NOT EXISTS ci_runs (
+            workflow_id TEXT PRIMARY KEY,
+            pipeline_id TEXT NOT NULL,
+            branch TEXT NOT NULL,
+            result TEXT NOT NULL,
+            created_at TEXT NOT NULL,
+            fetched_at TEXT NOT NULL DEFAULT (datetime('now'))
+          );
+
+          CREATE TABLE IF NOT EXISTS job_results (
+            job_id TEXT PRIMARY KEY,
+            workflow_id TEXT NOT NULL REFERENCES ci_runs(workflow_id),
+            job_name TEXT NOT NULL,
+            block_name TEXT NOT NULL,
+            result TEXT NOT NULL,
+            example_count INTEGER,
+            failure_count INTEGER,
+            seed INTEGER,
+            duration_seconds REAL,
+            fetched_at TEXT NOT NULL DEFAULT (datetime('now'))
+          );
+
+          CREATE TABLE IF NOT EXISTS test_failures (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            workflow_id TEXT NOT NULL REFERENCES ci_runs(workflow_id),
+            job_id TEXT NOT NULL REFERENCES job_results(job_id),
+            job_name TEXT NOT NULL,
+            spec_file TEXT NOT NULL,
+            line_number INTEGER NOT NULL,
+            description TEXT NOT NULL,
+            seed INTEGER NOT NULL,
+            branch TEXT NOT NULL,
+            failed_at TEXT NOT NULL,
+            UNIQUE(workflow_id, spec_file, line_number)
+          );
+
+          CREATE TABLE IF NOT EXISTS stress_runs (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            spec_location TEXT NOT NULL,
+            seed INTEGER,
+            iterations INTEGER NOT NULL,
+            passes INTEGER NOT NULL DEFAULT 0,
+            failures INTEGER NOT NULL DEFAULT 0,
+            ci_simulation INTEGER NOT NULL DEFAULT 0,
+            created_at TEXT NOT NULL DEFAULT (datetime('now'))
+          );
+
+          CREATE INDEX IF NOT EXISTS idx_test_failures_spec ON test_failures(spec_file, line_number);
+          CREATE INDEX IF NOT EXISTS idx_test_failures_branch ON test_failures(branch);
+          CREATE INDEX IF NOT EXISTS idx_ci_runs_branch ON ci_runs(branch);
+          CREATE INDEX IF NOT EXISTS idx_job_results_workflow ON job_results(workflow_id);
+
+          PRAGMA user_version = 1;
+        SQL
+      end
+    end
+  end
+end

data/lib/flaky/log_parser.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Flaky
+  class LogParser
+    Result = Data.define(:seed, :example_count, :failure_count, :duration_seconds, :failures)
+    Failure = Data.define(:spec_file, :line_number, :description)
+
+    def parse(raw_log)
+      log = normalize(raw_log)
+
+      Result.new(
+        seed: extract_seed(log),
+        example_count: extract_example_count(log),
+        failure_count: extract_failure_count(log),
+        duration_seconds: extract_duration(log),
+        failures: extract_failures(log)
+      )
+    end
+
+    private
+
+    def normalize(text)
+      # 1. Strip ANSI escape codes
+      clean = text.gsub(/\e\[\d*;?\d*m/, "")
+      # 2. Rejoin lines broken by Semaphore's ~80-char wrapping.
+      #    Real RSpec blank lines (section separators) are preserved.
+      #    A wrapped line is one where the previous line doesn't end with
+      #    a logical boundary (blank line, "exit status:", prompt markers).
+      clean
+    end
+
+    def extract_seed(log)
+      log.scan(/Randomized with seed (\d+)/).flatten.last&.to_i
+    end
+
+    def extract_example_count(log)
+      # Allow whitespace (including newlines from wrapping) between number and "examples"
+      log.scan(/(\d+)\s+examples?/).flatten.last&.to_i
+    end
+
+    def extract_failure_count(log)
+      # Match "N failure(s)" allowing wrapped newlines; take the last occurrence
+      log.scan(/(\d+)\s+failures?/).flatten.last&.to_i || 0
+    end
+
+    def extract_duration(log)
+      # "Finished in N minutes N.N seconds" — may span wrapped lines
+      # Collapse the area around "Finished in" first
+      section = log[/Finished in.{0,80}/m]
+      return nil unless section
+
+      collapsed = section.gsub(/\s+/, " ")
+      match = collapsed.match(/Finished in\s+((\d+)\s+minutes?\s+)?(\d+(?:\.\d+)?)\s+seconds?/)
+      return nil unless match
+
+      seconds = match[3].to_f
+      seconds += match[2].to_i * 60 if match[2]
+      seconds
+    end
+
+    def extract_failures(log)
+      # Find section between "Failed examples:" and "Randomized with seed"
+      section = log[/Failed examples:\s*\n(.*?)(?=\nRandomized with seed)/m, 1]
+      return [] unless section
+
+      # Semaphore wraps lines at ~80 chars, splitting mid-word.
+      # Remove newlines directly (not replacing with space) to rejoin split tokens.
+      collapsed = section.delete("\n").squeeze(" ").strip
+
+      collapsed.scan(/rspec\s+\.\/(\S+):(\d+)\s+#\s+(.*?)(?=\s*rspec\s+\.\/|\s*$)/).map do |file, line, desc|
+        Failure.new(spec_file: file, line_number: line.to_i, description: desc.strip)
+      end
+    end
+  end
+end

data/lib/flaky/middleware/simulate_ci_latency.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Flaky
+  module Middleware
+    class SimulateCiLatency
+      # f1-standard-2: 2 vCPU, 4 GB RAM (shared tenant)
+      # Local Mac: 10+ cores, 16-64 GB RAM (dedicated)
+      # Empirically, CI system tests take ~2x longer than local.
+      # A 30ms delay per request approximates the difference.
+      DEFAULT_DELAY_MS = 30
+
+      def initialize(app, delay_ms: nil)
+        @app = app
+        @delay_ms = (delay_ms || ENV.fetch("FLAKY_LATENCY_MS", DEFAULT_DELAY_MS)).to_f
+      end
+
+      def call(env)
+        sleep(@delay_ms / 1000.0) if @delay_ms > 0
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/flaky/providers/base.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Flaky
+  module Providers
+    class Base
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      # Returns Array of Hashes:
+      #   { id:, pipeline_id:, branch:, result:, created_at: }
+      def fetch_workflows(age: "24h")
+        raise NotImplementedError
+      end
+
+      # Returns Array of Hashes:
+      #   { id:, name:, block_name:, result: }
+      def fetch_jobs(pipeline_id:)
+        raise NotImplementedError
+      end
+
+      # Returns raw log String for a job
+      def fetch_log(job_id:)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/flaky/providers/github_actions.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "base"
+
+module Flaky
+  module Providers
+    class GithubActions < Base
+      def fetch_workflows(age: "24h")
+        hours = parse_age_to_hours(age)
+        cutoff = Time.now - (hours * 3600)
+
+        output = run_cmd("gh run list --branch #{config.branch} --limit 100 --json databaseId,conclusion,createdAt,headBranch,workflowName")
+        runs = JSON.parse(output)
+
+        runs.filter_map do |run|
+          created = Time.parse(run["createdAt"])
+          next if created < cutoff
+
+          {
+            id: run["databaseId"].to_s,
+            pipeline_id: run["databaseId"].to_s,
+            branch: run["headBranch"],
+            result: map_conclusion(run["conclusion"]),
+            created_at: run["createdAt"]
+          }
+        end
+      end
+
+      def fetch_jobs(pipeline_id:)
+        output = run_cmd("gh run view #{pipeline_id} --json jobs")
+        data = JSON.parse(output)
+
+        (data["jobs"] || []).filter_map do |job|
+          name = job["name"]
+          next unless name.match?(/test/i)
+
+          {
+            id: job["databaseId"].to_s,
+            name: name,
+            block_name: name,
+            result: map_conclusion(job["conclusion"])
+          }
+        end
+      end
+
+      def fetch_log(job_id:)
+        run_cmd("gh run view --job #{job_id} --log")
+      end
+
+      private
+
+      def run_cmd(cmd)
+        output = `#{cmd} 2>/dev/null`
+        raise Error, "Command failed (exit #{$?.exitstatus}): #{cmd}" unless $?.success?
+        output
+      end
+
+      def parse_age_to_hours(age)
+        case age
+        when /(\d+)h/ then $1.to_i
+        when /(\d+)d/ then $1.to_i * 24
+        else 24
+        end
+      end
+
+      def map_conclusion(conclusion)
+        case conclusion
+        when "success" then "passed"
+        when "failure" then "failed"
+        else conclusion || "unknown"
+        end
+      end
+    end
+
+    Configuration.register_provider(:github_actions, GithubActions)
+  end
+end

data/lib/flaky/providers/semaphore.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "yaml"
+require_relative "base"
+
+module Flaky
+  module Providers
+    class Semaphore < Base
+      TEST_BLOCKS = ["Unit Tests", "System Tests"].freeze
+
+      def fetch_workflows(age: "24h")
+        output = run_cmd("sem get workflows -p #{config.project} --age #{age}")
+        lines = output.lines.drop(1) # skip header
+        lines.filter_map { |line| parse_workflow_line(line) }
+      end
+
+      def fetch_jobs(pipeline_id:)
+        output = run_cmd("sem get pipelines #{pipeline_id}")
+        data = YAML.safe_load(output, permitted_classes: [Date, Time])
+        blocks = data&.[]("blocks")
+        return [] unless blocks
+
+        blocks.flat_map do |block|
+          block_name = block["name"]
+          next [] unless TEST_BLOCKS.include?(block_name)
+
+          (block["jobs"] || []).map do |job|
+            {
+              id: job["jobid"],
+              name: job["name"],
+              block_name: block_name,
+              result: block["result"]
+            }
+          end
+        end
+      end
+
+      def fetch_log(job_id:)
+        run_cmd("sem logs #{job_id}")
+      end
+
+      private
+
+      def run_cmd(cmd)
+        output = `#{cmd} 2>/dev/null`
+        raise Error, "Command failed (exit #{$?.exitstatus}): #{cmd}" unless $?.success?
+        output
+      end
+
+      def parse_workflow_line(line)
+        parts = line.strip.split(/\s{2,}/)
+        return nil if parts.length < 4
+
+        {
+          id: parts[0],
+          pipeline_id: parts[1],
+          branch: parts[3],
+          created_at: parts[2]
+        }
+      end
+    end
+
+    Configuration.register_provider(:semaphore, Semaphore)
+  end
+end

data/lib/flaky/railtie.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Flaky
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path("tasks/flaky.rake", __dir__)
+    end
+
+    initializer "flaky.middleware" do |app|
+      if ENV["FLAKY_CI_SIMULATE"] && Rails.env.test?
+        require_relative "middleware/simulate_ci_latency"
+        app.middleware.use Flaky::Middleware::SimulateCiLatency
+      end
+    end
+  end
+end

data/lib/flaky/tasks/flaky.rake

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+namespace :flaky do
+  desc "Fetch recent CI results (age: duration, default 24h)"
+  task :fetch, [:age] => :environment do |_t, args|
+    require "flaky/commands/fetch"
+    age = args[:age] || "24h"
+    Flaky::Commands::Fetch.new(age: age).execute
+  end
+
+  desc "Rank flaky tests by failure frequency (since: days, default 30)"
+  task :rank, [:since] => :environment do |_t, args|
+    require "flaky/commands/rank"
+    since = (args[:since] || 30).to_i
+    Flaky::Commands::Rank.new(since_days: since).execute
+  end
+
+  desc "Show failure history for a spec (spec_location: file:line)"
+  task :history, [:spec_location] => :environment do |_t, args|
+    require "flaky/commands/history"
+    raise "Usage: rake flaky:history[path/to/spec.rb:42]" unless args[:spec_location]
+    Flaky::Commands::History.new(spec_location: args[:spec_location]).execute
+  end
+
+  desc "Stress test a spec (spec, iterations, seed, ci)"
+  task :stress, [:spec, :iterations, :seed, :ci] => :environment do |_t, args|
+    require "flaky/commands/stress"
+    raise "Usage: rake flaky:stress[path/to/spec.rb:42,20,12345,true]" unless args[:spec]
+    Flaky::Commands::Stress.new(
+      spec_location: args[:spec],
+      iterations: (args[:iterations] || 20).to_i,
+      seed: args[:seed]&.to_i,
+      ci_simulate: args[:ci] == "true"
+    ).execute
+  end
+
+  desc "Summary dashboard of flaky test status"
+  task report: :environment do
+    require "flaky/commands/report"
+    Flaky::Commands::Report.new.execute
+  end
+end

data/lib/flaky/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Flaky
+  VERSION = "0.1.0"
+end

data/lib/flaky.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "flaky/version"
+require_relative "flaky/configuration"
+require_relative "flaky/providers/semaphore"
+require_relative "flaky/providers/github_actions"
+require_relative "flaky/railtie" if defined?(Rails::Railtie)
+
+module Flaky
+  class Error < StandardError; end
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def provider
+      configuration.provider_instance
+    end
+
+    def register_provider(name, klass)
+      Configuration.register_provider(name, klass)
+    end
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: flaky-friend
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Flytedesk
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Fetches CI test results, stores failures in SQLite, ranks by frequency,
+  and reproduces under simulated CI conditions.
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/flaky.rb
+- lib/flaky/commands/fetch.rb
+- lib/flaky/commands/history.rb
+- lib/flaky/commands/rank.rb
+- lib/flaky/commands/report.rb
+- lib/flaky/commands/stress.rb
+- lib/flaky/configuration.rb
+- lib/flaky/database.rb
+- lib/flaky/log_parser.rb
+- lib/flaky/middleware/simulate_ci_latency.rb
+- lib/flaky/providers/base.rb
+- lib/flaky/providers/github_actions.rb
+- lib/flaky/providers/semaphore.rb
+- lib/flaky/railtie.rb
+- lib/flaky/tasks/flaky.rake
+- lib/flaky/version.rb
+homepage: https://github.com/Flytedesk/flaky
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Track, rank, and reproduce flaky CI test failures
+test_files: []