coatepec 0.5.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be24375e5f36c63e0511dc268bacecd57a54a33a94f5d44540925927c49f6655
-  data.tar.gz: 0e80f68351e4e814e9e4486f7e02d8d3f70512edaa7e9806c6baa3091111e779
+  metadata.gz: ddcc044bcc53590ba19d988b0db8ea0aea4efd7db102bd6718196e61ab9ab820
+  data.tar.gz: 1b8b779b5bb59068a7fdd5fc90e7d44b900d50f1918e1d68877a9ccad24bf1e7
 SHA512:
-  metadata.gz: 297b7eb82d12ece3991893ed1052650648d37fb86b438356257e67e90f69260297b5c3f4b390a2ce25860e2e06a8487dc1597bb39219c3f13ac550901cc92e94
-  data.tar.gz: dbe1f843c872f42c20d7df9ee4226af83200b9c9457bb2a9b3aba185d22b7c5c8878fa3237a57dc4dfb4526c499e4c75cb1cb5c3135262aa24dd71fc4ea8805b
+  metadata.gz: 56fce0755730ab151c81fb43f88a28d6e081b242c3e9bb31fac71578c290e864efbc5e5313702a4c015b95a661cdd601f1701979c98dbbdae1a0cfa06e0be2fa
+  data.tar.gz: 4101b3595de972478ac414f3c019b413ec991cfe1d8ea0fbb251010fad6d699eaeb849818a1c4dbcfe813b47b41af220b68a7f971efe287dbbb8c598d8ff9763

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.6.0
+
+- Add `rails_spec_flaky_check`: runs a spec selection multiple times with
+  independently random seeds (RSpec's `--seed` is equivalent to `--order
+  rand:SEED`) and reports which examples' pass/fail status was
+  inconsistent across runs -- separately from examples that failed every
+  run (`consistently_failing`, not flaky) and examples that passed every
+  run (omitted). Each round's seed is included in the response so a
+  specific divergence can be reproduced with a plain `rails_spec_run`
+  call. `rails_spec_run` itself is unchanged by this release.
+
 ## 0.5.2
 
 - Fix `WorkerManager` getting permanently stuck treating a dead worker as

data/README.md

@@ -130,6 +130,7 @@ a no-op.
 | `rails_runtime_restart` | `{}` | Unconditionally respawns the worker, discarding its warm boot |
 | `rails_routes` | `query?`, `limit?` (1..200, default 50), `offset?` | Case-insensitive filter across name/verb/path/controller/action |
 | `rails_model` | `name` (constant path, e.g. `Widget` or `Admin::Widget`) | ActiveRecord models only; columns, associations, validators, enums -- no row data |
+| `rails_spec_flaky_check` | `paths`, `example?`, `timeout_seconds?` (per round, 1..900), `runs?` (2..20, default 5) | Runs the selection `runs` times with a fresh random seed each round; reports examples whose status was inconsistent across runs |
 
 ### Example queries
 
@@ -156,6 +157,30 @@ a no-op.
   rather than introspecting it (a nonexistent constant raises `model_not_found`
   instead) -- the tool only ever reflects on `ActiveRecord::Base` descendants.
 
+`rails_spec_flaky_check`:
+
+- "Is this spec flaky?" -- `rails_spec_flaky_check(paths: ["spec/models/widget_spec.rb"])`
+  runs it 5 times (default), each with an independently random seed
+  (equivalent to RSpec's `--order rand:SEED`), and reports any example
+  whose pass/fail status wasn't the same every time under `flaky_examples`
+  -- distinct from `consistently_failing` (fails every run: broken, not
+  flaky) and examples that passed every run (omitted -- nothing to report).
+- Each round's seed is included in the response (`rounds[].seed`), so a
+  specific divergence can be reproduced afterward with a plain
+  `rails_spec_run(seed: <that seed>)`.
+- `timeout_seconds` is a **per-round** budget, not a total; `runs *
+  timeout_seconds` is capped at 1800s combined (`flaky_check_budget_exceeded`
+  if exceeded) since this tool can run for a while.
+- Selections over 500 examples are capped per round (the same limit
+  `rails_spec_run` already has), and each round samples a different subset,
+  since execution order varies by design -- for suites this large, narrow
+  `paths`/`example` rather than passing a very broad directory selection.
+- `statuses[]` only aligns positionally with `rounds[]` when no round
+  crashed -- a round whose process itself failed contributes no entry to
+  `statuses[]` (though it still appears in `rounds[]`), so treat positional
+  correspondence as best-effort, not guaranteed, when a round's `status` in
+  `rounds[]` looks like an outright crash rather than a normal pass/fail.
+
 The warm test worker forces Rails' reload-checking on for its own boot,
 regardless of the target app's own `test.rb` setting (which disables it by
 default) -- so editing a model file takes effect on the next tool call

data/lib/coatepec/mcp/tools.rb

@@ -94,6 +94,47 @@ module Coatepec
       end
     end
 
+    # The `rails_spec_flaky_check` MCP tool: runs targeted RSpec examples
+    # multiple times with independently random seeds and reports which
+    # examples' pass/fail status was inconsistent across rounds. Separate
+    # tool from rails_spec_run for the same reason rails_spec_profile is --
+    # see docs/superpowers/specs/2026-08-13-flaky-spec-detection-design.md.
+    class FlakyCheckTool < ::MCP::Tool
+      tool_name "rails_spec_flaky_check"
+      description "Run targeted RSpec examples multiple times with random seeds to detect order-dependent or " \
+                  "intermittent flakiness, reporting which examples' status was inconsistent across runs"
+      annotations(read_only_hint: false, destructive_hint: true, idempotent_hint: false, open_world_hint: true)
+      input_schema(
+        properties: {
+          paths: { type: "array", items: { type: "string" }, minItems: 1, maxItems: 100 },
+          example: { type: %w[string null] },
+          timeout_seconds: { type: "integer", minimum: 1, maximum: 900 },
+          runs: { type: "integer", minimum: 2, maximum: 20 }
+        },
+        required: ["paths"],
+        additionalProperties: false
+      )
+
+      class << self
+        def call(paths:, server_context:, example: nil, timeout_seconds: 120, runs: 5)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          data = server_context[:worker_manager].check_flaky(
+            paths: paths, example: example, timeout_seconds: timeout_seconds, runs: runs
+          )
+          Response.ok(data: data, meta: meta_for(server_context, started_at))
+        rescue Coatepec::Error => e
+          Response.error(e)
+        end
+
+        private
+
+        def meta_for(server_context, started_at)
+          duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
+          { project_root: server_context[:project_root], environment: "test", duration_ms: duration_ms }
+        end
+      end
+    end
+
     # The `rails_routes` MCP tool: lists/filters/paginates the target
     # Rails app's routes.
     class RoutesTool < ::MCP::Tool

data/lib/coatepec/mcp.rb

@@ -13,14 +13,14 @@ require_relative "mcp/tools"
 
 module Coatepec
   # Wires the `rails_spec_run`, `rails_runtime_status`, `rails_runtime_restart`,
-  # `rails_routes`, and `rails_model` tools into an `::MCP::Server` instance
-  # backed by the given project's worker manager.
+  # `rails_spec_flaky_check`, `rails_routes`, and `rails_model` tools into an
+  # `::MCP::Server` instance backed by the given project's worker manager.
   module MCP
     def self.build_server(project:, worker_manager:)
       ::MCP::Server.new(
         name: "coatepec",
         version: Coatepec::VERSION,
-        tools: [SpecRunTool, RuntimeStatusTool, RuntimeRestartTool, RoutesTool, ModelTool],
+        tools: [SpecRunTool, RuntimeStatusTool, RuntimeRestartTool, FlakyCheckTool, RoutesTool, ModelTool],
         server_context: { worker_manager: worker_manager, project_root: project.root }
       )
     end

data/lib/coatepec/spec/flaky_checker.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Coatepec
+  module Spec
+    # Runs a spec selection multiple times, each round with an
+    # independently random seed (RSpec's --seed N is equivalent to --order
+    # rand:N, confirmed against rspec-core's own option_parser.rb), and
+    # groups per-example pass/fail status across rounds by the example's
+    # own id -- stable regardless of execution order, since it's derived
+    # from declaration position/nesting, not from --order. Composes Runner
+    # unchanged, exactly like FactoryProfRunner: every round is a plain,
+    # ordinary Runner#run call with a different seed, no strategy override
+    # needed (unlike FactoryProfRunner, this feature has no load-time
+    # activation trick to fight -- see
+    # docs/superpowers/specs/2026-08-13-flaky-spec-detection-design.md).
+    class FlakyChecker
+      DEFAULT_RUNS = 5
+      MAX_TOTAL_SECONDS = 1800 # timeout_seconds * runs must not exceed this
+      MAX_ITEMS = 200 # same bound rails_model/FactoryProfRunner use elsewhere
+
+      def initialize(project_root, rails_runtime: nil)
+        @runner = Runner.new(project_root, rails_runtime: rails_runtime)
+      end
+
+      def call(paths:, example: nil, timeout_seconds: Runner::DEFAULT_TIMEOUT, runs: DEFAULT_RUNS)
+        validate_budget!(timeout_seconds, runs)
+        rounds = Array.new(runs) { run_one_round(paths, example, timeout_seconds) }
+        build_report(rounds)
+      end
+
+      private
+
+      def validate_budget!(timeout_seconds, runs)
+        return if timeout_seconds * runs <= MAX_TOTAL_SECONDS
+
+        raise Coatepec::Error.new(
+          :flaky_check_budget_exceeded,
+          "timeout_seconds (#{timeout_seconds}) * runs (#{runs}) exceeds the " \
+          "#{MAX_TOTAL_SECONDS}s combined budget -- lower one or both"
+        )
+      end
+
+      def run_one_round(paths, example, timeout_seconds)
+        seed = SecureRandom.random_number(65_536)
+        result = @runner.run(
+          paths: paths, example: example, seed: seed, fail_fast: false,
+          timeout_seconds: timeout_seconds
+        )
+        { seed: seed, status: result[:status], examples: result[:examples] }
+      end
+
+      def build_report(rounds)
+        by_id = group_examples_by_id(rounds)
+        {
+          runs: rounds.size,
+          rounds: rounds.map { |r| { seed: r[:seed], status: r[:status] } },
+          flaky_examples: classify(by_id) { |statuses| statuses.uniq.size > 1 },
+          consistently_failing: classify(by_id) { |statuses| statuses.uniq == ["failed"] }
+        }
+      end
+
+      # Rounds whose RSpec process itself crashed/timed out (not a specific
+      # example failing -- the whole run killed before its JSON summary was
+      # ever written) contribute no per-example data here. Excluded rather
+      # than counted as "every example failed", which would misrepresent a
+      # dead process as evidence against examples that never actually ran;
+      # the round itself is still visible in the `rounds:` summary.
+      def group_examples_by_id(rounds)
+        rounds.each_with_object({}) do |round, acc|
+          round[:examples].each { |ex| (acc[ex[:id]] ||= { meta: ex, statuses: [] })[:statuses] << ex[:status] }
+        end
+      end
+
+      def classify(by_id)
+        by_id.values.select { |v| yield(v[:statuses]) }.first(MAX_ITEMS).map do |v|
+          v[:meta].slice(:id, :description, :file_path, :line_number).merge(
+            statuses: v[:statuses],
+            pass_count: v[:statuses].count("passed"),
+            failure_count: v[:statuses].count("failed")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/coatepec/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coatepec
-  VERSION = "0.5.2"
+  VERSION = "0.6.0"
 end

data/lib/coatepec/worker/server.rb

@@ -42,6 +42,7 @@ module Coatepec
         case command
         when "status" then @runtime.status
         when "spec_run" then handle_spec_run(args)
+        when "flaky_check" then handle_flaky_check(args)
         when "routes" then handle_routes(args)
         when "model" then handle_model(args)
         else
@@ -53,6 +54,10 @@ module Coatepec
         Spec::Runner.new(@project_root, rails_runtime: @runtime).run(**args.transform_keys(&:to_sym))
       end
 
+      def handle_flaky_check(args)
+        Spec::FlakyChecker.new(@project_root, rails_runtime: @runtime).call(**args.transform_keys(&:to_sym))
+      end
+
       def handle_routes(args)
         Introspection::Routes.new(**args.transform_keys(&:to_sym)).call
       end

data/lib/coatepec/worker_manager.rb

@@ -28,6 +28,18 @@ module Coatepec
       )
     end
 
+    def check_flaky(paths:, example:, timeout_seconds:, runs:)
+      dispatch(
+        "flaky_check",
+        { paths: paths, example: example, timeout_seconds: timeout_seconds, runs: runs },
+        # Each round costs timeout_seconds *plus* real per-round overhead (process
+        # termination/spawn/result-parsing -- roughly 0.6s+ per round from
+        # ProcessStrategy#terminate's own escalation sleeps alone), so the flat
+        # base slack below is scaled by an extra ~2s per round on top of it.
+        timeout: (timeout_seconds * runs) + (2 * runs) + 10
+      )
+    end
+
     def routes(query: nil, limit: 50, offset: 0)
       dispatch("routes", { query: query, limit: limit, offset: offset }, timeout: 30)
     end

data/lib/coatepec.rb

@@ -18,6 +18,7 @@ require_relative "coatepec/spec/fork_strategy"
 require_relative "coatepec/spec/spawn_strategy"
 require_relative "coatepec/spec/guarded_fork_strategy"
 require_relative "coatepec/spec/runner"
+require_relative "coatepec/spec/flaky_checker"
 require_relative "coatepec/worker/server"
 require_relative "coatepec/worker_manager"
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: coatepec
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Enrique Mogollan
 bindir: exe
 cert_chain: []
-date: 2026-08-10 00:00:00.000000000 Z
+date: 2026-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -74,6 +74,7 @@ files:
 - lib/coatepec/project.rb
 - lib/coatepec/project_config.rb
 - lib/coatepec/protocol.rb
+- lib/coatepec/spec/flaky_checker.rb
 - lib/coatepec/spec/fork_strategy.rb
 - lib/coatepec/spec/guarded_fork_strategy.rb
 - lib/coatepec/spec/path_policy.rb