coatepec 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 161aafede82c7279701f75a51cc7976a6ef7bbbadff7bc22a7bc04dc050e1780
-  data.tar.gz: 2b867df0d495bb573e1e00ad9a94f9aca800fa7045bbf6965fb0825acfc2683f
+  metadata.gz: ddcc044bcc53590ba19d988b0db8ea0aea4efd7db102bd6718196e61ab9ab820
+  data.tar.gz: 1b8b779b5bb59068a7fdd5fc90e7d44b900d50f1918e1d68877a9ccad24bf1e7
 SHA512:
-  metadata.gz: d93777d12dac89c55450377259fc7d206bd6ef077766395f1397045372a58a7203bf986bbb9a525fc89b8510159a7fd9d0c3a057c321219edd95d6f23caa7acf
-  data.tar.gz: 3e580288a7c4ef9b0e707909967c28293ce8bc8b749e8c11d03903faef69ef80587c4a8307e13e769633d521b6245a05f806859d124a40346a9a0120b56e3e9b
+  metadata.gz: 56fce0755730ab151c81fb43f88a28d6e081b242c3e9bb31fac71578c290e864efbc5e5313702a4c015b95a661cdd601f1701979c98dbbdae1a0cfa06e0be2fa
+  data.tar.gz: 4101b3595de972478ac414f3c019b413ec991cfe1d8ea0fbb251010fad6d699eaeb849818a1c4dbcfe813b47b41af220b68a7f971efe287dbbb8c598d8ff9763

data/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 0.6.0
+
+- Add `rails_spec_flaky_check`: runs a spec selection multiple times with
+  independently random seeds (RSpec's `--seed` is equivalent to `--order
+  rand:SEED`) and reports which examples' pass/fail status was
+  inconsistent across runs -- separately from examples that failed every
+  run (`consistently_failing`, not flaky) and examples that passed every
+  run (omitted). Each round's seed is included in the response so a
+  specific divergence can be reproduced with a plain `rails_spec_run`
+  call. `rails_spec_run` itself is unchanged by this release.
+
+## 0.5.2
+
+- Fix `WorkerManager` getting permanently stuck treating a dead worker as
+  alive: `Worker::Client#alive?` used `Process.kill(0, pid)`, which returns
+  true for an unreaped zombie, and the one existing auto-restart-and-retry
+  path only caught `Worker::Client::DisconnectedError`, not a raw
+  `Errno::EPIPE`/`SystemCallError` from writing to a dead worker's pipe.
+  Every call after a worker died this way failed identically, forever --
+  since stdio MCP servers have no lighter reconnect in Claude Code, this
+  previously required a full session restart to recover from.
+- Add `rails_runtime_restart`: unconditionally tears down and respawns the
+  test worker, for whatever the automatic detection above doesn't catch.
+- Fix RSpec's own "(files took N seconds to load)" reporting growing
+  across every `rails_spec_run` call made through the same warm,
+  forked-per-run worker -- it was measuring time since the worker first
+  booted, not time this run's files took to load, because `rspec-core`
+  freezes that timestamp once per process and Coatepec never reset it on
+  reuse.
+
 ## 0.5.1
 
 - Fix `rails_model` crashing outright for a model with a `has_one`/

data/README.md

@@ -127,8 +127,10 @@ a no-op.
 |---|---|---|
 | `rails_spec_run` | `paths: string[1..100]`, `example?`, `seed?`, `fail_fast?`, `timeout_seconds?` (1..900, default 120) | Isolated per run; output capped at 256 KiB per stream |
 | `rails_runtime_status` | `{}` | Reports Ruby/Rails versions, worker PID, boot_id, lifecycle state |
+| `rails_runtime_restart` | `{}` | Unconditionally respawns the worker, discarding its warm boot |
 | `rails_routes` | `query?`, `limit?` (1..200, default 50), `offset?` | Case-insensitive filter across name/verb/path/controller/action |
 | `rails_model` | `name` (constant path, e.g. `Widget` or `Admin::Widget`) | ActiveRecord models only; columns, associations, validators, enums -- no row data |
+| `rails_spec_flaky_check` | `paths`, `example?`, `timeout_seconds?` (per round, 1..900), `runs?` (2..20, default 5) | Runs the selection `runs` times with a fresh random seed each round; reports examples whose status was inconsistent across runs |
 
 ### Example queries
 
@@ -155,6 +157,30 @@ a no-op.
   rather than introspecting it (a nonexistent constant raises `model_not_found`
   instead) -- the tool only ever reflects on `ActiveRecord::Base` descendants.
 
+`rails_spec_flaky_check`:
+
+- "Is this spec flaky?" -- `rails_spec_flaky_check(paths: ["spec/models/widget_spec.rb"])`
+  runs it 5 times (default), each with an independently random seed
+  (equivalent to RSpec's `--order rand:SEED`), and reports any example
+  whose pass/fail status wasn't the same every time under `flaky_examples`
+  -- distinct from `consistently_failing` (fails every run: broken, not
+  flaky) and examples that passed every run (omitted -- nothing to report).
+- Each round's seed is included in the response (`rounds[].seed`), so a
+  specific divergence can be reproduced afterward with a plain
+  `rails_spec_run(seed: <that seed>)`.
+- `timeout_seconds` is a **per-round** budget, not a total; `runs *
+  timeout_seconds` is capped at 1800s combined (`flaky_check_budget_exceeded`
+  if exceeded) since this tool can run for a while.
+- Selections over 500 examples are capped per round (the same limit
+  `rails_spec_run` already has), and each round samples a different subset,
+  since execution order varies by design -- for suites this large, narrow
+  `paths`/`example` rather than passing a very broad directory selection.
+- `statuses[]` only aligns positionally with `rounds[]` when no round
+  crashed -- a round whose process itself failed contributes no entry to
+  `statuses[]` (though it still appears in `rounds[]`), so treat positional
+  correspondence as best-effort, not guaranteed, when a round's `status` in
+  `rounds[]` looks like an outright crash rather than a normal pass/fail.
+
 The warm test worker forces Rails' reload-checking on for its own boot,
 regardless of the target app's own `test.rb` setting (which disables it by
 default) -- so editing a model file takes effect on the next tool call
@@ -178,6 +204,13 @@ across a commit that touches the Gemfile, since the sidecar is managed by
 your MCP client rather than by Coatepec itself: restart your MCP client (or
 however it manages the Coatepec process) to pick up the change.
 
+A dead *worker* (as opposed to a dead sidecar) recovers on its own: the
+next tool call detects it and transparently boots a fresh one before
+retrying, whether the worker exited outright or a request to it failed
+with a broken pipe. If you want a fresh worker without waiting for a
+failure -- or one keeps recurring -- call `rails_runtime_restart` directly;
+it always respawns, even if the current worker looks healthy.
+
 ## Security boundary
 
 No eval, console, SQL/record access, shell, Rake, or file-write tool. Spec

data/lib/coatepec/mcp/tools.rb

@@ -69,6 +69,72 @@ module Coatepec
       end
     end
 
+    # The `rails_runtime_restart` MCP tool: unconditionally tears down and
+    # respawns the test worker, discarding its warm Rails boot. A manual
+    # escape hatch alongside WorkerManager's own automatic dead-worker
+    # detection, for whatever failure mode that detection doesn't catch.
+    class RuntimeRestartTool < ::MCP::Tool
+      tool_name "rails_runtime_restart"
+      description "Tear down and respawn the Coatepec test worker, discarding its warm Rails boot " \
+                  "(use if rails_spec_run/rails_runtime_status keep failing and a fresh worker is needed)"
+      annotations(read_only_hint: false, destructive_hint: true, idempotent_hint: false, open_world_hint: false)
+      input_schema(properties: {}, required: [], additionalProperties: false)
+
+      class << self
+        def call(server_context:)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          data = server_context[:worker_manager].restart!
+          duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
+          Response.ok(data: data,
+                      meta: { project_root: server_context[:project_root], environment: "test",
+                              duration_ms: duration_ms })
+        rescue Coatepec::Error => e
+          Response.error(e)
+        end
+      end
+    end
+
+    # The `rails_spec_flaky_check` MCP tool: runs targeted RSpec examples
+    # multiple times with independently random seeds and reports which
+    # examples' pass/fail status was inconsistent across rounds. Separate
+    # tool from rails_spec_run for the same reason rails_spec_profile is --
+    # see docs/superpowers/specs/2026-08-13-flaky-spec-detection-design.md.
+    class FlakyCheckTool < ::MCP::Tool
+      tool_name "rails_spec_flaky_check"
+      description "Run targeted RSpec examples multiple times with random seeds to detect order-dependent or " \
+                  "intermittent flakiness, reporting which examples' status was inconsistent across runs"
+      annotations(read_only_hint: false, destructive_hint: true, idempotent_hint: false, open_world_hint: true)
+      input_schema(
+        properties: {
+          paths: { type: "array", items: { type: "string" }, minItems: 1, maxItems: 100 },
+          example: { type: %w[string null] },
+          timeout_seconds: { type: "integer", minimum: 1, maximum: 900 },
+          runs: { type: "integer", minimum: 2, maximum: 20 }
+        },
+        required: ["paths"],
+        additionalProperties: false
+      )
+
+      class << self
+        def call(paths:, server_context:, example: nil, timeout_seconds: 120, runs: 5)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          data = server_context[:worker_manager].check_flaky(
+            paths: paths, example: example, timeout_seconds: timeout_seconds, runs: runs
+          )
+          Response.ok(data: data, meta: meta_for(server_context, started_at))
+        rescue Coatepec::Error => e
+          Response.error(e)
+        end
+
+        private
+
+        def meta_for(server_context, started_at)
+          duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
+          { project_root: server_context[:project_root], environment: "test", duration_ms: duration_ms }
+        end
+      end
+    end
+
     # The `rails_routes` MCP tool: lists/filters/paginates the target
     # Rails app's routes.
     class RoutesTool < ::MCP::Tool

data/lib/coatepec/mcp.rb

@@ -12,15 +12,15 @@ require_relative "mcp/response"
 require_relative "mcp/tools"
 
 module Coatepec
-  # Wires the `rails_spec_run`, `rails_runtime_status`, `rails_routes`, and
-  # `rails_model` tools into an `::MCP::Server` instance backed by the given
-  # project's worker manager.
+  # Wires the `rails_spec_run`, `rails_runtime_status`, `rails_runtime_restart`,
+  # `rails_spec_flaky_check`, `rails_routes`, and `rails_model` tools into an
+  # `::MCP::Server` instance backed by the given project's worker manager.
   module MCP
     def self.build_server(project:, worker_manager:)
       ::MCP::Server.new(
         name: "coatepec",
         version: Coatepec::VERSION,
-        tools: [SpecRunTool, RuntimeStatusTool, RoutesTool, ModelTool],
+        tools: [SpecRunTool, RuntimeStatusTool, RuntimeRestartTool, FlakyCheckTool, RoutesTool, ModelTool],
         server_context: { worker_manager: worker_manager, project_root: project.root }
       )
     end

data/lib/coatepec/spec/flaky_checker.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Coatepec
+  module Spec
+    # Runs a spec selection multiple times, each round with an
+    # independently random seed (RSpec's --seed N is equivalent to --order
+    # rand:N, confirmed against rspec-core's own option_parser.rb), and
+    # groups per-example pass/fail status across rounds by the example's
+    # own id -- stable regardless of execution order, since it's derived
+    # from declaration position/nesting, not from --order. Composes Runner
+    # unchanged, exactly like FactoryProfRunner: every round is a plain,
+    # ordinary Runner#run call with a different seed, no strategy override
+    # needed (unlike FactoryProfRunner, this feature has no load-time
+    # activation trick to fight -- see
+    # docs/superpowers/specs/2026-08-13-flaky-spec-detection-design.md).
+    class FlakyChecker
+      DEFAULT_RUNS = 5
+      MAX_TOTAL_SECONDS = 1800 # timeout_seconds * runs must not exceed this
+      MAX_ITEMS = 200 # same bound rails_model/FactoryProfRunner use elsewhere
+
+      def initialize(project_root, rails_runtime: nil)
+        @runner = Runner.new(project_root, rails_runtime: rails_runtime)
+      end
+
+      def call(paths:, example: nil, timeout_seconds: Runner::DEFAULT_TIMEOUT, runs: DEFAULT_RUNS)
+        validate_budget!(timeout_seconds, runs)
+        rounds = Array.new(runs) { run_one_round(paths, example, timeout_seconds) }
+        build_report(rounds)
+      end
+
+      private
+
+      def validate_budget!(timeout_seconds, runs)
+        return if timeout_seconds * runs <= MAX_TOTAL_SECONDS
+
+        raise Coatepec::Error.new(
+          :flaky_check_budget_exceeded,
+          "timeout_seconds (#{timeout_seconds}) * runs (#{runs}) exceeds the " \
+          "#{MAX_TOTAL_SECONDS}s combined budget -- lower one or both"
+        )
+      end
+
+      def run_one_round(paths, example, timeout_seconds)
+        seed = SecureRandom.random_number(65_536)
+        result = @runner.run(
+          paths: paths, example: example, seed: seed, fail_fast: false,
+          timeout_seconds: timeout_seconds
+        )
+        { seed: seed, status: result[:status], examples: result[:examples] }
+      end
+
+      def build_report(rounds)
+        by_id = group_examples_by_id(rounds)
+        {
+          runs: rounds.size,
+          rounds: rounds.map { |r| { seed: r[:seed], status: r[:status] } },
+          flaky_examples: classify(by_id) { |statuses| statuses.uniq.size > 1 },
+          consistently_failing: classify(by_id) { |statuses| statuses.uniq == ["failed"] }
+        }
+      end
+
+      # Rounds whose RSpec process itself crashed/timed out (not a specific
+      # example failing -- the whole run killed before its JSON summary was
+      # ever written) contribute no per-example data here. Excluded rather
+      # than counted as "every example failed", which would misrepresent a
+      # dead process as evidence against examples that never actually ran;
+      # the round itself is still visible in the `rounds:` summary.
+      def group_examples_by_id(rounds)
+        rounds.each_with_object({}) do |round, acc|
+          round[:examples].each { |ex| (acc[ex[:id]] ||= { meta: ex, statuses: [] })[:statuses] << ex[:status] }
+        end
+      end
+
+      def classify(by_id)
+        by_id.values.select { |v| yield(v[:statuses]) }.first(MAX_ITEMS).map do |v|
+          v[:meta].slice(:id, :description, :file_path, :line_number).merge(
+            statuses: v[:statuses],
+            pass_count: v[:statuses].count("passed"),
+            failure_count: v[:statuses].count("failed")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/coatepec/spec/fork_strategy.rb

@@ -14,6 +14,15 @@ module Coatepec
           redirect_output(out_w, err_w)
           # Forked children must not share the parent's live DB sockets.
           ActiveRecord::Base.connection_handler.clear_all_connections! if defined?(ActiveRecord::Base)
+          # RSpec freezes its own "load started at" timestamp once, at the moment
+          # rspec/core.rb is first required -- in this architecture, that's when
+          # the long-lived warm worker booted, not when THIS run started. Every
+          # forked child inherits that frozen timestamp via copy-on-write, so
+          # RSpec's own "(files took N seconds to load)" reporting would
+          # otherwise measure "time since the worker booted" and grow across
+          # every run for as long as the worker stays warm. Reset it fresh
+          # before each run.
+          RSpec.configuration.start_time = RSpec::Core::Time.now
 
           status = RSpec::Core::Runner.run(full_args, $stderr, $stdout)
           $stdout.flush

data/lib/coatepec/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coatepec
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

data/lib/coatepec/worker/client.rb

@@ -32,9 +32,20 @@ module Coatepec
       def alive?
         return false unless @pid
 
+        # A dead-but-unreaped worker (crash, the OS reclaiming a long-idle
+        # process, anything) is a zombie: Process.kill(0, pid) below would still
+        # succeed against it, since it still holds a process-table entry. Reap it
+        # here so staleness is detected instead of reported as "alive" forever --
+        # nothing else calls Process.wait on this pid except #stop, which only
+        # WorkerManager#restart_worker! reaches, and only once #alive? itself
+        # already says false.
+        _pid, status = Process.waitpid2(@pid, Process::WNOHANG)
+        @pid = nil if status
+        return false unless @pid
+
         Process.kill(0, @pid)
         true
-      rescue Errno::ESRCH
+      rescue Errno::ESRCH, Errno::ECHILD
         false
       end
 

data/lib/coatepec/worker/server.rb

@@ -42,6 +42,7 @@ module Coatepec
         case command
         when "status" then @runtime.status
         when "spec_run" then handle_spec_run(args)
+        when "flaky_check" then handle_flaky_check(args)
         when "routes" then handle_routes(args)
         when "model" then handle_model(args)
         else
@@ -53,6 +54,10 @@ module Coatepec
         Spec::Runner.new(@project_root, rails_runtime: @runtime).run(**args.transform_keys(&:to_sym))
       end
 
+      def handle_flaky_check(args)
+        Spec::FlakyChecker.new(@project_root, rails_runtime: @runtime).call(**args.transform_keys(&:to_sym))
+      end
+
       def handle_routes(args)
         Introspection::Routes.new(**args.transform_keys(&:to_sym)).call
       end

data/lib/coatepec/worker_manager.rb

@@ -28,6 +28,18 @@ module Coatepec
       )
     end
 
+    def check_flaky(paths:, example:, timeout_seconds:, runs:)
+      dispatch(
+        "flaky_check",
+        { paths: paths, example: example, timeout_seconds: timeout_seconds, runs: runs },
+        # Each round costs timeout_seconds *plus* real per-round overhead (process
+        # termination/spawn/result-parsing -- roughly 0.6s+ per round from
+        # ProcessStrategy#terminate's own escalation sleeps alone), so the flat
+        # base slack below is scaled by an extra ~2s per round on top of it.
+        timeout: (timeout_seconds * runs) + (2 * runs) + 10
+      )
+    end
+
     def routes(query: nil, limit: 50, offset: 0)
       dispatch("routes", { query: query, limit: limit, offset: offset }, timeout: 30)
     end
@@ -40,6 +52,27 @@ module Coatepec
       @lock.synchronize { @client&.stop }
     end
 
+    # Unconditionally tears down and respawns the worker, bypassing the usual
+    # "only restart if #ensure_worker! thinks it's needed" check. Manual escape
+    # hatch for rails_runtime_restart, independent of whatever #perform's
+    # rescue below already recovers from automatically.
+    #
+    # Still honors the sidecar-restart invariant: if the Gemfile changed since
+    # boot, a worker-only respawn would silently adopt the new bundle in the
+    # worker while the parent MCP process stays on the old one, so that case
+    # raises :sidecar_restart_required instead of restarting. The check runs
+    # against the *pre-existing* snapshot, before restart_worker! re-baselines
+    # it -- same ordering #dispatch uses. Nothing to compare against on the
+    # very first call.
+    def restart!
+      @lock.synchronize do
+        raise_sidecar_restart_required! if sidecar_restart_required?
+
+        restart_worker!
+        status
+      end
+    end
+
     private
 
     def dispatch(command, args, timeout:, retried: false)
@@ -57,26 +90,33 @@ module Coatepec
 
     def perform(command, args, timeout:, retried:)
       @client.request(command, args, timeout: timeout)
-    rescue Worker::Client::DisconnectedError
-      raise Coatepec::Error.new(:worker_disconnected, "Worker disconnected") if retried
+    rescue Worker::Client::DisconnectedError, SystemCallError, IOError => e
+      raise Coatepec::Error.new(:worker_disconnected, "Worker disconnected: #{e.message}") if retried
 
       restart_worker!
       dispatch(command, args, timeout: timeout, retried: true)
     end
 
     def check_for_restart!
-      reason = @change_detector.restart_reason(@snapshot)
-      case reason
+      case @change_detector.restart_reason(@snapshot)
       when :sidecar_restart_required
-        raise Coatepec::Error.new(
-          :sidecar_restart_required,
-          "Gemfile changed; restart your MCP client to restart Coatepec and pick up the change"
-        )
+        raise_sidecar_restart_required!
       when :worker_restart_required
         restart_worker!
       end
     end
 
+    def sidecar_restart_required?
+      @snapshot && @change_detector.restart_reason(@snapshot) == :sidecar_restart_required
+    end
+
+    def raise_sidecar_restart_required!
+      raise Coatepec::Error.new(
+        :sidecar_restart_required,
+        "Gemfile changed; restart your MCP client to restart Coatepec and pick up the change"
+      )
+    end
+
     def ensure_worker!
       restart_worker! unless @client&.alive?
     end

data/lib/coatepec.rb

@@ -18,6 +18,7 @@ require_relative "coatepec/spec/fork_strategy"
 require_relative "coatepec/spec/spawn_strategy"
 require_relative "coatepec/spec/guarded_fork_strategy"
 require_relative "coatepec/spec/runner"
+require_relative "coatepec/spec/flaky_checker"
 require_relative "coatepec/worker/server"
 require_relative "coatepec/worker_manager"
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: coatepec
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0
 platform: ruby
 authors:
 - Enrique Mogollan
 bindir: exe
 cert_chain: []
-date: 2026-08-07 00:00:00.000000000 Z
+date: 2026-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -74,6 +74,7 @@ files:
 - lib/coatepec/project.rb
 - lib/coatepec/project_config.rb
 - lib/coatepec/protocol.rb
+- lib/coatepec/spec/flaky_checker.rb
 - lib/coatepec/spec/fork_strategy.rb
 - lib/coatepec/spec/guarded_fork_strategy.rb
 - lib/coatepec/spec/path_policy.rb