polyrun 2.2.4 → 2.2.6

data/lib/polyrun/partition/suite.rb

@@ -0,0 +1,116 @@
+module Polyrun
+  module Partition
+    # Chooses the parallel runner suite for bare +polyrun+ and validates command vs path lists.
+    module Suite
+      module_function
+
+      # Resolve suite for default CLI dispatch.
+      # Order: explicit +partition.suite+ (not auto) → infer from +paths_file+ → +detect_auto_suite+.
+      # Returns +{ suite: :rspec|:minitest|:quick }+ or +{ error: "..." }+.
+      def resolve_default(cwd: Dir.pwd, partition: {})
+        configured = configured_suite(partition)
+        return {error: "unknown partition.suite: #{configured.inspect} (expected auto, rspec, minitest, quick)"} unless
+          %w[auto rspec minitest quick].include?(configured)
+
+        inferred = infer_from_paths_file(cwd, partition)
+        return inferred if inferred.is_a?(Hash) && inferred[:error]
+
+        if configured != "auto"
+          suite = configured.to_sym
+          if inferred && inferred != suite
+            return {
+              error: "partition.suite is #{suite} but paths_file infers #{inferred} " \
+                     "(fix partition.suite or the paths file)"
+            }
+          end
+          return {suite: suite}
+        end
+
+        return {suite: inferred} if inferred
+
+        auto = Paths.detect_auto_suite(cwd)
+        return {suite: auto} if auto
+
+        {
+          error: "no tests found (spec/**/*_spec.rb, test/**/*_test.rb, or Polyrun quick files). See polyrun help."
+        }
+      end
+
+      def infer_from_command(cmd)
+        tokens = Array(cmd).map(&:to_s)
+        return nil if tokens.empty?
+
+        return :rspec if tokens.any? { |t| File.basename(t) == "rspec" }
+        return :quick if tokens.include?("quick") && tokens.any? { |t| File.basename(t).include?("polyrun") }
+        return :minitest if minitest_command?(tokens)
+
+        nil
+      end
+
+      def command_mismatch_message(items, cmd)
+        path_suite = Paths.infer_suite_from_paths(Array(items))
+        cmd_suite = infer_from_command(cmd)
+
+        if path_suite == :invalid
+          return nil unless cmd_suite
+
+          return "mixing _spec.rb and _test.rb paths is not supported with a #{cmd_suite} command " \
+                 "(use a suite-agnostic custom command, or split the path list)"
+        end
+
+        return nil if path_suite.nil?
+        return nil if cmd_suite.nil?
+        return nil if path_suite == cmd_suite
+
+        "paths infer #{path_suite} but command looks like #{cmd_suite} " \
+          "(use parallel-#{path_suite}, fix partition.paths_file, or pass a matching command after --)"
+      end
+
+      def configured_suite(partition)
+        suite = (partition["suite"] || partition[:suite] || "auto").to_s.downcase
+        suite.empty? ? "auto" : suite
+      end
+      private_class_method :configured_suite
+
+      def infer_from_paths_file(cwd, partition)
+        paths_file = partition["paths_file"] || partition[:paths_file]
+        return nil unless paths_file
+
+        abs = File.expand_path(paths_file.to_s, cwd)
+        return nil unless File.file?(abs)
+
+        items = Paths.read_lines(abs)
+        inferred = Paths.infer_suite_from_paths(items)
+        return {error: "paths_file mixes _spec.rb and _test.rb (not supported in one run)"} if inferred == :invalid
+
+        inferred
+      end
+      private_class_method :infer_from_paths_file
+
+      def minitest_command?(tokens)
+        return true if tokens.any? { |t| File.basename(t) == "rails" } && tokens.include?("test")
+        return true if ruby_with_test_load_path?(tokens)
+
+        false
+      end
+      private_class_method :minitest_command?
+
+      def ruby_with_test_load_path?(tokens)
+        tokens.each_with_index do |tok, i|
+          next unless File.basename(tok) == "ruby"
+
+          rest = tokens[(i + 1)..] || []
+          rest.each_with_index do |arg, j|
+            return true if arg == "-Itest"
+            return true if arg == "-I" && rest[j + 1] == "test"
+            next unless arg.start_with?("-I") && arg.length > 2
+
+            return true if arg[2..].split(/[=:]/).include?("test")
+          end
+        end
+        false
+      end
+      private_class_method :ruby_with_test_load_path?
+    end
+  end
+end

data/lib/polyrun/partition/timing_diagnostics.rb

@@ -16,8 +16,9 @@ module Polyrun
         item_keys = items.map { |p| lookup_key(p, root, g) }
         cost_keys = costs&.keys || []
 
+        item_keys_set = item_keys.to_set
+        stale = cost_keys.reject { |k| item_keys_set.include?(k) }
         if g == :example
-          item_keys_set = item_keys.to_set
           cost_keys_set = cost_keys.to_set
           cost_file_keys_set = cost_keys.map { |k| file_from_locator(k) }.uniq.to_set
           known = item_keys.count do |ik|
@@ -26,11 +27,9 @@ module Polyrun
           missing = item_keys.reject do |ik|
             cost_keys_set.include?(ik) || cost_file_keys_set.include?(file_from_locator(ik))
           end
-          stale = cost_keys.reject { |k| item_keys_set.include?(k) }
         else
           known = item_keys.count { |k| costs&.key?(k) }
           missing = item_keys.reject { |k| costs&.key?(k) }
-          stale = cost_keys.reject { |k| item_keys.include?(k) }
         end
         total = item_keys.size
 

data/lib/polyrun/queue/file_store.rb

@@ -39,25 +39,38 @@ module Polyrun
         batch_size = Integer(batch_size)
         raise Polyrun::Error, "batch_size must be >= 1" if batch_size < 1
 
-        lease_id = SecureRandom.uuid
+        lease_id = nil
         batch = []
         with_lock do
           meta = load_meta!
           batch = take_pending_batch!(meta, batch_size)
-          leases = read_leases
-          leases[lease_id] = {
-            "worker_id" => worker_id.to_s,
-            "paths" => batch,
-            "claimed_at" => Time.now.utc.iso8601
-          }
-          write_meta!(meta)
-          write_leases!(leases)
-          append_ledger(
-            "CLAIM" => lease_id,
-            "worker_id" => worker_id.to_s,
-            "paths" => batch,
-            "pending_remaining" => meta["pending_count"]
-          )
+          unless batch.empty?
+            lease_id = SecureRandom.uuid
+            now = Time.now.utc.iso8601
+            write_intent!(
+              "op" => "claim",
+              "lease_id" => lease_id,
+              "worker_id" => worker_id.to_s,
+              "paths" => batch
+            )
+            leases = read_leases
+            leases[lease_id] = {
+              "worker_id" => worker_id.to_s,
+              "paths" => batch,
+              "claimed_at" => now,
+              "heartbeat_at" => now,
+              "attempts" => 0
+            }
+            write_meta!(meta)
+            write_leases!(leases)
+            clear_intent!
+            append_ledger(
+              "CLAIM" => lease_id,
+              "worker_id" => worker_id.to_s,
+              "paths" => batch,
+              "pending_remaining" => meta["pending_count"]
+            )
+          end
         end
         {"lease_id" => lease_id, "paths" => batch}
       end
@@ -72,19 +85,73 @@ module Polyrun
             raise Polyrun::Error, "lease worker mismatch"
           end
 
-          leases.delete(lease_id)
-          write_leases!(leases)
-
           paths = lease["paths"] || []
+          write_intent!(
+            "op" => "ack",
+            "lease_id" => lease_id,
+            "worker_id" => worker_id.to_s,
+            "paths" => paths
+          )
+          # done.jsonl before removing the lease so a crash still has durable completion evidence.
+          append_done_lines!(paths)
           meta = load_meta!
           meta["done_count"] = Integer(meta["done_count"]) + paths.size
-          append_done_lines!(paths)
           write_meta!(meta)
+          leases.delete(lease_id)
+          write_leases!(leases)
+          clear_intent!
           append_ledger("ACK" => lease_id, "worker_id" => worker_id.to_s, "paths" => paths)
         end
         true
       end
 
+      # Marks a subset of a lease's paths done; removes the lease when none remain.
+      def ack_paths!(lease_id:, worker_id:, paths:)
+        paths = Array(paths).map(&:to_s)
+        raise Polyrun::Error, "ack_paths! requires at least one path" if paths.empty?
+
+        with_lock do
+          leases = read_leases
+          lease = leases[lease_id]
+          raise Polyrun::Error, "unknown lease: #{lease_id}" unless lease
+
+          if lease["worker_id"].to_s != worker_id.to_s
+            raise Polyrun::Error, "lease worker mismatch"
+          end
+
+          held = Array(lease["paths"]).map(&:to_s)
+          unknown = paths - held
+          raise Polyrun::Error, "ack_paths! paths not on lease: #{unknown.join(", ")}" unless unknown.empty?
+
+          write_intent!(
+            "op" => "ack_paths",
+            "lease_id" => lease_id,
+            "worker_id" => worker_id.to_s,
+            "paths" => paths
+          )
+          apply_ack_side_effects!(leases, lease_id, lease, paths, worker_id)
+          clear_intent!
+        end
+        true
+      end
+
+      def heartbeat!(lease_id:, worker_id:)
+        with_lock do
+          leases = read_leases
+          lease = leases[lease_id]
+          raise Polyrun::Error, "unknown lease: #{lease_id}" unless lease
+
+          if lease["worker_id"].to_s != worker_id.to_s
+            raise Polyrun::Error, "lease worker mismatch"
+          end
+
+          lease["heartbeat_at"] = Time.now.utc.iso8601
+          leases[lease_id] = lease
+          write_leases!(leases)
+        end
+        true
+      end
+
       def status(detailed: false)
         with_lock do
           meta = load_meta!
@@ -111,13 +178,23 @@ module Polyrun
           leases.each do |lease_id, lease|
             if reclaim_lease?(lease, older_than: older_than, worker_id: worker_id)
               paths = lease["paths"] || []
-              return_paths_to_pending!(meta, paths)
+              bumped = lease.merge("attempts" => Integer(lease["attempts"] || 0) + 1)
+              write_intent!(
+                "op" => "reclaim",
+                "lease_id" => lease_id,
+                "worker_id" => lease["worker_id"],
+                "paths" => paths,
+                "attempts" => bumped["attempts"]
+              )
+              quarantine_or_return_paths!(meta, paths, bumped)
               reclaimed_paths += paths.size
               append_ledger(
                 "RECLAIM" => lease_id,
                 "worker_id" => lease["worker_id"],
-                "paths" => paths
+                "paths" => paths,
+                "attempts" => bumped["attempts"]
               )
+              clear_intent!
             else
               keep[lease_id] = lease
             end
@@ -135,14 +212,28 @@ module Polyrun
           lease = leases[lease_id]
           return 0 unless lease
 
+          paths = lease["paths"] || []
+          bumped = lease.merge("attempts" => Integer(lease["attempts"] || 0) + 1)
+          write_intent!(
+            "op" => "reclaim_lease",
+            "lease_id" => lease_id,
+            "worker_id" => lease["worker_id"],
+            "paths" => paths,
+            "attempts" => bumped["attempts"]
+          )
           leases.delete(lease_id)
           write_leases!(leases)
           meta = load_meta!
-          paths = lease["paths"] || []
-          return_paths_to_pending!(meta, paths)
+          quarantine_or_return_paths!(meta, paths, bumped)
           write_meta!(meta)
           reclaimed = paths.size
-          append_ledger("RECLAIM" => lease_id, "worker_id" => lease["worker_id"], "paths" => paths)
+          append_ledger(
+            "RECLAIM" => lease_id,
+            "worker_id" => lease["worker_id"],
+            "paths" => paths,
+            "attempts" => bumped["attempts"]
+          )
+          clear_intent!
         end
         reclaimed
       end
@@ -154,8 +245,10 @@ module Polyrun
           return false
         end
         if older_than
-          claimed = Time.parse(lease["claimed_at"].to_s)
-          return (Time.now - claimed) >= older_than
+          stamp = lease["heartbeat_at"].to_s
+          stamp = lease["claimed_at"].to_s if stamp.empty?
+          touched = Time.parse(stamp)
+          return (Time.now - touched) >= older_than
         end
         !!worker_id
       rescue ArgumentError
@@ -165,13 +258,16 @@ module Polyrun
       def lease_details(leases)
         now = Time.now
         leases.map do |lease_id, lease|
-          claimed = Time.parse(lease["claimed_at"].to_s)
+          stamp = lease["heartbeat_at"].to_s
+          stamp = lease["claimed_at"].to_s if stamp.empty?
+          touched = Time.parse(stamp)
           {
             "lease_id" => lease_id,
             "worker_id" => lease["worker_id"],
             "paths_count" => (lease["paths"] || []).size,
             "claimed_at" => lease["claimed_at"],
-            "age_seconds" => (now - claimed).round(1)
+            "heartbeat_at" => lease["heartbeat_at"],
+            "age_seconds" => (now - touched).round(1)
           }
         rescue ArgumentError
           {
@@ -179,6 +275,7 @@ module Polyrun
             "worker_id" => lease["worker_id"],
             "paths_count" => (lease["paths"] || []).size,
             "claimed_at" => lease["claimed_at"],
+            "heartbeat_at" => lease["heartbeat_at"],
             "age_seconds" => nil
           }
         end
@@ -228,6 +325,7 @@ module Polyrun
         FileUtils.mkdir_p(@root)
         File.open(lock_path, File::CREAT | File::RDWR) do |f|
           f.flock(File::LOCK_EX)
+          recover_intent!
           yield
         end
       end
@@ -297,9 +395,13 @@ module Polyrun
         File.open(ledger_path, "a") { |f| f.write(line) }
       end
 
-      def atomic_write(path, body)
+      def atomic_write(path, body, fsync: false)
         tmp = "#{path}.tmp.#{$$}"
-        File.write(tmp, body)
+        File.open(tmp, File::WRONLY | File::CREAT | File::TRUNC) do |io|
+          io.write(body)
+          io.flush
+          io.fsync if fsync
+        end
         File.rename(tmp, path)
       end
     end
@@ -307,4 +409,5 @@ module Polyrun
 end
 
 require_relative "file_store_pending"
+require_relative "file_store_intent"
 # rubocop:enable Polyrun/FileLength, Metrics/ClassLength

data/lib/polyrun/queue/file_store_intent.rb

@@ -0,0 +1,188 @@
+# rubocop:disable Metrics/ModuleLength -- claim/ack/reclaim intent recovery beside FileStore
+module Polyrun
+  module Queue
+    class FileStore
+      INTENT_BASENAME = "intent.json"
+
+      private
+
+      def intent_path
+        File.join(@root, INTENT_BASENAME)
+      end
+
+      def write_intent!(payload)
+        atomic_write(intent_path, JSON.generate(payload), fsync: true)
+      end
+
+      def clear_intent!
+        File.unlink(intent_path)
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def recover_intent!
+        return unless File.file?(intent_path)
+
+        intent = JSON.parse(File.read(intent_path))
+        case intent["op"].to_s
+        when "claim"
+          recover_claim_intent!(intent)
+        when "ack", "ack_paths"
+          recover_ack_intent!(intent)
+        when "reclaim", "reclaim_lease"
+          recover_reclaim_intent!(intent)
+        else
+          Polyrun::Log.warn "polyrun queue: dropping unknown intent op=#{intent["op"].inspect}"
+        end
+        clear_intent!
+      rescue JSON::ParserError, TypeError
+        Polyrun::Log.warn "polyrun queue: dropping corrupt intent at #{intent_path}"
+        clear_intent!
+      end
+
+      def recover_claim_intent!(intent)
+        lease_id = intent["lease_id"].to_s
+        leases = read_leases
+        return if leases.key?(lease_id)
+
+        paths = Array(intent["paths"])
+        meta = load_meta!
+        return_paths_to_pending!(meta, paths)
+        write_meta!(meta)
+        append_ledger(
+          "RECOVER_CLAIM" => lease_id,
+          "worker_id" => intent["worker_id"],
+          "paths" => paths
+        )
+      end
+
+      def recover_ack_intent!(intent)
+        lease_id = intent["lease_id"].to_s
+        leases = read_leases
+        lease = leases[lease_id]
+        return unless lease
+
+        paths = Array(intent["paths"])
+        paths = Array(lease["paths"]) if paths.empty?
+        apply_ack_side_effects!(leases, lease_id, lease, paths, intent["worker_id"])
+      end
+
+      def recover_reclaim_intent!(intent)
+        lease_id = intent["lease_id"].to_s
+        leases = read_leases
+        lease = leases[lease_id]
+        return unless lease
+
+        paths = Array(intent["paths"])
+        paths = Array(lease["paths"]) if paths.empty?
+        bumped = lease.merge("attempts" => intent["attempts"] || lease["attempts"])
+        leases.delete(lease_id)
+        write_leases!(leases)
+        meta = load_meta!
+        quarantine_or_return_paths!(meta, paths, bumped)
+        write_meta!(meta)
+        append_ledger(
+          "RECOVER_RECLAIM" => lease_id,
+          "worker_id" => lease["worker_id"],
+          "paths" => paths,
+          "attempts" => bumped["attempts"]
+        )
+      end
+
+      def apply_ack_side_effects!(leases, lease_id, lease, paths, worker_id)
+        append_done_lines!(paths)
+        clear_path_attempts!(paths)
+        meta = load_meta!
+        meta["done_count"] = Integer(meta["done_count"]) + paths.size
+        write_meta!(meta)
+        remaining = Array(lease["paths"]).map(&:to_s) - paths.map(&:to_s)
+        if remaining.empty?
+          leases.delete(lease_id)
+        else
+          lease["paths"] = remaining
+          leases[lease_id] = lease
+        end
+        write_leases!(leases)
+        append_ledger("ACK" => lease_id, "worker_id" => worker_id.to_s, "paths" => paths)
+      end
+
+      def quarantine_or_return_paths!(meta, paths, lease)
+        attempts_h = read_path_attempts
+        to_pending = []
+        poisoned = []
+        max_attempts = queue_max_attempts
+        Array(paths).each do |path|
+          key = path.to_s
+          n = Integer(attempts_h[key] || 0) + 1
+          attempts_h[key] = n
+          if max_attempts.positive? && n >= max_attempts
+            poisoned << [key, n]
+          else
+            to_pending << key
+          end
+        end
+        write_path_attempts!(attempts_h)
+        poisoned.each do |key, n|
+          append_failed_lines!(
+            [key],
+            "worker_id" => lease["worker_id"],
+            "attempts" => n
+          )
+          append_ledger("POISON" => lease["worker_id"], "attempts" => n, "paths" => [key])
+        end
+        return_paths_to_pending!(meta, to_pending)
+      end
+
+      def clear_path_attempts!(paths)
+        attempts_h = read_path_attempts
+        Array(paths).each { |p| attempts_h.delete(p.to_s) }
+        write_path_attempts!(attempts_h)
+      end
+
+      def queue_max_attempts
+        raw = ENV["POLYRUN_QUEUE_MAX_ATTEMPTS"].to_s.strip
+        return 3 if raw.empty?
+
+        n = Integer(raw, exception: false)
+        (n && n >= 1) ? n : 3
+      end
+
+      def path_attempts_path
+        File.join(@root, "path_attempts.json")
+      end
+
+      def read_path_attempts
+        return {} unless File.file?(path_attempts_path)
+
+        data = JSON.parse(File.read(path_attempts_path))
+        data.is_a?(Hash) ? data : {}
+      rescue JSON::ParserError
+        {}
+      end
+
+      def write_path_attempts!(hash)
+        atomic_write(path_attempts_path, JSON.generate(hash))
+      end
+
+      def failed_path
+        File.join(@root, "failed.jsonl")
+      end
+
+      def append_failed_lines!(paths, lease)
+        return if paths.empty?
+
+        File.open(failed_path, "a") do |io|
+          paths.each do |p|
+            io.puts(JSON.generate(
+              "path" => p.to_s,
+              "worker_id" => lease["worker_id"],
+              "attempts" => Integer(lease["attempts"] || 0),
+              "at" => Time.now.utc.iso8601
+            ))
+          end
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ModuleLength

data/lib/polyrun/queue/worker_loop.rb

@@ -1,6 +1,9 @@
 module Polyrun
   module Queue
     # Claim / run / ack loop for +polyrun run-queue+ workers (forked or direct).
+    #
+    # Successful batches stay one command invocation. On failure, paths are re-run one at a time so
+    # already-passing paths can be acked without reclaiming them with a poison neighbor.
     module WorkerLoop
       module_function
 
@@ -12,18 +15,25 @@ module Polyrun
           paths = claim["paths"] || []
           break if paths.empty?
 
+          store.heartbeat!(lease_id: claim["lease_id"], worker_id: worker_id)
           code = run_batch(cmd, paths)
           if code == 0
             store.ack!(lease_id: claim["lease_id"], worker_id: worker_id)
             batches_ok += 1
-          elsif on_failure.to_s == "requeue"
-            store.reclaim_lease!(claim["lease_id"])
-            batches_fail += 1
-            return {ok: batches_ok, fail: batches_fail, exit_code: 1}
-          else
-            batches_fail += 1
-            return {ok: batches_ok, fail: batches_fail, exit_code: code.zero? ? 1 : code}
+            next
           end
+
+          outcome = isolate_failed_batch!(
+            store: store,
+            worker_id: worker_id,
+            lease_id: claim["lease_id"],
+            paths: paths,
+            cmd: cmd,
+            on_failure: on_failure
+          )
+          batches_ok += outcome[:ok]
+          batches_fail += outcome[:fail]
+          return {ok: batches_ok, fail: batches_fail, exit_code: outcome[:exit_code]} if outcome[:stop]
         end
         {ok: batches_ok, fail: batches_fail, exit_code: 0}
       rescue Polyrun::Error => e
@@ -34,6 +44,30 @@ module Polyrun
       def run_batch(cmd, paths)
         system(*cmd, *paths) ? 0 : ($?.exitstatus || 1)
       end
+
+      def isolate_failed_batch!(store:, worker_id:, lease_id:, paths:, cmd:, on_failure:)
+        ok = 0
+        paths.each do |path|
+          store.heartbeat!(lease_id: lease_id, worker_id: worker_id)
+          code = run_batch(cmd, [path])
+          if code == 0
+            store.ack_paths!(lease_id: lease_id, worker_id: worker_id, paths: [path])
+            ok += 1
+            next
+          end
+
+          if on_failure.to_s == "requeue"
+            store.reclaim_lease!(lease_id)
+            return {ok: ok, fail: 1, exit_code: 1, stop: true}
+          end
+
+          return {ok: ok, fail: 1, exit_code: code.zero? ? 1 : code, stop: true}
+        end
+
+        # Every path passed on the isolating re-run; lease should already be empty.
+        {ok: ok.positive? ? 1 : 0, fail: 0, exit_code: 0, stop: false}
+      end
+      private_class_method :isolate_failed_batch!
     end
   end
 end

data/lib/polyrun/spec_quality/config.rb

@@ -53,7 +53,7 @@ module Polyrun
         path = File.expand_path(path)
         return {} unless File.file?(path)
 
-        data = YAML.load_file(path)
+        data = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true)
         data.is_a?(Hash) ? data : {}
       end
 

data/lib/polyrun/spec_quality/profile.rb

@@ -41,7 +41,6 @@ module Polyrun
 
           delta = numeric(a) - numeric(b)
           out[k] = delta if delta.positive? || k.start_with?("cpu")
-          out[k] = delta
         end
         out
       end