polyrun 2.2.5 → 2.2.6

data/lib/polyrun/queue/file_store.rb

@@ -39,25 +39,38 @@ module Polyrun
         batch_size = Integer(batch_size)
         raise Polyrun::Error, "batch_size must be >= 1" if batch_size < 1
 
-        lease_id = SecureRandom.uuid
+        lease_id = nil
         batch = []
         with_lock do
           meta = load_meta!
           batch = take_pending_batch!(meta, batch_size)
-          leases = read_leases
-          leases[lease_id] = {
-            "worker_id" => worker_id.to_s,
-            "paths" => batch,
-            "claimed_at" => Time.now.utc.iso8601
-          }
-          write_meta!(meta)
-          write_leases!(leases)
-          append_ledger(
-            "CLAIM" => lease_id,
-            "worker_id" => worker_id.to_s,
-            "paths" => batch,
-            "pending_remaining" => meta["pending_count"]
-          )
+          unless batch.empty?
+            lease_id = SecureRandom.uuid
+            now = Time.now.utc.iso8601
+            write_intent!(
+              "op" => "claim",
+              "lease_id" => lease_id,
+              "worker_id" => worker_id.to_s,
+              "paths" => batch
+            )
+            leases = read_leases
+            leases[lease_id] = {
+              "worker_id" => worker_id.to_s,
+              "paths" => batch,
+              "claimed_at" => now,
+              "heartbeat_at" => now,
+              "attempts" => 0
+            }
+            write_meta!(meta)
+            write_leases!(leases)
+            clear_intent!
+            append_ledger(
+              "CLAIM" => lease_id,
+              "worker_id" => worker_id.to_s,
+              "paths" => batch,
+              "pending_remaining" => meta["pending_count"]
+            )
+          end
         end
         {"lease_id" => lease_id, "paths" => batch}
       end
@@ -72,19 +85,73 @@ module Polyrun
             raise Polyrun::Error, "lease worker mismatch"
           end
 
-          leases.delete(lease_id)
-          write_leases!(leases)
-
           paths = lease["paths"] || []
+          write_intent!(
+            "op" => "ack",
+            "lease_id" => lease_id,
+            "worker_id" => worker_id.to_s,
+            "paths" => paths
+          )
+          # done.jsonl before removing the lease so a crash still has durable completion evidence.
+          append_done_lines!(paths)
           meta = load_meta!
           meta["done_count"] = Integer(meta["done_count"]) + paths.size
-          append_done_lines!(paths)
           write_meta!(meta)
+          leases.delete(lease_id)
+          write_leases!(leases)
+          clear_intent!
           append_ledger("ACK" => lease_id, "worker_id" => worker_id.to_s, "paths" => paths)
         end
         true
       end
 
+      # Marks a subset of a lease's paths done; removes the lease when none remain.
+      def ack_paths!(lease_id:, worker_id:, paths:)
+        paths = Array(paths).map(&:to_s)
+        raise Polyrun::Error, "ack_paths! requires at least one path" if paths.empty?
+
+        with_lock do
+          leases = read_leases
+          lease = leases[lease_id]
+          raise Polyrun::Error, "unknown lease: #{lease_id}" unless lease
+
+          if lease["worker_id"].to_s != worker_id.to_s
+            raise Polyrun::Error, "lease worker mismatch"
+          end
+
+          held = Array(lease["paths"]).map(&:to_s)
+          unknown = paths - held
+          raise Polyrun::Error, "ack_paths! paths not on lease: #{unknown.join(", ")}" unless unknown.empty?
+
+          write_intent!(
+            "op" => "ack_paths",
+            "lease_id" => lease_id,
+            "worker_id" => worker_id.to_s,
+            "paths" => paths
+          )
+          apply_ack_side_effects!(leases, lease_id, lease, paths, worker_id)
+          clear_intent!
+        end
+        true
+      end
+
+      def heartbeat!(lease_id:, worker_id:)
+        with_lock do
+          leases = read_leases
+          lease = leases[lease_id]
+          raise Polyrun::Error, "unknown lease: #{lease_id}" unless lease
+
+          if lease["worker_id"].to_s != worker_id.to_s
+            raise Polyrun::Error, "lease worker mismatch"
+          end
+
+          lease["heartbeat_at"] = Time.now.utc.iso8601
+          leases[lease_id] = lease
+          write_leases!(leases)
+        end
+        true
+      end
+
       def status(detailed: false)
         with_lock do
           meta = load_meta!
@@ -111,13 +178,23 @@ module Polyrun
           leases.each do |lease_id, lease|
             if reclaim_lease?(lease, older_than: older_than, worker_id: worker_id)
               paths = lease["paths"] || []
-              return_paths_to_pending!(meta, paths)
+              bumped = lease.merge("attempts" => Integer(lease["attempts"] || 0) + 1)
+              write_intent!(
+                "op" => "reclaim",
+                "lease_id" => lease_id,
+                "worker_id" => lease["worker_id"],
+                "paths" => paths,
+                "attempts" => bumped["attempts"]
+              )
+              quarantine_or_return_paths!(meta, paths, bumped)
               reclaimed_paths += paths.size
               append_ledger(
                 "RECLAIM" => lease_id,
                 "worker_id" => lease["worker_id"],
-                "paths" => paths
+                "paths" => paths,
+                "attempts" => bumped["attempts"]
               )
+              clear_intent!
             else
               keep[lease_id] = lease
             end
@@ -135,14 +212,28 @@ module Polyrun
           lease = leases[lease_id]
           return 0 unless lease
 
+          paths = lease["paths"] || []
+          bumped = lease.merge("attempts" => Integer(lease["attempts"] || 0) + 1)
+          write_intent!(
+            "op" => "reclaim_lease",
+            "lease_id" => lease_id,
+            "worker_id" => lease["worker_id"],
+            "paths" => paths,
+            "attempts" => bumped["attempts"]
+          )
           leases.delete(lease_id)
           write_leases!(leases)
           meta = load_meta!
-          paths = lease["paths"] || []
-          return_paths_to_pending!(meta, paths)
+          quarantine_or_return_paths!(meta, paths, bumped)
           write_meta!(meta)
           reclaimed = paths.size
-          append_ledger("RECLAIM" => lease_id, "worker_id" => lease["worker_id"], "paths" => paths)
+          append_ledger(
+            "RECLAIM" => lease_id,
+            "worker_id" => lease["worker_id"],
+            "paths" => paths,
+            "attempts" => bumped["attempts"]
+          )
+          clear_intent!
         end
         reclaimed
       end
@@ -154,8 +245,10 @@ module Polyrun
           return false
         end
         if older_than
-          claimed = Time.parse(lease["claimed_at"].to_s)
-          return (Time.now - claimed) >= older_than
+          stamp = lease["heartbeat_at"].to_s
+          stamp = lease["claimed_at"].to_s if stamp.empty?
+          touched = Time.parse(stamp)
+          return (Time.now - touched) >= older_than
         end
         !!worker_id
       rescue ArgumentError
@@ -165,13 +258,16 @@ module Polyrun
       def lease_details(leases)
         now = Time.now
         leases.map do |lease_id, lease|
-          claimed = Time.parse(lease["claimed_at"].to_s)
+          stamp = lease["heartbeat_at"].to_s
+          stamp = lease["claimed_at"].to_s if stamp.empty?
+          touched = Time.parse(stamp)
           {
             "lease_id" => lease_id,
             "worker_id" => lease["worker_id"],
             "paths_count" => (lease["paths"] || []).size,
             "claimed_at" => lease["claimed_at"],
-            "age_seconds" => (now - claimed).round(1)
+            "heartbeat_at" => lease["heartbeat_at"],
+            "age_seconds" => (now - touched).round(1)
           }
         rescue ArgumentError
           {
@@ -179,6 +275,7 @@ module Polyrun
             "worker_id" => lease["worker_id"],
             "paths_count" => (lease["paths"] || []).size,
             "claimed_at" => lease["claimed_at"],
+            "heartbeat_at" => lease["heartbeat_at"],
             "age_seconds" => nil
           }
         end
@@ -228,6 +325,7 @@ module Polyrun
         FileUtils.mkdir_p(@root)
         File.open(lock_path, File::CREAT | File::RDWR) do |f|
           f.flock(File::LOCK_EX)
+          recover_intent!
           yield
         end
       end
@@ -297,9 +395,13 @@ module Polyrun
         File.open(ledger_path, "a") { |f| f.write(line) }
       end
 
-      def atomic_write(path, body)
+      def atomic_write(path, body, fsync: false)
         tmp = "#{path}.tmp.#{$$}"
-        File.write(tmp, body)
+        File.open(tmp, File::WRONLY | File::CREAT | File::TRUNC) do |io|
+          io.write(body)
+          io.flush
+          io.fsync if fsync
+        end
         File.rename(tmp, path)
       end
     end
@@ -307,4 +409,5 @@ module Polyrun
 end
 
 require_relative "file_store_pending"
+require_relative "file_store_intent"
 # rubocop:enable Polyrun/FileLength, Metrics/ClassLength

data/lib/polyrun/queue/file_store_intent.rb

@@ -0,0 +1,188 @@
+# rubocop:disable Metrics/ModuleLength -- claim/ack/reclaim intent recovery beside FileStore
+module Polyrun
+  module Queue
+    class FileStore
+      INTENT_BASENAME = "intent.json"
+
+      private
+
+      def intent_path
+        File.join(@root, INTENT_BASENAME)
+      end
+
+      def write_intent!(payload)
+        atomic_write(intent_path, JSON.generate(payload), fsync: true)
+      end
+
+      def clear_intent!
+        File.unlink(intent_path)
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def recover_intent!
+        return unless File.file?(intent_path)
+
+        intent = JSON.parse(File.read(intent_path))
+        case intent["op"].to_s
+        when "claim"
+          recover_claim_intent!(intent)
+        when "ack", "ack_paths"
+          recover_ack_intent!(intent)
+        when "reclaim", "reclaim_lease"
+          recover_reclaim_intent!(intent)
+        else
+          Polyrun::Log.warn "polyrun queue: dropping unknown intent op=#{intent["op"].inspect}"
+        end
+        clear_intent!
+      rescue JSON::ParserError, TypeError
+        Polyrun::Log.warn "polyrun queue: dropping corrupt intent at #{intent_path}"
+        clear_intent!
+      end
+
+      def recover_claim_intent!(intent)
+        lease_id = intent["lease_id"].to_s
+        leases = read_leases
+        return if leases.key?(lease_id)
+
+        paths = Array(intent["paths"])
+        meta = load_meta!
+        return_paths_to_pending!(meta, paths)
+        write_meta!(meta)
+        append_ledger(
+          "RECOVER_CLAIM" => lease_id,
+          "worker_id" => intent["worker_id"],
+          "paths" => paths
+        )
+      end
+
+      def recover_ack_intent!(intent)
+        lease_id = intent["lease_id"].to_s
+        leases = read_leases
+        lease = leases[lease_id]
+        return unless lease
+
+        paths = Array(intent["paths"])
+        paths = Array(lease["paths"]) if paths.empty?
+        apply_ack_side_effects!(leases, lease_id, lease, paths, intent["worker_id"])
+      end
+
+      def recover_reclaim_intent!(intent)
+        lease_id = intent["lease_id"].to_s
+        leases = read_leases
+        lease = leases[lease_id]
+        return unless lease
+
+        paths = Array(intent["paths"])
+        paths = Array(lease["paths"]) if paths.empty?
+        bumped = lease.merge("attempts" => intent["attempts"] || lease["attempts"])
+        leases.delete(lease_id)
+        write_leases!(leases)
+        meta = load_meta!
+        quarantine_or_return_paths!(meta, paths, bumped)
+        write_meta!(meta)
+        append_ledger(
+          "RECOVER_RECLAIM" => lease_id,
+          "worker_id" => lease["worker_id"],
+          "paths" => paths,
+          "attempts" => bumped["attempts"]
+        )
+      end
+
+      def apply_ack_side_effects!(leases, lease_id, lease, paths, worker_id)
+        append_done_lines!(paths)
+        clear_path_attempts!(paths)
+        meta = load_meta!
+        meta["done_count"] = Integer(meta["done_count"]) + paths.size
+        write_meta!(meta)
+        remaining = Array(lease["paths"]).map(&:to_s) - paths.map(&:to_s)
+        if remaining.empty?
+          leases.delete(lease_id)
+        else
+          lease["paths"] = remaining
+          leases[lease_id] = lease
+        end
+        write_leases!(leases)
+        append_ledger("ACK" => lease_id, "worker_id" => worker_id.to_s, "paths" => paths)
+      end
+
+      def quarantine_or_return_paths!(meta, paths, lease)
+        attempts_h = read_path_attempts
+        to_pending = []
+        poisoned = []
+        max_attempts = queue_max_attempts
+        Array(paths).each do |path|
+          key = path.to_s
+          n = Integer(attempts_h[key] || 0) + 1
+          attempts_h[key] = n
+          if max_attempts.positive? && n >= max_attempts
+            poisoned << [key, n]
+          else
+            to_pending << key
+          end
+        end
+        write_path_attempts!(attempts_h)
+        poisoned.each do |key, n|
+          append_failed_lines!(
+            [key],
+            "worker_id" => lease["worker_id"],
+            "attempts" => n
+          )
+          append_ledger("POISON" => lease["worker_id"], "attempts" => n, "paths" => [key])
+        end
+        return_paths_to_pending!(meta, to_pending)
+      end
+
+      def clear_path_attempts!(paths)
+        attempts_h = read_path_attempts
+        Array(paths).each { |p| attempts_h.delete(p.to_s) }
+        write_path_attempts!(attempts_h)
+      end
+
+      def queue_max_attempts
+        raw = ENV["POLYRUN_QUEUE_MAX_ATTEMPTS"].to_s.strip
+        return 3 if raw.empty?
+
+        n = Integer(raw, exception: false)
+        (n && n >= 1) ? n : 3
+      end
+
+      def path_attempts_path
+        File.join(@root, "path_attempts.json")
+      end
+
+      def read_path_attempts
+        return {} unless File.file?(path_attempts_path)
+
+        data = JSON.parse(File.read(path_attempts_path))
+        data.is_a?(Hash) ? data : {}
+      rescue JSON::ParserError
+        {}
+      end
+
+      def write_path_attempts!(hash)
+        atomic_write(path_attempts_path, JSON.generate(hash))
+      end
+
+      def failed_path
+        File.join(@root, "failed.jsonl")
+      end
+
+      def append_failed_lines!(paths, lease)
+        return if paths.empty?
+
+        File.open(failed_path, "a") do |io|
+          paths.each do |p|
+            io.puts(JSON.generate(
+              "path" => p.to_s,
+              "worker_id" => lease["worker_id"],
+              "attempts" => Integer(lease["attempts"] || 0),
+              "at" => Time.now.utc.iso8601
+            ))
+          end
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ModuleLength

data/lib/polyrun/queue/worker_loop.rb

@@ -1,6 +1,9 @@
 module Polyrun
   module Queue
     # Claim / run / ack loop for +polyrun run-queue+ workers (forked or direct).
+    #
+    # Successful batches stay one command invocation. On failure, paths are re-run one at a time so
+    # already-passing paths can be acked without reclaiming them with a poison neighbor.
     module WorkerLoop
       module_function
 
@@ -12,18 +15,25 @@ module Polyrun
           paths = claim["paths"] || []
           break if paths.empty?
 
+          store.heartbeat!(lease_id: claim["lease_id"], worker_id: worker_id)
           code = run_batch(cmd, paths)
           if code == 0
             store.ack!(lease_id: claim["lease_id"], worker_id: worker_id)
             batches_ok += 1
-          elsif on_failure.to_s == "requeue"
-            store.reclaim_lease!(claim["lease_id"])
-            batches_fail += 1
-            return {ok: batches_ok, fail: batches_fail, exit_code: 1}
-          else
-            batches_fail += 1
-            return {ok: batches_ok, fail: batches_fail, exit_code: code.zero? ? 1 : code}
+            next
           end
+
+          outcome = isolate_failed_batch!(
+            store: store,
+            worker_id: worker_id,
+            lease_id: claim["lease_id"],
+            paths: paths,
+            cmd: cmd,
+            on_failure: on_failure
+          )
+          batches_ok += outcome[:ok]
+          batches_fail += outcome[:fail]
+          return {ok: batches_ok, fail: batches_fail, exit_code: outcome[:exit_code]} if outcome[:stop]
         end
         {ok: batches_ok, fail: batches_fail, exit_code: 0}
       rescue Polyrun::Error => e
@@ -34,6 +44,30 @@ module Polyrun
       def run_batch(cmd, paths)
         system(*cmd, *paths) ? 0 : ($?.exitstatus || 1)
       end
+
+      def isolate_failed_batch!(store:, worker_id:, lease_id:, paths:, cmd:, on_failure:)
+        ok = 0
+        paths.each do |path|
+          store.heartbeat!(lease_id: lease_id, worker_id: worker_id)
+          code = run_batch(cmd, [path])
+          if code == 0
+            store.ack_paths!(lease_id: lease_id, worker_id: worker_id, paths: [path])
+            ok += 1
+            next
+          end
+
+          if on_failure.to_s == "requeue"
+            store.reclaim_lease!(lease_id)
+            return {ok: ok, fail: 1, exit_code: 1, stop: true}
+          end
+
+          return {ok: ok, fail: 1, exit_code: code.zero? ? 1 : code, stop: true}
+        end
+
+        # Every path passed on the isolating re-run; lease should already be empty.
+        {ok: ok.positive? ? 1 : 0, fail: 0, exit_code: 0, stop: false}
+      end
+      private_class_method :isolate_failed_batch!
     end
   end
 end

data/lib/polyrun/spec_quality/config.rb

@@ -53,7 +53,7 @@ module Polyrun
         path = File.expand_path(path)
         return {} unless File.file?(path)
 
-        data = YAML.load_file(path)
+        data = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true)
         data.is_a?(Hash) ? data : {}
       end
 

data/lib/polyrun/spec_quality/profile.rb

@@ -41,7 +41,6 @@ module Polyrun
 
           delta = numeric(a) - numeric(b)
           out[k] = delta if delta.positive? || k.start_with?("cpu")
-          out[k] = delta
         end
         out
       end

data/lib/polyrun/spec_quality/report.rb

@@ -158,13 +158,7 @@ module Polyrun
           violations << "zero_hit_examples=#{analysis[:zero_hit].size} max=#{max_zero}"
         end
 
-        min_unique = cfg["minimum_unique_lines_per_example"]
-        if min_unique
-          bad = analysis[:zero_hit].keys
-          if bad.size.positive?
-            violations << "examples_below_minimum_unique_lines=#{bad.size} minimum=#{min_unique}"
-          end
-        end
+        violations.concat(minimum_unique_line_violations(merged, cfg))
 
         max_hot = cfg["max_hot_line_overlap"]
         if max_hot
@@ -175,6 +169,21 @@ module Polyrun
         violations
       end
 
+      def minimum_unique_line_violations(merged, cfg)
+        violations = []
+        min_unique = cfg["minimum_unique_lines_per_example"]
+        if min_unique
+          threshold = min_unique.to_i
+          examples = merged["examples"] || {}
+          bad = examples.select { |_loc, row| row["unique_lines"].to_i < threshold }.keys
+          if bad.size.positive?
+            violations << "examples_below_minimum_unique_lines=#{bad.size} minimum=#{min_unique}"
+          end
+        end
+
+        violations
+      end
+
       def emit_warnings!(merged, cfg = {})
         analyze(merged, cfg).fetch(:line_churn, {}).each do |loc, row|
           Polyrun::Log.warn "polyrun spec-quality line_churn: #{loc} churn=#{row["line_churn"]}"

data/lib/polyrun/templates/rails_prepare.polyrun.yml

@@ -22,7 +22,9 @@ prepare:
 #   merge_spec_quality: true
 #   merge_spec_quality_output: coverage/polyrun-spec-quality.json
 
-# Uncomment and edit when using Postgres template + per-shard DB names from polyrun env:
+# Uncomment and edit when using Postgres template + per-shard DB names from polyrun env.
+# CloneShards / polyrun start auto provision is PostgreSQL-only. For mysql2 (or other adapters),
+# keep template_db + shard_db_pattern for URL shaping, set start.databases: false, and provision shards in prepare.
 # databases:
 #   template_db: my_app_template
 #   shard_db_pattern: "my_app_test_%{shard}"

data/lib/polyrun/version.rb

@@ -1,3 +1,3 @@
 module Polyrun
-  VERSION = "2.2.5"
+  VERSION = "2.2.6"
 end

data/lib/polyrun/worker_output.rb

@@ -55,20 +55,37 @@ module Polyrun
       out_read, out_write = IO.pipe
       err_read, err_write = IO.pipe
 
-      pid =
-        if hook_cfg.worker_hooks? && !Polyrun::Hooks.disabled?
-          Process.spawn(
-            child_env,
-            "sh",
-            "-c",
-            hook_cfg.build_worker_shell_script(cmd, paths),
-            out: out_write,
-            err: err_write
-          )
-        else
-          # Array(cmd) so a String command is one argv element (not character-splatted).
-          Process.spawn(child_env, *Array(cmd), *Array(paths), out: out_write, err: err_write)
+      begin
+        pid =
+          if hook_cfg.worker_hooks? && !Polyrun::Hooks.disabled?
+            Process.spawn(
+              child_env,
+              "sh",
+              "-c",
+              hook_cfg.build_worker_shell_script(cmd, paths),
+              out: out_write,
+              err: err_write,
+              pgroup: true
+            )
+          else
+            # Array(cmd) so a String command is one argv element (not character-splatted).
+            Process.spawn(
+              child_env,
+              *Array(cmd),
+              *Array(paths),
+              out: out_write,
+              err: err_write,
+              pgroup: true
+            )
+          end
+      rescue
+        [out_read, out_write, err_read, err_write].each do |io|
+          io.close unless io.closed?
+        rescue IOError
+          nil
         end
+        raise
+      end
 
       out_write.close
       err_write.close

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polyrun
 version: !ruby/object:Gem::Version
-  version: 2.2.5
+  version: 2.2.6
 platform: ruby
 authors:
 - Andrei Makarov
@@ -321,6 +321,7 @@ files:
 - lib/polyrun/process_stdio.rb
 - lib/polyrun/queue/duration.rb
 - lib/polyrun/queue/file_store.rb
+- lib/polyrun/queue/file_store_intent.rb
 - lib/polyrun/queue/file_store_pending.rb
 - lib/polyrun/queue/worker_loop.rb
 - lib/polyrun/quick.rb