minitest-distributed 0.2.13 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1605e583ce4427da1909789380c1ac20ffe629a5f1ddf96452f93e45d50188f2
-  data.tar.gz: 052ba1210bee26b878052bb28207b800a67682f023e6fdcc1e03bebaea758376
+  metadata.gz: 5d4be59a24cfaf9122de65b8b5919dda05f39bcecd0a6bf9998c44f085edabe5
+  data.tar.gz: 4672adf9de55fde54385954316756688b27c0f37ba6318b73b1288d89af5b6d9
 SHA512:
-  metadata.gz: fb550faaa06a888561a3557b10f35fc867ca341e6e4008a49eb5c5137aa79b3490e6a5b24444d67c85ffc79c2e683aff7a21f948e7867b1b78bbafeccdebe9b8
-  data.tar.gz: ad60584710485b804af453696912a2676d56905e69642ead265111689ea10212768925f5c48017fd3c6da7547a94e8450b181dd2f89b5db82dc480d79d843d10
+  metadata.gz: 40768ad960753cef68edfc3cdcba75fa86c6203b0612f4eb35d387ead059d745d40ffeffaacdc6bac6487a7f3795de1be17c72cd8f2262b1d9ff8b34e6e44b8b
+  data.tar.gz: 536c77381692428674e69e3492903ae60acaf455e2b320686f5eee26f2ad88c80614f4c7dd1a769379fdecfa7c66eefe545e702dbaf704bfc06d2a2f4755c8c6

data/README.md

@@ -81,6 +81,30 @@ them to fail.
 - `--worker-id=IDENTIFIER` or `ENV[MINITEST_WORKER_ID]`: The ID of the worker,
   which should be unique to the cluster. We will default to a UUID if this is
   not set, which generally is fine.
+- `--key-ttl=SECONDS` or `ENV[MINITEST_KEY_TTL_SECONDS]` (default: 86400, i.e.
+  24 hours). The expiry applied to every Redis key a run owns, refreshed on
+  every write. A run's statistics deliberately outlive the run itself so retry
+  mode can read them, and this is what eventually reclaims them. Raise it if you
+  retry runs more than a day later. Once these keys expire, a later invocation
+  with the same run ID performs a full run rather than a selective retry. Do not
+  set it below the duration of a run: required state expiring mid-run aborts the
+  run to avoid reporting incomplete results. Active workers must use the same
+  TTL. A later retry may increase the retained run's TTL, but cannot decrease it
+  while that state exists, because doing so could invalidate another worker's
+  completion-grace wait.
+- `--stall-timeout=SECONDS` or `ENV[MINITEST_STALL_TIMEOUT_SECONDS]` (default:
+  300, i.e. 5 minutes). After this long without processing a batch or observing
+  the run counters change, inspect the Redis stream. If it has no pending or
+  undelivered tests but `acks` does not equal `size`, abort with a diagnostic
+  instead of waiting silently forever. A producer heartbeat keeps slow test
+  discovery from being mistaken for a stall. Keep this comfortably above the
+  normal end-of-run wait for your suite.
+- `--completion-grace=SECONDS` or `ENV[MINITEST_COMPLETION_GRACE_SECONDS]`
+  (default: 30). Before reusing a recently completed run ID, wait this long for
+  late workers from the previous cohort to finish. The key TTL must exceed this
+  grace period by at least one second so the retained retry snapshot cannot
+  expire while a worker is waiting. Lower this only when your CI guarantees
+  tighter worker-start synchronization.
 - `--exclude-file=PATH_TO_FILE`: Specify a file of tests to be excluded
   from running. The file should include test identifiers seperated by
   newlines.
@@ -88,6 +112,12 @@ them to fail.
   the test run. The file should include test identifiers seperated by
   newlines.
 
+Version 0.3 uses the versioned Redis namespace `minitest/v3/...`; it does not
+share streams or counters with the legacy 0.2 protocol. All workers in one
+cohort should still use the same gem version. Mixed-version cohorts run in
+separate namespaces and can duplicate test execution, but cannot mutate each
+other's coordinator state.
+
 **Limitations**
 
 **Parallel tests not supported:** Minitest comes bundled with a parallel test

data/lib/minitest/distributed/configuration.rb

@@ -11,6 +11,14 @@ module Minitest
       DEFAULT_MAX_ATTEMPTS = 1
       DEFAULT_TEST_TIMEOUT_SECONDS = 30.0 # seconds
 
+      # Expiry applied to every Redis key the coordinator writes, refreshed on
+      # every write, so a finished run's bookkeeping does not live in the
+      # coordinator forever.
+      DEFAULT_KEY_TTL_SECONDS = 86_400 # 24 hours
+      DEFAULT_STALL_TIMEOUT_SECONDS = 300.0 # 5 minutes
+      DEFAULT_COMPLETION_GRACE_SECONDS = 30.0
+      MIN_COMPLETION_GRACE_TTL_MARGIN_SECONDS = 1.0
+
       class << self
         extend T::Sig
 
@@ -24,6 +32,11 @@ module Minitest
             test_batch_size: Integer(env["MINITEST_TEST_BATCH_SIZE"] || DEFAULT_BATCH_SIZE),
             max_attempts: Integer(env["MINITEST_MAX_ATTEMPTS"] || DEFAULT_MAX_ATTEMPTS),
             max_failures: (max_failures_env = env["MINITEST_MAX_FAILURES"]) ? Integer(max_failures_env) : nil,
+            key_ttl_seconds: Integer(env["MINITEST_KEY_TTL_SECONDS"] || DEFAULT_KEY_TTL_SECONDS),
+            stall_timeout_seconds: Float(env["MINITEST_STALL_TIMEOUT_SECONDS"] || DEFAULT_STALL_TIMEOUT_SECONDS),
+            completion_grace_seconds: Float(
+              env["MINITEST_COMPLETION_GRACE_SECONDS"] || DEFAULT_COMPLETION_GRACE_SECONDS,
+            ),
           )
         end
 
@@ -83,6 +96,18 @@ module Minitest
             configuration.shuffle_suites = enabled
           end
 
+          opts.on("--key-ttl=SECONDS", "Expiry for the coordinator's Redis keys, refreshed on every write") do |ttl|
+            configuration.key_ttl_seconds = Integer(ttl)
+          end
+
+          opts.on("--stall-timeout=SECONDS", "Abort a drained Redis queue that has stopped making progress") do |timeout|
+            configuration.stall_timeout_seconds = Float(timeout)
+          end
+
+          opts.on("--completion-grace=SECONDS", "Wait before reusing a recently completed run ID") do |grace|
+            configuration.completion_grace_seconds = Float(grace)
+          end
+
           configuration
         end
       end
@@ -102,9 +127,13 @@ module Minitest
       prop :exclude_file, T.nilable(String)
       prop :include_file, T.nilable(String)
       prop :shuffle_suites, T::Boolean, default: true
+      prop :key_ttl_seconds, Integer, default: DEFAULT_KEY_TTL_SECONDS
+      prop :stall_timeout_seconds, Float, default: DEFAULT_STALL_TIMEOUT_SECONDS
+      prop :completion_grace_seconds, Float, default: DEFAULT_COMPLETION_GRACE_SECONDS
 
       sig { returns(Coordinators::CoordinatorInterface) }
       def coordinator
+        validate!
         @coordinator ||= T.let(
           case coordinator_uri.scheme
           when "redis"
@@ -117,6 +146,23 @@ module Minitest
           T.nilable(Coordinators::CoordinatorInterface),
         )
       end
+
+      sig { void }
+      def validate!
+        raise ArgumentError, "key_ttl_seconds must be greater than zero" unless key_ttl_seconds.positive?
+
+        unless stall_timeout_seconds.positive? && stall_timeout_seconds.finite?
+          raise ArgumentError, "stall_timeout_seconds must be finite and greater than zero"
+        end
+
+        unless completion_grace_seconds.positive? && completion_grace_seconds.finite?
+          raise ArgumentError, "completion_grace_seconds must be finite and greater than zero"
+        end
+
+        if key_ttl_seconds < completion_grace_seconds + MIN_COMPLETION_GRACE_TTL_MARGIN_SECONDS
+          raise ArgumentError, "key_ttl_seconds must exceed completion_grace_seconds by at least one second"
+        end
+      end
     end
   end
 end

data/lib/minitest/distributed/coordinators/coordinator_interface.rb

@@ -21,6 +21,12 @@ module Minitest
         sig { abstract.returns(T::Boolean) }
         def aborted?; end
 
+        sig { abstract.returns(T::Boolean) }
+        def valid_combined_results?; end
+
+        sig { abstract.returns(T::Boolean) }
+        def persisted_truncation?; end
+
         sig { abstract.params(test_selector: TestSelector).void }
         def produce(test_selector:); end
 

data/lib/minitest/distributed/coordinators/memory_coordinator.rb

@@ -39,6 +39,16 @@ module Minitest
           @aborted
         end
 
+        sig { override.returns(T::Boolean) }
+        def valid_combined_results?
+          combined_results.valid?
+        end
+
+        sig { override.returns(T::Boolean) }
+        def persisted_truncation?
+          combined_results.abort?
+        end
+
         sig { override.params(test_selector: TestSelector).void }
         def produce(test_selector:)
           if @leader.try_lock