cloudtasker 0.12.rc6 → 0.12.rc7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00be8d2e572e3129ef5330cdbb555aa0485edd15b191d5d8ccc5d28a98a0eab8
-  data.tar.gz: 79067d7953556f03e9a81ee9d977cf2734a9843df8f9a4613dbd8d81274bc599
+  metadata.gz: 96524dc4a6825a3760a462277f7b0a710d779521e759bce44374ea044dbd649d
+  data.tar.gz: 908f1497b7ad316549f4f347c363b183181f0e0a17a92bda3f50ac9c34e2247c
 SHA512:
-  metadata.gz: 99d9b7ba99390e6bc10e9ed917262782a7c47814df6412c847dedd06a8a06a6b9ad4eaf5ddbcde03534ec88f34fcb1cda03621a3c2a0202031d087e6cd915c5f
-  data.tar.gz: e15a6a167f15dbd8c390c7d393a80297b6cd805c85d4e5ca16f961ff147c909bebbfcf9c326708e7d7508caa3ff47b726f071e618e3979a1340848420311ddbb
+  metadata.gz: e18579d27321e09f1e997ad3fdd3e3e0d1d0c344cd5fc553b64f665937fbe510cd11957381eee1fee5c100e2f14d8390434f26c60307fec4ecf4a681b15885d8
+  data.tar.gz: 3b688838f1a518f2f5c7b914a590f035d8a828c2c2b67838173a1712b8765e2ce34cddb8ddf1214aaefc26ce8ec43b7cc1dbe4eb450ccb24feb9e06bcd19bac7

data/CHANGELOG.md

@@ -1,16 +1,18 @@
 # Changelog
 
-## Latest RC [v0.12.rc6](https://github.com/keypup-io/cloudtasker/tree/v0.12.rc6) (2021-03-31)
+## Latest RC [v0.12.rc7](https://github.com/keypup-io/cloudtasker/tree/v0.12.rc7) (2021-03-31)
 
-[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.11.0...v0.12.rc6)
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.11.0...v0.12.rc7)
 
 **Improvements:**
 - ActiveJob: do not double log errors (ActiveJob has its own error logging)
+- Cron jobs: Use Redis Sets instead of key pattern matching for resource listing
 - Error logging: Use worker logger so as to include context (job args etc.)
 - Error logging: Do not log exception and stack trace separately, combine them instead.
 - Batch callbacks: Retry jobs when completion callback fails
-- Redis: Use Redis Sets instead of key pattern matching for listing methods (Cron jobs and Local Server)
+- Batch state: use native Redis hashes to store batch state instead of a serialized hash in a string key
 - Batch progress: restrict calculation to direct children by default. Allow depth to be specified. Calculating progress using all tree jobs created significant delays on large batches.
+- Local server: Use Redis Sets instead of key pattern matching for resource listing
 - Worker: raise DeadWorkerError instead of MissingWorkerArgumentsError when arguments are missing. This is more consistent with what middlewares expect.
 
 **Fixed bugs:**

data/lib/cloudtasker/batch/job.rb

@@ -17,6 +17,10 @@ module Cloudtasker
       # because the jobs will be either retried or dropped
       IGNORED_ERRORED_CALLBACKS = %i[on_child_error on_child_dead].freeze
 
+      # The maximum number of seconds to wait for a batch state lock
+      # to be acquired.
+      BATCH_MAX_LOCK_WAIT = 60
+
       #
       # Return the cloudtasker redis client
       #
@@ -176,7 +180,9 @@ module Cloudtasker
       # @return [Hash] The state  of each child worker.
       #
       def batch_state
-        redis.fetch(batch_state_gid)
+        migrate_batch_state_to_redis_hash
+
+        redis.hgetall(batch_state_gid)
       end
 
       #
@@ -208,6 +214,24 @@ module Cloudtasker
         )
       end
 
+      #
+      # This method migrates the batch state to be a Redis hash instead
+      # of a hash stored in a string key.
+      #
+      def migrate_batch_state_to_redis_hash
+        return unless redis.type(batch_state_gid) == 'string'
+
+        # Migrate batch state to Redis hash if it is still using a legacy string key
+        # We acquire a lock then check again
+        redis.with_lock(batch_state_gid, max_wait: BATCH_MAX_LOCK_WAIT) do
+          if redis.type(batch_state_gid) == 'string'
+            state = redis.fetch(batch_state_gid)
+            redis.del(batch_state_gid)
+            redis.hset(batch_state_gid, state) if state.any?
+          end
+        end
+      end
+
       #
       # Save the batch.
       #
@@ -218,8 +242,11 @@ module Cloudtasker
         # complete (success or failure).
         redis.write(batch_gid, worker.to_h)
 
+        # Stop there if no jobs to save
+        return if jobs.empty?
+
         # Save list of child workers
-        redis.write(batch_state_gid, jobs.map { |e| [e.job_id, 'scheduled'] }.to_h)
+        redis.hset(batch_state_gid, jobs.map { |e| [e.job_id, 'scheduled'] }.to_h)
       end
 
       #
@@ -228,28 +255,27 @@ module Cloudtasker
       # @param [String] job_id The batch id.
       # @param [String] status The status of the sub-batch.
       #
-      # @return [<Type>] <description>
-      #
       def update_state(batch_id, status)
-        redis.with_lock(batch_state_gid) do
-          state = batch_state
-          state[batch_id.to_sym] = status.to_s if state.key?(batch_id.to_sym)
-          redis.write(batch_state_gid, state)
+        migrate_batch_state_to_redis_hash
+
+        # Update the batch state batch_id entry with the new status
+        redis.with_lock("#{batch_state_gid}/#{batch_id}", max_wait: BATCH_MAX_LOCK_WAIT) do
+          redis.hset(batch_state_gid, batch_id, status) if redis.hexists(batch_state_gid, batch_id)
         end
       end
 
       #
       # Return true if all the child workers have completed.
       #
-      # @return [<Type>] <description>
+      # @return [Boolean] True if the batch is complete.
       #
       def complete?
-        redis.with_lock(batch_state_gid) do
-          state = redis.fetch(batch_state_gid)
-          return true unless state
+        migrate_batch_state_to_redis_hash
 
+        # Check that all child jobs have completed
+        redis.with_lock(batch_state_gid, max_wait: BATCH_MAX_LOCK_WAIT) do
           # Check that all children are complete
-          state.values.all? { |e| COMPLETION_STATUSES.include?(e) }
+          redis.hvals(batch_state_gid).all? { |e| COMPLETION_STATUSES.include?(e) }
         end
       end
 
@@ -331,11 +357,10 @@ module Cloudtasker
       # Remove all batch and sub-batch keys from Redis.
       #
       def cleanup
-        # Capture batch state
-        state = batch_state
+        migrate_batch_state_to_redis_hash
 
         # Delete child batches recursively
-        state.to_h.keys.each { |id| self.class.find(id)&.cleanup }
+        redis.hkeys(batch_state_gid).each { |id| self.class.find(id)&.cleanup }
 
         # Delete batch redis entries
         redis.del(batch_gid)
@@ -402,7 +427,7 @@ module Cloudtasker
         # Perform job
         yield
 
-        # Save batch (if child worker has been enqueued)
+        # Save batch (if child workers have been enqueued)
         setup
 
         # Complete batch

data/lib/cloudtasker/redis_client.rb

@@ -75,14 +75,18 @@ module Cloudtasker
     #   end
     #
     # @param [String] cache_key The cache key to access.
+    # @param [Integer] max_wait The number of seconds after which the lock will be cleared anyway.
     #
-    def with_lock(cache_key)
+    def with_lock(cache_key, max_wait: nil)
       return nil unless cache_key
 
+      # Set max wait
+      max_wait = (max_wait || LOCK_DURATION).to_i
+
       # Wait to acquire lock
       lock_key = [LOCK_KEY_PREFIX, cache_key].join('/')
       client.with do |conn|
-        sleep(LOCK_WAIT_DURATION) until conn.set(lock_key, true, nx: true, ex: LOCK_DURATION)
+        sleep(LOCK_WAIT_DURATION) until conn.set(lock_key, true, nx: true, ex: max_wait)
       end
 
       # yield content

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.12.rc6'
+  VERSION = '0.12.rc7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.12.rc6
+  version: 0.12.rc7
 platform: ruby
 authors:
 - Arnaud Lachaume