cloudtasker 0.12.rc8 → 0.12.rc9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ac7405c882f6cc73a1d18d5f0ffd4aaf1032d2b22973b03ac0c05b579321d47
-  data.tar.gz: ab4e12890ba7f1cf2f8cfde952a532f8563998352ba3c0ba348f8253c0f562b4
+  metadata.gz: 50cd6021ad7511d7b5b385a6086eb315d9381a4f43909a6e1e9a0defbc679ea4
+  data.tar.gz: dc16cc1330b14b46a5d6ade9f9c171981fd3aef5b3db67dd474e29548acc177f
 SHA512:
-  metadata.gz: 99a740769048aa11f4a7b2a3710ed98973c44c7d9fa5e059b37027f408d10e5c731cd01e28bbf7e0e2cc4df49290384fdbc073d4a7869d511af661d8c5363e1b
-  data.tar.gz: 4214ca35db358f7f027cad6fe0e36353034789c3969726484c2af50cc019a34ce03840718609aa6941ab5a8d1d6b312f174d884a9c88a2e112c4219f6e1a26d8
+  metadata.gz: 30feac9331b9e8113e71af6c44a20d2f47fcbd41652a0d345ab8808d272bd14271ebb4a98711e42b238f6202fa8023a37c543e0f4ec7d9f9f7a46cc2732d96aa
+  data.tar.gz: 9b70a3c0733b2c510fafe48232c65bb79f5ad4da6b90fbcff86c0736fee590fab5f89846df9c41855dc7084460425660ee42799975975775b2239b9b8997171a

data/CHANGELOG.md

@@ -6,15 +6,17 @@
 
 **Improvements:**
 - ActiveJob: do not double log errors (ActiveJob has its own error logging)
+- Batch callbacks: Retry jobs when completion callback fails
+- Batch state: use native Redis hashes to store batch state instead of a serialized hash in a string key
+- Batch progress: restrict calculation to direct children by default. Allow depth to be specified. Calculating progress using all tree jobs created significant delays on large batches.
+- Batch redis usage: cleanup batches as they get completed or become dead to avoid excessive redis usage with large batches.
 - Configuration: allow configuration of Cloud Tasks `dispatch deadline` at global and worker level
 - Cron jobs: Use Redis Sets instead of key pattern matching for resource listing
 - Error logging: Use worker logger so as to include context (job args etc.)
 - Error logging: Do not log exception and stack trace separately, combine them instead.
-- Batch callbacks: Retry jobs when completion callback fails
-- Batch state: use native Redis hashes to store batch state instead of a serialized hash in a string key
-- Batch progress: restrict calculation to direct children by default. Allow depth to be specified. Calculating progress using all tree jobs created significant delays on large batches.
 - Local server: Use Redis Sets instead of key pattern matching for resource listing
 - Worker: raise DeadWorkerError instead of MissingWorkerArgumentsError when arguments are missing. This is more consistent with what middlewares expect.
+- Worker redis usage: delete redis payload storage once the job is successful or dead instead of expiring the key.
 
 **Fixed bugs:**
 - Retries: Enforce job retry limit on job processing. There was an edge case where jobs could be retried indefinitely on batch callback errors.

data/lib/cloudtasker/batch/job.rb

@@ -311,8 +311,8 @@ module Cloudtasker
         # Propagate event
         parent_batch&.on_child_complete(self, status)
 
-        # The batch tree is complete. Cleanup the tree.
-        cleanup unless parent_batch
+        # The batch tree is complete. Cleanup the downstream tree.
+        cleanup
       end
 
       #

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.12.rc8'
+  VERSION = '0.12.rc9'
 end

data/lib/cloudtasker/worker_handler.rb

@@ -14,12 +14,6 @@ module Cloudtasker
     # payloads in Redis
     REDIS_PAYLOAD_NAMESPACE = 'payload'
 
-    # Arg payload cache keys get expired instead of deleted
-    # in case jobs are re-processed due to connection interruption
-    # (job is successful but Cloud Task considers it as failed due
-    # to network interruption)
-    ARGS_PAYLOAD_CLEANUP_TTL = 3600 # 1 hour
-
     #
     # Return a namespaced key
     #
@@ -100,16 +94,13 @@ module Cloudtasker
       # Yied worker
       resp = yield(worker)
 
-      # Schedule args payload deletion after job has been successfully processed
-      # Note: we expire the key instead of deleting it immediately in case the job
-      # succeeds but is considered as failed by Cloud Task due to network interruption.
-      # In such case the job is likely to be re-processed soon after.
-      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key && !worker.job_reenqueued
+      # Delete stored args payload if job has completed
+      redis.del(args_payload_key) if args_payload_key && !worker.job_reenqueued
 
       resp
     rescue DeadWorkerError => e
       # Delete stored args payload if job is dead
-      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
+      redis.del(args_payload_key) if args_payload_key
       log_execution_error(worker, e)
       raise(e)
     rescue StandardError => e

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.12.rc8
+  version: 0.12.rc9
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-06 00:00:00.000000000 Z
+date: 2021-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport