cloudtasker 0.12.rc8 → 0.12.0

data/gemfiles/rails_6.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    cloudtasker (0.12.rc8)
+    cloudtasker (0.12.0)
       activesupport
       connection_pool
       fugit
@@ -68,35 +68,35 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.1, >= 2.1.8)
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
-    appraisal (2.4.0)
+    appraisal (2.4.1)
       bundler
       rake
       thor (>= 0.14.0)
     ast (2.4.2)
-    async (1.28.9)
+    async (1.30.1)
       console (~> 1.10)
       nio4r (~> 2.3)
       timers (~> 4.1)
-    async-http (0.54.1)
-      async (~> 1.25)
-      async-io (~> 1.28)
-      async-pool (~> 0.2)
-      protocol-http (~> 0.21.0)
-      protocol-http1 (~> 0.13.0)
+    async-http (0.56.5)
+      async (>= 1.25)
+      async-io (>= 1.28)
+      async-pool (>= 0.2)
+      protocol-http (~> 0.22.0)
+      protocol-http1 (~> 0.14.0)
       protocol-http2 (~> 0.14.0)
-    async-http-faraday (0.9.0)
+    async-http-faraday (0.11.0)
       async-http (~> 0.42)
       faraday
-    async-io (1.30.2)
-      async (~> 1.14)
-    async-pool (0.3.5)
-      async (~> 1.25)
+    async-io (1.32.2)
+      async
+    async-pool (0.3.8)
+      async (>= 1.25)
     builder (3.2.4)
-    concurrent-ruby (1.1.8)
-    connection_pool (2.2.3)
-    console (1.10.2)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
+    console (1.13.1)
       fiber-local
     crack (0.4.5)
       rexml
@@ -105,18 +105,32 @@ GEM
     erubi (1.10.0)
     et-orbi (1.2.4)
       tzinfo
-    faraday (1.3.0)
+    faraday (1.7.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
       faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
       multipart-post (>= 1.2, < 3)
-      ruby2_keywords
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
     faraday-http-cache (2.2.0)
       faraday (>= 0.8)
+    faraday-httpclient (1.0.1)
     faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
     fiber-local (1.0.0)
-    fugit (1.4.4)
+    fugit (1.5.1)
       et-orbi (~> 1.1, >= 1.1.8)
       raabro (~> 1.4)
-    github_changelog_generator (1.16.1)
+    github_changelog_generator (1.16.4)
       activesupport
       async (>= 1.25.0)
       async-http-faraday
@@ -125,9 +139,8 @@ GEM
       octokit (~> 4.6)
       rainbow (>= 2.2.1)
       rake (>= 10.0)
-      retriable (~> 3.0)
-    globalid (0.4.2)
-      activesupport (>= 4.2.0)
+    globalid (0.5.2)
+      activesupport (>= 5.0)
     google-cloud-tasks (1.5.1)
       google-gax (~> 1.8)
       googleapis-common-protos (>= 1.3.9, < 2.0)
@@ -139,22 +152,22 @@ GEM
       googleauth (~> 0.9)
       grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.15.7)
+    google-protobuf (3.17.3)
     googleapis-common-protos (1.3.11)
       google-protobuf (~> 3.14)
       googleapis-common-protos-types (>= 1.0.6, < 2.0)
       grpc (~> 1.27)
-    googleapis-common-protos-types (1.0.6)
+    googleapis-common-protos-types (1.1.0)
       google-protobuf (~> 3.14)
-    googleauth (0.16.1)
+    googleauth (0.17.0)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.14)
-    grpc (1.36.0)
-      google-protobuf (~> 3.14)
+    grpc (1.38.0)
+      google-protobuf (~> 3.15)
       googleapis-common-protos-types (~> 1.0)
     grpc-google-iam-v1 (0.6.11)
       google-protobuf (~> 3.14)
@@ -164,8 +177,8 @@ GEM
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
-    jwt (2.2.2)
-    loofah (2.9.0)
+    jwt (2.2.3)
+    loofah (2.12.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -178,25 +191,25 @@ GEM
       nokogiri (~> 1)
       rake
     mini_mime (1.1.0)
-    mini_portile2 (2.5.0)
+    mini_portile2 (2.6.1)
     minitest (5.14.4)
     multi_json (1.15.0)
     multipart-post (2.1.1)
-    nio4r (2.5.7)
-    nokogiri (1.11.2)
-      mini_portile2 (~> 2.5.0)
+    nio4r (2.5.8)
+    nokogiri (1.12.3)
+      mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
-    octokit (4.20.0)
+    octokit (4.21.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
     os (1.1.1)
     parallel (1.20.1)
-    parser (3.0.0.0)
+    parser (3.0.2.0)
       ast (~> 2.4.1)
     protocol-hpack (1.4.2)
-    protocol-http (0.21.0)
-    protocol-http1 (0.13.2)
-      protocol-http (~> 0.19)
+    protocol-http (0.22.5)
+    protocol-http1 (0.14.1)
+      protocol-http (~> 0.22)
     protocol-http2 (0.14.2)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.18)
@@ -224,7 +237,7 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.3.0)
+    rails-html-sanitizer (1.4.1)
       loofah (~> 2.3)
     railties (6.0.0)
       actionpack (= 6.0.0)
@@ -233,8 +246,8 @@ GEM
       rake (>= 0.8.7)
       thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
-    rake (13.0.3)
-    redis (4.2.5)
+    rake (13.0.6)
+    redis (4.4.0)
     retriable (3.1.2)
     rexml (3.2.5)
     rly (0.2.3)
@@ -251,7 +264,7 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-rails (5.0.1)
+    rspec-rails (5.0.2)
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       railties (>= 5.2)
@@ -270,11 +283,11 @@ GEM
     rubocop-rspec (1.37.0)
       rubocop (>= 0.68.1)
     ruby-progressbar (1.11.0)
-    ruby2_keywords (0.0.4)
+    ruby2_keywords (0.0.5)
     sawyer (0.8.2)
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
-    semantic_logger (4.7.4)
+    semantic_logger (4.8.2)
       concurrent-ruby (~> 1.0)
     signet (0.15.0)
       addressable (~> 2.3)
@@ -296,11 +309,11 @@ GEM
     tzinfo (1.2.9)
       thread_safe (~> 0.1)
     unicode-display_width (1.6.1)
-    webmock (3.12.2)
-      addressable (>= 2.3.6)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    websocket-driver (0.7.3)
+    websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     zeitwerk (2.4.2)

data/lib/cloudtasker/backend/redis_task.rb

@@ -39,7 +39,7 @@ module Cloudtasker
       def self.all
         if redis.exists?(key)
           # Use Schedule Set if available
-          redis.smembers(key).map { |id| find(id) }
+          redis.smembers(key).map { |id| find(id) }.compact
         else
           # Fallback to redis key matching and migrate tasks
           # to use Task Set instead.

data/lib/cloudtasker/batch/extension/worker.rb

@@ -6,7 +6,7 @@ module Cloudtasker
       # Include batch related methods onto Cloudtasker::Worker
       # See: Cloudtasker::Batch::Middleware#configure
       module Worker
-        attr_accessor :batch
+        attr_accessor :batch, :parent_batch
       end
     end
   end

data/lib/cloudtasker/batch/job.rb

@@ -73,8 +73,12 @@ module Cloudtasker
         # Load extension if not loaded already on the worker class
         worker.class.include(Extension::Worker) unless worker.class <= Extension::Worker
 
-        # Add batch capability
+        # Add batch and parent batch to worker
         worker.batch = new(worker)
+        worker.parent_batch = worker.batch.parent_batch
+
+        # Return the batch
+        worker.batch
       end
 
       #
@@ -259,9 +263,7 @@ module Cloudtasker
         migrate_batch_state_to_redis_hash
 
         # Update the batch state batch_id entry with the new status
-        redis.with_lock("#{batch_state_gid}/#{batch_id}", max_wait: BATCH_MAX_LOCK_WAIT) do
-          redis.hset(batch_state_gid, batch_id, status) if redis.hexists(batch_state_gid, batch_id)
-        end
+        redis.hset(batch_state_gid, batch_id, status) if redis.hexists(batch_state_gid, batch_id)
       end
 
       #
@@ -273,10 +275,7 @@ module Cloudtasker
         migrate_batch_state_to_redis_hash
 
         # Check that all child jobs have completed
-        redis.with_lock(batch_state_gid, max_wait: BATCH_MAX_LOCK_WAIT) do
-          # Check that all children are complete
-          redis.hvals(batch_state_gid).all? { |e| COMPLETION_STATUSES.include?(e) }
-        end
+        redis.hvals(batch_state_gid).all? { |e| COMPLETION_STATUSES.include?(e) }
       end
 
       #
@@ -311,8 +310,8 @@ module Cloudtasker
         # Propagate event
         parent_batch&.on_child_complete(self, status)
 
-        # The batch tree is complete. Cleanup the tree.
-        cleanup unless parent_batch
+        # The batch tree is complete. Cleanup the downstream tree.
+        cleanup
       end
 
       #
@@ -427,8 +426,11 @@ module Cloudtasker
         # Perform job
         yield
 
-        # Save batch (if child workers have been enqueued)
-        setup
+        # Save batch if child jobs added
+        setup if jobs.any?
+
+        # Save parent batch if batch expanded
+        parent_batch&.setup if parent_batch&.jobs&.any?
 
         # Complete batch
         complete(:completed)

data/lib/cloudtasker/config.rb

@@ -7,7 +7,8 @@ module Cloudtasker
   class Config
     attr_accessor :redis, :store_payloads_in_redis
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
-                :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries, :dispatch_deadline
+                :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries,
+                :dispatch_deadline, :on_error, :on_dead
 
     # Max Cloud Task size in bytes
     MAX_TASK_SIZE = 100 * 1024 # 100 KB
@@ -51,6 +52,9 @@ module Cloudtasker
     MIN_DISPATCH_DEADLINE = 15 # seconds
     MAX_DISPATCH_DEADLINE = 30 * 60 # 30 minutes
 
+    # Default on_error Proc
+    DEFAULT_ON_ERROR = ->(error, worker) {}
+
     # The number of times jobs will be attempted before declaring them dead.
     #
     # With the default retry configuration (maxDoublings = 16 and minBackoff = 0.100s)
@@ -229,11 +233,31 @@ module Cloudtasker
     # @return [String] The cloudtasker secret
     #
     def secret
-      @secret || (
+      @secret ||= (
         defined?(Rails) && Rails.application.credentials&.dig(:secret_key_base)
       ) || raise(StandardError, SECRET_MISSING_ERROR)
     end
 
+    #
+    # Return a Proc invoked whenever a worker runtime error is raised.
+    # See Cloudtasker::WorkerHandler.with_worker_handling
+    #
+    # @return [Proc] A Proc handler
+    #
+    def on_error
+      @on_error || DEFAULT_ON_ERROR
+    end
+
+    #
+    # Return a Proc invoked whenever a worker DeadWorkerError is raised.
+    # See Cloudtasker::WorkerHandler.with_worker_handling
+    #
+    # @return [Proc] A Proc handler
+    #
+    def on_dead
+      @on_dead || DEFAULT_ON_ERROR
+    end
+
     #
     # Return the chain of client middlewares.
     #

data/lib/cloudtasker/local_server.rb

@@ -84,7 +84,7 @@ module Cloudtasker
 
       # Deliver task
       begin
-        Thread.current['task'].deliver
+        Thread.current['task']&.deliver
       rescue Errno::EBADF, Errno::ECONNREFUSED => e
         raise(e) unless Thread.current['attempts'] < 3
 

data/lib/cloudtasker/unique_job/job.rb

@@ -149,25 +149,18 @@ module Cloudtasker
       # if taken by another job.
       #
       def lock!
-        redis.with_lock(unique_gid) do
-          locked_id = redis.get(unique_gid)
+        lock_acquired = redis.set(unique_gid, id, nx: true, ex: lock_ttl)
+        lock_already_acquired = !lock_acquired && redis.get(unique_gid) == id
 
-          # Abort job lock process if lock is already taken by another job
-          raise(LockError, locked_id) if locked_id && locked_id != id
-
-          # Take job lock if the lock is currently free
-          redis.set(unique_gid, id, ex: lock_ttl) unless locked_id
-        end
+        raise(LockError) unless lock_acquired || lock_already_acquired
       end
 
       #
       # Delete the job lock.
       #
       def unlock!
-        redis.with_lock(unique_gid) do
-          locked_id = redis.get(unique_gid)
-          redis.del(unique_gid) if locked_id == id
-        end
+        locked_id = redis.get(unique_gid)
+        redis.del(unique_gid) if locked_id == id
       end
     end
   end

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.12.rc8'
+  VERSION = '0.12.0'
 end

data/lib/cloudtasker/worker_handler.rb

@@ -14,12 +14,6 @@ module Cloudtasker
     # payloads in Redis
     REDIS_PAYLOAD_NAMESPACE = 'payload'
 
-    # Arg payload cache keys get expired instead of deleted
-    # in case jobs are re-processed due to connection interruption
-    # (job is successful but Cloud Task considers it as failed due
-    # to network interruption)
-    ARGS_PAYLOAD_CLEANUP_TTL = 3600 # 1 hour
-
     #
     # Return a namespaced key
     #
@@ -100,20 +94,19 @@ module Cloudtasker
       # Yied worker
       resp = yield(worker)
 
-      # Schedule args payload deletion after job has been successfully processed
-      # Note: we expire the key instead of deleting it immediately in case the job
-      # succeeds but is considered as failed by Cloud Task due to network interruption.
-      # In such case the job is likely to be re-processed soon after.
-      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key && !worker.job_reenqueued
+      # Delete stored args payload if job has completed
+      redis.del(args_payload_key) if args_payload_key && !worker.job_reenqueued
 
       resp
     rescue DeadWorkerError => e
       # Delete stored args payload if job is dead
-      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
+      redis.del(args_payload_key) if args_payload_key
       log_execution_error(worker, e)
+      Cloudtasker.config.on_dead.call(e, worker)
       raise(e)
     rescue StandardError => e
       log_execution_error(worker, e)
+      Cloudtasker.config.on_error.call(e, worker)
       raise(e)
     end