cloudtasker 0.11.1 → 0.12.rc1

data/gemfiles/rails_6.0.gemfile.lock

@@ -68,36 +68,16 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.1, >= 2.1.8)
-    addressable (2.8.0)
+    addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    appraisal (2.4.1)
+    appraisal (2.4.0)
       bundler
       rake
       thor (>= 0.14.0)
     ast (2.4.2)
-    async (1.30.1)
-      console (~> 1.10)
-      nio4r (~> 2.3)
-      timers (~> 4.1)
-    async-http (0.56.5)
-      async (>= 1.25)
-      async-io (>= 1.28)
-      async-pool (>= 0.2)
-      protocol-http (~> 0.22.0)
-      protocol-http1 (~> 0.14.0)
-      protocol-http2 (~> 0.14.0)
-    async-http-faraday (0.11.0)
-      async-http (~> 0.42)
-      faraday
-    async-io (1.32.2)
-      async
-    async-pool (0.3.8)
-      async (>= 1.25)
     builder (3.2.4)
-    concurrent-ruby (1.1.9)
-    connection_pool (2.2.5)
-    console (1.13.1)
-      fiber-local
+    concurrent-ruby (1.1.8)
+    connection_pool (2.2.3)
     crack (0.4.5)
       rexml
     crass (1.0.6)
@@ -105,42 +85,26 @@ GEM
     erubi (1.10.0)
     et-orbi (1.2.4)
       tzinfo
-    faraday (1.7.0)
-      faraday-em_http (~> 1.0)
-      faraday-em_synchrony (~> 1.0)
-      faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0.1)
+    faraday (1.3.0)
       faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.1)
-      faraday-patron (~> 1.0)
-      faraday-rack (~> 1.0)
       multipart-post (>= 1.2, < 3)
-      ruby2_keywords (>= 0.0.4)
-    faraday-em_http (1.0.0)
-    faraday-em_synchrony (1.0.0)
-    faraday-excon (1.1.0)
+      ruby2_keywords
     faraday-http-cache (2.2.0)
       faraday (>= 0.8)
-    faraday-httpclient (1.0.1)
     faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.2.0)
-    faraday-patron (1.0.0)
-    faraday-rack (1.0.0)
-    fiber-local (1.0.0)
-    fugit (1.5.1)
+    fugit (1.4.2)
       et-orbi (~> 1.1, >= 1.1.8)
       raabro (~> 1.4)
-    github_changelog_generator (1.16.4)
+    github_changelog_generator (1.15.2)
       activesupport
-      async (>= 1.25.0)
-      async-http-faraday
       faraday-http-cache
       multi_json
       octokit (~> 4.6)
       rainbow (>= 2.2.1)
       rake (>= 10.0)
-    globalid (0.5.2)
-      activesupport (>= 5.0)
+      retriable (~> 3.0)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
     google-cloud-tasks (1.5.1)
       google-gax (~> 1.8)
       googleapis-common-protos (>= 1.3.9, < 2.0)
@@ -152,33 +116,33 @@ GEM
       googleauth (~> 0.9)
       grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.17.3)
+    google-protobuf (3.15.5)
     googleapis-common-protos (1.3.11)
       google-protobuf (~> 3.14)
       googleapis-common-protos-types (>= 1.0.6, < 2.0)
       grpc (~> 1.27)
-    googleapis-common-protos-types (1.1.0)
+    googleapis-common-protos-types (1.0.6)
       google-protobuf (~> 3.14)
-    googleauth (0.17.0)
+    googleauth (0.16.0)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.14)
-    grpc (1.38.0)
-      google-protobuf (~> 3.15)
+    grpc (1.36.0)
+      google-protobuf (~> 3.14)
       googleapis-common-protos-types (~> 1.0)
     grpc-google-iam-v1 (0.6.11)
       google-protobuf (~> 3.14)
       googleapis-common-protos (>= 1.3.11, < 2.0)
       grpc (~> 1.27)
     hashdiff (1.0.1)
-    i18n (1.8.10)
+    i18n (1.8.9)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
-    jwt (2.2.3)
-    loofah (2.12.0)
+    jwt (2.2.2)
+    loofah (2.9.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -187,32 +151,23 @@ GEM
       mimemagic (~> 0.3.2)
     memoist (0.16.2)
     method_source (1.0.0)
-    mimemagic (0.3.10)
-      nokogiri (~> 1)
-      rake
-    mini_mime (1.1.1)
-    mini_portile2 (2.6.1)
+    mimemagic (0.3.5)
+    mini_mime (1.0.2)
+    mini_portile2 (2.5.0)
     minitest (5.14.4)
     multi_json (1.15.0)
     multipart-post (2.1.1)
-    nio4r (2.5.8)
-    nokogiri (1.12.3)
-      mini_portile2 (~> 2.6.1)
+    nio4r (2.5.7)
+    nokogiri (1.11.2)
+      mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
-    octokit (4.21.0)
+    octokit (4.20.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
     os (1.1.1)
     parallel (1.20.1)
-    parser (3.0.2.0)
+    parser (3.0.0.0)
       ast (~> 2.4.1)
-    protocol-hpack (1.4.2)
-    protocol-http (0.22.5)
-    protocol-http1 (0.14.2)
-      protocol-http (~> 0.22)
-    protocol-http2 (0.14.2)
-      protocol-hpack (~> 1.4)
-      protocol-http (~> 0.18)
     public_suffix (4.0.6)
     raabro (1.4.0)
     racc (1.5.2)
@@ -237,7 +192,7 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.2)
+    rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
     railties (6.0.0)
       actionpack (= 6.0.0)
@@ -246,10 +201,10 @@ GEM
       rake (>= 0.8.7)
       thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
-    rake (13.0.6)
-    redis (4.4.0)
+    rake (13.0.3)
+    redis (4.2.5)
     retriable (3.1.2)
-    rexml (3.2.5)
+    rexml (3.2.4)
     rly (0.2.3)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -264,7 +219,7 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-rails (5.0.2)
+    rspec-rails (5.0.0)
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       railties (>= 5.2)
@@ -283,11 +238,11 @@ GEM
     rubocop-rspec (1.37.0)
       rubocop (>= 0.68.1)
     ruby-progressbar (1.11.0)
-    ruby2_keywords (0.0.5)
+    ruby2_keywords (0.0.4)
     sawyer (0.8.2)
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
-    semantic_logger (4.8.2)
+    semantic_logger (4.7.4)
       concurrent-ruby (~> 1.0)
     signet (0.15.0)
       addressable (~> 2.3)
@@ -305,15 +260,14 @@ GEM
     thor (1.1.0)
     thread_safe (0.3.6)
     timecop (0.9.4)
-    timers (4.3.3)
     tzinfo (1.2.9)
       thread_safe (~> 0.1)
     unicode-display_width (1.6.1)
-    webmock (3.14.0)
-      addressable (>= 2.8.0)
+    webmock (3.12.1)
+      addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    websocket-driver (0.7.5)
+    websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     zeitwerk (2.4.2)

data/lib/cloudtasker/backend/redis_task.rb

@@ -23,14 +23,12 @@ module Cloudtasker
       #
       # Return a namespaced key.
       #
-      # @param [String, Symbol] val The key to namespace
+      # @param [String, Symbol, nil] val The key to namespace
       #
       # @return [String] The namespaced key.
       #
-      def self.key(val)
-        return nil if val.nil?
-
-        [to_s.underscore, val.to_s].join('/')
+      def self.key(val = nil)
+        [to_s.underscore, val].compact.map(&:to_s).join('/')
       end
 
       #
@@ -39,9 +37,17 @@ module Cloudtasker
       # @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks.
       #
       def self.all
-        redis.search(key('*')).map do |gid|
-          payload = redis.fetch(gid)
-          new(payload.merge(id: gid.sub(key(''), '')))
+        if redis.exists?(key)
+          # Use Schedule Set if available
+          redis.smembers(key).map { |id| find(id) }
+        else
+          # Fallback to redis key matching and migrate tasks
+          # to use Task Set instead.
+          redis.search(key('*')).map do |gid|
+            task_id = gid.sub(key(''), '')
+            redis.sadd(key, task_id)
+            find(task_id)
+          end
         end
       end
 
@@ -82,6 +88,7 @@ module Cloudtasker
 
         # Save job
         redis.write(key(id), payload)
+        redis.sadd(key, id)
         new(payload.merge(id: id))
       end
 
@@ -105,6 +112,7 @@ module Cloudtasker
       # @param [String] id The task id.
       #
       def self.delete(id)
+        redis.srem(key, id)
         redis.del(key(id))
       end
 
@@ -176,7 +184,7 @@ module Cloudtasker
       # Remove the task from the queue.
       #
       def destroy
-        redis.del(gid)
+        self.class.delete(id)
       end
 
       #

data/lib/cloudtasker/batch/job.rb

@@ -13,6 +13,10 @@ module Cloudtasker
       # List of statuses triggering a completion callback
       COMPLETION_STATUSES = %w[completed dead].freeze
 
+      # These callbacks do not need to raise errors on their own
+      # because the jobs will be either retried or dropped
+      IGNORED_ERRORED_CALLBACKS = %i[on_child_error on_child_dead].freeze
+
       #
       # Return the cloudtasker redis client
       #
@@ -250,8 +254,8 @@ module Cloudtasker
       end
 
       #
-      # Run worker callback in a controlled environment to
-      # avoid interruption of the callback flow.
+      # Run worker callback. The error and dead callbacks get
+      # silenced should they raise an error.
       #
       # @param [String, Symbol] callback The callback to run.
       # @param [Array<any>] *args The callback arguments.
@@ -261,9 +265,15 @@ module Cloudtasker
       def run_worker_callback(callback, *args)
         worker.try(callback, *args)
       rescue StandardError => e
-        Cloudtasker.logger.error("Error running callback #{callback}: #{e}")
-        Cloudtasker.logger.error(e.backtrace.join("\n"))
-        nil
+        # There is no point in retrying jobs due to failure callbacks failing
+        # Only completion callbacks will trigger a re-run of the job because
+        # these do matter for batch completion
+        raise(e) unless IGNORED_ERRORED_CALLBACKS.include?(callback)
+
+        # Log error instead
+        worker.logger.error(
+          ["Callback #{callback} failed to run. Skipping to preserve error flow.", e, e.backtrace].flatten.join("\n")
+        )
       end
 
       #
@@ -275,7 +285,7 @@ module Cloudtasker
 
         # Propagate event
         parent_batch&.on_child_complete(self, status)
-      ensure
+
         # The batch tree is complete. Cleanup the tree.
         cleanup unless parent_batch
       end

data/lib/cloudtasker/cron/schedule.rb

@@ -21,14 +21,12 @@ module Cloudtasker
       #
       # Return a namespaced key.
       #
-      # @param [String, Symbol] val The key to namespace
+      # @param [String, Symbol, nil] val The key to namespace
       #
       # @return [String] The namespaced key.
       #
-      def self.key(val)
-        return nil if val.nil?
-
-        [to_s.underscore, val.to_s].join('/')
+      def self.key(val = nil)
+        [to_s.underscore, val].compact.map(&:to_s).join('/')
       end
 
       #
@@ -37,8 +35,17 @@ module Cloudtasker
       # @return [Array<Cloudtasker::Batch::Schedule>] The list of stored schedules.
       #
       def self.all
-        redis.search(key('*')).map do |gid|
-          find(gid.sub(key(''), ''))
+        if redis.exists?(key)
+          # Use Schedule Set if available
+          redis.smembers(key).map { |id| find(id) }
+        else
+          # Fallback to redis key matching and migrate schedules
+          # to use Schedule Set instead.
+          redis.search(key('*')).map do |gid|
+            schedule_id = gid.sub(key(''), '')
+            redis.sadd(key, schedule_id)
+            find(schedule_id)
+          end
         end
       end
 
@@ -90,7 +97,7 @@ module Cloudtasker
       end
 
       #
-      # Destroy a schedule by id.
+      # Delete a schedule by id.
       #
       # @param [String] id The schedule id.
       #
@@ -101,6 +108,7 @@ module Cloudtasker
 
           # Delete task and stored schedule
           CloudTask.delete(schedule.task_id) if schedule.task_id
+          redis.srem(key, schedule.id)
           redis.del(schedule.gid)
         end
       end
@@ -270,6 +278,7 @@ module Cloudtasker
 
         # Save schedule
         config_was_changed = config_changed?
+        redis.sadd(self.class.key, id)
         redis.write(gid, to_h)
 
         # Stop there if backend does not need update

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.11.1'
+  VERSION = '0.12.rc1'
 end

data/lib/cloudtasker/worker.rb

@@ -311,13 +311,25 @@ module Cloudtasker
     end
 
     #
-    # Return true if the job has excceeded its maximum number
-    # of retries
+    # Return true if the job must declared dead upon raising
+    # an error.
+    #
+    # @return [Boolean] True if the job must die on error.
+    #
+    def job_must_die?
+      job_retries >= job_max_retries
+    end
+
+    #
+    # Return true if the job has strictly excceeded its maximum number
+    # of retries.
+    #
+    # Used a preemptive filter when running the job.
     #
     # @return [Boolean] True if the job is dead
     #
     def job_dead?
-      job_retries >= job_max_retries
+      job_retries > job_max_retries
     end
 
     #
@@ -332,11 +344,35 @@ module Cloudtasker
       (perform_ended_at - perform_started_at).ceil(3)
     end
 
+    #
+    # Run worker callback.
+    #
+    # @param [String, Symbol] callback The callback to run.
+    # @param [Array<any>] *args The callback arguments.
+    #
+    # @return [any] The callback return value
+    #
+    def run_callback(callback, *args)
+      try(callback, *args)
+    end
+
     #=============================
     # Private
     #=============================
     private
 
+    #
+    # Flag the worker as dead by invoking the on_dead hook
+    # and raising a DeadWorkerError
+    #
+    # @param [Exception, nil] error An optional exception to be passed to the DeadWorkerError.
+    #
+    def flag_as_dead(error = nil)
+      run_callback(:on_dead, error || DeadWorkerError.new)
+    ensure
+      raise(DeadWorkerError, error)
+    end
+
     #
     # Execute the worker perform method through the middleware chain.
     #
@@ -346,6 +382,9 @@ module Cloudtasker
       self.perform_started_at = Time.now
 
       Cloudtasker.config.server_middleware.invoke(self) do
+        # Immediately abort the job if it is already dead
+        flag_as_dead if job_dead?
+
         begin
           # Abort if arguments are missing. This may happen with redis arguments storage
           # if Cloud Tasks times out on a job but the job still succeeds
@@ -356,12 +395,11 @@ module Cloudtasker
           # Perform the job
           perform(*job_args)
         rescue StandardError => e
-          try(:on_error, e)
-          return raise(e) unless job_dead?
+          run_callback(:on_error, e)
+          return raise(e) unless job_must_die?
 
           # Flag job as dead
-          try(:on_dead, e)
-          raise(DeadWorkerError, e)
+          flag_as_dead(e)
         end
       end
     ensure

data/lib/cloudtasker/worker_handler.rb

@@ -45,6 +45,28 @@ module Cloudtasker
       end
     end
 
+    #
+    # Log error on execution failure.
+    #
+    # @param [Cloudtasker::Worker, nil] worker The worker.
+    # @param [Exception] error The error to log.
+    #
+    # @void
+    #
+    def self.log_execution_error(worker, error)
+      # ActiveJob has its own error logging. No need to double log the error.
+      # Note: we use string matching instead of class matching as
+      # ActiveJob::QueueAdapters::CloudtaskerAdapter::JobWrapper might not be loaded
+      return if worker.class.to_s =~ /^ActiveJob::/
+
+      # Choose logger to use based on context
+      # Worker will be nil on InvalidWorkerError - in that case we use generic logging
+      logger = worker&.logger || Cloudtasker.logger
+
+      # Log error
+      logger.error([error, error.backtrace].flatten.join("\n"))
+    end
+
     #
     # Execute a task worker from a task payload
     #
@@ -88,6 +110,10 @@ module Cloudtasker
     rescue DeadWorkerError, MissingWorkerArgumentsError => e
       # Delete stored args payload if job is dead
       redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
+      log_execution_error(worker, e)
+      raise(e)
+    rescue StandardError => e
+      log_execution_error(worker, e)
       raise(e)
     end
 

data/lib/cloudtasker.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'active_support/core_ext/string/inflections'
-require 'active_support/core_ext/object/try'
 
 require 'cloudtasker/version'
 require 'cloudtasker/config'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.rc1
 platform: ruby
 authors:
 - Arnaud Lachaume
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-08-25 00:00:00.000000000 Z
+date: 2021-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -324,6 +324,7 @@ files:
 - README.md
 - Rakefile
 - _config.yml
+- app/controllers/cloudtasker/application_controller.rb
 - app/controllers/cloudtasker/worker_controller.rb
 - bin/console
 - bin/setup
@@ -415,7 +416,7 @@ metadata:
   homepage_uri: https://github.com/keypup-io/cloudtasker
   source_code_uri: https://github.com/keypup-io/cloudtasker
   changelog_uri: https://github.com/keypup-io/cloudtasker/master/tree/CHANGELOG.md
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -426,12 +427,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.0
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Background jobs for Ruby using Google Cloud Tasks (beta)
 test_files: []