cloudtasker 0.9.5 → 0.10.rc1

data/gemfiles/rails_6.0.gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: ..
   specs:
-    cloudtasker (0.9.4)
+    cloudtasker (0.10.rc1)
       activesupport
       fugit
-      google-cloud-tasks (~> 1.0)
+      google-cloud-tasks
       jwt
       redis
 
@@ -66,79 +66,40 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.1, >= 2.1.8)
-    addressable (2.8.0)
+    addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    appraisal (2.4.1)
+    appraisal (2.2.0)
       bundler
       rake
       thor (>= 0.14.0)
-    ast (2.4.2)
-    async (1.30.1)
-      console (~> 1.10)
-      nio4r (~> 2.3)
-      timers (~> 4.1)
-    async-http (0.56.5)
-      async (>= 1.25)
-      async-io (>= 1.28)
-      async-pool (>= 0.2)
-      protocol-http (~> 0.22.0)
-      protocol-http1 (~> 0.14.0)
-      protocol-http2 (~> 0.14.0)
-    async-http-faraday (0.11.0)
-      async-http (~> 0.42)
-      faraday
-    async-io (1.32.2)
-      async
-    async-pool (0.3.8)
-      async (>= 1.25)
+    ast (2.4.0)
     builder (3.2.4)
-    concurrent-ruby (1.1.9)
-    console (1.13.1)
-      fiber-local
-    crack (0.4.5)
-      rexml
+    concurrent-ruby (1.1.6)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
     crass (1.0.6)
-    diff-lcs (1.4.4)
-    erubi (1.10.0)
-    et-orbi (1.2.4)
+    diff-lcs (1.3)
+    erubi (1.9.0)
+    et-orbi (1.2.3)
       tzinfo
-    faraday (1.7.0)
-      faraday-em_http (~> 1.0)
-      faraday-em_synchrony (~> 1.0)
-      faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0.1)
-      faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.1)
-      faraday-patron (~> 1.0)
-      faraday-rack (~> 1.0)
+    faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
-      ruby2_keywords (>= 0.0.4)
-    faraday-em_http (1.0.0)
-    faraday-em_synchrony (1.0.0)
-    faraday-excon (1.1.0)
-    faraday-http-cache (2.2.0)
-      faraday (>= 0.8)
-    faraday-httpclient (1.0.1)
-    faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.2.0)
-    faraday-patron (1.0.0)
-    faraday-rack (1.0.0)
-    fiber-local (1.0.0)
-    fugit (1.5.1)
+    faraday-http-cache (2.0.0)
+      faraday (~> 0.8)
+    fugit (1.3.3)
       et-orbi (~> 1.1, >= 1.1.8)
-      raabro (~> 1.4)
-    github_changelog_generator (1.16.4)
+      raabro (~> 1.1)
+    github_changelog_generator (1.15.0)
       activesupport
-      async (>= 1.25.0)
-      async-http-faraday
       faraday-http-cache
       multi_json
       octokit (~> 4.6)
       rainbow (>= 2.2.1)
       rake (>= 10.0)
-    globalid (0.5.2)
-      activesupport (>= 5.0)
-    google-cloud-tasks (1.5.1)
+      retriable (~> 3.0)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
+    google-cloud-tasks (1.4.0)
       google-gax (~> 1.8)
       googleapis-common-protos (>= 1.3.9, < 2.0)
       googleapis-common-protos-types (>= 1.0.4, < 2.0)
@@ -149,33 +110,32 @@ GEM
       googleauth (~> 0.9)
       grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.17.3)
-    googleapis-common-protos (1.3.11)
-      google-protobuf (~> 3.14)
-      googleapis-common-protos-types (>= 1.0.6, < 2.0)
-      grpc (~> 1.27)
-    googleapis-common-protos-types (1.1.0)
-      google-protobuf (~> 3.14)
-    googleauth (0.17.0)
+    google-protobuf (3.11.4)
+    googleapis-common-protos (1.3.9)
+      google-protobuf (~> 3.0)
+      googleapis-common-protos-types (~> 1.0)
+      grpc (~> 1.0)
+    googleapis-common-protos-types (1.0.4)
+      google-protobuf (~> 3.0)
+    googleauth (0.11.0)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.14)
-    grpc (1.38.0)
-      google-protobuf (~> 3.15)
+      signet (~> 0.12)
+    grpc (1.27.0)
+      google-protobuf (~> 3.11)
       googleapis-common-protos-types (~> 1.0)
-    grpc-google-iam-v1 (0.6.11)
-      google-protobuf (~> 3.14)
-      googleapis-common-protos (>= 1.3.11, < 2.0)
-      grpc (~> 1.27)
+    grpc-google-iam-v1 (0.6.9)
+      googleapis-common-protos (>= 1.3.1, < 2.0)
+      grpc (~> 1.0)
     hashdiff (1.0.1)
-    i18n (1.8.10)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
-    jwt (2.2.3)
-    loofah (2.12.0)
+    jwt (2.2.1)
+    loofah (2.4.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -183,37 +143,26 @@ GEM
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
     memoist (0.16.2)
-    method_source (1.0.0)
-    mimemagic (0.3.10)
-      nokogiri (~> 1)
-      rake
-    mini_mime (1.1.0)
-    mini_portile2 (2.6.1)
-    minitest (5.14.4)
-    multi_json (1.15.0)
+    method_source (0.9.2)
+    mimemagic (0.3.4)
+    mini_mime (1.0.2)
+    mini_portile2 (2.4.0)
+    minitest (5.14.0)
+    multi_json (1.14.1)
     multipart-post (2.1.1)
-    nio4r (2.5.8)
-    nokogiri (1.12.3)
-      mini_portile2 (~> 2.6.1)
-      racc (~> 1.4)
-    octokit (4.21.0)
+    nio4r (2.5.2)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
+    octokit (4.16.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
-    os (1.1.1)
-    parallel (1.20.1)
-    parser (3.0.2.0)
-      ast (~> 2.4.1)
-    protocol-hpack (1.4.2)
-    protocol-http (0.22.5)
-    protocol-http1 (0.14.1)
-      protocol-http (~> 0.22)
-    protocol-http2 (0.14.2)
-      protocol-hpack (~> 1.4)
-      protocol-http (~> 0.18)
-    public_suffix (4.0.6)
-    raabro (1.4.0)
-    racc (1.5.2)
-    rack (2.2.3)
+    os (1.0.1)
+    parallel (1.19.1)
+    parser (2.7.0.4)
+      ast (~> 2.4.0)
+    public_suffix (4.0.3)
+    raabro (1.1.6)
+    rack (2.2.2)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails (6.0.0)
@@ -234,7 +183,7 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.1)
+    rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
     railties (6.0.0)
       actionpack (= 6.0.0)
@@ -243,31 +192,31 @@ GEM
       rake (>= 0.8.7)
       thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
-    rake (13.0.6)
-    redis (4.4.0)
-    rexml (3.2.5)
+    rake (13.0.1)
+    redis (4.1.3)
+    retriable (3.1.2)
     rly (0.2.3)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-rails (5.0.2)
-      actionpack (>= 5.2)
-      activesupport (>= 5.2)
-      railties (>= 5.2)
-      rspec-core (~> 3.10)
-      rspec-expectations (~> 3.10)
-      rspec-mocks (~> 3.10)
-      rspec-support (~> 3.10)
-    rspec-support (3.10.2)
+      rspec-support (~> 3.9.0)
+    rspec-rails (3.9.0)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
     rubocop (0.76.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
@@ -277,41 +226,38 @@ GEM
       unicode-display_width (>= 1.4.0, < 1.7)
     rubocop-rspec (1.37.0)
       rubocop (>= 0.68.1)
-    ruby-progressbar (1.11.0)
-    ruby2_keywords (0.0.5)
+    ruby-progressbar (1.10.1)
+    safe_yaml (1.0.5)
     sawyer (0.8.2)
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
-    semantic_logger (4.8.2)
-      concurrent-ruby (~> 1.0)
-    signet (0.15.0)
+    signet (0.13.0)
       addressable (~> 2.3)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
-    sprockets (4.0.2)
+    sprockets (4.0.0)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.2)
+    sprockets-rails (3.2.1)
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
-    thor (1.1.0)
+    thor (1.0.1)
     thread_safe (0.3.6)
-    timecop (0.9.4)
-    timers (4.3.3)
-    tzinfo (1.2.9)
+    timecop (0.9.1)
+    tzinfo (1.2.6)
       thread_safe (~> 0.1)
     unicode-display_width (1.6.1)
-    webmock (3.14.0)
-      addressable (>= 2.8.0)
+    webmock (3.8.2)
+      addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    websocket-driver (0.7.5)
+    websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.5)
-    zeitwerk (2.4.2)
+    websocket-extensions (0.1.4)
+    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby
@@ -327,10 +273,9 @@ DEPENDENCIES
   rspec-rails
   rubocop (= 0.76.0)
   rubocop-rspec (= 1.37.0)
-  semantic_logger
   sqlite3
   timecop
   webmock
 
 BUNDLED WITH
-   2.2.9
+   2.1.4

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -146,7 +146,7 @@ module Cloudtasker
       #
       def self.delete(id)
         client.delete_task(id)
-      rescue Google::Gax::RetryError, GRPC::NotFound
+      rescue Google::Gax::RetryError, GRPC::NotFound, Google::Gax::PermissionDeniedError
         nil
       end
 

data/lib/cloudtasker/backend/memory_task.rb

@@ -7,6 +7,7 @@ module Cloudtasker
     # Manage local tasks pushed to memory.
     # Used for testing.
     class MemoryTask
+      attr_accessor :job_retries
       attr_reader :id, :http_request, :schedule_time, :queue
 
       #
@@ -18,17 +19,6 @@ module Cloudtasker
         @queue ||= []
       end
 
-      #
-      # Return the workers currently in the queue.
-      #
-      # @param [String] worker_class_name Filter jobs on worker class name.
-      #
-      # @return [Array<Cloudtasker::Worker] The list of workers
-      #
-      def self.jobs(worker_class_name = nil)
-        all(worker_class_name).map(&:worker)
-      end
-
       #
       # Run all Tasks in the queue. Optionally filter which tasks to run based
       # on the worker class name.
@@ -116,11 +106,12 @@ module Cloudtasker
       # @param [Hash] http_request The HTTP request content.
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       #
-      def initialize(id:, http_request:, schedule_time: nil, queue: nil)
+      def initialize(id:, http_request:, schedule_time: nil, queue: nil, job_retries: 0)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
         @queue = queue
+        @job_retries = job_retries || 0
       end
 
       #
@@ -155,26 +146,20 @@ module Cloudtasker
         }
       end
 
-      #
-      # Return the worker attached to this task.
-      #
-      # @return [Cloudtasker::Worker] The task worker.
-      #
-      def worker
-        @worker ||= Worker.from_hash(payload)
-      end
-
       #
       # Execute the task.
       #
       # @return [Any] The return value of the worker perform method.
       #
       def execute
-        resp = worker.execute
+        # Execute worker
+        resp = WorkerHandler.with_worker_handling(payload, &:execute)
+
+        # Delete task
         self.class.delete(id)
         resp
       rescue StandardError
-        worker.job_retries += 1
+        self.job_retries += 1
       end
 
       #

data/lib/cloudtasker/config.rb

@@ -5,7 +5,7 @@ require 'logger'
 module Cloudtasker
   # Holds cloudtasker configuration. See Cloudtasker#configure
   class Config
-    attr_accessor :redis
+    attr_accessor :redis, :store_payloads_in_redis
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
                 :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
 
@@ -54,6 +54,21 @@ module Cloudtasker
       Please specify a secret in the cloudtasker initializer or add Rails secret_key_base in your credentials
     DOC
 
+    #
+    # Return the threshold above which job arguments must be stored
+    # in Redis instead of being sent to the backend as part of the job
+    # payload.
+    #
+    # Return nil if redis payload storage is disabled.
+    #
+    # @return [Integer, nil] The threshold above which payloads will be stored in Redis.
+    #
+    def redis_payload_storage_threshold
+      return nil unless store_payloads_in_redis
+
+      store_payloads_in_redis.respond_to?(:to_i) ? store_payloads_in_redis.to_i : 0
+    end
+
     #
     # The number of times jobs will be retried. This number of
     # retries does not include failures due to the application being unreachable.

data/lib/cloudtasker/testing.rb

@@ -114,10 +114,10 @@ module Cloudtasker
       #
       # Return all jobs related to this worker class.
       #
-      # @return [Array<Cloudtasker::Worker] The list of workers
+      # @return [Array<Cloudtasker::Backend::MemoryTask>] The list of tasks
       #
       def jobs
-        Backend::MemoryTask.jobs(to_s)
+        Backend::MemoryTask.all(to_s)
       end
 
       #

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.9.5'
+  VERSION = '0.10.rc1'
 end

data/lib/cloudtasker/worker_handler.rb

@@ -10,16 +10,113 @@ module Cloudtasker
     # Alrogith used to sign the verification token
     JWT_ALG = 'HS256'
 
+    # Sub-namespace to use for redis keys when storing
+    # payloads in Redis
+    REDIS_PAYLOAD_NAMESPACE = 'payload'
+
+    # Arg payload cache keys get expired instead of deleted
+    # in case jobs are re-processed due to connection interruption
+    # (job is successful but Cloud Task considers it as failed due
+    # to network interruption)
+    ARGS_PAYLOAD_CLEANUP_TTL = 3600 # 1 hour
+
+    #
+    # Return a namespaced key
+    #
+    # @param [String, Symbol] val The key to namespace
+    #
+    # @return [String] The namespaced key.
+    #
+    def self.key(val)
+      return nil if val.nil?
+
+      [to_s.underscore, val.to_s].join('/')
+    end
+
+    #
+    # Return the cloudtasker redis client
+    #
+    # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
+    #
+    def self.redis
+      @redis ||= begin
+        require 'cloudtasker/redis_client'
+        RedisClient.new
+      end
+    end
+
     #
     # Execute a task worker from a task payload
     #
-    # @param [Hash] payload The Cloud Task payload.
+    # @param [Hash] input_payload The Cloud Task payload.
     #
     # @return [Any] The return value of the worker perform method.
     #
-    def self.execute_from_payload!(payload)
+    def self.execute_from_payload!(input_payload)
+      with_worker_handling(input_payload, &:execute)
+    end
+
+    # TODO: do not delete redis payload if job has been re-enqueued
+    # worker.job_reenqueued
+    #
+    # Idea: change with_worker_handling to with_worker_handling and build the worker
+    # inside the with_worker_handling block.
+    #
+    # Local middleware used to retrieve the job arg payload from cache
+    # if a arg payload reference is present.
+    #
+    # @param [Hash] payload The full job payload
+    #
+    # @yield [Hash] The actual payload to use to process the job.
+    #
+    # @return [Any] The block result
+    #
+    def self.with_worker_handling(input_payload)
+      # Extract payload information
+      extracted_payload = extract_payload(input_payload)
+      payload = extracted_payload[:payload]
+      args_payload_key = extracted_payload[:args_payload_key]
+
+      # Build worker
       worker = Cloudtasker::Worker.from_hash(payload) || raise(InvalidWorkerError)
-      worker.execute
+
+      # Yied worker
+      resp = yield(worker)
+
+      # Schedule args payload deletion after job has been successfully processed
+      # Note: we expire the key instead of deleting it immediately in case the job
+      # succeeds but is considered as failed by Cloud Task due to network interruption.
+      # In such case the job is likely to be re-processed soon after.
+      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key && !worker.job_reenqueued
+
+      resp
+    rescue DeadWorkerError => e
+      # Delete stored args payload if job is dead
+      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
+      raise(e)
+    end
+
+    #
+    # Return the argument payload key (if present) along with the actual worker payload.
+    #
+    # If the payload was stored in Redis then retrieve it.
+    #
+    # @return [Hash] Hash
+    #
+    def self.extract_payload(input_payload)
+      # Get references
+      payload = JSON.parse(input_payload.to_json, symbolize_names: true)
+      args_payload_id = payload.delete(:job_args_payload_id)
+      args_payload_key = args_payload_id ? key([REDIS_PAYLOAD_NAMESPACE, args_payload_id].join('/')) : nil
+
+      # Retrieve the actual worker args payload
+      args_payload = args_payload_key ? redis.fetch(args_payload_key) : payload[:job_args]
+
+      # Return the payload
+      {
+        args_payload_key: args_payload_key,
+        payload: payload.merge(job_args: args_payload)
+      }
     end
 
     #
@@ -51,6 +148,47 @@ module Cloudtasker
       }
     end
 
+    #
+    # Return true if the worker args must be stored in Redis.
+    #
+    # @return [Boolean] True if the payload must be stored in redis.
+    #
+    def store_payload_in_redis?
+      Cloudtasker.config.redis_payload_storage_threshold &&
+        worker.job_args.to_json.bytesize > (Cloudtasker.config.redis_payload_storage_threshold * 1024)
+    end
+
+    #
+    # Return the payload to use for job arguments. This payload
+    # is merged inside the #worker_payload.
+    #
+    # If the argument payload must be stored in Redis then returns:
+    # `{ job_args_payload_id: <worker_id> }`
+    #
+    # If the argument payload must be natively handled by the backend
+    # then returns:
+    # `{ job_args: [...] }`
+    #
+    # @return [Hash] The worker args payload.
+    #
+    def worker_args_payload
+      @worker_args_payload ||= begin
+        if store_payload_in_redis?
+          # Store payload in Redis
+          self.class.redis.write(
+            self.class.key([REDIS_PAYLOAD_NAMESPACE, worker.job_id].join('/')),
+            worker.job_args
+          )
+
+          # Return reference to args payload
+          { job_args_payload_id: worker.job_id }
+        else
+          # Return regular job args payload
+          { job_args: worker.job_args }
+        end
+      end
+    end
+
     #
     # Return the task payload that Google Task will eventually
     # send to the job processor.
@@ -68,9 +206,8 @@ module Cloudtasker
         worker: worker.job_class_name,
         job_queue: worker.job_queue,
         job_id: worker.job_id,
-        job_args: worker.job_args,
         job_meta: worker.job_meta.to_h
-      }
+      }.merge(worker_args_payload)
     end
 
     #

data/lib/cloudtasker/worker_logger.rb

@@ -141,8 +141,7 @@ module Cloudtasker
     # @param [Proc] &block Optional context block.
     #
     def log_message(level, msg, &block)
-      # Merge log-specific context into worker-specific context
-      payload_block = ->(*_args) { log_block.call.merge(block&.call || {}) }
+      payload_block = block || log_block
 
       # ActiveSupport::Logger does not support passing a payload through a block on top
       # of a message.

data/lib/cloudtasker.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'active_support/core_ext/string/inflections'
-require 'active_support/core_ext/object/try'
 
 require 'cloudtasker/version'
 require 'cloudtasker/config'