cloudtasker 0.9.2 → 0.10.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f86bcf36c7219610dea98de24a9c19ac055a1d205637c94e8a80c8f4c8e7e32b
-  data.tar.gz: f0d86b959b129cb9e4f8c9620b52b5e4a54c190b3cf6be63b70bd1f50f5345c9
+  metadata.gz: d4cba7de3e429d612adf6c9c2f4424b6ef73db39d4db93b70804800300011e1b
+  data.tar.gz: 3775cdf3f16430cf8decd49dfc28be9e26f0ef6a63d45224bdc5ed11b13a86fc
 SHA512:
-  metadata.gz: ebbb141e2929c1dae0e501230b349a6eddfc69ece423c161011489641aeac2ba9f1a48f69abc28ca9a6261ca8da23b90c7fca25f13cd1513777013e961d5c9c4
-  data.tar.gz: 2b1254f1dd38d9fef70d68af840976e0720698e9ea49470c24e73f069da110b3f284db51f6e5823c2a1df5ae17c1f0d32c7d0dd8ac5e5e82420faa0f47e8f462
+  metadata.gz: 5e2e15dc54fad72e3508763855a99804b591126968ccbcaccd8211d51b8b1e28bf6d2907c746f2b14c53c3c065ce1eb06871f30b39419df1f3d7b8a4e1b1fded
+  data.tar.gz: a2808491a7251b5212587351deb84a99f688a62f7a54c3f5b9c8ebe3a3b6a1ca6adda6ffe898424069223cef8260f1a49c8a086010c0e59dedc5e50ed371e830

data/README.md

@@ -224,7 +224,7 @@ Cloudtasker.configure do |config|
   # 
   # config.max_retries = 10
 
-  # 
+  #
   # Specify the redis connection hash.
   #
   # This is ONLY required in development for the Cloudtasker local server and in
@@ -235,6 +235,24 @@ Cloudtasker.configure do |config|
   # Default: redis-rb connects to redis://127.0.0.1:6379/0
   #
   # config.redis = { url: 'redis://localhost:6379/5' }
+
+  #
+  # Set to true to store job arguments in Redis instead of sending arguments as part
+  # of the job payload to Google Cloud Tasks.
+  #
+  # This is useful if you expect to process jobs with payloads exceeding 100KB, which
+  # is the limit enforced by Google Cloud Tasks.
+  #
+  # You can set this configuration parameter to a KB value if you want to store jobs
+  # args in redis only if the JSONified arguments payload exceeds that threshold.
+  #
+  # Default: false
+  #
+  # Store all job payloads in Redis:
+  # config.store_payloads_in_redis = true
+  #
+  # Store all job payloads in Redis exceeding 50 KB:
+  # config.store_payloads_in_redis = 50
 end
 ```
 
@@ -639,6 +657,27 @@ Google Cloud Tasks enforces a limit of 100 KB for job payloads. Taking into acco
 
 Any excessive job payload (> 100 KB) will raise a `Cloudtasker::MaxTaskSizeExceededError`, both in production and development mode.
 
+#### Option 1: Use Cloudtasker optional support for payload storage in Redis
+Cloudtasker provides optional support for storing argument payloads in Redis instead of sending them to Google Cloud Tasks.
+
+To enable it simply put the following in your Cloudtasker initializer:
+```ruby
+# config/initializers/cloudtasker.rb
+
+Cloudtasker.configure do |config|
+  # Enable Redis support. Specify your redis connection
+  config.redis = { url: 'redis://localhost:6379/5' }
+
+  # Store all job payloads in Redis:
+  config.store_payloads_in_redis = true
+
+  # OR: store all job payloads in Redis exceeding 50 KB:
+  # config.store_payloads_in_redis = 50
+end
+```
+
+#### Option 2: Do it yourself solution
+
 If you feel that a job payload is going to get big, prefer to store the payload using a datastore (e.g. Redis) and pass a reference to the job to retrieve the payload inside your job `perform` method.
 
 E.g. Define a job like this

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -146,7 +146,7 @@ module Cloudtasker
       #
       def self.delete(id)
         client.delete_task(id)
-      rescue Google::Gax::RetryError, GRPC::NotFound
+      rescue Google::Gax::RetryError, GRPC::NotFound, Google::Gax::PermissionDeniedError
         nil
       end
 

data/lib/cloudtasker/backend/memory_task.rb

@@ -7,6 +7,7 @@ module Cloudtasker
     # Manage local tasks pushed to memory.
     # Used for testing.
     class MemoryTask
+      attr_accessor :job_retries
       attr_reader :id, :http_request, :schedule_time, :queue
 
       #
@@ -18,17 +19,6 @@ module Cloudtasker
         @queue ||= []
       end
 
-      #
-      # Return the workers currently in the queue.
-      #
-      # @param [String] worker_class_name Filter jobs on worker class name.
-      #
-      # @return [Array<Cloudtasker::Worker] The list of workers
-      #
-      def self.jobs(worker_class_name = nil)
-        all(worker_class_name).map(&:worker)
-      end
-
       #
       # Run all Tasks in the queue. Optionally filter which tasks to run based
       # on the worker class name.
@@ -116,11 +106,12 @@ module Cloudtasker
       # @param [Hash] http_request The HTTP request content.
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       #
-      def initialize(id:, http_request:, schedule_time: nil, queue: nil)
+      def initialize(id:, http_request:, schedule_time: nil, queue: nil, job_retries: 0)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
         @queue = queue
+        @job_retries = job_retries || 0
       end
 
       #
@@ -155,26 +146,20 @@ module Cloudtasker
         }
       end
 
-      #
-      # Return the worker attached to this task.
-      #
-      # @return [Cloudtasker::Worker] The task worker.
-      #
-      def worker
-        @worker ||= Worker.from_hash(payload)
-      end
-
       #
       # Execute the task.
       #
       # @return [Any] The return value of the worker perform method.
       #
       def execute
-        resp = worker.execute
+        # Execute worker
+        resp = WorkerHandler.with_worker_handling(payload, &:execute)
+
+        # Delete task
         self.class.delete(id)
         resp
       rescue StandardError
-        worker.job_retries += 1
+        self.job_retries += 1
       end
 
       #

data/lib/cloudtasker/config.rb

@@ -5,7 +5,7 @@ require 'logger'
 module Cloudtasker
   # Holds cloudtasker configuration. See Cloudtasker#configure
   class Config
-    attr_accessor :redis
+    attr_accessor :redis, :store_payloads_in_redis
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
                 :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
 
@@ -54,6 +54,21 @@ module Cloudtasker
       Please specify a secret in the cloudtasker initializer or add Rails secret_key_base in your credentials
     DOC
 
+    #
+    # Return the threshold above which job arguments must be stored
+    # in Redis instead of being sent to the backend as part of the job
+    # payload.
+    #
+    # Return nil if redis payload storage is disabled.
+    #
+    # @return [Integer, nil] The threshold above which payloads will be stored in Redis.
+    #
+    def redis_payload_storage_threshold
+      return nil unless store_payloads_in_redis
+
+      store_payloads_in_redis.respond_to?(:to_i) ? store_payloads_in_redis.to_i : 0
+    end
+
     #
     # The number of times jobs will be retried. This number of
     # retries does not include failures due to the application being unreachable.

data/lib/cloudtasker/testing.rb

@@ -114,10 +114,10 @@ module Cloudtasker
       #
       # Return all jobs related to this worker class.
       #
-      # @return [Array<Cloudtasker::Worker] The list of workers
+      # @return [Array<Cloudtasker::Backend::MemoryTask>] The list of tasks
       #
       def jobs
-        Backend::MemoryTask.jobs(to_s)
+        Backend::MemoryTask.all(to_s)
       end
 
       #

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.9.2'
+  VERSION = '0.10.rc1'
 end

data/lib/cloudtasker/worker_handler.rb

@@ -10,16 +10,113 @@ module Cloudtasker
     # Alrogith used to sign the verification token
     JWT_ALG = 'HS256'
 
+    # Sub-namespace to use for redis keys when storing
+    # payloads in Redis
+    REDIS_PAYLOAD_NAMESPACE = 'payload'
+
+    # Arg payload cache keys get expired instead of deleted
+    # in case jobs are re-processed due to connection interruption
+    # (job is successful but Cloud Task considers it as failed due
+    # to network interruption)
+    ARGS_PAYLOAD_CLEANUP_TTL = 3600 # 1 hour
+
+    #
+    # Return a namespaced key
+    #
+    # @param [String, Symbol] val The key to namespace
+    #
+    # @return [String] The namespaced key.
+    #
+    def self.key(val)
+      return nil if val.nil?
+
+      [to_s.underscore, val.to_s].join('/')
+    end
+
+    #
+    # Return the cloudtasker redis client
+    #
+    # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
+    #
+    def self.redis
+      @redis ||= begin
+        require 'cloudtasker/redis_client'
+        RedisClient.new
+      end
+    end
+
     #
     # Execute a task worker from a task payload
     #
-    # @param [Hash] payload The Cloud Task payload.
+    # @param [Hash] input_payload The Cloud Task payload.
     #
     # @return [Any] The return value of the worker perform method.
     #
-    def self.execute_from_payload!(payload)
+    def self.execute_from_payload!(input_payload)
+      with_worker_handling(input_payload, &:execute)
+    end
+
+    # TODO: do not delete redis payload if job has been re-enqueued
+    # worker.job_reenqueued
+    #
+    # Idea: change with_worker_handling to with_worker_handling and build the worker
+    # inside the with_worker_handling block.
+    #
+    # Local middleware used to retrieve the job arg payload from cache
+    # if a arg payload reference is present.
+    #
+    # @param [Hash] payload The full job payload
+    #
+    # @yield [Hash] The actual payload to use to process the job.
+    #
+    # @return [Any] The block result
+    #
+    def self.with_worker_handling(input_payload)
+      # Extract payload information
+      extracted_payload = extract_payload(input_payload)
+      payload = extracted_payload[:payload]
+      args_payload_key = extracted_payload[:args_payload_key]
+
+      # Build worker
       worker = Cloudtasker::Worker.from_hash(payload) || raise(InvalidWorkerError)
-      worker.execute
+
+      # Yied worker
+      resp = yield(worker)
+
+      # Schedule args payload deletion after job has been successfully processed
+      # Note: we expire the key instead of deleting it immediately in case the job
+      # succeeds but is considered as failed by Cloud Task due to network interruption.
+      # In such case the job is likely to be re-processed soon after.
+      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key && !worker.job_reenqueued
+
+      resp
+    rescue DeadWorkerError => e
+      # Delete stored args payload if job is dead
+      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
+      raise(e)
+    end
+
+    #
+    # Return the argument payload key (if present) along with the actual worker payload.
+    #
+    # If the payload was stored in Redis then retrieve it.
+    #
+    # @return [Hash] Hash
+    #
+    def self.extract_payload(input_payload)
+      # Get references
+      payload = JSON.parse(input_payload.to_json, symbolize_names: true)
+      args_payload_id = payload.delete(:job_args_payload_id)
+      args_payload_key = args_payload_id ? key([REDIS_PAYLOAD_NAMESPACE, args_payload_id].join('/')) : nil
+
+      # Retrieve the actual worker args payload
+      args_payload = args_payload_key ? redis.fetch(args_payload_key) : payload[:job_args]
+
+      # Return the payload
+      {
+        args_payload_key: args_payload_key,
+        payload: payload.merge(job_args: args_payload)
+      }
     end
 
     #
@@ -51,6 +148,47 @@ module Cloudtasker
       }
     end
 
+    #
+    # Return true if the worker args must be stored in Redis.
+    #
+    # @return [Boolean] True if the payload must be stored in redis.
+    #
+    def store_payload_in_redis?
+      Cloudtasker.config.redis_payload_storage_threshold &&
+        worker.job_args.to_json.bytesize > (Cloudtasker.config.redis_payload_storage_threshold * 1024)
+    end
+
+    #
+    # Return the payload to use for job arguments. This payload
+    # is merged inside the #worker_payload.
+    #
+    # If the argument payload must be stored in Redis then returns:
+    # `{ job_args_payload_id: <worker_id> }`
+    #
+    # If the argument payload must be natively handled by the backend
+    # then returns:
+    # `{ job_args: [...] }`
+    #
+    # @return [Hash] The worker args payload.
+    #
+    def worker_args_payload
+      @worker_args_payload ||= begin
+        if store_payload_in_redis?
+          # Store payload in Redis
+          self.class.redis.write(
+            self.class.key([REDIS_PAYLOAD_NAMESPACE, worker.job_id].join('/')),
+            worker.job_args
+          )
+
+          # Return reference to args payload
+          { job_args_payload_id: worker.job_id }
+        else
+          # Return regular job args payload
+          { job_args: worker.job_args }
+        end
+      end
+    end
+
     #
     # Return the task payload that Google Task will eventually
     # send to the job processor.
@@ -68,9 +206,8 @@ module Cloudtasker
         worker: worker.job_class_name,
         job_queue: worker.job_queue,
         job_id: worker.job_id,
-        job_args: worker.job_args,
         job_meta: worker.job_meta.to_h
-      }
+      }.merge(worker_args_payload)
     end
 
     #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.rc1
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-04 00:00:00.000000000 Z
+date: 2020-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -364,12 +364,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.9
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: Background jobs for Ruby using Google Cloud Tasks (beta)