cloudtasker 0.12.2 → 0.13.0

data/lib/cloudtasker/backend/google_cloud_task_v2.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require 'google/cloud/tasks'
+require 'google/protobuf/timestamp_pb'
+require 'retriable'
+
+module Cloudtasker
+  module Backend
+    # Manage tasks pushed to GCP Cloud Task
+    class GoogleCloudTaskV2
+      attr_accessor :gcp_task
+
+      #
+      # Create the queue configured in Cloudtasker if it does not already exist.
+      #
+      # @param [String] :name The queue name
+      # @param [Integer] :concurrency The queue concurrency
+      # @param [Integer] :retries The number of retries for the queue
+      #
+      # @return [Google::Cloud::Tasks::V2::Queue] The queue
+      #
+      def self.setup_queue(name: nil, concurrency: nil, retries: nil)
+        # Build full queue path
+        queue_name = name || Cloudtasker::Config::DEFAULT_JOB_QUEUE
+        full_queue_name = queue_path(queue_name)
+
+        # Try to get existing queue
+        client.get_queue(name: full_queue_name)
+      rescue Google::Cloud::NotFoundError
+        # Extract options
+        queue_concurrency = (concurrency || Cloudtasker::Config::DEFAULT_QUEUE_CONCURRENCY).to_i
+        queue_retries = (retries || Cloudtasker::Config::DEFAULT_QUEUE_RETRIES).to_i
+
+        # Create queue on 'not found' error
+        client.create_queue(
+          parent: client.location_path(project: config.gcp_project_id, location: config.gcp_location_id),
+          queue: {
+            name: full_queue_name,
+            retry_config: { max_attempts: queue_retries },
+            rate_limits: { max_concurrent_dispatches: queue_concurrency }
+          }
+        )
+      end
+
+      #
+      # Return the Google Cloud Task client.
+      #
+      # @return [Google::Cloud::Tasks::V2::CloudTasks::Client] The Google Cloud Task client.
+      #
+      def self.client
+        @client ||= ::Google::Cloud::Tasks.cloud_tasks
+      end
+
+      #
+      # Return the cloudtasker configuration. See Cloudtasker#configure.
+      #
+      # @return [Cloudtasker::Config] The library configuration.
+      #
+      def self.config
+        Cloudtasker.config
+      end
+
+      #
+      # Return the fully qualified path for the Cloud Task queue.
+      #
+      # @param [String] queue_name The relative name of the queue.
+      #
+      # @return [String] The queue path.
+      #
+      def self.queue_path(queue_name)
+        client.queue_path(
+          project: config.gcp_project_id,
+          location: config.gcp_location_id,
+          queue: [config.gcp_queue_prefix, queue_name].join('-')
+        )
+      end
+
+      #
+      # Return a protobuf timestamp specifying how to wait
+      # before running a task.
+      #
+      # @param [Integer, nil] schedule_time A unix timestamp.
+      #
+      # @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
+      #
+      def self.format_schedule_time(schedule_time)
+        return nil unless schedule_time
+
+        # Generate protobuf timestamp
+        Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
+      end
+
+      #
+      # Format the job payload sent to Cloud Tasks.
+      #
+      # @param [Hash] hash The worker payload.
+      #
+      # @return [Hash] The Cloud Task payloadd.
+      #
+      def self.format_task_payload(payload)
+        payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
+
+        # Format schedule time to Google Protobuf timestamp
+        payload[:schedule_time] = format_schedule_time(payload[:schedule_time])
+
+        # Encode job content to support UTF-8.
+        # Google Cloud Task expect content to be ASCII-8BIT compatible (binary)
+        payload[:http_request][:headers] ||= {}
+        payload[:http_request][:headers][Cloudtasker::Config::CONTENT_TYPE_HEADER] = 'text/json'
+        payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
+        payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
+
+        payload
+      end
+
+      #
+      # Find a task by id.
+      #
+      # @param [String] id The task id.
+      #
+      # @return [Cloudtasker::Backend::GoogleCloudTaskV2, nil] The retrieved task.
+      #
+      def self.find(id)
+        resp = with_gapi_retries { client.get_task(name: id) }
+        resp ? new(resp) : nil
+      rescue Google::Cloud::NotFoundError
+        # The ID does not exist
+        nil
+      end
+
+      #
+      # Create a new task.
+      #
+      # @param [Hash] payload The task payload.
+      #
+      # @return [Cloudtasker::Backend::GoogleCloudTaskV2, nil] The created task.
+      #
+      def self.create(payload)
+        payload = format_task_payload(payload)
+
+        # Infer full queue name
+        relative_queue = payload.delete(:queue)
+        full_queue = queue_path(relative_queue)
+
+        # Create task
+        resp = with_gapi_retries { client.create_task(parent: full_queue, task: payload) }
+        resp ? new(resp) : nil
+      end
+
+      #
+      # Delete a task by id.
+      #
+      # @param [String] id The id of the task.
+      #
+      def self.delete(id)
+        with_gapi_retries { client.delete_task(name: id) }
+      rescue Google::Cloud::NotFoundError, Google::Cloud::PermissionDeniedError
+        # The ID does not exist
+        nil
+      end
+
+      #
+      # Helper method encapsulating the retry strategy for Google API calls
+      #
+      def self.with_gapi_retries
+        Retriable.retriable(on: [Google::Cloud::UnavailableError], tries: 3) do
+          yield
+        end
+      end
+
+      #
+      # Build a new instance of the class.
+      #
+      # @param [Google::Cloud::Tasks::V2::Task] resp The GCP Cloud Task response
+      #
+      def initialize(gcp_task)
+        @gcp_task = gcp_task
+      end
+
+      #
+      # Return the relative queue (queue name minus prefix) the task is in.
+      #
+      # @return [String] The relative queue name
+      #
+      def relative_queue
+        gcp_task
+          .name
+          .match(%r{/queues/([^/]+)})
+          &.captures
+          &.first
+          &.sub("#{self.class.config.gcp_queue_prefix}-", '')
+      end
+
+      #
+      # Return a hash description of the task.
+      #
+      # @return [Hash] A hash description of the task.
+      #
+      def to_h
+        {
+          id: gcp_task.name,
+          http_request: gcp_task.to_h[:http_request],
+          schedule_time: gcp_task.to_h.dig(:schedule_time, :seconds).to_i,
+          retries: gcp_task.to_h[:response_count],
+          queue: relative_queue
+        }
+      end
+    end
+  end
+end

data/lib/cloudtasker/backend/memory_task.rb

@@ -62,7 +62,7 @@ module Cloudtasker
         payload = payload.merge(schedule_time: payload[:schedule_time].to_i)
 
         # Save task
-        task = new(payload.merge(id: id))
+        task = new(**payload.merge(id: id))
         queue << task
 
         # Execute task immediately if in testing and inline mode enabled

data/lib/cloudtasker/backend/redis_task.rb

@@ -89,7 +89,7 @@ module Cloudtasker
         # Save job
         redis.write(key(id), payload)
         redis.sadd(key, id)
-        new(payload.merge(id: id))
+        new(**payload.merge(id: id))
       end
 
       #
@@ -103,7 +103,7 @@ module Cloudtasker
         gid = key(id)
         return nil unless (payload = redis.fetch(gid))
 
-        new(payload.merge(id: id))
+        new(**payload.merge(id: id))
       end
 
       #
@@ -172,8 +172,12 @@ module Cloudtasker
       # Retry the task later.
       #
       # @param [Integer] interval The delay in seconds before retrying the task
+      # @param [Hash] opts Additional options
+      # @option opts [Boolean] :is_error Increase number of retries. Default to true.
       #
-      def retry_later(interval, is_error: true)
+      def retry_later(interval, opts = {})
+        is_error = opts.to_h.fetch(:is_error, true)
+
         redis.write(
           gid,
           retries: is_error ? retries + 1 : retries,

data/lib/cloudtasker/cloud_task.rb

@@ -8,7 +8,12 @@ module Cloudtasker
     #
     # The backend to use for cloud tasks.
     #
-    # @return [Cloudtasker::Backend::GoogleCloudTask, Cloudtasker::Backend::RedisTask] The cloud task backend.
+    # @return [
+    #   Backend::MemoryTask,
+    #   Cloudtasker::Backend::GoogleCloudTaskV1,
+    #   Cloudtasker::Backend::GoogleCloudTaskV2,
+    #   Cloudtasker::Backend::RedisTask
+    # ] The cloud task backend.
     #
     def self.backend
       # Re-evaluate backend every time if testing mode enabled
@@ -22,12 +27,45 @@ module Cloudtasker
           require 'cloudtasker/backend/redis_task'
           Backend::RedisTask
         else
-          require 'cloudtasker/backend/google_cloud_task'
-          Backend::GoogleCloudTask
+          gct_backend
         end
       end
     end
 
+    #
+    # Return the GoogleCloudTaskV* backend to use based on the version
+    # of the currently installed google-cloud-tasks gem
+    #
+    # @return [
+    #   Cloudtasker::Backend::GoogleCloudTaskV1,
+    #   Cloudtasker::Backend::GoogleCloudTaskV2
+    # ] The google cloud task backend.
+    #
+    def self.gct_backend
+      @gct_backend ||= begin
+        if !defined?(Google::Cloud::Tasks::VERSION) || Google::Cloud::Tasks::VERSION < '2'
+          require 'cloudtasker/backend/google_cloud_task_v1'
+          Backend::GoogleCloudTaskV1
+        else
+          require 'cloudtasker/backend/google_cloud_task_v2'
+          Backend::GoogleCloudTaskV2
+        end
+      end
+    end
+
+    #
+    # Create the google cloud task queue based on provided parameters if it does not exist already.
+    #
+    # @param [String] :name The queue name
+    # @param [Integer] :concurrency The queue concurrency
+    # @param [Integer] :retries The number of retries for the queue
+    #
+    # @return [Google::Cloud::Tasks::V2::Queue, Google::Cloud::Tasks::V2beta3::Queue] The queue
+    #
+    def self.setup_production_queue(**kwargs)
+      gct_backend.setup_queue(**kwargs)
+    end
+
     #
     # Find a cloud task by id.
     #
@@ -37,7 +75,7 @@ module Cloudtasker
     #
     def self.find(id)
       payload = backend.find(id)&.to_h
-      payload ? new(payload) : nil
+      payload ? new(**payload) : nil
     end
 
     #
@@ -51,7 +89,7 @@ module Cloudtasker
       raise MaxTaskSizeExceededError if payload.to_json.bytesize > Config::MAX_TASK_SIZE
 
       resp = backend.create(payload)&.to_h
-      resp ? new(resp) : nil
+      resp ? new(**resp) : nil
     end
 
     #

data/lib/cloudtasker/cron/schedule.rb

@@ -62,7 +62,7 @@ module Cloudtasker
       def self.load_from_hash!(hash)
         schedules = hash.map do |id, config|
           schedule_config = JSON.parse(config.to_json, symbolize_names: true).merge(id: id.to_s)
-          create(schedule_config)
+          create(**schedule_config)
         end
 
         # Remove existing schedules which are not part of the list
@@ -79,7 +79,7 @@ module Cloudtasker
       def self.create(**opts)
         redis.with_lock(key(opts[:id])) do
           config = find(opts[:id]).to_h.merge(opts)
-          new(config).tap(&:save)
+          new(**config).tap(&:save)
         end
       end
 
@@ -93,7 +93,7 @@ module Cloudtasker
       def self.find(id)
         return nil unless (schedule_config = redis.fetch(key(id)))
 
-        new(schedule_config)
+        new(**schedule_config)
       end
 
       #
@@ -251,9 +251,9 @@ module Cloudtasker
       #
       # Buld edit the object attributes.
       #
-      # @param [Hash] **opts The attributes to edit.
+      # @param [Hash] opts The attributes to edit.
       #
-      def assign_attributes(**opts)
+      def assign_attributes(opts)
         opts
           .select { |k, _| instance_variables.include?("@#{k}".to_sym) }
           .each { |k, v| instance_variable_set("@#{k}", v) }
@@ -262,9 +262,9 @@ module Cloudtasker
       #
       # Edit the object attributes and save the object in Redis.
       #
-      # @param [Hash] **opts The attributes to edit.
+      # @param [Hash] opts The attributes to edit.
       #
-      def update(**opts)
+      def update(opts)
         assign_attributes(opts)
         save
       end

data/lib/cloudtasker/redis_client.rb

@@ -132,22 +132,47 @@ module Cloudtasker
       list
     end
 
-    #
-    # Delegate all methods to the redis client.
-    #
-    # @param [String, Symbol] name The method to delegate.
-    # @param [Array<any>] *args The list of method arguments.
-    # @param [Proc] &block Block passed to the method.
-    #
-    # @return [Any] The method return value
-    #
-    def method_missing(name, *args, &block)
-      if Redis.method_defined?(name)
-        client.with { |c| c.send(name, *args, &block) }
-      else
-        super
+    # rubocop:disable Style/MissingRespondToMissing
+    if RUBY_VERSION < '3'
+      #
+      # Delegate all methods to the redis client.
+      # Old delegation method.
+      #
+      # @param [String, Symbol] name The method to delegate.
+      # @param [Array<any>] *args The list of method positional arguments.
+      # @param [Hash<any>] *kwargs The list of method keyword arguments.
+      # @param [Proc] &block Block passed to the method.
+      #
+      # @return [Any] The method return value
+      #
+      def method_missing(name, *args, &block)
+        if Redis.method_defined?(name)
+          client.with { |c| c.send(name, *args, &block) }
+        else
+          super
+        end
+      end
+    else
+      #
+      # Delegate all methods to the redis client.
+      # Ruby 3 delegation method style.
+      #
+      # @param [String, Symbol] name The method to delegate.
+      # @param [Array<any>] *args The list of method positional arguments.
+      # @param [Hash<any>] *kwargs The list of method keyword arguments.
+      # @param [Proc] &block Block passed to the method.
+      #
+      # @return [Any] The method return value
+      #
+      def method_missing(name, *args, **kwargs, &block)
+        if Redis.method_defined?(name)
+          client.with { |c| c.send(name, *args, **kwargs, &block) }
+        else
+          super
+        end
       end
     end
+    # rubocop:enable Style/MissingRespondToMissing
 
     #
     # Check if the class respond to a certain method.

data/lib/cloudtasker/unique_job/job.rb

@@ -14,10 +14,11 @@ module Cloudtasker
       # Build a new instance of the class.
       #
       # @param [Cloudtasker::Worker] worker The worker at hand
+      # @param [Hash] worker The worker options
       #
-      def initialize(worker, **kwargs)
+      def initialize(worker, opts = {})
         @worker = worker
-        @call_opts = kwargs
+        @call_opts = opts
       end
 
       #

data/lib/cloudtasker/unique_job/middleware/client.rb

@@ -3,9 +3,10 @@
 module Cloudtasker
   module UniqueJob
     module Middleware
+      # TODO: kwargs to job otherwise it won't get the time_at
       # Client middleware, invoked when jobs are scheduled
       class Client
-        def call(worker, **_kwargs)
+        def call(worker, _opts = {})
           Job.new(worker).lock_instance.schedule { yield }
         end
       end

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.12.2'
+  VERSION = '0.13.0'
 end

data/lib/cloudtasker/worker.rb

@@ -47,7 +47,7 @@ module Cloudtasker
       return nil unless worker_klass.include?(self)
 
       # Return instantiated worker
-      worker_klass.new(payload.slice(:job_queue, :job_args, :job_id, :job_meta, :job_retries, :task_id))
+      worker_klass.new(**payload.slice(:job_queue, :job_args, :job_id, :job_meta, :job_retries, :task_id))
     rescue NameError
       nil
     end
@@ -121,7 +121,7 @@ module Cloudtasker
       # @return [Cloudtasker::CloudTask] The Google Task response
       #
       def schedule(args: nil, time_in: nil, time_at: nil, queue: nil)
-        new(job_args: args, job_queue: queue).schedule({ interval: time_in, time_at: time_at }.compact)
+        new(job_args: args, job_queue: queue).schedule(**{ interval: time_in, time_at: time_at }.compact)
       end
 
       #
@@ -239,7 +239,7 @@ module Cloudtasker
     #
     def schedule(**args)
       # Evaluate when to schedule the job
-      time_at = schedule_time(args)
+      time_at = schedule_time(**args)
 
       # Schedule job through client middlewares
       Cloudtasker.config.client_middleware.invoke(self, time_at: time_at) do

data/lib/cloudtasker/worker_wrapper.rb

@@ -27,7 +27,7 @@ module Cloudtasker
     #
     def initialize(worker_name:, **opts)
       @worker_name = worker_name
-      super(opts)
+      super(**opts)
     end
 
     #

data/lib/tasks/setup_queue.rake

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'cloudtasker/backend/google_cloud_task'
 require 'cloudtasker/config'
+require 'cloudtasker/cloud_task'
 
 ENV['GOOGLE_AUTH_SUPPRESS_CREDENTIALS_WARNINGS'] ||= 'true'
 
@@ -11,7 +11,7 @@ namespace :cloudtasker do
     "concurrency=#{Cloudtasker::Config::DEFAULT_QUEUE_CONCURRENCY}, " \
     "retries=#{Cloudtasker::Config::DEFAULT_QUEUE_RETRIES})"
   task setup_queue: :environment do
-    puts Cloudtasker::Backend::GoogleCloudTask.setup_queue(
+    puts Cloudtasker::CloudTask.setup_production_queue(
       name: ENV['name'],
       concurrency: ENV['concurrency'],
       retries: ENV['retries']