cloudtasker 0.6.0 → 0.9.0

data/lib/cloudtasker/backend/memory_task.rb

@@ -7,7 +7,7 @@ module Cloudtasker
     # Manage local tasks pushed to memory.
     # Used for testing.
     class MemoryTask
-      attr_reader :id, :http_request, :schedule_time
+      attr_reader :id, :http_request, :schedule_time, :queue
 
       #
       # Return the task queue. A worker class name
@@ -116,10 +116,11 @@ module Cloudtasker
       # @param [Hash] http_request The HTTP request content.
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       #
-      def initialize(id:, http_request:, schedule_time: nil)
+      def initialize(id:, http_request:, schedule_time: nil, queue: nil)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
+        @queue = queue
       end
 
       #
@@ -149,7 +150,8 @@ module Cloudtasker
         {
           id: id,
           http_request: http_request,
-          schedule_time: schedule_time.to_i
+          schedule_time: schedule_time.to_i,
+          queue: queue
         }
       end
 

data/lib/cloudtasker/backend/redis_task.rb

@@ -7,17 +7,17 @@ module Cloudtasker
   module Backend
     # Manage local tasks pushed to Redis
     class RedisTask
-      attr_reader :id, :http_request, :schedule_time, :retries
+      attr_reader :id, :http_request, :schedule_time, :retries, :queue
 
       RETRY_INTERVAL = 20 # seconds
 
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -48,20 +48,26 @@ module Cloudtasker
       #
       # Reeturn all tasks ready to process.
       #
+      # @param [String] queue The queue to retrieve items from.
+      #
       # @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks ready to process.
       #
-      def self.ready_to_process
-        all.select { |e| e.schedule_time <= Time.now }
+      def self.ready_to_process(queue = nil)
+        list = all.select { |e| e.schedule_time <= Time.now }
+        list = list.select { |e| e.queue == queue } if queue
+        list
       end
 
       #
       # Retrieve and remove a task from the queue.
       #
+      # @param [String] queue The queue to retrieve items from.
+      #
       # @return [Cloudtasker::Backend::RedisTask] A task ready to process.
       #
-      def self.pop
+      def self.pop(queue = nil)
         redis.with_lock('cloudtasker/server') do
-          ready_to_process.first&.tap(&:destroy)
+          ready_to_process(queue).first&.tap(&:destroy)
         end
       end
 
@@ -110,11 +116,12 @@ module Cloudtasker
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       # @param [Integer] retries The number of times the job failed.
       #
-      def initialize(id:, http_request:, schedule_time: nil, retries: 0)
+      def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
         @retries = retries || 0
+        @queue = queue || Cloudtasker::Config::DEFAULT_JOB_QUEUE
       end
 
       #
@@ -136,7 +143,8 @@ module Cloudtasker
           id: id,
           http_request: http_request,
           schedule_time: schedule_time.to_i,
-          retries: retries
+          retries: retries,
+          queue: queue
         }
       end
 
@@ -155,10 +163,13 @@ module Cloudtasker
       # @param [Integer] interval The delay in seconds before retrying the task
       #
       def retry_later(interval, is_error: true)
-        redis.write(gid,
-                    retries: is_error ? retries + 1 : retries,
-                    http_request: http_request,
-                    schedule_time: (Time.now + interval).to_i)
+        redis.write(
+          gid,
+          retries: is_error ? retries + 1 : retries,
+          http_request: http_request,
+          schedule_time: (Time.now + interval).to_i,
+          queue: queue
+        )
       end
 
       #

data/lib/cloudtasker/batch/batch_progress.rb

@@ -77,7 +77,16 @@ module Cloudtasker
       # @return [Integer] The number of jobs pending.
       #
       def pending
-        total - completed - dead
+        total - done
+      end
+
+      #
+      # Return the number of jobs completed or dead.
+      #
+      # @return [Integer] The number of jobs done.
+      #
+      def done
+        completed + dead
       end
 
       #
@@ -88,7 +97,7 @@ module Cloudtasker
       def percent
         return 0 if total.zero?
 
-        pending.to_f / total
+        (done.to_f / total) * 100
       end
 
       #

data/lib/cloudtasker/batch/job.rb

@@ -16,10 +16,10 @@ module Cloudtasker
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -87,7 +87,7 @@ module Cloudtasker
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def redis
         self.class.redis
@@ -180,9 +180,23 @@ module Cloudtasker
       # @return [Array<Cloudtasker::Worker>] The updated list of jobs.
       #
       def add(worker_klass, *args)
+        add_to_queue(worker.job_queue, worker_klass, *args)
+      end
+
+      #
+      # Add a worker to the batch using a specific queue.
+      #
+      # @param [String, Symbol] queue The name of the queue
+      # @param [Class] worker_klass The worker class.
+      # @param [Array<any>] *args The worker arguments.
+      #
+      # @return [Array<Cloudtasker::Worker>] The updated list of jobs.
+      #
+      def add_to_queue(queue, worker_klass, *args)
         jobs << worker_klass.new(
           job_args: args,
-          job_meta: { key(:parent_id) => batch_id }
+          job_meta: { key(:parent_id) => batch_id },
+          job_queue: queue
         )
       end
 

data/lib/cloudtasker/cli.rb

@@ -70,7 +70,7 @@ module Cloudtasker
     #
     # Run the cloudtasker development server.
     #
-    def run
+    def run(opts = {})
       boot_system
 
       # Print banner
@@ -90,16 +90,17 @@ module Cloudtasker
       logger.info "[Cloudtasker/Server] Running in #{RUBY_DESCRIPTION}"
 
       # Wait for signals
-      wait_for_signal(self_read)
+      run_server(self_read, opts)
     end
 
     #
-    # Wait for signals and handle them.
+    # Run server and wait for signals.
     #
     # @param [IO] read_pipe Where to read signals.
+    # @param [Hash] opts Server options.
     #
-    def wait_for_signal(read_pipe)
-      local_server.start
+    def run_server(read_pipe, opts = {})
+      local_server.start(opts)
 
       while (readable_io = IO.select([read_pipe]))
         signal = readable_io.first[0].gets.strip

data/lib/cloudtasker/cloud_task.rb

@@ -3,7 +3,7 @@
 module Cloudtasker
   # An interface class to manage tasks on the backend (Cloud Task or Redis)
   class CloudTask
-    attr_accessor :id, :http_request, :schedule_time, :retries
+    attr_accessor :id, :http_request, :schedule_time, :retries, :queue
 
     #
     # The backend to use for cloud tasks.
@@ -69,12 +69,14 @@ module Cloudtasker
     # @param [Hash] http_request The content of the http request.
     # @param [Integer] schedule_time When to run the job (Unix timestamp)
     # @param [Integer] retries The number of times the job failed.
+    # @param [String] queue The queue the task is in.
     #
-    def initialize(id:, http_request:, schedule_time: nil, retries: 0)
+    def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
       @id = id
       @http_request = http_request
       @schedule_time = schedule_time
       @retries = retries || 0
+      @queue = queue
     end
 
     #

data/lib/cloudtasker/config.rb

@@ -7,15 +7,29 @@ module Cloudtasker
   class Config
     attr_accessor :redis
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
-                :gcp_queue_id, :processor_path, :logger, :mode, :max_retries
+                :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
 
     # Retry header in Cloud Task responses
     RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
 
+    # Content-Transfer-Encoding header in Cloud Task responses
+    ENCODING_HEADER = 'Content-Transfer-Encoding'
+
+    # Content Type
+    CONTENT_TYPE_HEADER = 'Content-Type'
+
+    # Authorization header
+    AUTHORIZATION_HEADER = 'Authorization'
+
     # Default values
     DEFAULT_LOCATION_ID = 'us-east1'
     DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
 
+    # Default queue values
+    DEFAULT_JOB_QUEUE = 'default'
+    DEFAULT_QUEUE_CONCURRENCY = 10
+    DEFAULT_QUEUE_RETRIES = -1 # unlimited
+
     # The number of times jobs will be attempted before declaring them dead
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
@@ -23,9 +37,10 @@ module Cloudtasker
       Missing host for processing.
       Please specify a processor hostname in form of `https://some-public-dns.example.com`'
     DOC
-    QUEUE_ID_MISSING_ERROR = <<~DOC
-      Missing GCP queue ID.
-      Please specify a queue ID in the form of `my-queue-id`. You can create a queue using the Google SDK via `gcloud tasks queues create my-queue-id`
+    QUEUE_PREFIX_MISSING_ERROR = <<~DOC
+      Missing GCP queue prefix.
+      Please specify a queue prefix in the form of `my-app`.
+      You can create a default queue using the Google SDK via `gcloud tasks queues create my-app-default`
     DOC
     PROJECT_ID_MISSING_ERROR = <<~DOC
       Missing GCP project ID.
@@ -95,8 +110,14 @@ module Cloudtasker
     def processor_host=(val)
       @processor_host = val
 
+      # Check if Rails supports host filtering
+      return unless val &&
+                    defined?(Rails) &&
+                    Rails.application.config.respond_to?(:hosts) &&
+                    Rails.application.config.hosts&.any?
+
       # Add processor host to the list of authorized hosts
-      Rails.application.config.hosts << val.gsub(%r{https?://}, '') if val && defined?(Rails)
+      Rails.application.config.hosts << val.gsub(%r{https?://}, '')
     end
 
     #
@@ -121,12 +142,12 @@ module Cloudtasker
     end
 
     #
-    # Return the ID of GCP queue where tasks will be added.
+    # Return the prefix used for queues.
     #
-    # @return [String] The ID of the processing queue.
+    # @return [String] The prefix of the processing queues.
     #
-    def gcp_queue_id
-      @gcp_queue_id || raise(StandardError, QUEUE_ID_MISSING_ERROR)
+    def gcp_queue_prefix
+      @gcp_queue_prefix || raise(StandardError, QUEUE_PREFIX_MISSING_ERROR)
     end
 
     #

data/lib/cloudtasker/cron/job.rb

@@ -105,10 +105,10 @@ module Cloudtasker
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #

data/lib/cloudtasker/cron/schedule.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
 require 'fugit'
+require 'cloudtasker/worker_wrapper'
 
 module Cloudtasker
   module Cron
     # Manage cron schedules
     class Schedule
-      attr_accessor :id, :cron, :worker, :task_id, :job_id
+      attr_accessor :id, :cron, :worker, :task_id, :job_id, :queue, :args
 
       # Key Namespace used for object saved under this class
       SUB_NAMESPACE = 'schedule'
@@ -14,10 +15,10 @@ module Cloudtasker
       #
       # Return the redis client.
       #
-      # @return [Class] The redis client
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -113,21 +114,25 @@ module Cloudtasker
       # @param [String] id The schedule id.
       # @param [String] cron The cron expression.
       # @param [Class] worker The worker class to run.
+      # @param [Array<any>] args The worker arguments.
+      # @param [String] queue The queue to use for the cron job.
       # @param [String] task_id The ID of the actual backend task.
       # @param [String] job_id The ID of the Cloudtasker worker.
       #
-      def initialize(id:, cron:, worker:, task_id: nil, job_id: nil)
+      def initialize(id:, cron:, worker:, **opts)
         @id = id
         @cron = cron
         @worker = worker
-        @task_id = task_id
-        @job_id = job_id
+        @args = opts[:args]
+        @queue = opts[:queue]
+        @task_id = opts[:task_id]
+        @job_id = opts[:job_id]
       end
 
       #
       # Return the redis client.
       #
-      # @return [Class] The redis client
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
       #
       def redis
         self.class.redis
@@ -191,7 +196,9 @@ module Cloudtasker
         {
           id: id,
           cron: cron,
-          worker: worker
+          worker: worker,
+          args: args,
+          queue: queue
         }
       end
 
@@ -201,13 +208,10 @@ module Cloudtasker
       # @return [Hash] The attributes hash.
       #
       def to_h
-        {
-          id: id,
-          cron: cron,
-          worker: worker,
+        to_config.merge(
           task_id: task_id,
           job_id: job_id
-        }
+        )
       end
 
       #
@@ -219,6 +223,15 @@ module Cloudtasker
         @cron_schedule ||= Fugit::Cron.parse(cron)
       end
 
+      #
+      # Return an instance of the underlying worker.
+      #
+      # @return [Cloudtasker::WorkerWrapper] The worker instance
+      #
+      def worker_instance
+        WorkerWrapper.new(worker_name: worker, job_args: args, job_queue: queue)
+      end
+
       #
       # Return the next time a job should run.
       #
@@ -279,7 +292,6 @@ module Cloudtasker
         CloudTask.delete(task_id) if task_id
 
         # Schedule worker
-        worker_instance = Object.const_get(worker).new
         Job.new(worker_instance).set(schedule_id: id).schedule!
       end
     end