cloudtasker 0.7.0 → 0.8.0

data/lib/cloudtasker/backend/redis_task.rb

@@ -7,17 +7,17 @@ module Cloudtasker
   module Backend
     # Manage local tasks pushed to Redis
     class RedisTask
-      attr_reader :id, :http_request, :schedule_time, :retries
+      attr_reader :id, :http_request, :schedule_time, :retries, :queue
 
       RETRY_INTERVAL = 20 # seconds
 
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -48,20 +48,26 @@ module Cloudtasker
       #
       # Reeturn all tasks ready to process.
       #
+      # @param [String] queue The queue to retrieve items from.
+      #
       # @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks ready to process.
       #
-      def self.ready_to_process
-        all.select { |e| e.schedule_time <= Time.now }
+      def self.ready_to_process(queue = nil)
+        list = all.select { |e| e.schedule_time <= Time.now }
+        list = list.select { |e| e.queue == queue } if queue
+        list
       end
 
       #
       # Retrieve and remove a task from the queue.
       #
+      # @param [String] queue The queue to retrieve items from.
+      #
       # @return [Cloudtasker::Backend::RedisTask] A task ready to process.
       #
-      def self.pop
+      def self.pop(queue = nil)
         redis.with_lock('cloudtasker/server') do
-          ready_to_process.first&.tap(&:destroy)
+          ready_to_process(queue).first&.tap(&:destroy)
         end
       end
 
@@ -110,11 +116,12 @@ module Cloudtasker
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       # @param [Integer] retries The number of times the job failed.
       #
-      def initialize(id:, http_request:, schedule_time: nil, retries: 0)
+      def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
         @retries = retries || 0
+        @queue = queue
       end
 
       #
@@ -136,7 +143,8 @@ module Cloudtasker
           id: id,
           http_request: http_request,
           schedule_time: schedule_time.to_i,
-          retries: retries
+          retries: retries,
+          queue: queue
         }
       end
 

data/lib/cloudtasker/batch/batch_progress.rb

@@ -77,7 +77,16 @@ module Cloudtasker
       # @return [Integer] The number of jobs pending.
       #
       def pending
-        total - completed - dead
+        total - done
+      end
+
+      #
+      # Return the number of jobs completed or dead.
+      #
+      # @return [Integer] The number of jobs done.
+      #
+      def done
+        completed + dead
       end
 
       #
@@ -88,7 +97,7 @@ module Cloudtasker
       def percent
         return 0 if total.zero?
 
-        pending.to_f / total
+        (done.to_f / total) * 100
       end
 
       #

data/lib/cloudtasker/batch/job.rb

@@ -16,10 +16,10 @@ module Cloudtasker
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -87,7 +87,7 @@ module Cloudtasker
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def redis
         self.class.redis
@@ -180,9 +180,23 @@ module Cloudtasker
       # @return [Array<Cloudtasker::Worker>] The updated list of jobs.
       #
       def add(worker_klass, *args)
+        add_to_queue(worker.job_queue, worker_klass, *args)
+      end
+
+      #
+      # Add a worker to the batch using a specific queue.
+      #
+      # @param [String, Symbol] queue The name of the queue
+      # @param [Class] worker_klass The worker class.
+      # @param [Array<any>] *args The worker arguments.
+      #
+      # @return [Array<Cloudtasker::Worker>] The updated list of jobs.
+      #
+      def add_to_queue(queue, worker_klass, *args)
         jobs << worker_klass.new(
           job_args: args,
-          job_meta: { key(:parent_id) => batch_id }
+          job_meta: { key(:parent_id) => batch_id },
+          job_queue: queue
         )
       end
 

data/lib/cloudtasker/cli.rb

@@ -70,7 +70,7 @@ module Cloudtasker
     #
     # Run the cloudtasker development server.
     #
-    def run
+    def run(opts = {})
       boot_system
 
       # Print banner
@@ -90,16 +90,17 @@ module Cloudtasker
       logger.info "[Cloudtasker/Server] Running in #{RUBY_DESCRIPTION}"
 
       # Wait for signals
-      wait_for_signal(self_read)
+      run_server(self_read, opts)
     end
 
     #
-    # Wait for signals and handle them.
+    # Run server and wait for signals.
     #
     # @param [IO] read_pipe Where to read signals.
+    # @param [Hash] opts Server options.
     #
-    def wait_for_signal(read_pipe)
-      local_server.start
+    def run_server(read_pipe, opts = {})
+      local_server.start(opts)
 
       while (readable_io = IO.select([read_pipe]))
         signal = readable_io.first[0].gets.strip

data/lib/cloudtasker/cloud_task.rb

@@ -3,7 +3,7 @@
 module Cloudtasker
   # An interface class to manage tasks on the backend (Cloud Task or Redis)
   class CloudTask
-    attr_accessor :id, :http_request, :schedule_time, :retries
+    attr_accessor :id, :http_request, :schedule_time, :retries, :queue
 
     #
     # The backend to use for cloud tasks.
@@ -69,12 +69,14 @@ module Cloudtasker
     # @param [Hash] http_request The content of the http request.
     # @param [Integer] schedule_time When to run the job (Unix timestamp)
     # @param [Integer] retries The number of times the job failed.
+    # @param [String] queue The queue the task is in.
     #
-    def initialize(id:, http_request:, schedule_time: nil, retries: 0)
+    def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
       @id = id
       @http_request = http_request
       @schedule_time = schedule_time
       @retries = retries || 0
+      @queue = queue
     end
 
     #

data/lib/cloudtasker/config.rb

@@ -7,7 +7,7 @@ module Cloudtasker
   class Config
     attr_accessor :redis
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
-                :gcp_queue_id, :processor_path, :logger, :mode, :max_retries
+                :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
 
     # Retry header in Cloud Task responses
     RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
@@ -16,6 +16,11 @@ module Cloudtasker
     DEFAULT_LOCATION_ID = 'us-east1'
     DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
 
+    # Default queue values
+    DEFAULT_JOB_QUEUE = 'default'
+    DEFAULT_QUEUE_CONCURRENCY = 10
+    DEFAULT_QUEUE_RETRIES = -1 # unlimited
+
     # The number of times jobs will be attempted before declaring them dead
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
@@ -23,9 +28,10 @@ module Cloudtasker
       Missing host for processing.
       Please specify a processor hostname in form of `https://some-public-dns.example.com`'
     DOC
-    QUEUE_ID_MISSING_ERROR = <<~DOC
-      Missing GCP queue ID.
-      Please specify a queue ID in the form of `my-queue-id`. You can create a queue using the Google SDK via `gcloud tasks queues create my-queue-id`
+    QUEUE_PREFIX_MISSING_ERROR = <<~DOC
+      Missing GCP queue prefix.
+      Please specify a queue prefix in the form of `my-app`.
+      You can create a default queue using the Google SDK via `gcloud tasks queues create my-app-default`
     DOC
     PROJECT_ID_MISSING_ERROR = <<~DOC
       Missing GCP project ID.
@@ -121,12 +127,12 @@ module Cloudtasker
     end
 
     #
-    # Return the ID of GCP queue where tasks will be added.
+    # Return the prefix used for queues.
     #
-    # @return [String] The ID of the processing queue.
+    # @return [String] The prefix of the processing queues.
     #
-    def gcp_queue_id
-      @gcp_queue_id || raise(StandardError, QUEUE_ID_MISSING_ERROR)
+    def gcp_queue_prefix
+      @gcp_queue_prefix || raise(StandardError, QUEUE_PREFIX_MISSING_ERROR)
     end
 
     #

data/lib/cloudtasker/cron/job.rb

@@ -105,10 +105,10 @@ module Cloudtasker
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #

data/lib/cloudtasker/cron/schedule.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
 require 'fugit'
+require 'cloudtasker/worker_wrapper'
 
 module Cloudtasker
   module Cron
     # Manage cron schedules
     class Schedule
-      attr_accessor :id, :cron, :worker, :task_id, :job_id
+      attr_accessor :id, :cron, :worker, :task_id, :job_id, :queue, :args
 
       # Key Namespace used for object saved under this class
       SUB_NAMESPACE = 'schedule'
@@ -14,10 +15,10 @@ module Cloudtasker
       #
       # Return the redis client.
       #
-      # @return [Class] The redis client
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -113,21 +114,25 @@ module Cloudtasker
       # @param [String] id The schedule id.
       # @param [String] cron The cron expression.
       # @param [Class] worker The worker class to run.
+      # @param [Array<any>] args The worker arguments.
+      # @param [String] queue The queue to use for the cron job.
       # @param [String] task_id The ID of the actual backend task.
       # @param [String] job_id The ID of the Cloudtasker worker.
       #
-      def initialize(id:, cron:, worker:, task_id: nil, job_id: nil)
+      def initialize(id:, cron:, worker:, **opts)
         @id = id
         @cron = cron
         @worker = worker
-        @task_id = task_id
-        @job_id = job_id
+        @args = opts[:args]
+        @queue = opts[:queue]
+        @task_id = opts[:task_id]
+        @job_id = opts[:job_id]
       end
 
       #
       # Return the redis client.
       #
-      # @return [Class] The redis client
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
       #
       def redis
         self.class.redis
@@ -191,7 +196,9 @@ module Cloudtasker
         {
           id: id,
           cron: cron,
-          worker: worker
+          worker: worker,
+          args: args,
+          queue: queue
         }
       end
 
@@ -201,13 +208,10 @@ module Cloudtasker
       # @return [Hash] The attributes hash.
       #
       def to_h
-        {
-          id: id,
-          cron: cron,
-          worker: worker,
+        to_config.merge(
           task_id: task_id,
           job_id: job_id
-        }
+        )
       end
 
       #
@@ -219,6 +223,15 @@ module Cloudtasker
         @cron_schedule ||= Fugit::Cron.parse(cron)
       end
 
+      #
+      # Return an instance of the underlying worker.
+      #
+      # @return [Cloudtasker::WorkerWrapper] The worker instance
+      #
+      def worker_instance
+        WorkerWrapper.new(worker_name: worker, job_args: args, job_queue: queue)
+      end
+
       #
       # Return the next time a job should run.
       #
@@ -279,7 +292,6 @@ module Cloudtasker
         CloudTask.delete(task_id) if task_id
 
         # Schedule worker
-        worker_instance = Object.const_get(worker).new
         Job.new(worker_instance).set(schedule_id: id).schedule!
       end
     end

data/lib/cloudtasker/local_server.rb

@@ -9,6 +9,9 @@ module Cloudtasker
     # Max number of task requests sent to the processing server
     CONCURRENCY = (ENV['CLOUDTASKER_CONCURRENCY'] || 5).to_i
 
+    # Default number of threads to allocate to process a specific queue
+    QUEUE_CONCURRENCY = 1
+
     #
     # Stop the local server.
     #
@@ -16,7 +19,7 @@ module Cloudtasker
       @done = true
 
       # Terminate threads and repush tasks
-      @threads&.each do |t|
+      @threads&.values&.flatten&.each do |t|
         t.terminate
         t['task']&.retry_later(0, is_error: false)
       end
@@ -28,11 +31,21 @@ module Cloudtasker
     #
     # Start the local server
     #
+    # @param [Hash] opts Server options.
+    #
     #
-    def start
+    def start(opts = {})
+      # Extract queues to process
+      queues = opts[:queues].to_a.any? ? opts[:queues] : [[nil, CONCURRENCY]]
+
+      # Display start banner
+      queue_labels = queues.map { |n, c| "#{n || 'all'}=#{c || QUEUE_CONCURRENCY}" }.join(' ')
+      Cloudtasker.logger.info("[Cloudtasker/Server] Processing queues: #{queue_labels}")
+
+      # Start processing queues
       @start ||= Thread.new do
         until @done
-          process_jobs
+          queues.each { |(n, c)| process_jobs(n, c) }
           sleep 1
         end
         Cloudtasker.logger.info('[Cloudtasker/Server] Local server exiting...')
@@ -43,31 +56,40 @@ module Cloudtasker
     # Process enqueued workers.
     #
     #
-    def process_jobs
-      @threads ||= []
+    def process_jobs(queue = nil, concurrency = nil)
+      @threads ||= {}
+      @threads[queue] ||= []
+      max_threads = (concurrency || QUEUE_CONCURRENCY).to_i
 
       # Remove any done thread
-      @threads.select!(&:alive?)
+      @threads[queue].select!(&:alive?)
 
       # Process tasks
-      while @threads.count < CONCURRENCY && (task = Cloudtasker::Backend::RedisTask.pop)
-        @threads << Thread.new do
-          Thread.current['task'] = task
-          Thread.current['attempts'] = 0
+      while @threads[queue].count < max_threads && (task = Cloudtasker::Backend::RedisTask.pop(queue))
+        @threads[queue] << Thread.new { process_task(task) }
+      end
+    end
 
-          # Deliver task
-          begin
-            Thread.current['task'].deliver
-          rescue Errno::ECONNREFUSED => e
-            raise(e) unless Thread.current['attempts'] < 3
+    #
+    # Process a given task
+    #
+    # @param [Cloudtasker::CloudTask] task The task to process
+    #
+    def process_task(task)
+      Thread.current['task'] = task
+      Thread.current['attempts'] = 0
 
-            # Retry on connection error, in case the web server is not
-            # started yet.
-            Thread.current['attempts'] += 1
-            sleep(3)
-            retry
-          end
-        end
+      # Deliver task
+      begin
+        Thread.current['task'].deliver
+      rescue Errno::ECONNREFUSED => e
+        raise(e) unless Thread.current['attempts'] < 3
+
+        # Retry on connection error, in case the web server is not
+        # started yet.
+        Thread.current['attempts'] += 1
+        sleep(3)
+        retry
       end
     end
   end