cloudtasker 0.3.0 → 0.8.0

data/docs/CRON_JOBS.md

@@ -9,7 +9,7 @@ The Cloudtasker cron job extension allows you to register workers to run at fixe
 You can schedule cron jobs by adding the following to your cloudtasker initializer:
 ```ruby
 # The cron job extension is optional and must be explicitly required
-require 'cloudtasker/cron_job'
+require 'cloudtasker/cron'
 
 Cloudtasker.configure do |config|
   # Specify your redis url.
@@ -18,18 +18,24 @@ Cloudtasker.configure do |config|
 end
 
 # Specify all your cron jobs below. This will synchronize your list of cron jobs (cron jobs previously created and not listed below will be removed).
-Cloudtasker::Cron::Schedule.load_from_hash!(
-  # Run job every minute
-  some_schedule_name: {
-    worker: 'SomeCronWorker',
-    cron: '* * * * *'
-  },
-  # Run job every hour on the fifteenth minute 
-  other_cron_schedule: {
-    worker: 'OtherCronWorker',
-    cron: '15 * * * *'
-  }
-)
+unless Rails.env.test?
+  Cloudtasker::Cron::Schedule.load_from_hash!(
+    # Run job every minute
+    some_schedule_name: {
+      worker: 'SomeCronWorker',
+      cron: '* * * * *'
+    },
+    # Run job every hour on the fifteenth minute 
+    other_cron_schedule: {
+      worker: 'OtherCronWorker',
+      cron: '15 * * * *',
+      queue: 'critical'
+      args:
+        - 'foo'
+        - 'bar
+    }
+  )
+end
 ```
 
 ## Using a configuration file
@@ -56,7 +62,7 @@ Then register the jobs inside your Cloudtasker initializer this way:
 # ... Cloudtasker configuration ...
 
 schedule_file = 'config/cloudtasker_cron.yml'
-if File.exist?(schedule_file)
+if File.exist?(schedule_file) && !Rails.env.test?
   Cloudtasker::Cron::Schedule.load_from_hash!(YAML.load_file(schedule_file))
 end
 ```

data/exe/cloudtasker

@@ -3,9 +3,21 @@
 
 require 'bundler/setup'
 require 'cloudtasker/cli'
+require 'optparse'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: cloudtasker [options]'
+
+  opts.on('-q QUEUE', '--queue=QUEUE', 'Queue to process and number of threads. ' \
+    "Examples: '-q critical' | '-q critical,2' | '-q critical,3 -q defaults,2'") do |o|
+    options[:queues] ||= []
+    options[:queues] << o.split(',')
+  end
+end.parse!
 
 begin
-  Cloudtasker::CLI.run
+  Cloudtasker::CLI.run(options)
 rescue StandardError => e
   raise e if $DEBUG
 

data/lib/cloudtasker.rb

@@ -47,5 +47,4 @@ module Cloudtasker
   end
 end
 
-require 'cloudtasker/railtie' if defined?(Rails)
 require 'cloudtasker/engine' if defined?(::Rails::Engine)

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -9,15 +9,28 @@ module Cloudtasker
       #
       # Create the queue configured in Cloudtasker if it does not already exist.
       #
+      # @param [String] queue_name The relative name of the queue.
+      #
       # @return [Google::Cloud::Tasks::V2beta3::Queue] The queue
       #
-      def self.setup_queue
-        client.get_queue(queue_path)
+      def self.setup_queue(**opts)
+        # Build full queue path
+        queue_name = opts[:name] || Cloudtasker::Config::DEFAULT_JOB_QUEUE
+        full_queue_name = queue_path(queue_name)
+
+        # Try to get existing queue
+        client.get_queue(full_queue_name)
       rescue Google::Gax::RetryError
+        # Extract options
+        concurrency = (opts[:concurrency] || Cloudtasker::Config::DEFAULT_QUEUE_CONCURRENCY).to_i
+        retries = (opts[:retries] || Cloudtasker::Config::DEFAULT_QUEUE_RETRIES).to_i
+
+        # Create queue on 'not found' error
         client.create_queue(
           client.location_path(config.gcp_project_id, config.gcp_location_id),
-          name: queue_path,
-          retry_config: { max_attempts: -1 }
+          name: full_queue_name,
+          retry_config: { max_attempts: retries },
+          rate_limits: { max_concurrent_dispatches: concurrency }
         )
       end
 
@@ -42,13 +55,15 @@ module Cloudtasker
       #
       # Return the fully qualified path for the Cloud Task queue.
       #
+      # @param [String] queue_name The relative name of the queue.
+      #
       # @return [String] The queue path.
       #
-      def self.queue_path
+      def self.queue_path(queue_name)
         client.queue_path(
           config.gcp_project_id,
           config.gcp_location_id,
-          config.gcp_queue_id
+          [config.gcp_queue_prefix, queue_name].join('-')
         )
       end
 
@@ -94,8 +109,11 @@ module Cloudtasker
           schedule_time: format_schedule_time(payload[:schedule_time])
         ).compact
 
+        # Extract relative queue name
+        relative_queue = payload.delete(:queue)
+
         # Create task
-        resp = client.create_task(queue_path, payload)
+        resp = client.create_task(queue_path(relative_queue), payload)
         resp ? new(resp) : nil
       rescue Google::Gax::RetryError
         nil
@@ -121,6 +139,20 @@ module Cloudtasker
         @gcp_task = gcp_task
       end
 
+      #
+      # Return the relative queue (queue name minus prefix) the task is in.
+      #
+      # @return [String] The relative queue name
+      #
+      def relative_queue
+        gcp_task
+          .name
+          .match(%r{/queues/([^/]+)})
+          &.captures
+          &.first
+          &.sub("#{self.class.config.gcp_queue_prefix}-", '')
+      end
+
       #
       # Return a hash description of the task.
       #
@@ -131,7 +163,8 @@ module Cloudtasker
           id: gcp_task.name,
           http_request: gcp_task.to_h[:http_request],
           schedule_time: gcp_task.to_h.dig(:schedule_time, :seconds).to_i,
-          retries: gcp_task.to_h[:response_count]
+          retries: gcp_task.to_h[:response_count],
+          queue: relative_queue
         }
       end
     end

data/lib/cloudtasker/backend/memory_task.rb

@@ -7,7 +7,7 @@ module Cloudtasker
     # Manage local tasks pushed to memory.
     # Used for testing.
     class MemoryTask
-      attr_reader :id, :http_request, :schedule_time
+      attr_reader :id, :http_request, :schedule_time, :queue
 
       #
       # Return the task queue. A worker class name
@@ -116,10 +116,11 @@ module Cloudtasker
       # @param [Hash] http_request The HTTP request content.
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       #
-      def initialize(id:, http_request:, schedule_time: nil)
+      def initialize(id:, http_request:, schedule_time: nil, queue: nil)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
+        @queue = queue
       end
 
       #
@@ -149,7 +150,8 @@ module Cloudtasker
         {
           id: id,
           http_request: http_request,
-          schedule_time: schedule_time.to_i
+          schedule_time: schedule_time.to_i,
+          queue: queue
         }
       end
 

data/lib/cloudtasker/backend/redis_task.rb

@@ -1,22 +1,23 @@
 # frozen_string_literal: true
 
 require 'cloudtasker/redis_client'
+require 'net/http'
 
 module Cloudtasker
   module Backend
     # Manage local tasks pushed to Redis
     class RedisTask
-      attr_reader :id, :http_request, :schedule_time, :retries
+      attr_reader :id, :http_request, :schedule_time, :retries, :queue
 
       RETRY_INTERVAL = 20 # seconds
 
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -47,20 +48,26 @@ module Cloudtasker
       #
       # Reeturn all tasks ready to process.
       #
+      # @param [String] queue The queue to retrieve items from.
+      #
       # @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks ready to process.
       #
-      def self.ready_to_process
-        all.select { |e| e.schedule_time <= Time.now }
+      def self.ready_to_process(queue = nil)
+        list = all.select { |e| e.schedule_time <= Time.now }
+        list = list.select { |e| e.queue == queue } if queue
+        list
       end
 
       #
       # Retrieve and remove a task from the queue.
       #
+      # @param [String] queue The queue to retrieve items from.
+      #
       # @return [Cloudtasker::Backend::RedisTask] A task ready to process.
       #
-      def self.pop
+      def self.pop(queue = nil)
         redis.with_lock('cloudtasker/server') do
-          ready_to_process.first&.tap(&:destroy)
+          ready_to_process(queue).first&.tap(&:destroy)
         end
       end
 
@@ -109,11 +116,12 @@ module Cloudtasker
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       # @param [Integer] retries The number of times the job failed.
       #
-      def initialize(id:, http_request:, schedule_time: nil, retries: 0)
+      def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
         @retries = retries || 0
+        @queue = queue
       end
 
       #
@@ -135,7 +143,8 @@ module Cloudtasker
           id: id,
           http_request: http_request,
           schedule_time: schedule_time.to_i,
-          retries: retries
+          retries: retries,
+          queue: queue
         }
       end
 

data/lib/cloudtasker/batch/batch_progress.rb

@@ -77,7 +77,16 @@ module Cloudtasker
       # @return [Integer] The number of jobs pending.
       #
       def pending
-        total - completed - dead
+        total - done
+      end
+
+      #
+      # Return the number of jobs completed or dead.
+      #
+      # @return [Integer] The number of jobs done.
+      #
+      def done
+        completed + dead
       end
 
       #
@@ -88,7 +97,7 @@ module Cloudtasker
       def percent
         return 0 if total.zero?
 
-        pending.to_f / total
+        (done.to_f / total) * 100
       end
 
       #

data/lib/cloudtasker/batch/job.rb

@@ -7,7 +7,8 @@ module Cloudtasker
       attr_reader :worker
 
       # Key Namespace used for object saved under this class
-      SUB_NAMESPACE = 'job'
+      JOBS_NAMESPACE = 'jobs'
+      STATES_NAMESPACE = 'states'
 
       # List of statuses triggering a completion callback
       COMPLETION_STATUSES = %w[completed dead].freeze
@@ -15,10 +16,10 @@ module Cloudtasker
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -32,7 +33,7 @@ module Cloudtasker
         return nil unless worker_id
 
         # Retrieve related worker
-        payload = redis.fetch(key(worker_id))
+        payload = redis.fetch(key("#{JOBS_NAMESPACE}/#{worker_id}"))
         worker = Cloudtasker::Worker.from_hash(payload)
         return nil unless worker
 
@@ -86,7 +87,7 @@ module Cloudtasker
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def redis
         self.class.redis
@@ -140,7 +141,7 @@ module Cloudtasker
       # @return [String] The worker namespaced id.
       #
       def batch_gid
-        key(batch_id)
+        key("#{JOBS_NAMESPACE}/#{batch_id}")
       end
 
       #
@@ -149,7 +150,7 @@ module Cloudtasker
       # @return [String] The batch state namespaced id.
       #
       def batch_state_gid
-        [batch_gid, 'state'].join('/')
+        key("#{STATES_NAMESPACE}/#{batch_id}")
       end
 
       #
@@ -179,9 +180,23 @@ module Cloudtasker
       # @return [Array<Cloudtasker::Worker>] The updated list of jobs.
       #
       def add(worker_klass, *args)
+        add_to_queue(worker.job_queue, worker_klass, *args)
+      end
+
+      #
+      # Add a worker to the batch using a specific queue.
+      #
+      # @param [String, Symbol] queue The name of the queue
+      # @param [Class] worker_klass The worker class.
+      # @param [Array<any>] *args The worker arguments.
+      #
+      # @return [Array<Cloudtasker::Worker>] The updated list of jobs.
+      #
+      def add_to_queue(queue, worker_klass, *args)
         jobs << worker_klass.new(
           job_args: args,
-          job_meta: { key(:parent_id) => batch_id }
+          job_meta: { key(:parent_id) => batch_id },
+          job_queue: queue
         )
       end
 
@@ -371,7 +386,7 @@ module Cloudtasker
         setup
 
         # Complete batch
-        complete(:success)
+        complete(:completed)
       rescue DeadWorkerError => e
         complete(:dead)
         raise(e)

data/lib/cloudtasker/cli.rb

@@ -70,7 +70,7 @@ module Cloudtasker
     #
     # Run the cloudtasker development server.
     #
-    def run
+    def run(opts = {})
       boot_system
 
       # Print banner
@@ -90,16 +90,17 @@ module Cloudtasker
       logger.info "[Cloudtasker/Server] Running in #{RUBY_DESCRIPTION}"
 
       # Wait for signals
-      wait_for_signal(self_read)
+      run_server(self_read, opts)
     end
 
     #
-    # Wait for signals and handle them.
+    # Run server and wait for signals.
     #
     # @param [IO] read_pipe Where to read signals.
+    # @param [Hash] opts Server options.
     #
-    def wait_for_signal(read_pipe)
-      local_server.start
+    def run_server(read_pipe, opts = {})
+      local_server.start(opts)
 
       while (readable_io = IO.select([read_pipe]))
         signal = readable_io.first[0].gets.strip

data/lib/cloudtasker/cloud_task.rb

@@ -3,7 +3,7 @@
 module Cloudtasker
   # An interface class to manage tasks on the backend (Cloud Task or Redis)
   class CloudTask
-    attr_accessor :id, :http_request, :schedule_time, :retries
+    attr_accessor :id, :http_request, :schedule_time, :retries, :queue
 
     #
     # The backend to use for cloud tasks.
@@ -69,12 +69,14 @@ module Cloudtasker
     # @param [Hash] http_request The content of the http request.
     # @param [Integer] schedule_time When to run the job (Unix timestamp)
     # @param [Integer] retries The number of times the job failed.
+    # @param [String] queue The queue the task is in.
     #
-    def initialize(id:, http_request:, schedule_time: nil, retries: 0)
+    def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
       @id = id
       @http_request = http_request
       @schedule_time = schedule_time
       @retries = retries || 0
+      @queue = queue
     end
 
     #