cloudtasker 0.4.0 → 0.8.1

data/lib/cloudtasker/cli.rb

@@ -70,7 +70,7 @@ module Cloudtasker
     #
     # Run the cloudtasker development server.
     #
-    def run
+    def run(opts = {})
       boot_system
 
       # Print banner
@@ -90,16 +90,17 @@ module Cloudtasker
       logger.info "[Cloudtasker/Server] Running in #{RUBY_DESCRIPTION}"
 
       # Wait for signals
-      wait_for_signal(self_read)
+      run_server(self_read, opts)
     end
 
     #
-    # Wait for signals and handle them.
+    # Run server and wait for signals.
     #
     # @param [IO] read_pipe Where to read signals.
+    # @param [Hash] opts Server options.
     #
-    def wait_for_signal(read_pipe)
-      local_server.start
+    def run_server(read_pipe, opts = {})
+      local_server.start(opts)
 
       while (readable_io = IO.select([read_pipe]))
         signal = readable_io.first[0].gets.strip

data/lib/cloudtasker/cloud_task.rb

@@ -3,7 +3,7 @@
 module Cloudtasker
   # An interface class to manage tasks on the backend (Cloud Task or Redis)
   class CloudTask
-    attr_accessor :id, :http_request, :schedule_time, :retries
+    attr_accessor :id, :http_request, :schedule_time, :retries, :queue
 
     #
     # The backend to use for cloud tasks.
@@ -69,12 +69,14 @@ module Cloudtasker
     # @param [Hash] http_request The content of the http request.
     # @param [Integer] schedule_time When to run the job (Unix timestamp)
     # @param [Integer] retries The number of times the job failed.
+    # @param [String] queue The queue the task is in.
     #
-    def initialize(id:, http_request:, schedule_time: nil, retries: 0)
+    def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
       @id = id
       @http_request = http_request
       @schedule_time = schedule_time
       @retries = retries || 0
+      @queue = queue
     end
 
     #

data/lib/cloudtasker/config.rb

@@ -7,7 +7,7 @@ module Cloudtasker
   class Config
     attr_accessor :redis
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
-                :gcp_queue_id, :processor_path, :logger, :mode, :max_retries
+                :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
 
     # Retry header in Cloud Task responses
     RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
@@ -16,6 +16,11 @@ module Cloudtasker
     DEFAULT_LOCATION_ID = 'us-east1'
     DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
 
+    # Default queue values
+    DEFAULT_JOB_QUEUE = 'default'
+    DEFAULT_QUEUE_CONCURRENCY = 10
+    DEFAULT_QUEUE_RETRIES = -1 # unlimited
+
     # The number of times jobs will be attempted before declaring them dead
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
@@ -23,9 +28,10 @@ module Cloudtasker
       Missing host for processing.
       Please specify a processor hostname in form of `https://some-public-dns.example.com`'
     DOC
-    QUEUE_ID_MISSING_ERROR = <<~DOC
-      Missing GCP queue ID.
-      Please specify a queue ID in the form of `my-queue-id`. You can create a queue using the Google SDK via `gcloud tasks queues create my-queue-id`
+    QUEUE_PREFIX_MISSING_ERROR = <<~DOC
+      Missing GCP queue prefix.
+      Please specify a queue prefix in the form of `my-app`.
+      You can create a default queue using the Google SDK via `gcloud tasks queues create my-app-default`
     DOC
     PROJECT_ID_MISSING_ERROR = <<~DOC
       Missing GCP project ID.
@@ -95,8 +101,11 @@ module Cloudtasker
     def processor_host=(val)
       @processor_host = val
 
+      # Check if Rails supports host filtering
+      return unless val && defined?(Rails) && Rails.application.config.respond_to?(:hosts)
+
       # Add processor host to the list of authorized hosts
-      Rails.application.config.hosts << val.gsub(%r{https?://}, '') if val && defined?(Rails)
+      Rails.application.config.hosts << val.gsub(%r{https?://}, '')
     end
 
     #
@@ -121,12 +130,12 @@ module Cloudtasker
     end
 
     #
-    # Return the ID of GCP queue where tasks will be added.
+    # Return the prefix used for queues.
     #
-    # @return [String] The ID of the processing queue.
+    # @return [String] The prefix of the processing queues.
     #
-    def gcp_queue_id
-      @gcp_queue_id || raise(StandardError, QUEUE_ID_MISSING_ERROR)
+    def gcp_queue_prefix
+      @gcp_queue_prefix || raise(StandardError, QUEUE_PREFIX_MISSING_ERROR)
     end
 
     #

data/lib/cloudtasker/cron/job.rb

@@ -105,10 +105,10 @@ module Cloudtasker
       #
       # Return the cloudtasker redis client
       #
-      # @return [Class] The redis client.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
       #
       def redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #

data/lib/cloudtasker/cron/schedule.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
 require 'fugit'
+require 'cloudtasker/worker_wrapper'
 
 module Cloudtasker
   module Cron
     # Manage cron schedules
     class Schedule
-      attr_accessor :id, :cron, :worker, :task_id, :job_id
+      attr_accessor :id, :cron, :worker, :task_id, :job_id, :queue, :args
 
       # Key Namespace used for object saved under this class
       SUB_NAMESPACE = 'schedule'
@@ -14,10 +15,10 @@ module Cloudtasker
       #
       # Return the redis client.
       #
-      # @return [Class] The redis client
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
       #
       def self.redis
-        RedisClient
+        @redis ||= RedisClient.new
       end
 
       #
@@ -72,8 +73,10 @@ module Cloudtasker
       # @return [Cloudtasker::Cron::Schedule] The schedule instance.
       #
       def self.create(**opts)
-        config = find(opts[:id]).to_h.merge(opts)
-        new(config).tap(&:save)
+        redis.with_lock(key(opts[:id])) do
+          config = find(opts[:id]).to_h.merge(opts)
+          new(config).tap(&:save)
+        end
       end
 
       #
@@ -95,12 +98,14 @@ module Cloudtasker
       # @param [String] id The schedule id.
       #
       def self.delete(id)
-        schedule = find(id)
-        return false unless schedule
+        redis.with_lock(key(id)) do
+          schedule = find(id)
+          return false unless schedule
 
-        # Delete task and stored schedule
-        CloudTask.delete(schedule.task_id) if schedule.task_id
-        redis.del(schedule.gid)
+          # Delete task and stored schedule
+          CloudTask.delete(schedule.task_id) if schedule.task_id
+          redis.del(schedule.gid)
+        end
       end
 
       #
@@ -109,21 +114,25 @@ module Cloudtasker
       # @param [String] id The schedule id.
       # @param [String] cron The cron expression.
       # @param [Class] worker The worker class to run.
+      # @param [Array<any>] args The worker arguments.
+      # @param [String] queue The queue to use for the cron job.
       # @param [String] task_id The ID of the actual backend task.
       # @param [String] job_id The ID of the Cloudtasker worker.
       #
-      def initialize(id:, cron:, worker:, task_id: nil, job_id: nil)
+      def initialize(id:, cron:, worker:, **opts)
         @id = id
         @cron = cron
         @worker = worker
-        @task_id = task_id
-        @job_id = job_id
+        @args = opts[:args]
+        @queue = opts[:queue]
+        @task_id = opts[:task_id]
+        @job_id = opts[:job_id]
       end
 
       #
       # Return the redis client.
       #
-      # @return [Class] The redis client
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
       #
       def redis
         self.class.redis
@@ -187,7 +196,9 @@ module Cloudtasker
         {
           id: id,
           cron: cron,
-          worker: worker
+          worker: worker,
+          args: args,
+          queue: queue
         }
       end
 
@@ -197,13 +208,10 @@ module Cloudtasker
       # @return [Hash] The attributes hash.
       #
       def to_h
-        {
-          id: id,
-          cron: cron,
-          worker: worker,
+        to_config.merge(
           task_id: task_id,
           job_id: job_id
-        }
+        )
       end
 
       #
@@ -215,6 +223,15 @@ module Cloudtasker
         @cron_schedule ||= Fugit::Cron.parse(cron)
       end
 
+      #
+      # Return an instance of the underlying worker.
+      #
+      # @return [Cloudtasker::WorkerWrapper] The worker instance
+      #
+      def worker_instance
+        WorkerWrapper.new(worker_name: worker, job_args: args, job_queue: queue)
+      end
+
       #
       # Return the next time a job should run.
       #
@@ -275,7 +292,6 @@ module Cloudtasker
         CloudTask.delete(task_id) if task_id
 
         # Schedule worker
-        worker_instance = Object.const_get(worker).new
         Job.new(worker_instance).set(schedule_id: id).schedule!
       end
     end

data/lib/cloudtasker/local_server.rb

@@ -9,6 +9,9 @@ module Cloudtasker
     # Max number of task requests sent to the processing server
     CONCURRENCY = (ENV['CLOUDTASKER_CONCURRENCY'] || 5).to_i
 
+    # Default number of threads to allocate to process a specific queue
+    QUEUE_CONCURRENCY = 1
+
     #
     # Stop the local server.
     #
@@ -16,7 +19,7 @@ module Cloudtasker
       @done = true
 
       # Terminate threads and repush tasks
-      @threads&.each do |t|
+      @threads&.values&.flatten&.each do |t|
         t.terminate
         t['task']&.retry_later(0, is_error: false)
       end
@@ -28,11 +31,21 @@ module Cloudtasker
     #
     # Start the local server
     #
+    # @param [Hash] opts Server options.
+    #
     #
-    def start
+    def start(opts = {})
+      # Extract queues to process
+      queues = opts[:queues].to_a.any? ? opts[:queues] : [[nil, CONCURRENCY]]
+
+      # Display start banner
+      queue_labels = queues.map { |n, c| "#{n || 'all'}=#{c || QUEUE_CONCURRENCY}" }.join(' ')
+      Cloudtasker.logger.info("[Cloudtasker/Server] Processing queues: #{queue_labels}")
+
+      # Start processing queues
       @start ||= Thread.new do
         until @done
-          process_jobs
+          queues.each { |(n, c)| process_jobs(n, c) }
           sleep 1
         end
         Cloudtasker.logger.info('[Cloudtasker/Server] Local server exiting...')
@@ -43,31 +56,40 @@ module Cloudtasker
     # Process enqueued workers.
     #
     #
-    def process_jobs
-      @threads ||= []
+    def process_jobs(queue = nil, concurrency = nil)
+      @threads ||= {}
+      @threads[queue] ||= []
+      max_threads = (concurrency || QUEUE_CONCURRENCY).to_i
 
       # Remove any done thread
-      @threads.select!(&:alive?)
+      @threads[queue].select!(&:alive?)
 
       # Process tasks
-      while @threads.count < CONCURRENCY && (task = Cloudtasker::Backend::RedisTask.pop)
-        @threads << Thread.new do
-          Thread.current['task'] = task
-          Thread.current['attempts'] = 0
+      while @threads[queue].count < max_threads && (task = Cloudtasker::Backend::RedisTask.pop(queue))
+        @threads[queue] << Thread.new { process_task(task) }
+      end
+    end
 
-          # Deliver task
-          begin
-            Thread.current['task'].deliver
-          rescue Errno::ECONNREFUSED => e
-            raise(e) unless Thread.current['attempts'] < 3
+    #
+    # Process a given task
+    #
+    # @param [Cloudtasker::CloudTask] task The task to process
+    #
+    def process_task(task)
+      Thread.current['task'] = task
+      Thread.current['attempts'] = 0
 
-            # Retry on connection error, in case the web server is not
-            # started yet.
-            Thread.current['attempts'] += 1
-            sleep(3)
-            retry
-          end
-        end
+      # Deliver task
+      begin
+        Thread.current['task'].deliver
+      rescue Errno::ECONNREFUSED => e
+        raise(e) unless Thread.current['attempts'] < 3
+
+        # Retry on connection error, in case the web server is not
+        # started yet.
+        Thread.current['attempts'] += 1
+        sleep(3)
+        retry
       end
     end
   end

data/lib/cloudtasker/redis_client.rb

@@ -4,11 +4,9 @@ require 'redis'
 
 module Cloudtasker
   # A wrapper with helper methods for redis
-  module RedisClient
-    module_function
-
+  class RedisClient
     # Suffix added to cache keys when locking them
-    LOCK_KEY_SUFFIX = 'lock'
+    LOCK_KEY_PREFIX = 'cloudtasker/lock'
 
     #
     # Return the underlying redis client.
@@ -50,9 +48,10 @@ module Cloudtasker
     # Acquire a lock on a cache entry.
     #
     # @example
-    #   RedisClient.with_lock('foo')
-    #     content = RedisClient.fetch('foo')
-    #     RedisClient.set(content.merge(bar: 'bar).to_json)
+    #   redis = RedisClient.new
+    #   redis.with_lock('foo')
+    #     content = redis.fetch('foo')
+    #     redis.set(content.merge(bar: 'bar).to_json)
     #   end
     #
     # @param [String] cache_key The cache key to access.
@@ -61,7 +60,7 @@ module Cloudtasker
       return nil unless cache_key
 
       # Wait to acquire lock
-      lock_key = [cache_key, LOCK_KEY_SUFFIX].join('/')
+      lock_key = [LOCK_KEY_PREFIX, cache_key].join('/')
       true until client.setnx(lock_key, true)
 
       # yield content

data/lib/cloudtasker/unique_job/job.rb

@@ -100,10 +100,10 @@ module Cloudtasker
       #
       # Return the Cloudtasker redis client.
       #
-      # @return [Class] The Cloudtasker::RedisClient wrapper.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
       #
       def redis
-        Cloudtasker::RedisClient
+        @redis ||= Cloudtasker::RedisClient.new
       end
 
       #

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.4.0'
+  VERSION = '0.8.1'
 end

data/lib/cloudtasker/worker.rb

@@ -6,6 +6,7 @@ module Cloudtasker
     # Add class method to including class
     def self.included(base)
       base.extend(ClassMethods)
+      base.attr_writer :job_queue
       base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries
     end
 
@@ -32,8 +33,9 @@ module Cloudtasker
     # @return [Cloudtasker::Worker, nil] The instantiated worker.
     #
     def self.from_hash(hash)
-      # Symbolize payload keys
+      # Symbolize metadata keys and stringify job arguments
       payload = JSON.parse(hash.to_json, symbolize_names: true)
+      payload[:job_args] = JSON.parse(payload[:job_args].to_json)
 
       # Extract worker parameters
       klass_name = payload&.dig(:worker)
@@ -44,7 +46,7 @@ module Cloudtasker
       return nil unless worker_klass.include?(self)
 
       # Return instantiated worker
-      worker_klass.new(payload.slice(:job_args, :job_id, :job_meta, :job_retries))
+      worker_klass.new(payload.slice(:job_queue, :job_args, :job_id, :job_meta, :job_retries))
     rescue NameError
       nil
     end
@@ -80,7 +82,7 @@ module Cloudtasker
       # @return [Cloudtasker::CloudTask] The Google Task response
       #
       def perform_async(*args)
-        perform_in(nil, *args)
+        schedule(args: args)
       end
 
       #
@@ -92,7 +94,7 @@ module Cloudtasker
       # @return [Cloudtasker::CloudTask] The Google Task response
       #
       def perform_in(interval, *args)
-        new(job_args: args).schedule(interval: interval)
+        schedule(args: args, time_in: interval)
       end
 
       #
@@ -104,7 +106,21 @@ module Cloudtasker
       # @return [Cloudtasker::CloudTask] The Google Task response
       #
       def perform_at(time_at, *args)
-        new(job_args: args).schedule(time_at: time_at)
+        schedule(args: args, time_at: time_at)
+      end
+
+      #
+      # Enqueue a worker with explicity options.
+      #
+      # @param [Array<any>] args The job arguments.
+      # @param [Time, Integer] time_in The delay in seconds.
+      # @param [Time, Integer] time_at The time at which the job should run.
+      # @param [String, Symbol] queue The queue on which the worker should run.
+      #
+      # @return [Cloudtasker::CloudTask] The Google Task response
+      #
+      def schedule(args: nil, time_in: nil, time_at: nil, queue: nil)
+        new(job_args: args, job_queue: queue).schedule({ interval: time_in, time_at: time_at }.compact)
       end
 
       #
@@ -123,11 +139,30 @@ module Cloudtasker
     # @param [Array<any>] job_args The list of perform args.
     # @param [String] job_id A unique ID identifying this job.
     #
-    def initialize(job_args: [], job_id: nil, job_meta: {}, job_retries: 0)
-      @job_args = job_args
+    def initialize(job_queue: nil, job_args: nil, job_id: nil, job_meta: {}, job_retries: 0)
+      @job_args = job_args || []
       @job_id = job_id || SecureRandom.uuid
       @job_meta = MetaStore.new(job_meta)
       @job_retries = job_retries || 0
+      @job_queue = job_queue
+    end
+
+    #
+    # Return the class name of the worker.
+    #
+    # @return [String] The class name.
+    #
+    def job_class_name
+      self.class.to_s
+    end
+
+    #
+    # Return the queue to use for this worker.
+    #
+    # @return [String] The name of queue.
+    #
+    def job_queue
+      (@job_queue ||= self.class.cloudtasker_options_hash[:queue] || Config::DEFAULT_JOB_QUEUE).to_s
     end
 
     #
@@ -198,10 +233,10 @@ module Cloudtasker
     # Return a new instance of the worker with the same args and metadata
     # but with a different id.
     #
-    # @return [<Type>] <description>
+    # @return [Cloudtasker::Worker] <description>
     #
     def new_instance
-      self.class.new(job_args: job_args, job_meta: job_meta)
+      self.class.new(job_queue: job_queue, job_args: job_args, job_meta: job_meta)
     end
 
     #
@@ -215,7 +250,8 @@ module Cloudtasker
         job_id: job_id,
         job_args: job_args,
         job_meta: job_meta.to_h,
-        job_retries: job_retries
+        job_retries: job_retries,
+        job_queue: job_queue
       }
     end