cloudtasker 0.6.0 → 0.9.0

data/lib/cloudtasker/local_server.rb

@@ -9,6 +9,9 @@ module Cloudtasker
     # Max number of task requests sent to the processing server
     CONCURRENCY = (ENV['CLOUDTASKER_CONCURRENCY'] || 5).to_i
 
+    # Default number of threads to allocate to process a specific queue
+    QUEUE_CONCURRENCY = 1
+
     #
     # Stop the local server.
     #
@@ -16,7 +19,7 @@ module Cloudtasker
       @done = true
 
       # Terminate threads and repush tasks
-      @threads&.each do |t|
+      @threads&.values&.flatten&.each do |t|
         t.terminate
         t['task']&.retry_later(0, is_error: false)
       end
@@ -28,11 +31,21 @@ module Cloudtasker
     #
     # Start the local server
     #
+    # @param [Hash] opts Server options.
+    #
     #
-    def start
+    def start(opts = {})
+      # Extract queues to process
+      queues = opts[:queues].to_a.any? ? opts[:queues] : [[nil, CONCURRENCY]]
+
+      # Display start banner
+      queue_labels = queues.map { |n, c| "#{n || 'all'}=#{c || QUEUE_CONCURRENCY}" }.join(' ')
+      Cloudtasker.logger.info("[Cloudtasker/Server] Processing queues: #{queue_labels}")
+
+      # Start processing queues
       @start ||= Thread.new do
         until @done
-          process_jobs
+          queues.each { |(n, c)| process_jobs(n, c) }
           sleep 1
         end
         Cloudtasker.logger.info('[Cloudtasker/Server] Local server exiting...')
@@ -43,31 +56,40 @@ module Cloudtasker
     # Process enqueued workers.
     #
     #
-    def process_jobs
-      @threads ||= []
+    def process_jobs(queue = nil, concurrency = nil)
+      @threads ||= {}
+      @threads[queue] ||= []
+      max_threads = (concurrency || QUEUE_CONCURRENCY).to_i
 
       # Remove any done thread
-      @threads.select!(&:alive?)
+      @threads[queue].select!(&:alive?)
 
       # Process tasks
-      while @threads.count < CONCURRENCY && (task = Cloudtasker::Backend::RedisTask.pop)
-        @threads << Thread.new do
-          Thread.current['task'] = task
-          Thread.current['attempts'] = 0
+      while @threads[queue].count < max_threads && (task = Cloudtasker::Backend::RedisTask.pop(queue))
+        @threads[queue] << Thread.new { process_task(task) }
+      end
+    end
 
-          # Deliver task
-          begin
-            Thread.current['task'].deliver
-          rescue Errno::ECONNREFUSED => e
-            raise(e) unless Thread.current['attempts'] < 3
+    #
+    # Process a given task
+    #
+    # @param [Cloudtasker::CloudTask] task The task to process
+    #
+    def process_task(task)
+      Thread.current['task'] = task
+      Thread.current['attempts'] = 0
 
-            # Retry on connection error, in case the web server is not
-            # started yet.
-            Thread.current['attempts'] += 1
-            sleep(3)
-            retry
-          end
-        end
+      # Deliver task
+      begin
+        Thread.current['task'].deliver
+      rescue Errno::ECONNREFUSED => e
+        raise(e) unless Thread.current['attempts'] < 3
+
+        # Retry on connection error, in case the web server is not
+        # started yet.
+        Thread.current['attempts'] += 1
+        sleep(3)
+        retry
       end
     end
   end

data/lib/cloudtasker/redis_client.rb

@@ -4,19 +4,21 @@ require 'redis'
 
 module Cloudtasker
   # A wrapper with helper methods for redis
-  module RedisClient
-    module_function
-
+  class RedisClient
     # Suffix added to cache keys when locking them
     LOCK_KEY_PREFIX = 'cloudtasker/lock'
 
+    def self.client
+      @client ||= Redis.new(Cloudtasker.config.redis || {})
+    end
+
     #
     # Return the underlying redis client.
     #
     # @return [Redis] The redis client.
     #
     def client
-      @client ||= Redis.new(Cloudtasker.config.redis || {})
+      @client ||= self.class.client
     end
 
     #
@@ -50,9 +52,10 @@ module Cloudtasker
     # Acquire a lock on a cache entry.
     #
     # @example
-    #   RedisClient.with_lock('foo')
-    #     content = RedisClient.fetch('foo')
-    #     RedisClient.set(content.merge(bar: 'bar).to_json)
+    #   redis = RedisClient.new
+    #   redis.with_lock('foo')
+    #     content = redis.fetch('foo')
+    #     redis.set(content.merge(bar: 'bar).to_json)
     #   end
     #
     # @param [String] cache_key The cache key to access.

data/lib/cloudtasker/unique_job/job.rb

@@ -100,10 +100,10 @@ module Cloudtasker
       #
       # Return the Cloudtasker redis client.
       #
-      # @return [Class] The Cloudtasker::RedisClient wrapper.
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
       #
       def redis
-        Cloudtasker::RedisClient
+        @redis ||= Cloudtasker::RedisClient.new
       end
 
       #

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.6.0'
+  VERSION = '0.9.0'
 end

data/lib/cloudtasker/worker.rb

@@ -6,6 +6,7 @@ module Cloudtasker
     # Add class method to including class
     def self.included(base)
       base.extend(ClassMethods)
+      base.attr_writer :job_queue
       base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries
     end
 
@@ -32,8 +33,9 @@ module Cloudtasker
     # @return [Cloudtasker::Worker, nil] The instantiated worker.
     #
     def self.from_hash(hash)
-      # Symbolize payload keys
+      # Symbolize metadata keys and stringify job arguments
       payload = JSON.parse(hash.to_json, symbolize_names: true)
+      payload[:job_args] = JSON.parse(payload[:job_args].to_json)
 
       # Extract worker parameters
       klass_name = payload&.dig(:worker)
@@ -44,7 +46,7 @@ module Cloudtasker
       return nil unless worker_klass.include?(self)
 
       # Return instantiated worker
-      worker_klass.new(payload.slice(:job_args, :job_id, :job_meta, :job_retries))
+      worker_klass.new(payload.slice(:job_queue, :job_args, :job_id, :job_meta, :job_retries))
     rescue NameError
       nil
     end
@@ -80,7 +82,7 @@ module Cloudtasker
       # @return [Cloudtasker::CloudTask] The Google Task response
       #
       def perform_async(*args)
-        perform_in(nil, *args)
+        schedule(args: args)
       end
 
       #
@@ -92,7 +94,7 @@ module Cloudtasker
       # @return [Cloudtasker::CloudTask] The Google Task response
       #
       def perform_in(interval, *args)
-        new(job_args: args).schedule(interval: interval)
+        schedule(args: args, time_in: interval)
       end
 
       #
@@ -104,7 +106,21 @@ module Cloudtasker
       # @return [Cloudtasker::CloudTask] The Google Task response
       #
       def perform_at(time_at, *args)
-        new(job_args: args).schedule(time_at: time_at)
+        schedule(args: args, time_at: time_at)
+      end
+
+      #
+      # Enqueue a worker with explicity options.
+      #
+      # @param [Array<any>] args The job arguments.
+      # @param [Time, Integer] time_in The delay in seconds.
+      # @param [Time, Integer] time_at The time at which the job should run.
+      # @param [String, Symbol] queue The queue on which the worker should run.
+      #
+      # @return [Cloudtasker::CloudTask] The Google Task response
+      #
+      def schedule(args: nil, time_in: nil, time_at: nil, queue: nil)
+        new(job_args: args, job_queue: queue).schedule({ interval: time_in, time_at: time_at }.compact)
       end
 
       #
@@ -123,11 +139,30 @@ module Cloudtasker
     # @param [Array<any>] job_args The list of perform args.
     # @param [String] job_id A unique ID identifying this job.
     #
-    def initialize(job_args: [], job_id: nil, job_meta: {}, job_retries: 0)
-      @job_args = job_args
+    def initialize(job_queue: nil, job_args: nil, job_id: nil, job_meta: {}, job_retries: 0)
+      @job_args = job_args || []
       @job_id = job_id || SecureRandom.uuid
       @job_meta = MetaStore.new(job_meta)
       @job_retries = job_retries || 0
+      @job_queue = job_queue
+    end
+
+    #
+    # Return the class name of the worker.
+    #
+    # @return [String] The class name.
+    #
+    def job_class_name
+      self.class.to_s
+    end
+
+    #
+    # Return the queue to use for this worker.
+    #
+    # @return [String] The name of queue.
+    #
+    def job_queue
+      (@job_queue ||= self.class.cloudtasker_options_hash[:queue] || Config::DEFAULT_JOB_QUEUE).to_s
     end
 
     #
@@ -198,10 +233,10 @@ module Cloudtasker
     # Return a new instance of the worker with the same args and metadata
     # but with a different id.
     #
-    # @return [<Type>] <description>
+    # @return [Cloudtasker::Worker] <description>
     #
     def new_instance
-      self.class.new(job_args: job_args, job_meta: job_meta)
+      self.class.new(job_queue: job_queue, job_args: job_args, job_meta: job_meta)
     end
 
     #
@@ -215,7 +250,8 @@ module Cloudtasker
         job_id: job_id,
         job_args: job_args,
         job_meta: job_meta.to_h,
-        job_retries: job_retries
+        job_retries: job_retries,
+        job_queue: job_queue
       }
     end
 

data/lib/cloudtasker/worker_handler.rb

@@ -5,7 +5,7 @@ require 'google/cloud/tasks'
 module Cloudtasker
   # Build, serialize and schedule tasks on the processing backend.
   class WorkerHandler
-    attr_reader :worker, :job_args
+    attr_reader :worker
 
     # Alrogith used to sign the verification token
     JWT_ALG = 'HS256'
@@ -42,11 +42,12 @@ module Cloudtasker
           http_method: 'POST',
           url: Cloudtasker.config.processor_url,
           headers: {
-            'Content-Type' => 'application/json',
-            'Authorization' => "Bearer #{Authenticator.verification_token}"
+            Cloudtasker::Config::CONTENT_TYPE_HEADER => 'application/json',
+            Cloudtasker::Config::AUTHORIZATION_HEADER => "Bearer #{Authenticator.verification_token}"
           },
           body: worker_payload.to_json
-        }
+        },
+        queue: worker.job_queue
       }
     end
 
@@ -64,7 +65,8 @@ module Cloudtasker
     #
     def worker_payload
       @worker_payload ||= {
-        worker: worker.class.to_s,
+        worker: worker.job_class_name,
+        job_queue: worker.job_queue,
         job_id: worker.job_id,
         job_args: worker.job_args,
         job_meta: worker.job_meta.to_h

data/lib/cloudtasker/worker_logger.rb

@@ -11,7 +11,7 @@ module Cloudtasker
     end
 
     # Only log the job meta information by default (exclude arguments)
-    DEFAULT_CONTEXT_PROCESSOR = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta) }
+    DEFAULT_CONTEXT_PROCESSOR = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta, :job_queue) }
 
     #
     # Build a new instance of the class.

data/lib/cloudtasker/worker_wrapper.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/worker'
+
+module Cloudtasker
+  # A worker class used to schedule jobs without actually
+  # instantiating the worker class. This is useful for middlewares
+  # needing to enqueue jobs in a Rails initializer. Rails 6 complains
+  # about instantiating workers in an iniitializer because of autoloading
+  # in zeitwerk mode.
+  #
+  # Downside of this wrapper: any cloudtasker_options specified on on the
+  # worker_class will be ignored.
+  #
+  # See: https://github.com/rails/rails/issues/36363
+  #
+  class WorkerWrapper
+    include Worker
+
+    attr_accessor :worker_name
+
+    #
+    # Build a new instance of the class.
+    #
+    # @param [String] worker_class The name of the worker class.
+    # @param [Hash] **opts The worker arguments.
+    #
+    def initialize(worker_name:, **opts)
+      @worker_name = worker_name
+      super(opts)
+    end
+
+    #
+    # Override parent. Return the underlying worker class name.
+    #
+    # @return [String] The worker class.
+    #
+    def job_class_name
+      worker_name
+    end
+
+    #
+    # Return a new instance of the worker with the same args and metadata
+    # but with a different id.
+    #
+    # @return [Cloudtasker::WorkerWrapper] <description>
+    #
+    def new_instance
+      self.class.new(worker_name: worker_name, job_queue: job_queue, job_args: job_args, job_meta: job_meta)
+    end
+  end
+end

data/lib/tasks/setup_queue.rake

@@ -1,10 +1,20 @@
 # frozen_string_literal: true
 
 require 'cloudtasker/backend/google_cloud_task'
+require 'cloudtasker/config'
+
+ENV['GOOGLE_AUTH_SUPPRESS_CREDENTIALS_WARNINGS'] ||= 'true'
 
 namespace :cloudtasker do
-  desc 'Setup the Cloud Task queue'
+  desc 'Setup a Cloud Task queue. (default options: ' \
+    "name=#{Cloudtasker::Config::DEFAULT_JOB_QUEUE}, " \
+    "concurrency=#{Cloudtasker::Config::DEFAULT_QUEUE_CONCURRENCY}, " \
+    "retries=#{Cloudtasker::Config::DEFAULT_QUEUE_RETRIES})"
   task setup_queue: :environment do
-    Cloudtasker::Backend::GoogleCloudTask.setup_queue
+    puts Cloudtasker::Backend::GoogleCloudTask.setup_queue(
+      name: ENV['name'],
+      concurrency: ENV['concurrency'],
+      retries: ENV['retries']
+    )
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-25 00:00:00.000000000 Z
+date: 2020-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -248,7 +248,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Background jobs for Ruby using Google Cloud Tasks (alpha)
+description: Background jobs for Ruby using Google Cloud Tasks (beta)
 email:
 - arnaud.lachaume@keypup.io
 executables:
@@ -264,10 +264,10 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
+- _config.yml
 - app/controllers/cloudtasker/application_controller.rb
 - app/controllers/cloudtasker/worker_controller.rb
 - bin/console
@@ -322,7 +322,6 @@ files:
 - lib/cloudtasker/local_server.rb
 - lib/cloudtasker/meta_store.rb
 - lib/cloudtasker/middleware/chain.rb
-- lib/cloudtasker/railtie.rb
 - lib/cloudtasker/redis_client.rb
 - lib/cloudtasker/testing.rb
 - lib/cloudtasker/unique_job.rb
@@ -344,6 +343,7 @@ files:
 - lib/cloudtasker/worker.rb
 - lib/cloudtasker/worker_handler.rb
 - lib/cloudtasker/worker_logger.rb
+- lib/cloudtasker/worker_wrapper.rb
 - lib/tasks/setup_queue.rake
 homepage: https://github.com/keypup-io/cloudtasker
 licenses:
@@ -371,5 +371,5 @@ rubyforge_project:
 rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
-summary: Background jobs for Ruby using Google Cloud Tasks (alpha)
+summary: Background jobs for Ruby using Google Cloud Tasks (beta)
 test_files: []