cloudtasker 0.2.0 → 0.7.0

data/lib/cloudtasker/config.rb

@@ -1,15 +1,24 @@
 # frozen_string_literal: true
 
+require 'logger'
+
 module Cloudtasker
   # Holds cloudtasker configuration. See Cloudtasker#configure
   class Config
     attr_accessor :redis
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
-                :gcp_queue_id, :processor_host, :processor_path
+                :gcp_queue_id, :processor_path, :logger, :mode, :max_retries
+
+    # Retry header in Cloud Task responses
+    RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
 
+    # Default values
     DEFAULT_LOCATION_ID = 'us-east1'
     DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
 
+    # The number of times jobs will be attempted before declaring them dead
+    DEFAULT_MAX_RETRY_ATTEMPTS = 25
+
     PROCESSOR_HOST_MISSING = <<~DOC
       Missing host for processing.
       Please specify a processor hostname in form of `https://some-public-dns.example.com`'
@@ -27,6 +36,46 @@ module Cloudtasker
       Please specify a secret in the cloudtasker initializer or add Rails secret_key_base in your credentials
     DOC
 
+    #
+    # The number of times jobs will be retried. This number of
+    # retries does not include failures due to the application being unreachable.
+    #
+    #
+    # @return [Integer] The number of retries
+    #
+    def max_retries
+      @max_retries ||= DEFAULT_MAX_RETRY_ATTEMPTS
+    end
+
+    #
+    # The operating mode.
+    #   - :production => process tasks via GCP Cloud Task.
+    #   - :development => process tasks locally via Redis.
+    #
+    # @return [<Type>] <description>
+    #
+    def mode
+      @mode ||= environment == 'development' ? :development : :production
+    end
+
+    #
+    # Return the current environment.
+    #
+    # @return [String] The environment name.
+    #
+    def environment
+      ENV['CLOUDTASKER_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    end
+
+    #
+    # Return the Cloudtasker logger.
+    #
+    # @return [Logger, any] The cloudtasker logger.
+    #
+    def logger
+      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
+    end
+
     #
     # Return the full URL of the processor. Worker payloads will be sent
     # to this URL.
@@ -37,6 +86,19 @@ module Cloudtasker
       File.join(processor_host, processor_path)
     end
 
+    #
+    # Set the processor host. In the context of Rails the host will
+    # also be added to the list of authorized Rails hosts.
+    #
+    # @param [String] val The processor host to set.
+    #
+    def processor_host=(val)
+      @processor_host = val
+
+      # Add processor host to the list of authorized hosts
+      Rails.application.config.hosts << val.gsub(%r{https?://}, '') if val && defined?(Rails)
+    end
+
     #
     # The hostname of the application processing the workers. The hostname must
     # be reachable from Cloud Task.
@@ -93,7 +155,7 @@ module Cloudtasker
     #
     def secret
       @secret || (
-        defined?(Rails) && Rails.application.credentials&.secret_key_base
+        defined?(Rails) && Rails.application.credentials&.dig(:secret_key_base)
       ) || raise(StandardError, SECRET_MISSING_ERROR)
     end
 

data/lib/cloudtasker/cron/job.rb

@@ -179,8 +179,8 @@ module Cloudtasker
         next_worker = worker.new_instance.tap { |e| e.job_meta.set(key(:time_at), next_time.iso8601) }
 
         # Schedule next worker
-        resp = next_worker.schedule(time_at: next_time)
-        cron_schedule.update(task_id: resp.name, job_id: next_worker.job_id)
+        task = next_worker.schedule(time_at: next_time)
+        cron_schedule.update(task_id: task.id, job_id: next_worker.job_id)
       end
 
       #

data/lib/cloudtasker/cron/schedule.rb

@@ -56,7 +56,7 @@ module Cloudtasker
       #
       def self.load_from_hash!(hash)
         schedules = hash.map do |id, config|
-          schedule_config = JSON.parse(config.to_json, symbolize_names: true).merge(id: id)
+          schedule_config = JSON.parse(config.to_json, symbolize_names: true).merge(id: id.to_s)
           create(schedule_config)
         end
 
@@ -72,8 +72,10 @@ module Cloudtasker
       # @return [Cloudtasker::Cron::Schedule] The schedule instance.
       #
       def self.create(**opts)
-        config = find(opts[:id]).to_h.merge(opts)
-        new(config).tap(&:save)
+        redis.with_lock(key(opts[:id])) do
+          config = find(opts[:id]).to_h.merge(opts)
+          new(config).tap(&:save)
+        end
       end
 
       #
@@ -95,12 +97,14 @@ module Cloudtasker
       # @param [String] id The schedule id.
       #
       def self.delete(id)
-        schedule = find(id)
-        return false unless schedule
+        redis.with_lock(key(id)) do
+          schedule = find(id)
+          return false unless schedule
 
-        # Delete task and stored schedule
-        Task.delete(schedule.task_id) if schedule.task_id
-        redis.del(schedule.gid)
+          # Delete task and stored schedule
+          CloudTask.delete(schedule.task_id) if schedule.task_id
+          redis.del(schedule.gid)
+        end
       end
 
       #
@@ -252,17 +256,27 @@ module Cloudtasker
       # then any existing cloud task is removed and a task is recreated.
       #
       def save(update_task: true)
-        return false unless valid? && changed?
+        return false unless valid?
 
         # Save schedule
         config_was_changed = config_changed?
         redis.write(gid, to_h)
 
         # Stop there if backend does not need update
-        return true unless update_task && config_was_changed
+        return true unless update_task && (config_was_changed || !task_id || !CloudTask.find(task_id))
+
+        # Update backend
+        persist_cloud_task
+      end
 
+      private
+
+      #
+      # Update the task in backend.
+      #
+      def persist_cloud_task
         # Delete previous instance
-        Task.delete(task_id) if task_id
+        CloudTask.delete(task_id) if task_id
 
         # Schedule worker
         worker_instance = Object.const_get(worker).new

data/lib/cloudtasker/dead_worker_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class DeadWorkerError < StandardError
+  end
+end

data/lib/cloudtasker/local_server.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/backend/redis_task'
+
+module Cloudtasker
+  # Process jobs stored in Redis.
+  # Only to be used in development.
+  class LocalServer
+    # Max number of task requests sent to the processing server
+    CONCURRENCY = (ENV['CLOUDTASKER_CONCURRENCY'] || 5).to_i
+
+    #
+    # Stop the local server.
+    #
+    def stop
+      @done = true
+
+      # Terminate threads and repush tasks
+      @threads&.each do |t|
+        t.terminate
+        t['task']&.retry_later(0, is_error: false)
+      end
+
+      # Wait for main server to be done
+      sleep 1 while @start&.alive?
+    end
+
+    #
+    # Start the local server
+    #
+    #
+    def start
+      @start ||= Thread.new do
+        until @done
+          process_jobs
+          sleep 1
+        end
+        Cloudtasker.logger.info('[Cloudtasker/Server] Local server exiting...')
+      end
+    end
+
+    #
+    # Process enqueued workers.
+    #
+    #
+    def process_jobs
+      @threads ||= []
+
+      # Remove any done thread
+      @threads.select!(&:alive?)
+
+      # Process tasks
+      while @threads.count < CONCURRENCY && (task = Cloudtasker::Backend::RedisTask.pop)
+        @threads << Thread.new do
+          Thread.current['task'] = task
+          Thread.current['attempts'] = 0
+
+          # Deliver task
+          begin
+            Thread.current['task'].deliver
+          rescue Errno::ECONNREFUSED => e
+            raise(e) unless Thread.current['attempts'] < 3
+
+            # Retry on connection error, in case the web server is not
+            # started yet.
+            Thread.current['attempts'] += 1
+            sleep(3)
+            retry
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker/railtie.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Rails extensions
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load 'tasks/setup_queue.rake'
+    end
+  end
+end

data/lib/cloudtasker/redis_client.rb

@@ -8,7 +8,7 @@ module Cloudtasker
     module_function
 
     # Suffix added to cache keys when locking them
-    LOCK_KEY_SUFFIX = 'lock'
+    LOCK_KEY_PREFIX = 'cloudtasker/lock'
 
     #
     # Return the underlying redis client.
@@ -61,7 +61,7 @@ module Cloudtasker
       return nil unless cache_key
 
       # Wait to acquire lock
-      lock_key = [cache_key, LOCK_KEY_SUFFIX].join('/')
+      lock_key = [LOCK_KEY_PREFIX, cache_key].join('/')
       true until client.setnx(lock_key, true)
 
       # yield content

data/lib/cloudtasker/testing.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/backend/memory_task'
+
+module Cloudtasker
+  # Enable/Disable test mode for Cloudtasker
+  module Testing
+    module_function
+
+    #
+    # Set the test mode, either permanently or
+    # temporarily (via block).
+    #
+    # @param [Symbol] mode The test mode.
+    #
+    # @return [Symbol] The test mode.
+    #
+    def switch_test_mode(mode)
+      if block_given?
+        current_mode = @test_mode
+        begin
+          @test_mode = mode
+          yield
+        ensure
+          @test_mode = current_mode
+        end
+      else
+        @test_mode = mode
+      end
+    end
+
+    #
+    # Set cloudtasker to real mode temporarily
+    #
+    # @param [Proc] &block The block to run in real mode
+    #
+    def enable!(&block)
+      switch_test_mode(:enabled, &block)
+    end
+
+    #
+    # Set cloudtasker to fake mode temporarily
+    #
+    # @param [Proc] &block The block to run in fake mode
+    #
+    def fake!(&block)
+      switch_test_mode(:fake, &block)
+    end
+
+    #
+    # Set cloudtasker to inline mode temporarily
+    #
+    # @param [Proc] &block The block to run in inline mode
+    #
+    def inline!(&block)
+      switch_test_mode(:inline, &block)
+    end
+
+    #
+    # Return true if Cloudtasker is enabled.
+    #
+    def enabled?
+      !@test_mode || @test_mode == :enabled
+    end
+
+    #
+    # Return true if Cloudtasker is in fake mode.
+    #
+    # @return [Boolean] True if jobs must be processed through drain calls.
+    #
+    def fake?
+      @test_mode == :fake
+    end
+
+    #
+    # Return true if Cloudtasker is in inline mode.
+    #
+    # @return [Boolean] True if jobs are run inline.
+    #
+    def inline?
+      @test_mode == :inline
+    end
+
+    #
+    # Return true if tasks should be managed in memory.
+    #
+    # @return [Boolean] True if jobs are managed in memory.
+    #
+    def in_memory?
+      !enabled?
+    end
+  end
+
+  # Add extra methods for testing purpose
+  module Worker
+    #
+    # Clear all jobs.
+    #
+    def self.clear_all
+      Backend::MemoryTask.clear
+    end
+
+    #
+    # Run all the jobs.
+    #
+    # @return [Array<any>] The return values of the workers perform method.
+    #
+    def self.drain_all
+      Backend::MemoryTask.drain
+    end
+
+    # Module class methods
+    module ClassMethods
+      #
+      # Return all jobs related to this worker class.
+      #
+      # @return [Array<Cloudtasker::Worker] The list of workers
+      #
+      def jobs
+        Backend::MemoryTask.jobs(to_s)
+      end
+
+      #
+      # Run all jobs related to this worker class.
+      #
+      # @return [Array<any>] The return values of the workers perform method.
+      #
+      def drain
+        Backend::MemoryTask.drain(to_s)
+      end
+    end
+  end
+end

data/lib/cloudtasker/unique_job/job.rb

@@ -40,7 +40,7 @@ module Cloudtasker
         @lock_instance ||=
           begin
             # Infer lock class and get instance
-            lock_name = options[:lock] || options['lock']
+            lock_name = options[:lock]
             lock_klass = Lock.const_get(lock_name.to_s.split('_').collect(&:capitalize).join)
             lock_klass.new(self)
           rescue NameError

data/lib/cloudtasker/unique_job/lock/base_lock.rb

@@ -43,7 +43,7 @@ module Cloudtasker
           @conflict_instance ||=
             begin
               # Infer lock class and get instance
-              strategy_name = options[:on_conflict] || options['on_conflict']
+              strategy_name = options[:on_conflict]
               strategy_klass = ConflictStrategy.const_get(strategy_name.to_s.split('_').collect(&:capitalize).join)
               strategy_klass.new(job)
             rescue NameError