cloudtasker-tonix 0.1.0

data/lib/cloudtasker/backend/memory_task.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Backend
+    # Manage local tasks pushed to memory.
+    # Used for testing.
+    class MemoryTask
+      attr_accessor :job_retries
+      attr_reader :id, :http_request, :schedule_time, :queue
+
+      #
+      # Return true if we are in test inline execution mode.
+      #
+      # @return [Boolean] True if inline mode enabled.
+      #
+      def self.inline_mode?
+        defined?(Cloudtasker::Testing) && Cloudtasker::Testing.inline?
+      end
+
+      #
+      # Return true if errors must be raised immediately
+      #
+      # @return [Boolean] True if raise error mode is enabled.
+      #
+      def self.raise_errors?
+        defined?(Cloudtasker::Testing) && Cloudtasker::Testing.raise_errors?
+      end
+
+      #
+      # Return the task queue. A worker class name
+      #
+      # @return [Array<Hash>] <description>
+      #
+      def self.queue
+        @queue ||= []
+      end
+
+      #
+      # Run all Tasks in the queue. Optionally filter which tasks to run based
+      # on the worker class name.
+      #
+      # @param [String] worker_class_name Run tasks for a specific worker class name.
+      #
+      # @return [Array<any>] The return values of the workers perform method.
+      #
+      def self.drain(worker_class_name = nil)
+        all(worker_class_name).map(&:execute)
+      end
+
+      #
+      # Return all enqueued tasks. A worker class name can be specified
+      # to filter the returned results.
+      #
+      # @param [String] worker_class_name Filter tasks on worker class name.
+      #
+      # @return [Array<Cloudtasker::Backend::MemoryTask>] All the tasks
+      #
+      def self.all(worker_class_name = nil)
+        # Always return a copy of the queue so that tasks can safely be removed
+        # (via #execute) without tampering with the enumerator
+        if worker_class_name
+          queue.select { |e| e.worker_class_name == worker_class_name }
+        else
+          queue.dup
+        end
+      end
+
+      #
+      # Push a job to the queue.
+      #
+      # @param [Hash] payload The Cloud Task payload.
+      #
+      def self.create(payload)
+        id = payload[:id] || SecureRandom.uuid
+        payload = payload.merge(schedule_time: payload[:schedule_time].to_i)
+
+        # Save task
+        task = new(**payload, id: id)
+        queue << task
+
+        # Execute task immediately if in testing and inline mode enabled
+        task.execute if inline_mode?
+
+        task
+      end
+
+      #
+      # Get a task by id.
+      #
+      # @param [String] id The id of the task.
+      #
+      # @return [Cloudtasker::Backend::MemoryTask, nil] The task.
+      #
+      def self.find(id)
+        queue.find { |e| e.id == id }
+      end
+
+      #
+      # Delete a task by id.
+      #
+      # @param [String] id The task id.
+      #
+      def self.delete(id)
+        queue.reject! { |e| e.id == id }
+      end
+
+      #
+      # Clear the queue.
+      #
+      # @param [String] worker_class_name Filter jobs on worker class name.
+      #
+      # @return [Array<Cloudtasker::Backend::MemoryTask>] The updated queue
+      #
+      def self.clear(worker_class_name = nil)
+        if worker_class_name
+          queue.reject! { |e| e.worker_class_name == worker_class_name }
+        else
+          queue.clear
+        end
+      end
+
+      #
+      # Build a new instance of the class.
+      #
+      # @param [String] id The ID of the task.
+      # @param [Hash] http_request The HTTP request content.
+      # @param [Integer] schedule_time When to run the task (Unix timestamp)
+      #
+      def initialize(id:, http_request:, schedule_time: nil, queue: nil, job_retries: 0, **_xargs)
+        @id = id
+        @http_request = http_request
+        @schedule_time = Time.at(schedule_time || 0)
+        @queue = queue
+        @job_retries = job_retries || 0
+      end
+
+      #
+      # Return task payload.
+      #
+      # @return [Hash] The task payload.
+      #
+      def payload
+        @payload ||= JSON.parse(http_request[:body], symbolize_names: true)
+      end
+
+      #
+      # Return the worker class from the task payload.
+      #
+      # @return [String] The task worker class name.
+      #
+      def worker_class_name
+        payload[:worker]
+      end
+
+      #
+      # Return a hash description of the task.
+      #
+      # @return [Hash] A hash description of the task.
+      #
+      def to_h
+        {
+          id: id,
+          http_request: http_request,
+          schedule_time: schedule_time.to_i,
+          queue: queue
+        }
+      end
+
+      #
+      # Execute the task.
+      #
+      # @return [Any] The return value of the worker perform method.
+      #
+      def execute
+        # Execute worker
+        worker_payload = payload.merge(job_retries: job_retries, task_id: id)
+        resp = WorkerHandler.with_worker_handling(worker_payload, &:execute)
+
+        # Delete task
+        self.class.delete(id)
+        resp
+      rescue DeadWorkerError => e
+        self.class.delete(id)
+        raise(e) if self.class.raise_errors?
+      rescue StandardError => e
+        self.job_retries += 1
+        raise(e) if self.class.raise_errors?
+      end
+
+      #
+      # Equality operator.
+      #
+      # @param [Any] other The object to compare.
+      #
+      # @return [Boolean] True if the object is equal.
+      #
+      def ==(other)
+        other.is_a?(self.class) && other.id == id
+      end
+    end
+  end
+end

data/lib/cloudtasker/backend/redis_task.rb

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/redis_client'
+require 'net/http'
+
+module Cloudtasker
+  module Backend
+    # Manage local tasks pushed to Redis
+    class RedisTask
+      attr_reader :id, :http_request, :schedule_time, :retries, :queue, :dispatch_deadline
+
+      RETRY_INTERVAL = Config::LOCAL_SERVER_RETRY_DELAY
+
+      #
+      # Return the Cloudtasker redis client
+      #
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
+      #
+      def self.redis
+        @redis ||= RedisClient.new
+      end
+
+      #
+      # Return a namespaced key.
+      #
+      # @param [String, Symbol, nil] val The key to namespace
+      #
+      # @return [String] The namespaced key.
+      #
+      def self.key(val = nil)
+        [to_s.underscore, val].compact.map(&:to_s).join('/')
+      end
+
+      #
+      # Return all tasks stored in Redis.
+      #
+      # @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks.
+      #
+      def self.all
+        if redis.exists?(key)
+          # Use Schedule Set if available
+          redis.smembers(key).map { |id| find(id) }.compact
+        else
+          # Fallback to redis key matching and migrate tasks
+          # to use Task Set instead.
+          redis.search(key('*')).map do |gid|
+            task_id = gid.sub(key(''), '')
+            redis.sadd(key, [task_id])
+            find(task_id)
+          end
+        end
+      end
+
+      #
+      # Return all tasks ready to process.
+      #
+      # @param [String] queue The queue to retrieve items from.
+      #
+      # @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks ready to process.
+      #
+      def self.ready_to_process(queue = nil)
+        list = all.select { |e| e.schedule_time <= Time.now }
+        list = list.select { |e| e.queue == queue } if queue
+        list
+      end
+
+      #
+      # Retrieve and remove a task from the queue.
+      #
+      # @param [String] queue The queue to retrieve items from.
+      #
+      # @return [Cloudtasker::Backend::RedisTask] A task ready to process.
+      #
+      def self.pop(queue = nil)
+        redis.with_lock('cloudtasker/server') do
+          ready_to_process(queue).first&.tap(&:destroy)
+        end
+      end
+
+      #
+      # Push a job to the queue.
+      #
+      # @param [Hash] payload The Cloud Task payload.
+      #
+      def self.create(payload)
+        id = SecureRandom.uuid
+        payload = payload.merge(schedule_time: payload[:schedule_time].to_i)
+
+        # Save job
+        redis.write(key(id), payload)
+        redis.sadd(key, [id])
+        new(**payload, id: id)
+      end
+
+      #
+      # Get a task by id.
+      #
+      # @param [String] id The id of the task.
+      #
+      # @return [Cloudtasker::Backend::RedisTask, nil] The task.
+      #
+      def self.find(id)
+        gid = key(id)
+        return nil unless (payload = redis.fetch(gid))
+
+        new(**payload, id: id)
+      end
+
+      #
+      # Delete a task by id.
+      #
+      # @param [String] id The task id.
+      #
+      def self.delete(id)
+        redis.srem(key, [id])
+        redis.del(key(id))
+      end
+
+      #
+      # Build a new instance of the class.
+      #
+      # @param [String] id The ID of the task.
+      # @param [Hash] http_request The HTTP request content.
+      # @param [Integer] schedule_time When to run the task (Unix timestamp)
+      # @param [Integer] retries The number of times the job failed.
+      # @param [Integer] dispatch_deadline The dispatch_deadline in seconds.
+      #
+      def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil, dispatch_deadline: nil)
+        @id = id
+        @http_request = http_request
+        @schedule_time = Time.at(schedule_time || 0)
+        @retries = retries || 0
+        @queue = queue || Config::DEFAULT_JOB_QUEUE
+        @dispatch_deadline = dispatch_deadline || Config::DEFAULT_DISPATCH_DEADLINE
+      end
+
+      #
+      # Return the redis client.
+      #
+      # @return [Class] The RedisClient.
+      #
+      def redis
+        self.class.redis
+      end
+
+      #
+      # Return a hash description of the task.
+      #
+      # @return [Hash] A hash description of the task.
+      #
+      def to_h
+        {
+          id: id,
+          http_request: http_request,
+          schedule_time: schedule_time.to_i,
+          retries: retries,
+          queue: queue,
+          dispatch_deadline: dispatch_deadline
+        }
+      end
+
+      #
+      # Return the namespaced task id
+      #
+      # @return [<Type>] The namespaced task id
+      #
+      def gid
+        self.class.key(id)
+      end
+
+      #
+      # Retry the task later.
+      #
+      # @param [Integer] interval The delay in seconds before retrying the task
+      # @param [Hash] opts Additional options
+      # @option opts [Boolean] :is_error Increase number of retries. Default to true.
+      #
+      def retry_later(interval, opts = {})
+        is_error = opts.to_h.fetch(:is_error, true)
+
+        redis.write(
+          gid,
+          retries: is_error ? retries + 1 : retries,
+          http_request: http_request,
+          schedule_time: (Time.now + interval).to_i,
+          queue: queue,
+          dispatch_deadline: dispatch_deadline
+        )
+        redis.sadd(self.class.key, [id])
+      end
+
+      #
+      # Remove the task from the queue.
+      #
+      def destroy
+        self.class.delete(id)
+      end
+
+      #
+      # Deliver the task to the processing endpoint.
+      #
+      def deliver
+        Cloudtasker.logger.info(format_log_message('Processing task...'))
+
+        # Send request
+        resp = http_client.request(request_content)
+
+        # Delete task if successful
+        if resp.code.to_s =~ /20\d/
+          destroy
+          Cloudtasker.logger.info(format_log_message('Task handled successfully'))
+        else
+          retry_later(RETRY_INTERVAL)
+          Cloudtasker.logger.info(format_log_message("Task failure - Retry in #{RETRY_INTERVAL} seconds..."))
+        end
+
+        resp
+      rescue Errno::ECONNREFUSED
+        retry_later(RETRY_INTERVAL)
+        Cloudtasker.logger.info(format_log_message("Processor not ready - Retry in #{RETRY_INTERVAL} seconds..."))
+      rescue Net::ReadTimeout
+        retry_later(RETRY_INTERVAL)
+        Cloudtasker.logger.info(
+          format_log_message(
+            "Task deadline exceeded (#{dispatch_deadline}s) - Retry in #{RETRY_INTERVAL} seconds..."
+          )
+        )
+      end
+
+      #
+      # Equality operator.
+      #
+      # @param [Any] other The object to compare.
+      #
+      # @return [Boolean] True if the object is equal.
+      #
+      def ==(other)
+        other.is_a?(self.class) && other.id == id
+      end
+
+      private
+
+      #
+      # Format a log message
+      #
+      # @param [String] msg The message to log.
+      #
+      # @return [String] The formatted message
+      #
+      def format_log_message(msg)
+        "[Cloudtasker/Server][#{id}] #{msg}"
+      end
+
+      #
+      # Return the HTTP client.
+      #
+      # @return [Net::HTTP] The http_client.
+      #
+      def http_client
+        @http_client ||=
+          begin
+            uri = URI(http_request[:url])
+            http = Net::HTTP.new(uri.host, uri.port).tap { |e| e.read_timeout = dispatch_deadline }
+            http.use_ssl = true if uri.instance_of?(URI::HTTPS)
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless Cloudtasker.config.local_server_ssl_verify
+            http
+          end
+      end
+
+      #
+      # Return the HTTP request to send
+      #
+      # @return [Net::HTTP::Post] The http request
+      #
+      def request_content
+        @request_content ||= begin
+          uri = URI(http_request[:url])
+          req = Net::HTTP::Post.new(uri.path, http_request[:headers])
+
+          # Add task headers
+          req[Cloudtasker::Config::TASK_ID_HEADER] = id
+          req[Cloudtasker::Config::RETRY_HEADER] = retries
+
+          # Set job payload
+          req.body = http_request[:body]
+          req
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker/batch/batch_progress.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'fugit'
+
+module Cloudtasker
+  module Batch
+    # Capture the progress of a batch
+    class BatchProgress
+      attr_reader :batches
+
+      #
+      # Build a new instance of the class.
+      #
+      # @param [Array<Cloudtasker::Batch::Job>] batches The batches to consider
+      #
+      def initialize(batches = [])
+        @batches = batches
+      end
+
+      # Count the number of items in a given status
+
+      #
+      # Count the number of items in a given status
+      #
+      # @param [String] status The status to count
+      #
+      # @return [Integer] The number of jobs in the status
+      #
+      def count(status = 'all')
+        batches.sum { |e| e.batch_state_count(status) }
+      end
+
+      #
+      # Return the total number jobs.
+      #
+      # @return [Integer] The number number of jobs.
+      #
+      def total
+        count
+      end
+
+      #
+      # Return the number of completed jobs.
+      #
+      # @return [Integer] The number of completed jobs.
+      #
+      def completed
+        @completed ||= count('completed')
+      end
+
+      #
+      # Return the number of scheduled jobs.
+      #
+      # @return [Integer] The number of scheduled jobs.
+      #
+      def scheduled
+        @scheduled ||= count('scheduled')
+      end
+
+      #
+      # Return the number of processing jobs.
+      #
+      # @return [Integer] The number of processing jobs.
+      #
+      def processing
+        @processing ||= count('processing')
+      end
+
+      #
+      # Return the number of jobs with errors.
+      #
+      # @return [Integer] The number of errored jobs.
+      #
+      def errored
+        @errored ||= count('errored')
+      end
+
+      #
+      # Return the number of dead jobs.
+      #
+      # @return [Integer] The number of dead jobs.
+      #
+      def dead
+        @dead ||= count('dead')
+      end
+
+      #
+      # Return the number of jobs not completed yet.
+      #
+      # @return [Integer] The number of jobs pending.
+      #
+      def pending
+        total - done
+      end
+
+      #
+      # Return the number of jobs completed or dead.
+      #
+      # @return [Integer] The number of jobs done.
+      #
+      def done
+        completed + dead
+      end
+
+      #
+      # Return the batch progress percentage.
+      #
+      # A `min_total` can be specified to linearize the calculation, while jobs get added at
+      # the start of the batch.
+      #
+      # Similarly a `smoothing` parameter can be specified to add a constant to the total
+      # and linearize the calculation, which becomes: `done / (total + smoothing)`
+      #
+      # @param [Integer] min_total The minimum for the total number of jobs
+      # @param [Integer] smoothing An additive smoothing for the total number of jobs
+      #
+      # @return [Float] The progress percentage.
+      #
+      def percent(min_total: 0, smoothing: 0)
+        # Get the total value to use
+        actual_total = [min_total, total + smoothing].max
+
+        # Abort if we cannot divide
+        return 0 if actual_total.zero?
+
+        # Calculate progress
+        (done.to_f / actual_total) * 100
+      end
+
+      #
+      # Add a batch progress to another one.
+      #
+      # @param [Cloudtasker::Batch::BatchProgress] progress The progress to add.
+      #
+      # @return [Cloudtasker::Batch::BatchProgress] The sum of the two batch progresses.
+      #
+      def +(other)
+        self.class.new(batches + other.batches)
+      end
+    end
+  end
+end

data/lib/cloudtasker/batch/extension/worker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Batch
+    module Extension
+      # Include batch related methods onto Cloudtasker::Worker
+      # See: Cloudtasker::Batch::Middleware#configure
+      module Worker
+        attr_accessor :batch, :parent_batch
+      end
+    end
+  end
+end