cloudtasker 0.1.0 → 0.6.0

data/lib/cloudtasker/batch/job.rb

@@ -6,6 +6,13 @@ module Cloudtasker
     class Job
       attr_reader :worker
 
+      # Key Namespace used for object saved under this class
+      JOBS_NAMESPACE = 'jobs'
+      STATES_NAMESPACE = 'states'
+
+      # List of statuses triggering a completion callback
+      COMPLETION_STATUSES = %w[completed dead].freeze
+
       #
       # Return the cloudtasker redis client
       #
@@ -25,13 +32,13 @@ module Cloudtasker
       def self.find(worker_id)
         return nil unless worker_id
 
-        # Retrieve parent worker
-        parent_payload = redis.fetch(key(worker_id))
-        parent_worker = Cloudtasker::Worker.from_hash(parent_payload)
-        return nil unless parent_worker
+        # Retrieve related worker
+        payload = redis.fetch(key("#{JOBS_NAMESPACE}/#{worker_id}"))
+        worker = Cloudtasker::Worker.from_hash(payload)
+        return nil unless worker
 
         # Build batch job
-        self.for(parent_worker)
+        self.for(worker)
       end
 
       #
@@ -44,7 +51,7 @@ module Cloudtasker
       def self.key(val)
         return nil if val.nil?
 
-        [Config::KEY_NAMESPACE, val.to_s].join('/')
+        [to_s.underscore, val.to_s].join('/')
       end
 
       #
@@ -134,7 +141,7 @@ module Cloudtasker
       # @return [String] The worker namespaced id.
       #
       def batch_gid
-        key(batch_id)
+        key("#{JOBS_NAMESPACE}/#{batch_id}")
       end
 
       #
@@ -143,7 +150,7 @@ module Cloudtasker
       # @return [String] The batch state namespaced id.
       #
       def batch_state_gid
-        [batch_gid, 'state'].join('/')
+        key("#{STATES_NAMESPACE}/#{batch_id}")
       end
 
       #
@@ -220,24 +227,62 @@ module Cloudtasker
           return true unless state
 
           # Check that all children are complete
-          state.values.all? { |e| e == 'completed' }
+          state.values.all? { |e| COMPLETION_STATUSES.include?(e) }
         end
       end
 
+      #
+      # Run worker callback in a controlled environment to
+      # avoid interruption of the callback flow.
+      #
+      # @param [String, Symbol] callback The callback to run.
+      # @param [Array<any>] *args The callback arguments.
+      #
+      # @return [any] The callback return value
+      #
+      def run_worker_callback(callback, *args)
+        worker.try(callback, *args)
+      rescue StandardError => e
+        Cloudtasker.logger.error("Error running callback #{callback}: #{e}")
+        Cloudtasker.logger.error(e.backtrace.join("\n"))
+        nil
+      end
+
+      #
+      # Callback invoked when the batch is complete
+      #
+      def on_complete(status = :completed)
+        # Invoke worker callback
+        run_worker_callback(:on_batch_complete) if status == :completed
+
+        # Propagate event
+        parent_batch&.on_child_complete(self, status)
+      ensure
+        # The batch tree is complete. Cleanup the tree.
+        cleanup unless parent_batch
+      end
+
       #
       # Callback invoked when a direct child batch is complete.
       #
       # @param [Cloudtasker::Batch::Job] child_batch The completed child batch.
       #
-      def on_child_complete(child_batch)
+      def on_child_complete(child_batch, status = :completed)
         # Update batch state
-        update_state(child_batch.batch_id, :completed)
+        update_state(child_batch.batch_id, status)
 
         # Notify the worker that a direct batch child worker has completed
-        worker.try(:on_child_complete, child_batch.worker)
+        case status
+        when :completed
+          run_worker_callback(:on_child_complete, child_batch.worker)
+        when :errored
+          run_worker_callback(:on_child_error, child_batch.worker)
+        when :dead
+          run_worker_callback(:on_child_dead, child_batch.worker)
+        end
 
         # Notify the parent batch that we are done with this batch
-        parent_batch&.on_child_complete(self) if complete?
+        on_complete if status != :errored && complete?
       end
 
       #
@@ -245,14 +290,31 @@ module Cloudtasker
       #
       # @param [Cloudtasker::Batch::Job] child_batch The completed child batch.
       #
-      def on_batch_node_complete(child_batch)
+      def on_batch_node_complete(child_batch, status = :completed)
+        return false unless status == :completed
+
         # Notify the worker that a batch node worker has completed
-        worker.try(:on_batch_node_complete, child_batch.worker)
+        run_worker_callback(:on_batch_node_complete, child_batch.worker)
 
         # Notify the parent batch that a node is complete
         parent_batch&.on_batch_node_complete(child_batch)
       end
 
+      #
+      # Remove all batch and sub-batch keys from Redis.
+      #
+      def cleanup
+        # Capture batch state
+        state = batch_state
+
+        # Delete child batches recursively
+        state.to_h.keys.each { |id| self.class.find(id)&.cleanup }
+
+        # Delete batch redis entries
+        redis.del(batch_gid)
+        redis.del(batch_state_gid)
+      end
+
       #
       # Calculate the progress of the batch.
       #
@@ -271,7 +333,7 @@ module Cloudtasker
       #
       # Save the batch and enqueue all child workers attached to it.
       #
-      # @return [Array<Google::Cloud::Tasks::V2beta3::Task>] The Google Task responses
+      # @return [Array<Cloudtasker::CloudTask>] The Google Task responses
       #
       def setup
         return true if jobs.empty?
@@ -286,17 +348,14 @@ module Cloudtasker
       #
       # Post-perform logic. The parent batch is notified if the job is complete.
       #
-      def complete
+      def complete(status = :completed)
         return true if reenqueued? || jobs.any?
 
         # Notify the parent batch that a child is complete
-        if complete?
-          worker.try(:on_batch_complete)
-          parent_batch&.on_child_complete(self)
-        end
+        on_complete(status) if complete?
 
         # Notify the parent that a batch node has completed
-        parent_batch&.on_batch_node_complete(self)
+        parent_batch&.on_batch_node_complete(self, status)
       end
 
       #
@@ -313,7 +372,13 @@ module Cloudtasker
         setup
 
         # Complete batch
-        complete
+        complete(:completed)
+      rescue DeadWorkerError => e
+        complete(:dead)
+        raise(e)
+      rescue StandardError => e
+        complete(:errored)
+        raise(e)
       end
     end
   end

data/lib/cloudtasker/batch/middleware.rb

@@ -3,7 +3,6 @@
 require 'cloudtasker/redis_client'
 
 require_relative 'extension/worker'
-require_relative 'config'
 require_relative 'batch_progress'
 require_relative 'job'
 

data/lib/cloudtasker/cli.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require 'cloudtasker'
+require 'cloudtasker/local_server'
+
+module Cloudtasker
+  # Cloudtasker executable logic
+  module CLI
+    module_function
+
+    #
+    # Return the current environment.
+    #
+    # @return [String] The environment name.
+    #
+    def environment
+      Cloudtasker.config.environment
+    end
+
+    #
+    # Return true if we are running in Rails.
+    #
+    # @return [Boolean] True if rails is loaded.
+    #
+    def rails_app?
+      defined?(::Rails)
+    end
+
+    #
+    # Return true if we are running in JRuby.
+    #
+    # @return [Boolean] True if JRuby is loaded.
+    #
+    def jruby?
+      defined?(::JRUBY_VERSION)
+    end
+
+    #
+    # Return the Cloudtasker logger
+    #
+    # @return [Logger] The Cloudtasker logger.
+    #
+    def logger
+      Cloudtasker.logger
+    end
+
+    #
+    # Return the local Cloudtasker server.
+    #
+    # @return [Cloudtasker::LocalServer] The local Cloudtasker server.
+    #
+    def local_server
+      @local_server ||= LocalServer.new
+    end
+
+    #
+    # Load Rails if defined
+    #
+    def boot_system
+      # Sync logs
+      STDOUT.sync = true
+
+      # Check for Rails
+      return false unless File.exist?('./config/environment.rb')
+
+      require 'rails'
+      require File.expand_path('./config/environment.rb')
+    end
+
+    #
+    # Run the cloudtasker development server.
+    #
+    def run
+      boot_system
+
+      # Print banner
+      environment == 'development' ? print_banner : print_non_dev_warning
+
+      # Print rails info
+      if rails_app?
+        logger.info "[Cloudtasker/Server] Booted Rails #{::Rails.version} application in #{environment} environment"
+      end
+
+      # Get internal read/write pip
+      self_read, self_write = IO.pipe
+
+      # Setup signals to trap
+      setup_signals(self_write)
+
+      logger.info "[Cloudtasker/Server] Running in #{RUBY_DESCRIPTION}"
+
+      # Wait for signals
+      wait_for_signal(self_read)
+    end
+
+    #
+    # Wait for signals and handle them.
+    #
+    # @param [IO] read_pipe Where to read signals.
+    #
+    def wait_for_signal(read_pipe)
+      local_server.start
+
+      while (readable_io = IO.select([read_pipe]))
+        signal = readable_io.first[0].gets.strip
+        handle_signal(signal)
+      end
+    rescue Interrupt
+      logger.info 'Shutting down'
+      local_server.stop
+      logger.info 'Stopped'
+    end
+
+    #
+    # Define which signals to trap
+    #
+    # @param [IO] write_pipe Where to write signals.
+    #
+    def setup_signals(write_pipe)
+      # Display signals on log output
+      sigs = %w[INT TERM TTIN TSTP]
+      # USR1 and USR2 don't work on the JVM
+      sigs << 'USR2' unless jruby?
+      sigs.each do |sig|
+        begin
+          trap(sig) { write_pipe.puts(sig) }
+        rescue ArgumentError
+          puts "Signal #{sig} not supported"
+        end
+      end
+    end
+
+    #
+    # Handle process signals
+    #
+    # @param [String] sig The signal.
+    #
+    def handle_signal(sig)
+      raise(Interrupt) if %w[INT TERM].include?(sig)
+    end
+
+    #
+    # Return the server banner
+    #
+    # @return [String] The server banner
+    #
+    def banner
+      <<~'TEXT'
+           ___ _                 _ _            _
+          / __\ | ___  _   _  __| | |_ __ _ ___| | _____ _ __
+         / /  | |/ _ \| | | |/ _` | __/ _` / __| |/ / _ \ '__|
+        / /___| | (_) | |_| | (_| | || (_| \__ \   <  __/ |
+        \____/|_|\___/ \__,_|\__,_|\__\__,_|___/_|\_\___|_|
+
+      TEXT
+    end
+
+    #
+    # Display a warning message when run in non-dev env.
+    #
+    # @return [<Type>] <description>
+    #
+    def print_non_dev_warning
+      puts "\e[31m"
+      puts non_dev_warning_message
+      puts "\e[0m"
+    end
+
+    #
+    # Return the message to display when users attempt to run
+    # the local development server in non-dev environments.
+    #
+    # @return [String] The warning message.
+    #
+    def non_dev_warning_message
+      <<~'TEXT'
+        ============================================ /!\ ====================================================
+        Your are running the Cloudtasker local development server in a NON-DEVELOPMENT environment.
+        This is not recommended as the the development server is not designed for production-like load.
+        If you need a job processing server to run yourself please use Sidekiq instead (https://sidekiq.org)
+        ============================================ /!\ ====================================================
+      TEXT
+    end
+
+    #
+    # Print the server banner
+    #
+    def print_banner
+      puts "\e[96m"
+      puts banner
+      puts "\e[0m"
+    end
+  end
+end

data/lib/cloudtasker/cloud_task.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # An interface class to manage tasks on the backend (Cloud Task or Redis)
+  class CloudTask
+    attr_accessor :id, :http_request, :schedule_time, :retries
+
+    #
+    # The backend to use for cloud tasks.
+    #
+    # @return [Cloudtasker::Backend::GoogleCloudTask, Cloudtasker::Backend::RedisTask] The cloud task backend.
+    #
+    def self.backend
+      # Re-evaluate backend every time if testing mode enabled
+      @backend = nil if defined?(Cloudtasker::Testing)
+
+      @backend ||= begin
+        if defined?(Cloudtasker::Testing) && Cloudtasker::Testing.in_memory?
+          require 'cloudtasker/backend/memory_task'
+          Backend::MemoryTask
+        elsif Cloudtasker.config.mode.to_sym == :development
+          require 'cloudtasker/backend/redis_task'
+          Backend::RedisTask
+        else
+          require 'cloudtasker/backend/google_cloud_task'
+          Backend::GoogleCloudTask
+        end
+      end
+    end
+
+    #
+    # Find a cloud task by id.
+    #
+    # @param [String] id The id of the task.
+    #
+    # @return [Cloudtasker::Cloudtask] The task.
+    #
+    def self.find(id)
+      payload = backend.find(id)&.to_h
+      payload ? new(payload) : nil
+    end
+
+    #
+    # Create a new cloud task.
+    #
+    # @param [Hash] payload Thee task payload
+    #
+    # @return [Cloudtasker::CloudTask] The created task.
+    #
+    def self.create(payload)
+      resp = backend.create(payload)&.to_h
+      resp ? new(resp) : nil
+    end
+
+    #
+    # Delete a cloud task by id.
+    #
+    # @param [String] id The task id.
+    #
+    def self.delete(id)
+      backend.delete(id)
+    end
+
+    #
+    # Build a new instance of the class using a backend response
+    # payload.
+    #
+    # @param [String] id The task id.
+    # @param [Hash] http_request The content of the http request.
+    # @param [Integer] schedule_time When to run the job (Unix timestamp)
+    # @param [Integer] retries The number of times the job failed.
+    #
+    def initialize(id:, http_request:, schedule_time: nil, retries: 0)
+      @id = id
+      @http_request = http_request
+      @schedule_time = schedule_time
+      @retries = retries || 0
+    end
+
+    #
+    # Equality operator.
+    #
+    # @param [Any] other The object to compare.
+    #
+    # @return [Boolean] True if the object is equal.
+    #
+    def ==(other)
+      other.is_a?(self.class) && other.id == id
+    end
+  end
+end