procrastinator 0.6.1 → 0.9.0

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/lib/procrastinator.rb

@@ -1,27 +1,33 @@
+# frozen_string_literal: true
+
 require 'procrastinator/version'
-require 'procrastinator/queue_worker'
+require 'procrastinator/task_meta_data'
 require 'procrastinator/task_worker'
-require 'procrastinator/environment'
-require 'logger'
-
+require 'procrastinator/queue'
+require 'procrastinator/queue_worker'
+require 'procrastinator/config'
+require 'procrastinator/queue_manager'
+require 'procrastinator/task'
+require 'procrastinator/scheduler'
+require 'procrastinator/loaders/csv_loader'
 
+require 'logger'
+require 'pathname'
+
+# Top-level module for the Procrastinator Gem.
+#
+# Call Procrastinator.setup with a block to initialize and run independent worker sub processes to complete tasks
+# asynchronously from your main application.
+#
+# Read the README for details.
+#
+# @author Robin Miller
+#
+# @see https://github.com/TenjinInc/procrastinator
 module Procrastinator
+   # rubocop:disable Style/ClassVars
    @@test_mode = false
 
-   def self.setup(&block)
-      raise ArgumentError.new('Procrastinator.setup must be given a block') if block.nil?
-
-      env = Environment.new(test_mode: @@test_mode)
-
-      yield(env)
-
-      raise RuntimeError.new('setup block must call #persister_factory on the environment') if env.persister.nil?
-      raise RuntimeError.new('setup block must call #define_queue on the environment') if env.queue_definitions.empty?
-      env.spawn_workers
-
-      env
-   end
-
    def self.test_mode=(value)
       @@test_mode = value
    end
@@ -29,4 +35,19 @@ module Procrastinator
    def self.test_mode
       @@test_mode
    end
+
+   # rubocop:enable Style/ClassVars
+
+   # Creates a configuration object and passes it into the given block.
+   #
+   # @yield the created configuration object
+   def self.setup(&block)
+      raise ArgumentError, 'Procrastinator.setup must be given a block' unless block_given?
+
+      config = Config.new
+
+      config.setup(@@test_mode, &block)
+
+      QueueManager.new(config).spawn_workers
+   end
 end

data/lib/procrastinator/config.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Procrastinator
+   # Configuration object (State Pattern) used to coordinate settings across
+   # various components within Procrastinator.
+   #
+   # All of its state is read-only, set using the methods in the DSL module below.
+   #
+   # @author Robin Miller
+   #
+   # @!attribute [r] :test_mode?
+   #    @return [Boolean] Whether test mode is enabled
+   # @!attribute [r] :queues
+   #    @return [Array] List of defined queues
+   # @!attribute [r] :context
+   #    @return [Object] Provided context object that will be forwarded to tasks
+   # @!attribute [r] :loader
+   #    @return [Object] Provided persistence strategy object to use for task I/O
+   # @!attribute [r] :log_dir
+   #    @return [Pathname] Directory to write log files in
+   # @!attribute [r] :log_level
+   #    @return [Integer] Logging level to use
+   # @!attribute [r] :prefix
+   #    @return [String] The prefix to prepend to process names
+   # @!attribute [r] :pid_dir
+   #    @return [Pathname] Directory to write process ID records in
+   class Config
+      attr_reader :queues, :log_dir, :log_level, :prefix, :test_mode, :context, :loader, :pid_dir
+      alias test_mode? test_mode
+
+      DEFAULT_LOG_DIRECTORY = 'log/'
+      DEFAULT_PID_DIRECTORY = 'pid/'
+
+      def initialize
+         @test_mode        = false
+         @queues           = []
+         @loader           = nil
+         @context          = nil
+         @subprocess_block = nil
+         @log_dir          = Pathname.new(DEFAULT_LOG_DIRECTORY)
+         @log_level        = Logger::INFO
+         @pid_dir          = Pathname.new(DEFAULT_PID_DIRECTORY)
+      end
+
+      # Collection of all of the methods intended for use within Procrastinator.setup
+      #
+      # @see Procrastinator
+      module DSL
+         # Assigns a task loader
+         # It should be called in an each_process block as well so that they get
+         # distinct resources (eg. DB connections) from the parent process.
+         def load_with(loader)
+            if loader.is_a? Hash
+               unless loader.key? :location
+                  raise ArgumentError, 'Must pass keyword :location if specifying a location for CSV file'
+               end
+
+               loader = Loader::CSVLoader.new(loader[:location])
+            end
+
+            raise MalformedTaskLoaderError, 'task loader cannot be nil' if loader.nil?
+
+            [:read, :create, :update, :delete].each do |method|
+               unless loader.respond_to? method
+                  raise MalformedTaskLoaderError, "task loader #{ loader.class } must respond to ##{ method }"
+               end
+            end
+
+            @loader = loader
+         end
+
+         def provide_context(context)
+            @context = context
+         end
+
+         # Accepts a block that will be executed on the queue sub-processes. Use it to control resource allocations.
+         def each_process(prefix: nil, pid_dir: DEFAULT_PID_DIRECTORY, &block)
+            @prefix           = prefix
+            @subprocess_block = block
+            @pid_dir          = Pathname.new(pid_dir)
+         end
+
+         def define_queue(name, task_class, properties = {})
+            raise ArgumentError, 'queue name cannot be nil' if name.nil?
+            raise ArgumentError, 'queue task class cannot be nil' if task_class.nil?
+
+            verify_task_class(task_class)
+
+            @queues << Queue.new(properties.merge(name: name, task_class: task_class))
+         end
+
+         def enable_test_mode
+            @test_mode = true
+         end
+
+         def log_inside(path)
+            @log_dir = path ? Pathname.new(path) : path
+         end
+
+         def log_at_level(lvl)
+            @log_level = lvl
+         end
+      end
+
+      include DSL
+
+      def setup(test_mode = false)
+         yield(self)
+
+         enable_test_mode if test_mode
+
+         load_with(Loader::CSVLoader.new) unless @loader
+
+         raise 'setup block must call #define_queue on the environment' if @queues.empty?
+
+         if @context && @queues.none? { |queue| queue.task_class.method_defined?(:context=) }
+            raise <<~ERROR
+               setup block called #provide_context, but no queue task classes import :context.
+
+               Add this to your Task classes that expect to receive the context:
+
+                  include Procrastinator::Task
+
+                  task_attr :context
+            ERROR
+         end
+
+         self
+      end
+
+      def queues_string
+         # it drops the colon if you call #to_s on a symbol, so we need to add it back
+         @queues.map { |queue| ":#{ queue.name }" }.join(', ')
+      end
+
+      def single_queue?
+         @queues.size == 1
+      end
+
+      def run_process_block
+         @subprocess_block&.call
+      end
+
+      def queue(name: nil)
+         if name
+            @queues.find do |q|
+               q.name == name
+            end
+         else
+            @queues.first
+         end
+      end
+
+      private
+
+      def verify_task_class(task_class)
+         unless task_class.method_defined? :run
+            raise MalformedTaskError, "task #{ task_class } does not support #run method"
+         end
+
+         # We're checking the interface compliance on init because it's one of those extremely rare cases where
+         # you'd want to know early because the sub-processes would crash async, which is harder to debug.
+         # It's a bit belt-and suspenders, but UX is important for devs, too. - robinetmiller
+         if task_class.method_defined?(:run) && task_class.instance_method(:run).arity.positive?
+            err = "task #{ task_class } cannot require parameters to its #run method"
+
+            raise MalformedTaskError, err
+         end
+
+         expected_arity = 1
+
+         [:success, :fail, :final_fail].each do |method_name|
+            next unless task_class.method_defined?(method_name)
+            next if task_class.instance_method(method_name).arity == expected_arity
+
+            err = "task #{ task_class } must accept #{ expected_arity } parameter to its ##{ method_name } method"
+
+            raise MalformedTaskError, err
+         end
+      end
+   end
+
+   class MalformedTaskLoaderError < StandardError
+   end
+end

data/lib/procrastinator/loaders/csv_loader.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'pathname'
+
+module Procrastinator
+   module Loader
+      # Simple Task I/O object that writes task information (ie. TaskMetaData attributes) to a CSV file.
+      #
+      # @author Robin Miller
+      class CSVLoader
+         # ordered
+         HEADERS = [:id, :queue, :run_at, :initial_run_at, :expire_at,
+                    :attempts, :last_fail_at, :last_error, :data].freeze
+
+         DEFAULT_FILE = 'procrastinator-tasks.csv'
+
+         def initialize(file_path = DEFAULT_FILE)
+            @path = Pathname.new(file_path)
+
+            if @path.directory? || @path.to_s.end_with?('/')
+               @path += DEFAULT_FILE
+            elsif @path.extname.empty?
+               @path = Pathname.new("#{ file_path }.csv")
+            end
+         end
+
+         def read
+            data = CSV.table(@path.to_s, force_quotes: false).to_a
+
+            headers = data.shift
+
+            data.collect do |d|
+               hash = Hash[headers.zip(d)]
+
+               hash[:data] = hash[:data].gsub('""', '"')
+
+               hash
+            end
+         end
+
+         def create(queue:, run_at:, initial_run_at:, expire_at:, data: '')
+            existing_data = begin
+               read
+            rescue Errno::ENOENT
+               []
+            end
+
+            max_id = existing_data.collect { |task| task[:id] }.max || 0
+
+            new_data = {
+                  id:             max_id + 1,
+                  queue:          queue,
+                  run_at:         run_at,
+                  initial_run_at: initial_run_at,
+                  expire_at:      expire_at,
+                  attempts:       0,
+                  data:           data
+            }
+
+            write(existing_data + [new_data])
+         end
+
+         def update(id, data)
+            existing_data = begin
+               read
+            rescue Errno::ENOENT
+               []
+            end
+
+            task_data = existing_data.find do |task|
+               task[:id] == id
+            end
+
+            task_data.merge!(data)
+
+            write(existing_data)
+         end
+
+         def delete(id)
+            existing_data = begin
+               read
+            rescue Errno::ENOENT
+               []
+            end
+
+            existing_data.delete_if do |task|
+               task[:id] == id
+            end
+
+            write(existing_data)
+         end
+
+         def write(data)
+            lines = data.collect do |d|
+               CSV.generate_line(d, headers: HEADERS, force_quotes: true)
+            end
+
+            @path.dirname.mkpath
+            @path.open('w') do |f|
+               f.puts HEADERS.join(',')
+               f.puts lines.join
+            end
+         end
+      end
+   end
+end

data/lib/procrastinator/queue.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Procrastinator
+   # A Queue defines how a certain type task will be processed.
+   #
+   # @author Robin Miller
+   #
+   # @!attribute [r] :name
+   #    @return [Symbol] The queue's identifier symbol
+   # @!attribute [r] :task_class
+   #    @return [Class] Class that defines the work to be done for jobs in this queue.
+   # @!attribute [r] :timeout
+   #    @return [Object] Duration (seconds) after which tasks in this queue should fail for taking too long.
+   # @!attribute [r] :max_attempts
+   #    @return [Object] Maximum number of attempts for tasks in this queue.
+   # @!attribute [r] :update_period
+   #    @return [Pathname] Delay (seconds) between reloads of tasks from the task loader.
+   # @!attribute [r] :max_tasks
+   #    @return [Pathname] The maximum number of tasks to run concurrently within a queue worker process.
+   class Queue
+      DEFAULT_TIMEOUT       = 3600 # in seconds; one hour total
+      DEFAULT_MAX_ATTEMPTS  = 20
+      DEFAULT_UPDATE_PERIOD = 10 # seconds
+      DEFAULT_MAX_TASKS     = 10
+
+      attr_reader :name, :task_class, :max_attempts, :timeout, :update_period, :max_tasks
+
+      # Timeout is in seconds
+      def initialize(name:,
+                     task_class:,
+                     max_attempts: DEFAULT_MAX_ATTEMPTS,
+                     timeout: DEFAULT_TIMEOUT,
+                     update_period: DEFAULT_UPDATE_PERIOD,
+                     max_tasks: DEFAULT_MAX_TASKS)
+         raise ArgumentError, ':name may not be nil' unless name
+         raise ArgumentError, ':task_class may not be nil' unless task_class
+
+         raise ArgumentError, 'Task class must be initializable' unless task_class.respond_to? :new
+
+         raise ArgumentError, 'timeout cannot be negative' if timeout&.negative?
+
+         @name          = name.to_s.strip.gsub(/[^A-Za-z0-9]+/, '_').to_sym
+         @task_class    = task_class
+         @max_attempts  = max_attempts
+         @timeout       = timeout
+         @update_period = update_period
+         @max_tasks     = max_tasks
+      end
+   end
+end

data/lib/procrastinator/queue_manager.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module Procrastinator
+   # Spawns and manages work queue subprocesses.
+   #
+   # This is where all of the multi-process logic should be kept to.
+   #
+   # @author Robin Miller
+   #
+   # @!attribute [r] :workers
+   #    @return [Hash] Maps the constructed QueueWorkers to their process ID.
+   class QueueManager
+      attr_reader :workers
+
+      def initialize(config)
+         @workers = {}
+         @config  = config
+         @logger  = start_log
+      end
+
+      # Shuts down any remaining old queue workers and spawns a new one for each queue defined in the config
+      #
+      # @return [Scheduler] a scheduler object that can be used to interact with the queues
+      def spawn_workers
+         scheduler = Scheduler.new(@config, self)
+
+         kill_old_workers
+
+         if ENV['PROCRASTINATOR_STOP']
+            @logger.warn('Cannot spawn queue workers because environment variable PROCRASTINATOR_STOP is set')
+         else
+            @config.queues.each do |queue|
+               spawn_worker(queue, scheduler: scheduler)
+            end
+         end
+
+         scheduler
+      end
+
+      # Produces a new QueueWorker for the given queue.
+      #
+      # If Test Mode is disabled in the config, then it will also fork a new independent process for that worker
+      # to work in.
+      #
+      # @param queue [Queue] the queue to build a worker for
+      # @param scheduler [Scheduler] an optional scheduler instance to pass to the worker
+      def spawn_worker(queue, scheduler: nil)
+         worker = QueueWorker.new(queue:     queue,
+                                  config:    @config,
+                                  scheduler: scheduler)
+         if @config.test_mode?
+            @workers[worker] = Process.pid
+         else
+            check_for_name(worker.long_name)
+
+            pid = fork
+
+            if pid
+               # === PARENT PROCESS ===
+               Process.detach(pid)
+               @workers[worker] = pid
+            else
+               deamonize(worker.long_name)
+
+               worker.work
+               shutdown_worker
+            end
+         end
+      end
+
+      def act(*queue_names)
+         unless @config.test_mode?
+            raise <<~ERR
+               Procrastinator.act called outside Test Mode.
+               Either use Procrastinator.spawn_workers or call #enable_test_mode in Procrastinator.setup.
+            ERR
+         end
+
+         workers = @workers.keys
+
+         if queue_names.empty?
+            workers.each(&:act)
+         else
+            queue_names.each do |name|
+               workers.find { |worker| worker.name == name }.act
+            end
+         end
+      end
+
+      private
+
+      def start_log
+         directory = @config.log_dir
+
+         return unless directory
+
+         log_path = directory + 'queue-manager.log'
+
+         directory.mkpath
+         File.open(log_path.to_path, 'a+') { |f| f.write '' }
+
+         logger = Logger.new(log_path.to_path)
+
+         logger.level = @config.log_level
+
+         # @logger.info(['',
+         #               '===================================',
+         #               "Started worker process, #{long_name}, to work off queue #{@queue.name}.",
+         #               "Worker pid=#{Process.pid}; parent pid=#{Process.ppid}.",
+         #               '==================================='].join("\n"))
+
+         logger
+      end
+
+      # Methods exclusive to the child process
+      module ChildMethods
+         def deamonize(name)
+            Process.daemon(true)
+            Process.setsid
+            srand
+            Process.setproctitle(name)
+            close_io
+
+            write_pid_file(Process.pid, name)
+
+            @config.run_process_block
+         end
+
+         # Make sure all input/output streams are closed
+         def close_io
+            stds = [$stdin, $stdout, $stderr]
+
+            # Part 1: close all IO objects (except for $stdin/$stdout/$stderr)
+            ObjectSpace.each_object(IO) do |io|
+               next if stds.include?(io)
+
+               begin
+                  io.close
+               rescue IOError
+                  next
+               end
+            end
+
+            # Part 2: redirect STD connections
+            stds.each do |io|
+               io.reopen '/dev/null'
+            end
+
+            # TODO: redirect OUT or ERR to logger?
+         end
+
+         # Wrapping #exit to allow for tests to easily stub out this behaviour.
+         # If #exit isn't prevented, the test framework will break,
+         # but #exit can't be directly stubbed either (because it's a required Kernel method)
+         def shutdown_worker
+            exit
+         end
+      end
+
+      # Methods exclusive to the main/parent process
+      module ParentMethods
+         def kill_old_workers
+            @config.pid_dir.mkpath
+
+            @config.pid_dir.each_child do |file|
+               pid = file.read.to_i
+
+               begin
+                  Process.kill('KILL', pid)
+                  @logger.info("Killing old worker process pid: #{ pid }")
+               rescue Errno::ESRCH
+                  @logger.info("Expected old worker process pid=#{ pid }, but none was found")
+               end
+
+               file.delete
+            end
+         end
+
+         def write_pid_file(pid, filename)
+            @config.pid_dir.mkpath
+
+            pid_file = @config.pid_dir + "#{ filename }.pid"
+
+            File.open(pid_file.to_path, 'w') do |f|
+               f.print(pid)
+            end
+         end
+
+         def check_for_name(name)
+            # better to use backticks so we can get the info and not spam user's stdout
+            warn <<~WARNING unless `pgrep -f #{ name }`.empty?
+               Warning: there is another process named "#{ name }". Use #each_process(prefix: '') in
+                        Procrastinator setup if you want to help yourself distinguish them.
+            WARNING
+         end
+      end
+
+      include ChildMethods
+      include ParentMethods
+   end
+end