procrastinator 0.9.0 → 1.0.0.pre.rc2

data/lib/procrastinator/queue_worker.rb

@@ -1,133 +1,88 @@
 # frozen_string_literal: true
 
 module Procrastinator
-   # A QueueWorker checks for tasks to run from the loader defined in the provided config and executes them,
-   # updating information in the task loader as necessary.
+   # A QueueWorker checks for tasks to run from the task store and executes them, updating information in the task
+   # store as necessary.
    #
    # @author Robin Miller
    class QueueWorker
       extend Forwardable
 
-      def_delegators :@queue, :name
+      def_delegators :@queue, :name, :next_task
 
       # expected methods for all persistence strategies
       PERSISTER_METHODS = [:read, :update, :delete].freeze
 
-      def initialize(queue:, config:, scheduler: nil)
-         @queue     = queue
-         @config    = config
-         @scheduler = scheduler
+      def initialize(queue:, config:)
+         raise ArgumentError, ':queue cannot be nil' if queue.nil?
+         raise ArgumentError, ':config cannot be nil' if config.nil?
 
-         @logger = nil
-      end
-
-      def work
-         start_log
+         @config = config
 
-         begin
-            loop do
-               sleep(@queue.update_period)
+         @queue = if queue.is_a? Symbol
+                     config.queue(name: queue)
+                  else
+                     queue
+                  end
 
-               act
-            end
-         rescue StandardError => e
-            raise if @config.test_mode? || !@logger
-
-            @logger.fatal(e)
-         end
+         @scheduler = Scheduler.new(config)
+         @logger    = Logger.new(StringIO.new)
       end
 
-      def act
-         persister = @config.loader
-
-         tasks = fetch_tasks(persister)
+      # Works on jobs forever
+      def work!
+         @logger = open_log!("#{ name }-queue-worker", @config)
+         @logger.info("Started worker thread to consume queue: #{ name }")
 
-         tasks.each do |metadata|
-            tw = build_worker(metadata)
+         loop do
+            sleep(@queue.update_period)
 
-            tw.work
-
-            if tw.successful?
-               persister.delete(metadata.id)
-            else
-               persister.update(metadata.id, tw.to_h.merge(queue: @queue.name.to_s))
-            end
+            work_one
          end
-      end
-
-      def long_name
-         name = "#{ @queue.name }-queue-worker"
-
-         name = "#{ @config.prefix }-#{ name }" if @config.prefix
+      rescue StandardError => e
+         @logger.fatal(e)
 
-         name
+         raise
       end
 
-      # Starts a log file and stores the logger within this queue worker.
-      #
-      # Separate from init because logging is context-dependent
-      def start_log
-         return if @logger || !@config.log_dir
+      # Performs exactly one task on the queue
+      def work_one
+         task = next_task(logger:    @logger,
+                          container: @config.container,
+                          scheduler: @scheduler) || return
 
-         @logger = Logger.new(log_target, level: @config.log_level)
-
-         msg = <<~MSG
-            ======================================================================
-            Started worker process, #{ long_name }, to work off queue #{ @queue.name }.
-            Worker pid=#{ Process.pid }; parent pid=#{ Process.ppid }.
-            ======================================================================
-         MSG
-
-         @logger.info("\n#{ msg }")
-      end
-
-      private
-
-      def build_worker(metadata)
-         start_log
-
-         TaskWorker.new(metadata:  metadata,
-                        queue:     @queue,
-                        scheduler: @scheduler,
-                        context:   @config.context,
-                        logger:    @logger)
-      end
-
-      def log_target
-         return $stdout if @config.test_mode?
-
-         log_path = @config.log_dir + "#{ long_name }.log"
-
-         write_log_file(log_path)
+         begin
+            task.run
 
-         log_path.to_path
-      end
+            @queue.delete(task.id)
+         rescue StandardError => e
+            task.fail(e)
 
-      def write_log_file(log_path)
-         @config.log_dir.mkpath
-         File.open(log_path.to_path, 'a+') do |f|
-            f.write ''
+            task_info = task.to_h
+            id        = task_info.delete(:id)
+            @queue.update(id, **task_info)
          end
       end
 
-      def fetch_tasks(persister)
-         tasks = persister.read(queue: @queue.name).map(&:to_h).reject { |t| t[:run_at].nil? }
+      def halt
+         @logger&.info("Halted worker on queue: #{ name }")
+         @logger&.close
+      end
 
-         tasks = sort_tasks(tasks)
+      # Starts a log file and returns the created Logger
+      def open_log!(name, config)
+         return @logger unless config.log_level
 
-         metas = tasks.collect do |t|
-            TaskMetaData.new(t.delete_if { |key| !TaskMetaData::EXPECTED_DATA.include?(key) })
-         end
+         log_path = config.log_dir / "#{ name }.log"
 
-         metas.select(&:runnable?)
-      end
+         config.log_dir.mkpath
+         FileUtils.touch(log_path)
 
-      def sort_tasks(tasks)
-         # shuffling and re-sorting to avoid worst case O(n^2) when receiving already sorted data
-         # on quicksort (which is default ruby sort). It is not unreasonable that the persister could return sorted
-         # results
-         # Ideally, we'd use a better algo than qsort for this, but this will do for now
-         tasks.shuffle.sort_by { |t| t[:run_at] }.first(@queue.max_tasks)
+         Logger.new(log_path.to_path,
+                    config.log_shift_age, config.log_shift_size,
+                    level:     config.log_level || Logger::FATAL,
+                    progname:  name,
+                    formatter: Config::DEFAULT_LOG_FORMATTER)
       end
    end
 

data/lib/procrastinator/rake/daemon_tasks.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'rake'
+
+module Procrastinator
+   module Rake
+      # RakeTask builder. Provide this in your Rakefile:
+      #
+      #    require 'procrastinator/rake/task'
+      #    Procrastinator::RakeTask.new('/var/run') do
+      #       # return your Procrastinator::Scheduler here or construct it using Procrastinator.config
+      #    end
+      #
+      class DaemonTasks
+         include ::Rake::Cloneable
+         include ::Rake::DSL
+
+         # Shorthand for DaemonTasks.new.define
+         #
+         # @param (see #define)
+         # @see DaemonTasks#define
+         def self.define(**args)
+            new.define(**args)
+         end
+
+         # Defines procrastinator:start and procrastinator:stop Rake tasks that operate on the given scheduler.
+         # If provided a block, that block will run in the daemon process.
+         #
+         # @param scheduler [Procrastinator::Scheduler]
+         # @param pid_path [Pathname, File, String, nil]
+         def define(scheduler:, pid_path: nil, &block)
+            pid_path = Scheduler::DaemonWorking.normalize_pid(pid_path)
+
+            namespace :procrastinator do
+               task :start do
+                  scheduler.work.daemonized!(pid_path, &block)
+               end
+
+               task :status do
+                  if Scheduler::DaemonWorking.running?(pid_path)
+                     warn "Procrastinator instance running (pid #{ File.read(pid_path) })"
+                  else
+                     warn "No Procrastinator instance detected for #{ pid_path }"
+                  end
+               end
+
+               task :stop do
+                  Scheduler::DaemonWorking.halt!(pid_path)
+               end
+            end
+         end
+      end
+   end
+end

data/lib/procrastinator/rake/tasks.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative 'daemon_tasks'