procrastinator 1.0.0.pre.rc4 → 1.0.0

data/RELEASE_NOTES.md

@@ -1,6 +1,6 @@
 # Release Notes
 
-## 1.0.0 (       )
+## 1.0.0 (2022-09-18)
 
 ### Major Changes
 
@@ -21,16 +21,19 @@
         * `Procrastinator::Config#run_process_block`
     * Removed use of envvar `PROCRASTINATOR_STOP`
     * `Procrastinator::QueueManager` is merged into `Procrastinator::Scheduler`
-    * Removed rake task to halt queue processes
+    * Removed rake task to halt individual queue processes
     * Renamed `Procrastinator::Config#provide_context` to `provide_container`
     * You must now call `Scheduler#work` on the result of `Procrastinator.config`
-    * Use a dedicated process monitor (like `monit`) instead in production environments
-    * Suuply a block to `daemonized!` to run code in the spawned process.
-* `max_tasks` is removed as it only added concurrency complexity
+    * Use a dedicated process monitor (like `monit`) instead in production environments to maintain uptime
+* `max_tasks` is removed as it only added concurrency complexity. Each queue worker only selects one task from only its
+  queue.
 * Data is now stored as JSON instead of YAML
 * Added with_store that applies its settings to its block
     * `load_with` has been removed
 * Removed `task_attr` and `Procrastinator::Task` module. Tasks is now duck-type checked for accessors instead.
+* Added Rake tasks to manage process daemon
+* Times are passed to Task Store as a Ruby Time object instead of an epoch time integer 
+* `#delay` is now `#defer`
 
 ### Minor Changes
 
@@ -38,6 +41,8 @@
 * Updated development gems
 * Logs now include the queue name in log lines
 * Logs can now set the shift size or age (like Ruby's Logger)
+* Log format is now tab-separated to align better and work with POSIX cut
+* General renaming of terms
 
 ### Bugfixes
 

data/Rakefile

@@ -2,7 +2,14 @@
 
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
+require 'yard'
 
 RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
+
+YARD::Rake::YardocTask.new do |t|
+   t.files = %w[lib/**/*.rb]
+   # t.options       = %w[--some-option]
+   t.stats_options = ['--list-undoc']
+end

data/lib/procrastinator/config.rb

@@ -25,10 +25,20 @@ module Procrastinator
    class Config
       attr_reader :queues, :log_dir, :log_level, :log_shift_age, :log_shift_size, :container
 
-      DEFAULT_LOG_DIRECTORY  = Pathname.new('log').freeze
-      DEFAULT_LOG_SHIFT_AGE  = 0
+      # Default directory to keep logs in.
+      DEFAULT_LOG_DIRECTORY = Pathname.new('log').freeze
+
+      # Default age to keep log files for.
+      # @see Logger
+      DEFAULT_LOG_SHIFT_AGE = 0
+
+      # Default max size to keep log files under.
+      # @see Logger
       DEFAULT_LOG_SHIFT_SIZE = 2 ** 20 # 1 MB
-      DEFAULT_LOG_FORMATTER  = proc do |severity, datetime, progname, msg|
+
+      # Default log formatter
+      # @see Logger
+      DEFAULT_LOG_FORMATTER = proc do |severity, datetime, progname, msg|
          [datetime.iso8601(8),
           severity,
           "#{ progname } (#{ Process.pid }):",
@@ -70,10 +80,29 @@ module Procrastinator
             @default_store = old_store
          end
 
+         # Defines the container to assign to each Task Handler's :container attribute.
+         #
+         # @param container [Object] the container
          def provide_container(container)
             @container = container
          end
 
+         # Defines a queue in the Procrastinator Scheduler.
+         #
+         # The Task Handler will be initialized for each task and assigned each of these attributes:
+         #    :container, :logger, :scheduler
+         #
+         # @param name [Symbol, String] the identifier to label the queue. Referenced in #defer calls
+         # @param task_class [Class] the Task Handler class.
+         # @param properties [Hash] Settings options object for this queue
+         # @option properties [Object] :store (Procrastinator::TaskStore::SimpleCommaStore)
+         #                                    Storage strategy for tasks in the queue.
+         # @option properties [Integer] :max_attempts (Procrastinator::Queue::DEFAULT_MAX_ATTEMPTS)
+         #                                            Maximum number of times a task may be attempted before giving up.
+         # @option properties [Integer] :timeout (Procrastinator::Queue::DEFAULT_TIMEOUT)
+         #                                       Maximum number of seconds to wait for a single task to complete.
+         # @option properties [Integer] :update_period (Procrastinator::Queue::DEFAULT_UPDATE_PERIOD)
+         #                                             Time to wait before checking for new tasks.
          def define_queue(name, task_class, properties = {})
             raise ArgumentError, 'queue name cannot be nil' if name.nil?
             raise ArgumentError, 'queue task class cannot be nil' if task_class.nil?
@@ -142,7 +171,9 @@ module Procrastinator
          end
       end
 
+      # Raised when there is an error in the setup configuration.
       class SetupError < RuntimeError
+         # Standard error message for when there is no queue defined
          ERR_NO_QUEUE = 'setup block must call #define_queue on the environment'
       end
    end

data/lib/procrastinator/logged_task.rb

@@ -14,10 +14,6 @@ module Procrastinator
    #
    # @see Task
    class LoggedTask < DelegateClass(Task)
-      # extend Forwardable
-      #
-      # def_delegators :@task, :id, :to_h
-
       attr_reader :logger
 
       alias task __getobj__
@@ -27,6 +23,7 @@ module Procrastinator
          @logger = logger || raise(ArgumentError, 'Logger cannot be nil')
       end
 
+      # (see Task#run)
       def run
          task.run
 
@@ -37,6 +34,7 @@ module Procrastinator
          end
       end
 
+      # @param (see Task#fail)
       def fail(error)
          hook = task.fail(error)
          begin

data/lib/procrastinator/queue.rb

@@ -18,8 +18,13 @@ module Procrastinator
    class Queue
       extend Forwardable
 
-      DEFAULT_TIMEOUT       = 3600 # in seconds; one hour total
-      DEFAULT_MAX_ATTEMPTS  = 20
+      # Default number of seconds to wait for a task to complete
+      DEFAULT_TIMEOUT = 3600 # in seconds; one hour total
+
+      # Default number of times to retry a task
+      DEFAULT_MAX_ATTEMPTS = 20
+
+      # Default amount of time between checks for new Tasks
       DEFAULT_UPDATE_PERIOD = 10 # seconds
 
       attr_reader :name, :max_attempts, :timeout, :update_period, :task_store, :task_class
@@ -54,6 +59,12 @@ module Procrastinator
          freeze
       end
 
+      # Constructs the next available task on the queue.
+      #
+      # @param logger [Logger] logger to provide to the constructed task handler
+      # @param container [Object, nil] container to provide to the constructed task handler
+      # @param scheduler [Procrastinator::Scheduler, nil] the scheduler to provide to the constructed task handler
+      # @return [LoggedTask, nil] A Task or nil if no task is found
       def next_task(logger: Logger.new(StringIO.new), container: nil, scheduler: nil)
          metadata = next_metas.find(&:runnable?)
 
@@ -67,6 +78,9 @@ module Procrastinator
          LoggedTask.new(task, logger: logger)
       end
 
+      # Fetch a task matching the given identifier
+      #
+      # @param identifier [Hash] attributes to match
       def fetch_task(identifier)
          identifier[:data] = JSON.dump(identifier[:data]) if identifier[:data]
 
@@ -78,6 +92,11 @@ module Procrastinator
          TaskMetaData.new(tasks.first.merge(queue: self))
       end
 
+      # Creates a task on the queue, saved using the Task Store strategy.
+      #
+      # @param run_at [Time] Earliest time to attempt running the task
+      # @param expire_at [Time, nil] Time after which the task will not be attempted
+      # @param data [Hash, String, Numeric, nil] The data to save
       def create(run_at:, expire_at:, data:)
          if data.nil? && expects_data?
             raise ArgumentError, "task #{ @task_class } expects to receive :data. Provide :data to #delay."
@@ -102,6 +121,7 @@ module Procrastinator
          @task_store.create(**create_data)
       end
 
+      # @return [Boolean] whether the task handler will accept data to be assigned via its :data attribute
       def expects_data?
          @task_class.method_defined?(:data=)
       end
@@ -136,6 +156,8 @@ module Procrastinator
 
       # Internal queue validator
       module QueueValidation
+         private
+
          def validate!
             verify_task_class!
             verify_task_store!
@@ -198,9 +220,11 @@ module Procrastinator
       include QueueValidation
    end
 
+   # Raised when a Task Handler does not conform to the expected API
    class MalformedTaskError < StandardError
    end
 
+   # Raised when a Task Store strategy does not conform to the expected API
    class MalformedTaskStoreError < RuntimeError
    end
 end

data/lib/procrastinator/queue_worker.rb

@@ -64,6 +64,7 @@ module Procrastinator
          end
       end
 
+      # Logs halting the queue
       def halt
          @logger&.info("Halted worker on queue: #{ name }")
          @logger&.close
@@ -86,6 +87,9 @@ module Procrastinator
       end
    end
 
+   # Raised when a Task Storage strategy is missing a required part of the API.
+   #
+   # @see TaskStore
    class MalformedTaskPersisterError < StandardError
    end
 end

data/lib/procrastinator/rake/daemon_tasks.rb

@@ -3,14 +3,20 @@
 require 'rake'
 
 module Procrastinator
+   # Rake tasks specific to Procrastinator.
+   #
+   # Provide this in your Rakefile:
+   #
+   #    require 'procrastinator/rake/task'
+   #    Procrastinator::RakeTask.new do
+   #       # return your Procrastinator::Scheduler here or construct it using Procrastinator.config
+   #    end
+   #
+   # And then you will be able to run rake tasks like:
+   #
+   #    bundle exec rake procrastinator:start
    module Rake
-      # RakeTask builder. Provide this in your Rakefile:
-      #
-      #    require 'procrastinator/rake/task'
-      #    Procrastinator::RakeTask.new('/var/run') do
-      #       # return your Procrastinator::Scheduler here or construct it using Procrastinator.config
-      #    end
-      #
+      # RakeTask builder class. Use DaemonTasks.define to generate the needed tasks.
       class DaemonTasks
          include ::Rake::Cloneable
          include ::Rake::DSL
@@ -26,7 +32,7 @@ module Procrastinator
          # Defines procrastinator:start and procrastinator:stop Rake tasks that operate on the given scheduler.
          #
          # @param pid_path [Pathname, File, String, nil] The pid file path
-         # @yieldreturn scheduler [Procrastinator::Scheduler]
+         # @yieldreturn [Procrastinator::Scheduler] Constructed Scheduler to use as basis for starting tasks
          #
          # @see Scheduler::DaemonWorking#daemonized!
          def define(pid_path: nil)
@@ -78,8 +84,9 @@ module Procrastinator
          def stop
             return unless Scheduler::DaemonWorking.running?(@pid_path)
 
+            pid = File.read(@pid_path)
             Scheduler::DaemonWorking.halt!(@pid_path)
-            warn "Procrastinator pid #{ File.read(@pid_path) } halted."
+            warn "Procrastinator pid #{ pid } halted."
          end
       end
    end

data/lib/procrastinator/rspec/matchers.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'rspec/expectations'
+
+# Determines if the given task store has a task that matches the expectation hash
+RSpec::Matchers.define :have_task do |expected_task|
+   match do |task_store|
+      task_store.read.any? do |task|
+         task_hash        = task.to_h
+         task_hash[:data] = JSON.parse(task_hash[:data], symbolize_names: true) unless task_hash[:data].empty?
+
+         expected_task.all? do |field, expected_value|
+            expected_value = case field
+                             when :queue
+                                expected_value.to_sym
+                             when :run_at, :initial_run_at, :expire_at, :last_fail_at
+                                Time.at(expected_value.to_i)
+                             else
+                                expected_value
+                             end
+
+            values_match? expected_value, task_hash[field]
+         end
+      end
+   end
+
+   description do
+      "have a task with properties #{ expected_task.collect { |k, v| "#{ k }=#{ v }" }.join(', ') }"
+   end
+end

data/lib/procrastinator/scheduler.rb

@@ -19,7 +19,7 @@ module Procrastinator
       # @param run_at [Time, Integer] Optional time when this task should be executed. Defaults to the current time.
       # @param data [Hash, Array, String, Integer] Optional simple data object to be provided to the task on execution.
       # @param expire_at [Time, Integer] Optional time when the task should be abandoned
-      def delay(queue_name = nil, data: nil, run_at: Time.now, expire_at: nil)
+      def defer(queue_name = nil, data: nil, run_at: Time.now, expire_at: nil)
          raise ArgumentError, <<~ERR unless queue_name.nil? || queue_name.is_a?(Symbol)
             must provide a queue name as the first argument. Received: #{ queue_name }
          ERR
@@ -29,13 +29,15 @@ module Procrastinator
          queue.create(run_at: run_at, expire_at: expire_at, data: data)
       end
 
+      alias delay defer
+
       # Alters an existing task to run at a new time, expire at a new time, or both.
       #
       # Call #to on the result and pass in the new :run_at and/or :expire_at.
       #
       # Example:
       #
-      # scheduler.reschedule(:alerts, data: {user_id: 5}).to(run_at: Time.now, expire_at: Time.now + 10)
+      #    scheduler.reschedule(:alerts, data: {user_id: 5}).to(run_at: Time.now, expire_at: Time.now + 10)
       #
       # The identifier can include any data field stored in the task loader. Often this is the information in :data.
       #
@@ -123,6 +125,7 @@ module Procrastinator
       #
       # @see WorkProxy
       module ThreadedWorking
+         # Program name. Used as default for pid file names and in logging.
          PROG_NAME = 'Procrastinator'
 
          # Work off jobs per queue, each in its own thread.
@@ -135,7 +138,7 @@ module Procrastinator
             begin
                @threads = spawn_threads
 
-               @logger.info "Procrastinator running. Process ID: #{ Process.pid }"
+               @logger.info "#{ PROG_NAME } running. Process ID: #{ Process.pid }"
                @threads.each do |thread|
                   thread.join(timeout)
                end
@@ -243,10 +246,13 @@ module Procrastinator
       #
       # @see WorkProxy
       module DaemonWorking
-         PID_EXT         = '.pid'
-         DEFAULT_PID_DIR = Pathname.new('/var/run/').freeze
+         # File extension for process ID files
+         PID_EXT = '.pid'
+
+         # Default directory to store PID files in.
+         DEFAULT_PID_DIR = Pathname.new('/tmp').freeze
 
-         # 15 chars is linux limit
+         # Maximum process name size. 15 chars is linux limit
          MAX_PROC_LEN = 15
 
          # Consumes the current process and turns it into a background daemon and proceed as #threaded.
@@ -290,6 +296,10 @@ module Procrastinator
             false
          end
 
+         # Raised when a process is already found to exist using the same pid_path
+         class ProcessExistsError < RuntimeError
+         end
+
          private
 
          def spawn_daemon(pid_path)
@@ -384,7 +394,4 @@ module Procrastinator
          end
       end
    end
-
-   class ProcessExistsError < RuntimeError
-   end
 end

data/lib/procrastinator/task.rb

@@ -21,6 +21,12 @@ module Procrastinator
          @handler  = handler
       end
 
+      # Executes the Task Handler's #run hook and records the attempt.
+      #
+      # If the #run hook completes successfully, the #success hook will also be executed, if defined.
+      #
+      # @raise [ExpiredError] when the task run_at is after the expired_at.
+      # @raise [AttemptsExhaustedError] when the task has been attempted more times than allowed by the queue settings.
       def run
          raise ExpiredError, "task is over its expiry time of #{ @metadata.expire_at.iso8601 }" if @metadata.expired?
 
@@ -45,19 +51,25 @@ module Procrastinator
          hook
       end
 
+      # Attempts to run the given optional event hook on the handler, catching any resultant errors to prevent the whole
+      # task from failing despite the actual work in #run completing.
       def try_hook(method, *params)
          @handler.send(method, *params) if @handler.respond_to? method
       rescue StandardError => e
          warn "#{ method.to_s.capitalize } hook error: #{ e.message }"
       end
 
+      # Convert the task into a human-legible string.
+      # @return [String] Including the queue name, id, and serialized data.
       def to_s
          "#{ @metadata.queue.name }##{ id } [#{ serialized_data }]"
       end
 
+      # Raised when a Task's run_at is beyond its expire_at
       class ExpiredError < RuntimeError
       end
 
+      # Raised when a Task's attempts has exceeded the max_attempts defined for its queue (if any).
       class AttemptsExhaustedError < RuntimeError
       end
    end

data/lib/procrastinator/task_meta_data.rb

@@ -50,6 +50,9 @@ module Procrastinator
          @data           = data ? JSON.parse(data, symbolize_names: true) : nil
       end
 
+      # Increases the number of attempts on this task by one, unless the limit has been reached.
+      #
+      # @raise [Task::AttemptsExhaustedError] when the number of attempts has exceeded the Queue's defined maximum.
       def add_attempt
          raise Task::AttemptsExhaustedError unless attempts_left?
 
@@ -72,22 +75,28 @@ module Procrastinator
          end
       end
 
+      # @return [Boolean] whether the task has attempts left and is not expired
       def retryable?
          attempts_left? && !expired?
       end
 
+      # @return [Boolean] whether the task is expired
       def expired?
          !@expire_at.nil? && @expire_at < Time.now
       end
 
+      # @return [Boolean] whether there are attempts left until the Queue's defined maximum is reached (if any)
       def attempts_left?
          @queue.max_attempts.nil? || @attempts < @queue.max_attempts
       end
 
+      # @return [Boolean] whether the task's run_at is exceeded
       def runnable?
          !@run_at.nil? && @run_at <= Time.now
       end
 
+      # @return [Boolean] whether the task's last execution completed successfully.
+      # @raise [RuntimeError] when the task has not been attempted yet or when it is expired
       def successful?
          raise 'you cannot check for success before running #work' if !expired? && @attempts <= 0
 
@@ -117,6 +126,7 @@ module Procrastinator
          @run_at += 30 + (@attempts ** 4) unless @run_at.nil?
       end
 
+      # @return [Hash] representation of the task metadata as a hash
       def to_h
          {id:             @id,
           queue:          @queue.name.to_s,
@@ -129,10 +139,12 @@ module Procrastinator
           data:           serialized_data}
       end
 
+      # @return [String] :data serialized as a JSON string
       def serialized_data
          JSON.dump(@data)
       end
 
+      # Resets the last failure time and error.
       def clear_fails
          @last_error   = nil
          @last_fail_at = nil

data/lib/procrastinator/task_store/file_transaction.rb

@@ -3,74 +3,72 @@
 require 'pathname'
 
 module Procrastinator
-   module TaskStore
-      # The general idea is that there may be two threads that need to do these actions on the same file:
-      #    thread A:   read
-      #    thread B:   read
-      #    thread A/B: write
-      #    thread A/B: write
-      #
-      # When this sequence happens, the second file write is based on old information and loses the info from
-      # the prior write. Using a global mutex per file path prevents this case.
-      #
-      # This situation can also occur with multi processing, so file locking is also used for solitary access.
-      # File locking is only advisory in some systems, though, so it may only work against other applications
-      # that request a lock.
-      #
-      # @author Robin Miller
-      class FileTransaction
-         # Holds the mutual exclusion locks for file paths by name
-         @file_mutex = {}
+   # The general idea is that there may be two threads that need to do these actions on the same file:
+   #    thread A:   read
+   #    thread B:   read
+   #    thread A/B: write
+   #    thread A/B: write
+   #
+   # When this sequence happens, the second file write is based on old information and loses the info from
+   # the prior write. Using a global mutex per file path prevents this case.
+   #
+   # This situation can also occur with multi processing, so file locking is also used for solitary access.
+   # File locking is only advisory in some systems, though, so it may only work against other applications
+   # that request a lock.
+   #
+   # @author Robin Miller
+   class FileTransaction
+      # Holds the mutual exclusion locks for file paths by name
+      @file_mutex = {}
 
-         class << self
-            attr_reader :file_mutex
-         end
+      class << self
+         attr_reader :file_mutex
+      end
 
-         def initialize(path)
-            @path = ensure_path(path)
-         end
+      def initialize(path)
+         @path = ensure_path(path)
+      end
 
-         # Alias for transact(writable: false)
-         def read(&block)
-            transact(writable: false, &block)
-         end
+      # Alias for transact(writable: false)
+      def read(&block)
+         transact(writable: false, &block)
+      end
 
-         # Alias for transact(writable: true)
-         def write(&block)
-            transact(writable: true, &block)
-         end
+      # Alias for transact(writable: true)
+      def write(&block)
+         transact(writable: true, &block)
+      end
 
-         # Completes the given block as an atomic transaction locked using a global mutex table.
-         # The block is provided the current file contents.
-         # The block's result is written to the file.
-         def transact(writable: false)
-            semaphore = FileTransaction.file_mutex[@path.to_s] ||= Mutex.new
+      # Completes the given block as an atomic transaction locked using a global mutex table.
+      # The block is provided the current file contents.
+      # The block's result is written to the file.
+      def transact(writable: false)
+         semaphore = FileTransaction.file_mutex[@path.to_s] ||= Mutex.new
 
-            semaphore.synchronize do
-               @path.open(writable ? 'r+' : 'r') do |file|
-                  file.flock(File::LOCK_EX)
+         semaphore.synchronize do
+            @path.open(writable ? 'r+' : 'r') do |file|
+               file.flock(File::LOCK_EX)
 
-                  yield_result = yield(file.read)
-                  if writable
-                     file.rewind
-                     file.write yield_result
-                     file.truncate(file.pos)
-                  end
-                  yield_result
+               yield_result = yield(file.read)
+               if writable
+                  file.rewind
+                  file.write yield_result
+                  file.truncate(file.pos)
                end
+               yield_result
             end
          end
+      end
 
-         private
+      private
 
-         def ensure_path(path)
-            path = Pathname.new path
-            unless path.exist?
-               path.dirname.mkpath
-               FileUtils.touch path
-            end
-            path
+      def ensure_path(path)
+         path = Pathname.new path
+         unless path.exist?
+            path.dirname.mkpath
+            FileUtils.touch path
          end
+         path
       end
    end
 end