procrastinator 0.6.1 → 1.0.0.pre.rc2

data/RELEASE_NOTES.md

@@ -0,0 +1,44 @@
+# Release Notes
+
+## 1.0.0 (       )
+
+### Major Changes
+
+* Minimum supported Ruby is now 2.4
+* Added generic `Procrastinator::Config#log_with`
+    * Removed `Procrastinator::Config#log_inside`
+    * Removed `Procrastinator::Config#log_at_level`
+    * falsey log level is now the control for whether logging occurs, instead of falsey log directory
+* Queues are managed as threads rather than sub processes
+    * These unnecessary methods no longer exist:
+        * `Procrastinator.test_mode`
+        * `Procrastinator::Config#enable_test_mode`
+        * `Procrastinator::Config#test_mode?`
+        * `Procrastinator::Config#test_mode`
+        * `Procrastinator::Config#prefix`
+        * `Procrastinator::Config#pid_dir`
+        * `Procrastinator::Config#each_process`
+        * `Procrastinator::Config#run_process_block`
+    * Removed use of envvar `PROCRASTINATOR_STOP`
+    * `Procrastinator::QueueManager` is merged into `Procrastinator::Scheduler`
+    * Removed rake task to halt queue processes
+    * Renamed `Procrastinator::Config#provide_context` to `provide_container`
+    * You must now call `Scheduler#work` on the result of `Procrastinator.config`
+    * Use a dedicated process monitor (like `monit`) instead in production environments
+    * Suuply a block to `daemonized!` to run code in the spawned process.
+* `max_tasks` is removed as it only added concurrency complexity
+* Data is now stored as JSON instead of YAML
+* Added with_store that applies its settings to its block
+    * `load_with` has been removed
+* Removed `task_attr` and `Procrastinator::Task` module. Tasks is now duck-type checked for accessors instead.
+
+### Minor Changes
+
+* Started release notes file
+* Updated development gems
+* Logs now include the queue name in log lines
+* Logs can now set the shift size or age (like Ruby's Logger)
+
+### Bugfixes
+
+* none 

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/lib/procrastinator/config.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module Procrastinator
+   # Configuration object (State Pattern) used to coordinate settings across
+   # various components within Procrastinator.
+   #
+   # It is immutable after init; use the config DSL in the configuration block to set its state.
+   #
+   # @author Robin Miller
+   #
+   # @!attribute [r] :queues
+   #    @return [Array] List of defined queues
+   # @!attribute [r] :container
+   #    @return [Object] Container object that will be forwarded to tasks
+   # @!attribute [r] :log_dir
+   #    @return [Pathname] Directory to write log files in
+   # @!attribute [r] :log_level
+   #    @return [Integer] Logging level to use
+   # @!attribute [r] :log_shift_age
+   #    @return [Integer] Number of previous files to keep (see Ruby Logger for details)
+   # @!attribute [r] :log_shift_size
+   #    @return [Integer] Filesize before rotating to a new logfile (see Ruby Logger for details)
+   class Config
+      attr_reader :queues, :log_dir, :log_level, :log_shift_age, :log_shift_size, :container
+
+      DEFAULT_LOG_DIRECTORY  = Pathname.new('log').freeze
+      DEFAULT_LOG_SHIFT_AGE  = 0
+      DEFAULT_LOG_SHIFT_SIZE = 2 ** 20 # 1 MB
+      DEFAULT_LOG_FORMATTER  = proc do |severity, datetime, progname, msg|
+         [datetime.iso8601(8),
+          severity,
+          "#{ progname } (#{ Process.pid }):",
+          msg].join("\t") << "\n"
+      end
+
+      def initialize
+         @queues         = []
+         @container      = nil
+         @log_dir        = DEFAULT_LOG_DIRECTORY
+         @log_level      = Logger::INFO
+         @log_shift_age  = DEFAULT_LOG_SHIFT_AGE
+         @log_shift_size = DEFAULT_LOG_SHIFT_SIZE
+
+         with_store(csv: TaskStore::SimpleCommaStore::DEFAULT_FILE) do
+            if block_given?
+               yield(self)
+               raise SetupError, SetupError::ERR_NO_QUEUE if @queues.empty?
+            end
+         end
+
+         @log_dir = @log_dir.expand_path
+
+         @queues.freeze
+         freeze
+      end
+
+      # Collection of all of the methods intended for use within Procrastinator.setup
+      #
+      # @see Procrastinator
+      module DSL
+         # Assigns a task loader
+         def with_store(store)
+            raise(ArgumentError, 'with_store must be provided a block') unless block_given?
+
+            old_store      = @default_store
+            @default_store = interpret_store(store)
+            yield
+            @default_store = old_store
+         end
+
+         def provide_container(container)
+            @container = container
+         end
+
+         def define_queue(name, task_class, properties = {})
+            raise ArgumentError, 'queue name cannot be nil' if name.nil?
+            raise ArgumentError, 'queue task class cannot be nil' if task_class.nil?
+
+            properties[:store] = interpret_store(properties[:store]) if properties.key? :store
+
+            @queues << Queue.new(**{name: name, task_class: task_class, store: @default_store}.merge(properties))
+         end
+
+         # Sets details of logging behaviour
+         #
+         # @param directory [Pathname,String] the directory to save logs within.
+         # @param level [Logger::UNKNOWN,Logger::FATAL,Logger::ERROR,Logger::WARN,Logger::INFO,Logger::DEBUG,Integer,Boolean] the Ruby Logger level to use. If falsey, no logging is performed.
+         # @param shift_age [Integer] number of old log files to keep (see Ruby Logger for details)
+         # @param shift_size [Integer] filesize before log is rotated to a fresh file (see Ruby Logger for details)
+         def log_with(directory: @log_dir, level: @log_level, shift_age: @log_shift_age, shift_size: @log_shift_size)
+            @log_dir        = directory ? Pathname.new(directory) : directory
+            @log_level      = level
+            @log_shift_age  = shift_age
+            @log_shift_size = shift_size
+         end
+      end
+
+      include DSL
+
+      def queue(name: nil)
+         queue = if name
+                    @queues.find do |q|
+                       q.name == name
+                    end
+                 else
+                    if name.nil? && @queues.length > 1
+                       raise ArgumentError,
+                             "queue must be specified when more than one is defined. #{ known_queues }"
+                    end
+
+                    @queues.first
+                 end
+
+         raise ArgumentError, "there is no :#{ name } queue registered. #{ known_queues }" unless queue
+
+         queue
+      end
+
+      private
+
+      def known_queues
+         "Known queues are: #{ @queues.map { |queue| ":#{ queue.name }" }.join(', ') }"
+      end
+
+      def interpret_store(store)
+         raise(ArgumentError, 'task store cannot be nil') if store.nil?
+
+         case store
+         when Hash
+            store_strategy = :csv
+            unless store.key? store_strategy
+               raise ArgumentError, "Must pass keyword :#{ store_strategy } if specifying a location for CSV file"
+            end
+
+            TaskStore::SimpleCommaStore.new(store[store_strategy])
+         when String, Pathname
+            TaskStore::SimpleCommaStore.new(store)
+         else
+            store
+         end
+      end
+
+      class SetupError < RuntimeError
+         ERR_NO_QUEUE = 'setup block must call #define_queue on the environment'
+      end
+   end
+end

data/lib/procrastinator/logged_task.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'ostruct'
+require 'timeout'
+# require 'forwardable'
+require 'delegate'
+require_relative 'task'
+
+module Procrastinator
+   # Task wrapper that adds logging to each step.
+   #
+   # @author Robin Miller
+   #
+   # @see Task
+   class LoggedTask < DelegateClass(Task)
+      # extend Forwardable
+      #
+      # def_delegators :@task, :id, :to_h
+
+      attr_reader :logger
+
+      alias task __getobj__
+
+      def initialize(task, logger: Logger.new(StringIO.new))
+         super task
+         @logger = logger || raise(ArgumentError, 'Logger cannot be nil')
+      end
+
+      def run
+         task.run
+
+         begin
+            @logger.info("Task completed: #{ task }")
+         rescue StandardError => e
+            warn "Task logging error: #{ e.message }"
+         end
+      end
+
+      def fail(error)
+         hook = task.fail(error)
+         begin
+            @logger.error("Task #{ hook }ed: #{ task }")
+         rescue StandardError => e
+            warn "Task logging error: #{ e.message }"
+         end
+         hook
+      end
+   end
+end

data/lib/procrastinator/queue.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Procrastinator
+   # A Queue defines how a certain type task will be processed.
+   #
+   # @author Robin Miller
+   #
+   # @!attribute [r] :name
+   #    @return [Symbol] The queue's identifier symbol
+   # @!attribute [r] :task_class
+   #    @return [Class] Class that defines the work to be done for jobs in this queue.
+   # @!attribute [r] :timeout
+   #    @return [Object] Duration (seconds) after which tasks in this queue should fail for taking too long.
+   # @!attribute [r] :max_attempts
+   #    @return [Object] Maximum number of attempts for tasks in this queue.
+   # @!attribute [r] :update_period
+   #    @return [Pathname] Delay (seconds) between reloads of tasks from the task store.
+   class Queue
+      extend Forwardable
+
+      DEFAULT_TIMEOUT       = 3600 # in seconds; one hour total
+      DEFAULT_MAX_ATTEMPTS  = 20
+      DEFAULT_UPDATE_PERIOD = 10 # seconds
+
+      attr_reader :name, :max_attempts, :timeout, :update_period, :task_store, :task_class
+
+      alias store task_store
+      alias storage task_store
+
+      def_delegators :@task_store, :read, :update, :delete
+
+      # Timeout is in seconds
+      def initialize(name:, task_class:,
+                     max_attempts: DEFAULT_MAX_ATTEMPTS,
+                     timeout: DEFAULT_TIMEOUT,
+                     update_period: DEFAULT_UPDATE_PERIOD,
+                     store: TaskStore::SimpleCommaStore.new)
+         raise ArgumentError, ':name cannot be nil' unless name
+
+         raise ArgumentError, ':task_class cannot be nil' unless task_class
+         raise ArgumentError, 'Task class must be initializable' unless task_class.respond_to? :new
+
+         raise ArgumentError, ':timeout cannot be negative' if timeout&.negative?
+
+         @name          = name.to_s.strip.gsub(/[^A-Za-z0-9]+/, '_').to_sym
+         @task_class    = task_class
+         @task_store    = store
+         @max_attempts  = max_attempts
+         @timeout       = timeout
+         @update_period = update_period
+
+         validate!
+
+         freeze
+      end
+
+      def next_task(logger: Logger.new(StringIO.new), container: nil, scheduler: nil)
+         metadata = next_metas.find(&:runnable?)
+
+         return nil unless metadata
+
+         task = Task.new(metadata, task_handler(data:      metadata.data,
+                                                container: container,
+                                                logger:    logger,
+                                                scheduler: scheduler))
+
+         LoggedTask.new(task, logger: logger)
+      end
+
+      def fetch_task(identifier)
+         identifier[:data] = JSON.dump(identifier[:data]) if identifier[:data]
+
+         tasks = read(**identifier)
+
+         raise "no task found matching #{ identifier }" if tasks.nil? || tasks.empty?
+         raise "too many (#{ tasks.size }) tasks match #{ identifier }. Found: #{ tasks }" if tasks.size > 1
+
+         TaskMetaData.new(tasks.first.merge(queue: self))
+      end
+
+      def create(run_at:, expire_at:, data:)
+         if data.nil? && expects_data?
+            raise ArgumentError, "task #{ @task_class } expects to receive :data. Provide :data to #delay."
+         end
+
+         unless data.nil? || expects_data?
+            raise MalformedTaskError, <<~ERROR
+               found unexpected :data argument. Either do not provide :data when scheduling a task,
+               or add this in the #{ @task_class } class definition:
+                     attr_accessor :data
+            ERROR
+         end
+
+         # TODO: shorten to using slice once updated to Ruby 2.5+
+         attrs = {queue: self, run_at: run_at, initial_run_at: run_at, expire_at: expire_at, data: JSON.dump(data)}
+
+         create_data = TaskMetaData.new(**attrs).to_h
+         create_data.delete(:id)
+         create_data.delete(:attempts)
+         create_data.delete(:last_fail_at)
+         create_data.delete(:last_error)
+         @task_store.create(**create_data)
+      end
+
+      def expects_data?
+         @task_class.method_defined?(:data=)
+      end
+
+      private
+
+      def task_handler(data: nil, container: nil, logger: nil, scheduler: nil)
+         handler           = @task_class.new
+         handler.data      = data if handler.respond_to?(:data=)
+         handler.container = container
+         handler.logger    = logger
+         handler.scheduler = scheduler
+         handler
+      end
+
+      def next_metas
+         tasks = read(queue: @name).reject { |t| t[:run_at].nil? }.collect do |t|
+            t.to_h.delete_if { |key| !TaskMetaData::EXPECTED_DATA.include?(key) }.merge(queue: self)
+         end
+
+         sort_tasks(tasks.collect { |t| TaskMetaData.new(**t) })
+      end
+
+      def sort_tasks(tasks)
+         # TODO: improve this
+         # shuffling and re-sorting to avoid worst case O(n^2) when receiving already sorted data
+         # on quicksort (which is default ruby sort). It is not unreasonable that the persister could return sorted
+         # results
+         # Ideally, we'd use a better algo than qsort for this, but this will do for now
+         tasks.shuffle.sort_by(&:run_at)
+      end
+
+      # Internal queue validator
+      module QueueValidation
+         def validate!
+            verify_task_class!
+            verify_task_store!
+         end
+
+         def verify_task_class!
+            verify_run_method!
+            verify_accessors!
+            verify_hooks!
+         end
+
+         # The interface compliance is checked on init because it's one of those rare cases where you want to know early;
+         # otherwise, you wouldn't know until task execution and that could be far in the future.
+         # UX is important for devs, too.
+         #    - R
+         def verify_run_method!
+            unless @task_class.method_defined? :run
+               raise MalformedTaskError, "task #{ @task_class } does not support #run method"
+            end
+
+            return unless @task_class.instance_method(:run).arity.positive?
+
+            raise MalformedTaskError, "task #{ @task_class } cannot require parameters to its #run method"
+         end
+
+         def verify_accessors!
+            [:logger, :container, :scheduler].each do |method_name|
+               next if @task_class.method_defined?(method_name) && @task_class.method_defined?("#{ method_name }=")
+
+               raise MalformedTaskError, <<~ERR
+                  Task handler is missing a #{ method_name } accessor. Add this to the #{ @task_class } class definition:
+                     attr_accessor :logger, :container, :scheduler
+               ERR
+            end
+         end
+
+         def verify_hooks!
+            expected_arity = 1
+
+            [:success, :fail, :final_fail].each do |method_name|
+               next unless @task_class.method_defined?(method_name)
+               next if @task_class.instance_method(method_name).arity == expected_arity
+
+               err = "task #{ @task_class } must accept #{ expected_arity } parameter to its ##{ method_name } method"
+
+               raise MalformedTaskError, err
+            end
+         end
+
+         def verify_task_store!
+            raise ArgumentError, ':store cannot be nil' if @task_store.nil?
+
+            [:read, :create, :update, :delete].each do |method|
+               unless @task_store.respond_to? method
+                  raise MalformedTaskStoreError, "task store #{ @task_store.class } must respond to ##{ method }"
+               end
+            end
+         end
+      end
+      include QueueValidation
+   end
+
+   class MalformedTaskError < StandardError
+   end
+
+   class MalformedTaskStoreError < RuntimeError
+   end
+end

data/lib/procrastinator/queue_worker.rb

@@ -1,116 +1,91 @@
+# frozen_string_literal: true
+
 module Procrastinator
+   # A QueueWorker checks for tasks to run from the task store and executes them, updating information in the task
+   # store as necessary.
+   #
+   # @author Robin Miller
    class QueueWorker
-      DEFAULT_TIMEOUT       = 3600 # in seconds; one hour total
-      DEFAULT_MAX_ATTEMPTS  = 20
-      DEFAULT_UPDATE_PERIOD = 10 # seconds
-      DEFAULT_MAX_TASKS     = 10
-
-      attr_reader :name, :timeout, :max_attempts, :update_period, :max_tasks
-
-      # Timeout is in seconds
-      def initialize(name:,
-                     persister:,
-                     log_dir: nil,
-                     log_level: Logger::INFO,
-                     max_attempts: DEFAULT_MAX_ATTEMPTS,
-                     timeout: DEFAULT_TIMEOUT,
-                     update_period: DEFAULT_UPDATE_PERIOD,
-                     max_tasks: DEFAULT_MAX_TASKS)
-         raise ArgumentError.new('Queue name may not be nil') unless name
-         raise ArgumentError.new('Persister may not be nil') unless persister
-
-         raise(MalformedTaskPersisterError.new('The supplied IO object must respond to #read_tasks')) unless persister.respond_to? :read_tasks
-         raise(MalformedTaskPersisterError.new('The supplied IO object must respond to #update_task')) unless persister.respond_to? :update_task
-         raise(MalformedTaskPersisterError.new('The supplied IO object must respond to #delete_task')) unless persister.respond_to? :delete_task
-
-         @name          = name.to_s.gsub(/\s/, '_').to_sym
-         @timeout       = timeout
-         @max_attempts  = max_attempts
-         @update_period = update_period
-         @max_tasks     = max_tasks
-         @persister     = persister
-         @log_dir       = log_dir
-         @log_level     = log_level
-
-         start_log
-      end
+      extend Forwardable
 
-      def work
-         begin
-            loop do
-               sleep(@update_period)
+      def_delegators :@queue, :name, :next_task
 
-               act
-            end
-         rescue StandardError => e
-            @logger.fatal(e)
-            # raise e
-         end
-      end
+      # expected methods for all persistence strategies
+      PERSISTER_METHODS = [:read, :update, :delete].freeze
+
+      def initialize(queue:, config:)
+         raise ArgumentError, ':queue cannot be nil' if queue.nil?
+         raise ArgumentError, ':config cannot be nil' if config.nil?
 
-      def act
-         # shuffling and re-sorting to avoid worst case O(n^2) on quicksort (which is default ruby sort)
-         # when receiving already sorted data. Ideally, we'd use a better algo, but this will do for now
-         tasks = @persister.read_tasks(@name).reject { |t| t[:run_at].nil? }.shuffle.sort_by { |t| t[:run_at] }
+         @config = config
 
-         tasks.first(@max_tasks).each do |task_data|
-            if Time.now.to_i >= task_data[:run_at].to_i
-               task_data.merge!(logger: @logger) if @logger
+         @queue = if queue.is_a? Symbol
+                     config.queue(name: queue)
+                  else
+                     queue
+                  end
+
+         @scheduler = Scheduler.new(config)
+         @logger    = Logger.new(StringIO.new)
+      end
 
-               tw = TaskWorker.new(task_data)
+      # Works on jobs forever
+      def work!
+         @logger = open_log!("#{ name }-queue-worker", @config)
+         @logger.info("Started worker thread to consume queue: #{ name }")
 
-               tw.work
+         loop do
+            sleep(@queue.update_period)
 
-               if tw.successful?
-                  @persister.delete_task(task_data[:id])
-               else
-                  @persister.update_task(tw.to_hash.merge(queue: @name))
-               end
-            end
+            work_one
          end
-      end
+      rescue StandardError => e
+         @logger.fatal(e)
 
-      def long_name
-         "#{@name}-queue-worker"
+         raise
       end
 
-      # Starts a log file and stores the logger within this queue worker.
-      #
-      # Separate from init because logging is context-dependent
-      def start_log
-         if @log_dir
-            log_path = Pathname.new("#{@log_dir}/#{long_name}.log")
-
-            log_path.dirname.mkpath
-            File.open(log_path.to_path, 'a+') do |f|
-               f.write ''
-            end
+      # Performs exactly one task on the queue
+      def work_one
+         task = next_task(logger:    @logger,
+                          container: @config.container,
+                          scheduler: @scheduler) || return
 
-            @logger = Logger.new(log_path.to_path)
+         begin
+            task.run
 
-            @logger.level = @log_level
+            @queue.delete(task.id)
+         rescue StandardError => e
+            task.fail(e)
 
-            @logger.info(['',
-                          '===================================',
-                          "Started worker process, #{long_name}, to work off queue #{@name}.",
-                          "Worker pid=#{Process.pid}; parent pid=#{Process.ppid}.",
-                          '==================================='].join("\n"))
+            task_info = task.to_h
+            id        = task_info.delete(:id)
+            @queue.update(id, **task_info)
          end
       end
 
-      # Logs a termination due to parent process termination
-      #
-      # == Parameters:
-      # @param ppid the parent's process id
-      # @param pid the child's process id
-      #
-      def log_parent_exit(ppid:, pid:)
-         raise RuntimeError.new('Cannot log when logger not defined. Call #start_log first.') unless @logger
+      def halt
+         @logger&.info("Halted worker on queue: #{ name }")
+         @logger&.close
+      end
+
+      # Starts a log file and returns the created Logger
+      def open_log!(name, config)
+         return @logger unless config.log_level
+
+         log_path = config.log_dir / "#{ name }.log"
+
+         config.log_dir.mkpath
+         FileUtils.touch(log_path)
 
-         @logger.error("Terminated worker process (pid=#{pid}) due to main process (ppid=#{ppid}) disappearing.")
+         Logger.new(log_path.to_path,
+                    config.log_shift_age, config.log_shift_size,
+                    level:     config.log_level || Logger::FATAL,
+                    progname:  name,
+                    formatter: Config::DEFAULT_LOG_FORMATTER)
       end
    end
 
    class MalformedTaskPersisterError < StandardError
    end
-end
+end