procrastinator 0.9.0 → 1.0.0.pre.rc2

data/lib/procrastinator/task_meta_data.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+require 'time'
+
 module Procrastinator
    # TaskMetaData objects are State Patterns that record information about the work done on a particular task.
    #
-   # It contains the specific information needed to run a task instance. Users define a task class, which describes
-   # the "how" of a task and TaskMetaData represents the "what" and "when".
+   # It contains the specific information needed to run a task instance. Users define a task handler class, which
+   # describes the "how" of a task and TaskMetaData represents the "what" and "when".
    #
    # It contains task-specific data, timing information, and error records.
    #
@@ -27,64 +29,63 @@ module Procrastinator
    # @!attribute [r] :last_fail_at
    #    @return [Integer] Linux epoch timestamp of when the last_error was recorded
    # @!attribute [r] :data
-   #    @return [String] User-provided data serialized to string using YAML
+   #    @return [String] App-provided JSON data
    class TaskMetaData
       # These are the attributes expected to be in the persistence mechanism
       EXPECTED_DATA = [:id, :run_at, :initial_run_at, :expire_at, :attempts, :last_error, :last_fail_at, :data].freeze
 
-      attr_reader(*EXPECTED_DATA)
+      attr_reader(*EXPECTED_DATA, :queue)
 
-      def initialize(id: nil,
-                     run_at: nil,
-                     initial_run_at: nil,
-                     expire_at: nil,
-                     attempts: 0,
-                     last_error: nil,
-                     last_fail_at: nil,
-                     data: nil)
+      def initialize(id: nil, queue: nil, data: nil,
+                     run_at: nil, initial_run_at: nil, expire_at: nil,
+                     attempts: 0, last_error: nil, last_fail_at: nil)
          @id             = id
-         @run_at         = run_at.nil? ? nil : run_at.to_i
-         @initial_run_at = initial_run_at.to_i
-         @expire_at      = expire_at.nil? ? nil : expire_at.to_i
-         @attempts       = attempts || 0
+         @queue          = queue || raise(ArgumentError, 'queue cannot be nil')
+         @run_at         = get_time(run_at)
+         @initial_run_at = get_time(initial_run_at) || @run_at
+         @expire_at      = get_time(expire_at)
+         @attempts       = (attempts || 0).to_i
          @last_error     = last_error
-         @last_fail_at   = last_fail_at
-         @data           = data ? YAML.safe_load(data, [Symbol, Date]) : nil
-      end
-
-      def init_task(queue)
-         @data ? queue.task_class.new(@data) : queue.task_class.new
+         @last_fail_at   = get_time(last_fail_at)
+         @data           = data ? JSON.parse(data, symbolize_names: true) : nil
       end
 
       def add_attempt
-         @attempts += 1
-      end
+         raise Task::AttemptsExhaustedError unless attempts_left?
 
-      def clear_fails
-         @last_error   = nil
-         @last_fail_at = nil
+         @attempts += 1
       end
 
-      def fail(msg, final: false)
-         @last_fail_at = Time.now.to_i
-         @last_error   = msg
-         @run_at       = nil if final
+      # Records a failure on this task
+      #
+      # @param error [StandardError] The error to record
+      def failure(error)
+         @last_fail_at = Time.now
+         @last_error   = %[Task failed: #{ error.message }\n#{ error.backtrace&.join("\n") }]
+
+         if retryable?
+            reschedule
+            :fail
+         else
+            @run_at = nil
+            :final_fail
+         end
       end
 
-      def final_fail?(queue)
-         too_many_fails?(queue) || expired?
+      def retryable?
+         attempts_left? && !expired?
       end
 
       def expired?
-         !@expire_at.nil? && Time.now.to_i > @expire_at
+         !@expire_at.nil? && @expire_at < Time.now
       end
 
-      def too_many_fails?(queue)
-         !queue.max_attempts.nil? && @attempts >= queue.max_attempts
+      def attempts_left?
+         @queue.max_attempts.nil? || @attempts < @queue.max_attempts
       end
 
       def runnable?
-         !(@run_at.nil? || Time.now.to_i < @run_at)
+         !@run_at.nil? && @run_at <= Time.now
       end
 
       def successful?
@@ -93,26 +94,32 @@ module Procrastinator
          !expired? && @last_error.nil? && @last_fail_at.nil?
       end
 
-      # TODO: This cop for ** is currently incorrect. This disable can be removed once they fix it.
-      # rubocop:disable Layout/SpaceAroundOperators
-      def reschedule
-         # (30 + n_attempts^4) seconds is chosen to rapidly expand
-         # but with the baseline of 30s to avoid hitting the disk too frequently.
-         @run_at += 30 + (@attempts ** 4) unless @run_at.nil?
-      end
+      # Updates the run and/or expiry time. If neither is provided, will reschedule based on the rescheduling
+      # calculation algorithm.
+      #
+      # @param run_at - the new time to run this task
+      # @param expire_at - the new time to expire this task
+      def reschedule(run_at: nil, expire_at: nil)
+         validate_run_at(run_at, expire_at)
 
-      # rubocop:enable Layout/SpaceAroundOperators
+         @expire_at = expire_at if expire_at
 
-      def serialized_data
-         YAML.dump(@data)
-      end
+         if run_at
+            @run_at = @initial_run_at = get_time(run_at)
+            clear_fails
+            @attempts = 0
+         end
 
-      def verify_expiry!
-         raise TaskExpiredError, "task is over its expiry time of #{ @expire_at }" if expired?
+         return if run_at || expire_at
+
+         # (30 + n_attempts^4) seconds is chosen to rapidly expand
+         # but with the baseline of 30s to avoid hitting the disk too frequently.
+         @run_at += 30 + (@attempts ** 4) unless @run_at.nil?
       end
 
       def to_h
          {id:             @id,
+          queue:          @queue.name.to_s,
           run_at:         @run_at,
           initial_run_at: @initial_run_at,
           expire_at:      @expire_at,
@@ -121,8 +128,45 @@ module Procrastinator
           last_error:     @last_error,
           data:           serialized_data}
       end
-   end
 
-   class TaskExpiredError < StandardError
+      def serialized_data
+         JSON.dump(@data)
+      end
+
+      def clear_fails
+         @last_error   = nil
+         @last_fail_at = nil
+      end
+
+      private
+
+      def get_time(data)
+         case data
+         when NilClass
+            nil
+         when Numeric
+            Time.at data
+         when String
+            Time.parse data
+         when Time
+            data
+         else
+            return data.to_time if data.respond_to? :to_time
+
+            raise ArgumentError, "Unknown data type: #{ data.class } (#{ data })"
+         end
+      end
+
+      def validate_run_at(run_at, expire_at)
+         return unless run_at
+
+         if expire_at && run_at > expire_at
+            raise ArgumentError, "new run_at (#{ run_at }) is later than new expire_at (#{ expire_at })"
+         end
+
+         return unless @expire_at && run_at > @expire_at
+
+         raise ArgumentError, "new run_at (#{ run_at }) is later than existing expire_at (#{ @expire_at })"
+      end
    end
 end

data/lib/procrastinator/task_store/file_transaction.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Procrastinator
+   module TaskStore
+      # The general idea is that there may be two threads that need to do these actions on the same file:
+      #    thread A:   read
+      #    thread B:   read
+      #    thread A/B: write
+      #    thread A/B: write
+      #
+      # When this sequence happens, the second file write is based on old information and loses the info from
+      # the prior write. Using a global mutex per file path prevents this case.
+      #
+      # This situation can also occur with multi processing, so file locking is also used for solitary access.
+      # File locking is only advisory in some systems, though, so it may only work against other applications
+      # that request a lock.
+      #
+      # @author Robin Miller
+      class FileTransaction
+         # Holds the mutual exclusion locks for file paths by name
+         @file_mutex = {}
+
+         class << self
+            attr_reader :file_mutex
+         end
+
+         def initialize(path)
+            @path = ensure_path(path)
+         end
+
+         # Alias for transact(writable: false)
+         def read(&block)
+            transact(writable: false, &block)
+         end
+
+         # Alias for transact(writable: true)
+         def write(&block)
+            transact(writable: true, &block)
+         end
+
+         # Completes the given block as an atomic transaction locked using a global mutex table.
+         # The block is provided the current file contents.
+         # The block's result is written to the file.
+         def transact(writable: false)
+            semaphore = FileTransaction.file_mutex[@path.to_s] ||= Mutex.new
+
+            semaphore.synchronize do
+               @path.open(writable ? 'r+' : 'r') do |file|
+                  file.flock(File::LOCK_EX)
+
+                  yield_result = yield(file.read)
+                  if writable
+                     file.rewind
+                     file.write yield_result
+                     file.truncate(file.pos)
+                  end
+                  yield_result
+               end
+            end
+         end
+
+         private
+
+         def ensure_path(path)
+            path = Pathname.new path
+            unless path.exist?
+               path.dirname.mkpath
+               FileUtils.touch path
+            end
+            path
+         end
+      end
+   end
+end

data/lib/procrastinator/task_store/simple_comma_store.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'pathname'
+
+module Procrastinator
+   module TaskStore
+      # Simple Task I/O adapter that writes task information (ie. TaskMetaData attributes) to a CSV file.
+      #
+      # SimpleCommaStore is not designed for efficiency or large loads (10,000+ tasks).
+      #
+      # For critical production environments, it is strongly recommended to use a more robust storage mechanism like a
+      # proper database.
+      #
+      # @author Robin Miller
+      class SimpleCommaStore
+         # ordered
+         HEADERS = [:id, :queue, :run_at, :initial_run_at, :expire_at,
+                    :attempts, :last_fail_at, :last_error, :data].freeze
+
+         EXT          = 'csv'
+         DEFAULT_FILE = Pathname.new("procrastinator-tasks.#{ EXT }").freeze
+
+         TIME_FIELDS = [:run_at, :initial_run_at, :expire_at, :last_fail_at].freeze
+
+         READ_CONVERTER = proc do |value, field_info|
+            if field_info.header == :data
+               value
+            elsif TIME_FIELDS.include? field_info.header
+               value.empty? ? nil : Time.parse(value)
+            else
+               begin
+                  Integer(value)
+               rescue ArgumentError
+                  value
+               end
+            end
+         end
+
+         attr_reader :path
+
+         def initialize(file_path = DEFAULT_FILE)
+            @path = Pathname.new(file_path)
+
+            if @path.directory? || @path.to_s.end_with?('/')
+               @path /= DEFAULT_FILE
+            elsif @path.extname.empty?
+               @path = @path.dirname / "#{ @path.basename }.csv"
+            end
+
+            @path = @path.expand_path
+
+            freeze
+         end
+
+         def read(filter = {})
+            FileTransaction.new(@path).read do |existing_data|
+               parse(existing_data).select do |row|
+                  filter.keys.all? do |key|
+                     row[key] == filter[key]
+                  end
+               end
+            end
+         end
+
+         # Saves a task to the CSV file.
+         #
+         # @param queue [String] queue name
+         # @param run_at [Time, nil] time to run the task at
+         # @param initial_run_at [Time, nil] first time to run the task at. Defaults to run_at.
+         # @param expire_at [Time, nil] time to expire the task
+         def create(queue:, run_at:, expire_at:, data: '', initial_run_at: nil)
+            FileTransaction.new(@path).write do |existing_data|
+               tasks  = parse(existing_data)
+               max_id = tasks.collect { |task| task[:id] }.max || 0
+
+               new_data = {
+                     id:             max_id + 1,
+                     queue:          queue,
+                     run_at:         run_at,
+                     initial_run_at: initial_run_at || run_at,
+                     expire_at:      expire_at,
+                     attempts:       0,
+                     data:           data
+               }
+
+               generate(tasks + [new_data])
+            end
+         end
+
+         def update(id, data)
+            FileTransaction.new(@path).write do |existing_data|
+               tasks     = parse(existing_data)
+               task_data = tasks.find do |task|
+                  task[:id] == id
+               end
+
+               task_data&.merge!(data)
+
+               generate(tasks)
+            end
+         end
+
+         def delete(id)
+            FileTransaction.new(@path).write do |file_content|
+               existing_data = parse(file_content)
+               generate(existing_data.reject { |task| task[:id] == id })
+            end
+         end
+
+         def generate(data)
+            lines = data.collect do |d|
+               TIME_FIELDS.each do |field|
+                  d[field] = d[field]&.iso8601
+               end
+               CSV.generate_line(d, headers: HEADERS, force_quotes: true).strip
+            end
+
+            lines.unshift(HEADERS.join(','))
+
+            lines.join("\n") << "\n"
+         end
+
+         private
+
+         def parse(csv_string)
+            data = CSV.parse(csv_string,
+                             headers:           true,
+                             header_converters: :symbol,
+                             skip_blanks:       true,
+                             converters:        READ_CONVERTER,
+                             force_quotes:      true).to_a
+
+            headers = data.shift || HEADERS
+
+            data = data.collect do |d|
+               headers.zip(d).to_h
+            end
+
+            correct_types(data)
+         end
+
+         def correct_types(data)
+            non_empty_keys = [:run_at, :expire_at, :attempts, :last_fail_at]
+
+            data.collect do |hash|
+               non_empty_keys.each do |key|
+                  hash.delete(key) if hash[key].is_a?(String) && hash[key].empty?
+               end
+
+               hash[:attempts] ||= 0
+
+               # hash[:data]  = (hash[:data] || '').gsub('""', '"')
+               hash[:queue] = hash[:queue].to_sym
+
+               hash
+            end
+         end
+      end
+   end
+end

data/lib/procrastinator/test/mocks.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Procrastinator
+   module Test
+      # Testing mock Task class
+      #
+      # You can use this like:
+      #
+      #    require 'procrastinator/rspec/mocks'
+      #    # ...
+      #    Procrastinator.config do |c|
+      #       c.define_queue :test_queue, Procrastinator::RSpec::MockTask
+      #    end
+      #
+      # @see MockDataTask for data-accepting tasks
+      class MockTask
+         attr_accessor :container, :logger, :scheduler
+
+         def run
+            @run = true
+         end
+
+         def run?
+            @run
+         end
+      end
+
+      # Data-accepting MockTask
+      #
+      # @see MockTask
+      class MockDataTask < MockTask
+         attr_accessor :data
+      end
+   end
+end

data/lib/procrastinator/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Procrastinator
-   VERSION = '0.9.0'
+   VERSION = '1.0.0-rc2'
 end

data/lib/procrastinator.rb

@@ -2,52 +2,37 @@
 
 require 'procrastinator/version'
 require 'procrastinator/task_meta_data'
-require 'procrastinator/task_worker'
+require 'procrastinator/logged_task'
 require 'procrastinator/queue'
 require 'procrastinator/queue_worker'
 require 'procrastinator/config'
-require 'procrastinator/queue_manager'
 require 'procrastinator/task'
 require 'procrastinator/scheduler'
-require 'procrastinator/loaders/csv_loader'
+require 'procrastinator/task_store/file_transaction'
+require 'procrastinator/task_store/simple_comma_store'
 
 require 'logger'
 require 'pathname'
 
 # Top-level module for the Procrastinator Gem.
 #
-# Call Procrastinator.setup with a block to initialize and run independent worker sub processes to complete tasks
-# asynchronously from your main application.
+# Call Procrastinator.setup with a block to configure task queues.
 #
-# Read the README for details.
+# See README for details.
 #
 # @author Robin Miller
 #
 # @see https://github.com/TenjinInc/procrastinator
 module Procrastinator
-   # rubocop:disable Style/ClassVars
-   @@test_mode = false
-
-   def self.test_mode=(value)
-      @@test_mode = value
-   end
-
-   def self.test_mode
-      @@test_mode
-   end
-
-   # rubocop:enable Style/ClassVars
-
    # Creates a configuration object and passes it into the given block.
    #
    # @yield the created configuration object
+   # @return [Scheduler] a scheduler object that can be used to interact with the queues
    def self.setup(&block)
-      raise ArgumentError, 'Procrastinator.setup must be given a block' unless block_given?
-
-      config = Config.new
+      raise ArgumentError, 'Procrastinator.setup must be given a block' unless block
 
-      config.setup(@@test_mode, &block)
+      config = Config.new(&block)
 
-      QueueManager.new(config).spawn_workers
+      Scheduler.new(config)
    end
 end

data/procrastinator.gemspec

@@ -10,23 +10,27 @@ Gem::Specification.new do |spec|
    spec.authors = ['Robin Miller']
    spec.email   = ['robin@tenjin.ca']
 
-   spec.summary     = 'For apps that put work off until later'
-   spec.description = 'A straightforward, customizable Ruby job queue with zero dependencies.'
+   spec.summary     = 'For apps to put off work until later'
+   spec.description = 'A flexible pure Ruby job queue. Tasks are reschedulable after failures.'
    spec.homepage    = 'https://github.com/TenjinInc/procrastinator'
    spec.license     = 'MIT'
+   spec.metadata    = {
+         'rubygems_mfa_required' => 'true'
+   }
 
    spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
    spec.bindir        = 'exe'
    spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
    spec.require_paths = ['lib']
 
-   spec.required_ruby_version = '>= 2.3'
+   spec.required_ruby_version = '>= 2.4'
 
-   spec.add_development_dependency 'bundler', '~> 1.11'
-   spec.add_development_dependency 'fakefs', '~> 0.10'
-   spec.add_development_dependency 'rake', '~> 12.3'
-   spec.add_development_dependency 'rspec', '~> 3.0'
-   spec.add_development_dependency 'rubocop', '~> 0.58'
-   spec.add_development_dependency 'simplecov', '~> 0.16.1'
+   spec.add_development_dependency 'bundler', '~> 2.3'
+   spec.add_development_dependency 'fakefs', '~> 1.8'
+   spec.add_development_dependency 'rake', '~> 13.0'
+   spec.add_development_dependency 'rspec', '~> 3.9'
+   spec.add_development_dependency 'rubocop', '~> 1.12'
+   spec.add_development_dependency 'rubocop-performance', '~> 1.10'
+   spec.add_development_dependency 'simplecov', '~> 0.18.0'
    spec.add_development_dependency 'timecop', '~> 0.9'
 end