procrastinator 0.6.1 → 0.9.0

data/lib/procrastinator/task_worker.rb

@@ -1,120 +1,100 @@
+# frozen_string_literal: true
+
 require 'yaml'
+require 'ostruct'
+require 'timeout'
 
 module Procrastinator
+   # Works on a given task by creating a new instance of the queue's task class and running the appropriate hooks.
+   #
+   # The behaviour outside of the actual user-defined task is guided by the provided metadata.
+   #
+   # @author Robin Miller
+   #
+   # @see TaskMetaData
    class TaskWorker
-      attr_reader :id, :run_at, :initial_run_at, :expire_at, :task, :attempts, :last_fail_at, :last_error
-
-      def initialize(id: nil,
-                     run_at: nil,
-                     initial_run_at: nil,
-                     expire_at: nil,
-                     attempts: 0,
-                     timeout: nil,
-                     max_attempts: nil,
-                     last_fail_at: nil,
-                     last_error: nil,
-                     task:,
-                     logger: Logger.new(StringIO.new))
-         @id             = id
-         @run_at         = run_at.nil? ? nil : run_at.to_i
-         @initial_run_at = initial_run_at.to_i
-         @expire_at      = expire_at.nil? ? nil : expire_at.to_i
-         @task           = YAML.load(task)
-         @attempts       = attempts || 0
-         @max_attempts   = max_attempts
-         @timeout        = timeout
-         @last_fail_at   = last_fail_at
-         @last_error     = last_error
-         @logger         = logger
-
-         raise(MalformedTaskError.new('given task does not support #run method')) unless @task.respond_to? :run
-         raise(ArgumentError.new('timeout cannot be negative')) if timeout && timeout < 0
+      extend Forwardable
+
+      def_delegators :@metadata,
+                     :id, :run_at, :initial_run_at, :expire_at,
+                     :attempts, :last_fail_at, :last_error,
+                     :data,
+                     :to_h, :successful?
+
+      def initialize(metadata:,
+                     queue:,
+                     logger: Logger.new(StringIO.new),
+                     context: nil,
+                     scheduler: nil)
+         @queue = queue
+
+         @metadata       = metadata
+         @task           = queue.task_class.new
+
+         @task.data      = @metadata.data if @task.respond_to?(:data=)
+         @task.context   = context if @task.respond_to?(:context=)
+         @task.logger    = logger if @task.respond_to?(:logger=)
+         @task.scheduler = scheduler if @task.respond_to?(:scheduler=)
+
+         @logger  = logger
+         @context = context
+
+         raise MalformedTaskError, "task #{ @task.class } does not support #run method" unless @task.respond_to? :run
       end
 
       def work
-         @attempts += 1
+         @metadata.add_attempt
 
          begin
-            raise(TaskExpiredError.new("task is over its expiry time of #{@expire_at}")) if expired?
+            @metadata.verify_expiry!
 
-            Timeout::timeout(@timeout) do
+            result = Timeout.timeout(@queue.timeout) do
                @task.run
             end
 
-            try_hook(:success, @logger)
+            @logger&.debug("Task completed: #{ @task.class } [#{ @metadata.serialized_data }]")
 
-            @logger.debug("Task completed: #{YAML.dump(@task)}")
+            @metadata.clear_fails
 
-            @last_error   = nil
-            @last_fail_at = nil
-         rescue StandardError => e
-            @last_fail_at = Time.now.to_i
-
-            if too_many_fails? || expired?
-               try_hook(:final_fail, @logger, e)
-
-               @run_at     = nil
-               @last_error = "#{expired? ? 'Task expired' : 'Task failed too many times'}: #{e.backtrace.join("\n")}"
-
-               @logger.debug("Task failed permanently: #{YAML.dump(@task)}")
+            try_hook(:success, result)
+         rescue StandardError => error
+            if @metadata.final_fail?(@queue)
+               handle_final_failure(error)
             else
-               try_hook(:fail, @logger, e)
-
-               @last_error = %Q[Task failed: #{e.message}\n#{e.backtrace.join("\n")}]
-               @logger.debug("Task failed: #{YAML.dump(@task)}")
-
-               reschedule
+               handle_failure(error)
             end
          end
       end
 
-      def successful?
-         if !expired? && @attempts <= 0
-            raise(RuntimeError, 'you cannot check for success before running #work')
-         end
+      private
 
-         @last_error.nil? && @last_fail_at.nil?
+      def try_hook(method, *params)
+         @task.send(method, *params) if @task.respond_to? method
+      rescue StandardError => e
+         warn "#{ method.to_s.capitalize } hook error: #{ e.message }"
       end
 
-      def too_many_fails?
-         !@max_attempts.nil? && @attempts >= @max_attempts
-      end
+      def handle_failure(error)
+         @metadata.fail(%[Task failed: #{ error.message }\n#{ error.backtrace.join("\n") }])
+         @logger&.debug("Task failed: #{ @queue.name } with #{ @metadata.serialized_data }")
 
-      def expired?
-         !@expire_at.nil? && Time.now.to_i > @expire_at
-      end
+         @metadata.reschedule
 
-      def to_hash
-         {id:             @id,
-          run_at:         @run_at,
-          initial_run_at: @initial_run_at,
-          expire_at:      @expire_at,
-          attempts:       @attempts,
-          last_fail_at:   @last_fail_at,
-          last_error:     @last_error,
-          task:           YAML.dump(@task)}
+         try_hook(:fail, error)
       end
 
-      private
-      def try_hook(method, *params)
-         begin
-            @task.send(method, *params) if @task.respond_to? method
-         rescue StandardError => e
-            $stderr.puts "#{method.to_s.capitalize} hook error: #{e.message}"
-         end
-      end
+      def handle_final_failure(error)
+         trace = error.backtrace.join("\n")
+         msg   = "#{ @metadata.expired? ? 'Task expired' : 'Task failed too many times' }: #{ trace }"
 
-      def reschedule
-         # (30 + n_attempts^4) seconds is chosen to rapidly expand
-         # but with the baseline of 30s to avoid hitting the disc too frequently.
+         @metadata.fail(msg, final: true)
 
-         @run_at += 30 + (@attempts**4)
-      end
-   end
+         @logger&.debug("Task failed permanently: #{ YAML.dump(@task) }")
 
-   class TaskExpiredError < StandardError
+         try_hook(:final_fail, error)
+      end
    end
 
    class MalformedTaskError < StandardError
    end
-end
+end

data/lib/procrastinator/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Procrastinator
-   VERSION = '0.6.1'
+   VERSION = '0.9.0'
 end

data/lib/rake/procrastinator_task.rb

@@ -0,0 +1,34 @@
+require 'rake'
+require 'pathname'
+
+namespace :procrastinator do
+   desc 'Halt all Procrastinator processes'
+   task :stop, [:pid_dir] do |task, args|
+      pid_dir = args[:pid_dir] || Procrastinator::Config::DEFAULT_PID_DIRECTORY
+
+      if !pid_dir.exist? || pid_dir.empty?
+         raise <<~ERR
+            Default PID directory does not exist or is empty. Run: 
+               rake procrastinator:stop[directory]
+            with the directory to search for pid files.
+         ERR
+      end
+
+      pid_dir.each_child do |file|
+         pid = file.read.to_i
+
+         begin
+            name = `ps -p #{pid} -o command`
+            print "Halting worker process #{name} (pid: #{ pid })... "
+            Process.kill('KILL', pid)
+            puts 'halted'
+         rescue Errno::ESRCH
+            warn "Expected worker process pid=#{ pid }, but none was found. Continuing."
+         end
+
+         file.delete
+      end
+
+      puts '<= procrastinator:stop executed'
+   end
+end

data/procrastinator.gemspec

@@ -1,5 +1,6 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'procrastinator/version'
 
@@ -9,8 +10,8 @@ Gem::Specification.new do |spec|
    spec.authors = ['Robin Miller']
    spec.email   = ['robin@tenjin.ca']
 
-   spec.summary     = %q{Delayed task queues made simple.}
-   spec.description = %q{A strightforward job queue that is not dependent on Rails or any particular database or persistence mechanism.}
+   spec.summary     = 'For apps that put work off until later'
+   spec.description = 'A straightforward, customizable Ruby job queue with zero dependencies.'
    spec.homepage    = 'https://github.com/TenjinInc/procrastinator'
    spec.license     = 'MIT'
 
@@ -19,12 +20,13 @@ Gem::Specification.new do |spec|
    spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
    spec.require_paths = ['lib']
 
-   spec.required_ruby_version = '> 2.0'
+   spec.required_ruby_version = '>= 2.3'
 
    spec.add_development_dependency 'bundler', '~> 1.11'
-   spec.add_development_dependency 'rake', '~> 10.0'
-   spec.add_development_dependency 'rspec', '~> 3.0'
-   spec.add_development_dependency 'timecop', '~> 0.8'
-   spec.add_development_dependency 'simplecov', '~> 0.11'
    spec.add_development_dependency 'fakefs', '~> 0.10'
+   spec.add_development_dependency 'rake', '~> 12.3'
+   spec.add_development_dependency 'rspec', '~> 3.0'
+   spec.add_development_dependency 'rubocop', '~> 0.58'
+   spec.add_development_dependency 'simplecov', '~> 0.16.1'
+   spec.add_development_dependency 'timecop', '~> 0.9'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: procrastinator
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Robin Miller
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-01 00:00:00.000000000 Z
+date: 2018-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -24,20 +24,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -53,49 +67,48 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: timecop
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.58'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.58'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 0.16.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.11'
+        version: 0.16.1
 - !ruby/object:Gem::Dependency
-  name: fakefs
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
-description: A strightforward job queue that is not dependent on Rails or any particular
-  database or persistence mechanism.
+        version: '0.9'
+description: A straightforward, customizable Ruby job queue with zero dependencies.
 email:
 - robin@tenjin.ca
 executables: []
@@ -104,6 +117,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
@@ -114,10 +128,17 @@ files:
 - bin/console
 - bin/setup
 - lib/procrastinator.rb
-- lib/procrastinator/environment.rb
+- lib/procrastinator/config.rb
+- lib/procrastinator/loaders/csv_loader.rb
+- lib/procrastinator/queue.rb
+- lib/procrastinator/queue_manager.rb
 - lib/procrastinator/queue_worker.rb
+- lib/procrastinator/scheduler.rb
+- lib/procrastinator/task.rb
+- lib/procrastinator/task_meta_data.rb
 - lib/procrastinator/task_worker.rb
 - lib/procrastinator/version.rb
+- lib/rake/procrastinator_task.rb
 - procrastinator.gemspec
 homepage: https://github.com/TenjinInc/procrastinator
 licenses:
@@ -129,9 +150,9 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '2.0'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -139,8 +160,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
-summary: Delayed task queues made simple.
+summary: For apps that put work off until later
 test_files: []

data/lib/procrastinator/environment.rb

@@ -1,148 +0,0 @@
-module Procrastinator
-   class Environment
-      attr_reader :persister, :queue_definitions, :queue_workers, :processes, :test_mode
-
-      DEFAULT_LOG_DIRECTORY = 'log/'
-
-      def initialize(test_mode: false)
-         @test_mode         = test_mode
-         @queue_definitions = {}
-         @queue_workers     = []
-         @processes         = []
-         @log_dir           = DEFAULT_LOG_DIRECTORY
-         @log_level         = Logger::INFO
-      end
-
-      def persister_factory(&block)
-         @persister_factory = block
-
-         build_persister
-      end
-
-      def define_queue(name, properties={})
-         raise ArgumentError.new('queue name cannot be nil') if name.nil?
-
-         @queue_definitions[name] = properties
-      end
-
-      def spawn_workers
-         if @test_mode
-            @queue_definitions.each do |name, props|
-               @queue_workers << QueueWorker.new(props.merge(name:      name,
-                                                             persister: @persister))
-            end
-         else
-            @queue_definitions.each do |name, props|
-               pid = fork do
-                  build_persister
-                  worker = QueueWorker.new(props.merge(name:      name,
-                                                       persister: @persister,
-                                                       log_dir:   @log_dir,
-                                                       log_level: @log_level))
-
-                  Process.setproctitle("#{@process_prefix ? "#{@process_prefix}-" : ''}#{worker.long_name}") # tODO: add an app name prefix
-
-                  monitor_parent(worker)
-
-                  worker.work
-               end
-
-               Process.detach(pid) unless pid.nil?
-               @processes << pid
-            end
-         end
-      end
-
-      def act(*queue_names)
-         unless @test_mode
-            raise RuntimeError.new('Procrastinator.act called outside Test Mode. Enable test mode by setting Procrastinator.test_mode = true before running setup')
-         end
-
-         if queue_names.empty?
-            @queue_workers.each do |worker|
-               worker.act
-            end
-         else
-            queue_names.each do |name|
-               @queue_workers.find { |worker| worker.name == name }.act
-            end
-         end
-      end
-
-      def delay(queue: nil, run_at: Time.now.to_i, expire_at: nil, task:)
-         raise ArgumentError.new('task may not be nil') if task.nil?
-         raise MalformedTaskError.new('the provided task does not support #run method') unless task.respond_to? :run
-
-         # We're checking these on init because it's one of those extremely rare cases where you'd want to know
-         # incredibly early, because of the sub-processing. It's a bit belt-and suspenders, but UX is important for
-         # devs, too.
-         [:success, :fail, :final_fail].each do |method_name|
-            if task.respond_to?(method_name) && task.method(method_name).arity <= 0
-               raise MalformedTaskError.new("the provided task must accept a parameter to its ##{method_name} method")
-            end
-         end
-
-         if queue.nil? && @queue_definitions.size > 1
-            raise ArgumentError.new("queue must be specified when more than one is registered. Defined queues are: #{queue_definitions.keys.map { |k| ':' + k.to_s }.join(', ')}")
-         else
-            queue ||= @queue_definitions.keys.first
-            raise ArgumentError.new(%Q{there is no "#{queue}" queue registered in this environment}) if @queue_definitions[queue].nil?
-         end
-
-         @persister.create_task(queue:          queue,
-                                run_at:         run_at.to_i,
-                                initial_run_at: run_at.to_i,
-                                expire_at:      expire_at.nil? ? nil : expire_at.to_i,
-                                task:           YAML.dump(task))
-      end
-
-      def enable_test_mode
-         @test_mode = true
-      end
-
-      def log_dir(path)
-         @log_dir = path
-      end
-
-      def log_level(lvl)
-         @log_level = lvl
-      end
-
-      def process_prefix(prefix)
-         @process_prefix = prefix
-      end
-
-      private
-      def monitor_parent(worker)
-         parent_pid = Process.ppid
-
-         heartbeat_thread = Thread.new(parent_pid) do |ppid|
-            loop do
-               begin
-                  Process.kill(0, ppid) # kill with 0 flag checks if the process exists & has permissions
-               rescue Errno::ESRCH
-                  worker.log_parent_exit(ppid: ppid, pid: Process.pid)
-                  exit
-               end
-
-               sleep(5)
-            end
-         end
-
-         heartbeat_thread.abort_on_exception = true
-      end
-
-      def build_persister
-         @persister = @persister_factory.call
-
-         raise ArgumentError.new('persister cannot be nil') if @persister.nil?
-
-         [:read_tasks, :create_task, :update_task, :delete_task].each do |method|
-            raise MalformedPersisterError.new("persister must repond to ##{method}") unless @persister.respond_to? method
-         end
-      end
-   end
-
-   class MalformedPersisterError < StandardError
-   end
-end