procrastinator 0.9.0 → 1.0.0.pre.rc3

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: procrastinator
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 1.0.0.pre.rc3
 platform: ruby
 authors:
 - Robin Miller
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-26 00:00:00.000000000 Z
+date: 2022-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,84 +16,98 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '1.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.58'
+        version: '1.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.58'
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.16.1
+        version: 0.18.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.16.1
+        version: 0.18.0
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -108,7 +122,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-description: A straightforward, customizable Ruby job queue with zero dependencies.
+description: A flexible pure Ruby job queue. Tasks are reschedulable after failures.
 email:
 - robin@tenjin.ca
 executables: []
@@ -124,26 +138,30 @@ files:
 - Gemfile
 - LICENSE.txt
 - README.md
+- RELEASE_NOTES.md
 - Rakefile
 - bin/console
 - bin/setup
 - lib/procrastinator.rb
 - lib/procrastinator/config.rb
-- lib/procrastinator/loaders/csv_loader.rb
+- lib/procrastinator/logged_task.rb
 - lib/procrastinator/queue.rb
-- lib/procrastinator/queue_manager.rb
 - lib/procrastinator/queue_worker.rb
+- lib/procrastinator/rake/daemon_tasks.rb
+- lib/procrastinator/rake/tasks.rb
 - lib/procrastinator/scheduler.rb
 - lib/procrastinator/task.rb
 - lib/procrastinator/task_meta_data.rb
-- lib/procrastinator/task_worker.rb
+- lib/procrastinator/task_store/file_transaction.rb
+- lib/procrastinator/task_store/simple_comma_store.rb
+- lib/procrastinator/test/mocks.rb
 - lib/procrastinator/version.rb
-- lib/rake/procrastinator_task.rb
 - procrastinator.gemspec
 homepage: https://github.com/TenjinInc/procrastinator
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -152,16 +170,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
-summary: For apps that put work off until later
+summary: For apps to put off work until later
 test_files: []

data/lib/procrastinator/loaders/csv_loader.rb

@@ -1,107 +0,0 @@
-# frozen_string_literal: true
-
-require 'csv'
-require 'pathname'
-
-module Procrastinator
-   module Loader
-      # Simple Task I/O object that writes task information (ie. TaskMetaData attributes) to a CSV file.
-      #
-      # @author Robin Miller
-      class CSVLoader
-         # ordered
-         HEADERS = [:id, :queue, :run_at, :initial_run_at, :expire_at,
-                    :attempts, :last_fail_at, :last_error, :data].freeze
-
-         DEFAULT_FILE = 'procrastinator-tasks.csv'
-
-         def initialize(file_path = DEFAULT_FILE)
-            @path = Pathname.new(file_path)
-
-            if @path.directory? || @path.to_s.end_with?('/')
-               @path += DEFAULT_FILE
-            elsif @path.extname.empty?
-               @path = Pathname.new("#{ file_path }.csv")
-            end
-         end
-
-         def read
-            data = CSV.table(@path.to_s, force_quotes: false).to_a
-
-            headers = data.shift
-
-            data.collect do |d|
-               hash = Hash[headers.zip(d)]
-
-               hash[:data] = hash[:data].gsub('""', '"')
-
-               hash
-            end
-         end
-
-         def create(queue:, run_at:, initial_run_at:, expire_at:, data: '')
-            existing_data = begin
-               read
-            rescue Errno::ENOENT
-               []
-            end
-
-            max_id = existing_data.collect { |task| task[:id] }.max || 0
-
-            new_data = {
-                  id:             max_id + 1,
-                  queue:          queue,
-                  run_at:         run_at,
-                  initial_run_at: initial_run_at,
-                  expire_at:      expire_at,
-                  attempts:       0,
-                  data:           data
-            }
-
-            write(existing_data + [new_data])
-         end
-
-         def update(id, data)
-            existing_data = begin
-               read
-            rescue Errno::ENOENT
-               []
-            end
-
-            task_data = existing_data.find do |task|
-               task[:id] == id
-            end
-
-            task_data.merge!(data)
-
-            write(existing_data)
-         end
-
-         def delete(id)
-            existing_data = begin
-               read
-            rescue Errno::ENOENT
-               []
-            end
-
-            existing_data.delete_if do |task|
-               task[:id] == id
-            end
-
-            write(existing_data)
-         end
-
-         def write(data)
-            lines = data.collect do |d|
-               CSV.generate_line(d, headers: HEADERS, force_quotes: true)
-            end
-
-            @path.dirname.mkpath
-            @path.open('w') do |f|
-               f.puts HEADERS.join(',')
-               f.puts lines.join
-            end
-         end
-      end
-   end
-end

data/lib/procrastinator/queue_manager.rb

@@ -1,201 +0,0 @@
-# frozen_string_literal: true
-
-module Procrastinator
-   # Spawns and manages work queue subprocesses.
-   #
-   # This is where all of the multi-process logic should be kept to.
-   #
-   # @author Robin Miller
-   #
-   # @!attribute [r] :workers
-   #    @return [Hash] Maps the constructed QueueWorkers to their process ID.
-   class QueueManager
-      attr_reader :workers
-
-      def initialize(config)
-         @workers = {}
-         @config  = config
-         @logger  = start_log
-      end
-
-      # Shuts down any remaining old queue workers and spawns a new one for each queue defined in the config
-      #
-      # @return [Scheduler] a scheduler object that can be used to interact with the queues
-      def spawn_workers
-         scheduler = Scheduler.new(@config, self)
-
-         kill_old_workers
-
-         if ENV['PROCRASTINATOR_STOP']
-            @logger.warn('Cannot spawn queue workers because environment variable PROCRASTINATOR_STOP is set')
-         else
-            @config.queues.each do |queue|
-               spawn_worker(queue, scheduler: scheduler)
-            end
-         end
-
-         scheduler
-      end
-
-      # Produces a new QueueWorker for the given queue.
-      #
-      # If Test Mode is disabled in the config, then it will also fork a new independent process for that worker
-      # to work in.
-      #
-      # @param queue [Queue] the queue to build a worker for
-      # @param scheduler [Scheduler] an optional scheduler instance to pass to the worker
-      def spawn_worker(queue, scheduler: nil)
-         worker = QueueWorker.new(queue:     queue,
-                                  config:    @config,
-                                  scheduler: scheduler)
-         if @config.test_mode?
-            @workers[worker] = Process.pid
-         else
-            check_for_name(worker.long_name)
-
-            pid = fork
-
-            if pid
-               # === PARENT PROCESS ===
-               Process.detach(pid)
-               @workers[worker] = pid
-            else
-               deamonize(worker.long_name)
-
-               worker.work
-               shutdown_worker
-            end
-         end
-      end
-
-      def act(*queue_names)
-         unless @config.test_mode?
-            raise <<~ERR
-               Procrastinator.act called outside Test Mode.
-               Either use Procrastinator.spawn_workers or call #enable_test_mode in Procrastinator.setup.
-            ERR
-         end
-
-         workers = @workers.keys
-
-         if queue_names.empty?
-            workers.each(&:act)
-         else
-            queue_names.each do |name|
-               workers.find { |worker| worker.name == name }.act
-            end
-         end
-      end
-
-      private
-
-      def start_log
-         directory = @config.log_dir
-
-         return unless directory
-
-         log_path = directory + 'queue-manager.log'
-
-         directory.mkpath
-         File.open(log_path.to_path, 'a+') { |f| f.write '' }
-
-         logger = Logger.new(log_path.to_path)
-
-         logger.level = @config.log_level
-
-         # @logger.info(['',
-         #               '===================================',
-         #               "Started worker process, #{long_name}, to work off queue #{@queue.name}.",
-         #               "Worker pid=#{Process.pid}; parent pid=#{Process.ppid}.",
-         #               '==================================='].join("\n"))
-
-         logger
-      end
-
-      # Methods exclusive to the child process
-      module ChildMethods
-         def deamonize(name)
-            Process.daemon(true)
-            Process.setsid
-            srand
-            Process.setproctitle(name)
-            close_io
-
-            write_pid_file(Process.pid, name)
-
-            @config.run_process_block
-         end
-
-         # Make sure all input/output streams are closed
-         def close_io
-            stds = [$stdin, $stdout, $stderr]
-
-            # Part 1: close all IO objects (except for $stdin/$stdout/$stderr)
-            ObjectSpace.each_object(IO) do |io|
-               next if stds.include?(io)
-
-               begin
-                  io.close
-               rescue IOError
-                  next
-               end
-            end
-
-            # Part 2: redirect STD connections
-            stds.each do |io|
-               io.reopen '/dev/null'
-            end
-
-            # TODO: redirect OUT or ERR to logger?
-         end
-
-         # Wrapping #exit to allow for tests to easily stub out this behaviour.
-         # If #exit isn't prevented, the test framework will break,
-         # but #exit can't be directly stubbed either (because it's a required Kernel method)
-         def shutdown_worker
-            exit
-         end
-      end
-
-      # Methods exclusive to the main/parent process
-      module ParentMethods
-         def kill_old_workers
-            @config.pid_dir.mkpath
-
-            @config.pid_dir.each_child do |file|
-               pid = file.read.to_i
-
-               begin
-                  Process.kill('KILL', pid)
-                  @logger.info("Killing old worker process pid: #{ pid }")
-               rescue Errno::ESRCH
-                  @logger.info("Expected old worker process pid=#{ pid }, but none was found")
-               end
-
-               file.delete
-            end
-         end
-
-         def write_pid_file(pid, filename)
-            @config.pid_dir.mkpath
-
-            pid_file = @config.pid_dir + "#{ filename }.pid"
-
-            File.open(pid_file.to_path, 'w') do |f|
-               f.print(pid)
-            end
-         end
-
-         def check_for_name(name)
-            # better to use backticks so we can get the info and not spam user's stdout
-            warn <<~WARNING unless `pgrep -f #{ name }`.empty?
-               Warning: there is another process named "#{ name }". Use #each_process(prefix: '') in
-                        Procrastinator setup if you want to help yourself distinguish them.
-            WARNING
-         end
-      end
-
-      include ChildMethods
-      include ParentMethods
-   end
-end

data/lib/procrastinator/task_worker.rb

@@ -1,100 +0,0 @@
-# frozen_string_literal: true
-
-require 'yaml'
-require 'ostruct'
-require 'timeout'
-
-module Procrastinator
-   # Works on a given task by creating a new instance of the queue's task class and running the appropriate hooks.
-   #
-   # The behaviour outside of the actual user-defined task is guided by the provided metadata.
-   #
-   # @author Robin Miller
-   #
-   # @see TaskMetaData
-   class TaskWorker
-      extend Forwardable
-
-      def_delegators :@metadata,
-                     :id, :run_at, :initial_run_at, :expire_at,
-                     :attempts, :last_fail_at, :last_error,
-                     :data,
-                     :to_h, :successful?
-
-      def initialize(metadata:,
-                     queue:,
-                     logger: Logger.new(StringIO.new),
-                     context: nil,
-                     scheduler: nil)
-         @queue = queue
-
-         @metadata       = metadata
-         @task           = queue.task_class.new
-
-         @task.data      = @metadata.data if @task.respond_to?(:data=)
-         @task.context   = context if @task.respond_to?(:context=)
-         @task.logger    = logger if @task.respond_to?(:logger=)
-         @task.scheduler = scheduler if @task.respond_to?(:scheduler=)
-
-         @logger  = logger
-         @context = context
-
-         raise MalformedTaskError, "task #{ @task.class } does not support #run method" unless @task.respond_to? :run
-      end
-
-      def work
-         @metadata.add_attempt
-
-         begin
-            @metadata.verify_expiry!
-
-            result = Timeout.timeout(@queue.timeout) do
-               @task.run
-            end
-
-            @logger&.debug("Task completed: #{ @task.class } [#{ @metadata.serialized_data }]")
-
-            @metadata.clear_fails
-
-            try_hook(:success, result)
-         rescue StandardError => error
-            if @metadata.final_fail?(@queue)
-               handle_final_failure(error)
-            else
-               handle_failure(error)
-            end
-         end
-      end
-
-      private
-
-      def try_hook(method, *params)
-         @task.send(method, *params) if @task.respond_to? method
-      rescue StandardError => e
-         warn "#{ method.to_s.capitalize } hook error: #{ e.message }"
-      end
-
-      def handle_failure(error)
-         @metadata.fail(%[Task failed: #{ error.message }\n#{ error.backtrace.join("\n") }])
-         @logger&.debug("Task failed: #{ @queue.name } with #{ @metadata.serialized_data }")
-
-         @metadata.reschedule
-
-         try_hook(:fail, error)
-      end
-
-      def handle_final_failure(error)
-         trace = error.backtrace.join("\n")
-         msg   = "#{ @metadata.expired? ? 'Task expired' : 'Task failed too many times' }: #{ trace }"
-
-         @metadata.fail(msg, final: true)
-
-         @logger&.debug("Task failed permanently: #{ YAML.dump(@task) }")
-
-         try_hook(:final_fail, error)
-      end
-   end
-
-   class MalformedTaskError < StandardError
-   end
-end

data/lib/rake/procrastinator_task.rb

@@ -1,34 +0,0 @@
-require 'rake'
-require 'pathname'
-
-namespace :procrastinator do
-   desc 'Halt all Procrastinator processes'
-   task :stop, [:pid_dir] do |task, args|
-      pid_dir = args[:pid_dir] || Procrastinator::Config::DEFAULT_PID_DIRECTORY
-
-      if !pid_dir.exist? || pid_dir.empty?
-         raise <<~ERR
-            Default PID directory does not exist or is empty. Run: 
-               rake procrastinator:stop[directory]
-            with the directory to search for pid files.
-         ERR
-      end
-
-      pid_dir.each_child do |file|
-         pid = file.read.to_i
-
-         begin
-            name = `ps -p #{pid} -o command`
-            print "Halting worker process #{name} (pid: #{ pid })... "
-            Process.kill('KILL', pid)
-            puts 'halted'
-         rescue Errno::ESRCH
-            warn "Expected worker process pid=#{ pid }, but none was found. Continuing."
-         end
-
-         file.delete
-      end
-
-      puts '<= procrastinator:stop executed'
-   end
-end