belated 0.4.2 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33b9abbd449108f850f65e0afaa92929c780b03856e9e151efb6cc33b84021d5
-  data.tar.gz: eb9a7142602bae2572454c227d057adf9f38397c1638dcc6f66f2cd16019b766
+  metadata.gz: 5824acbe80af6b8b64a09ab2a0ee62d204504f99edee65d426ab05c737d52da0
+  data.tar.gz: ed3ff585ac66e715e55afd143bcf63fa534bd3434c9f6e9b5d4e0960fa47d9c8
 SHA512:
-  metadata.gz: c50c1dfe80dceee431785ec150a0a5b6a7c220e18fc8c6a05069637d474d848719c5bfd05f0c0f4125e379348296ebb92823e71b985e76cccffe9e3080349822
-  data.tar.gz: 357053481de792ea44b832eade91895d5257074e3381333ddb58aaa48c20d08b6d02d6e84b57664218108ae2927d8247db7229f1a1fa91483a745a3202031a0d
+  metadata.gz: 7d42b2530bd69a3a523383d98ccceb7ef76406bcf4c20791d93231b31004a6c43a208781308ede75fb614c84281bac3a97394646df9f0225a09de244030ff485
+  data.tar.gz: 62a9d873677daba7273ba182c2c0470f9ef9188d8a370d35fa94a7a45676abdbb4a26898299b886e06496f5bab85e9f740276e62f9dd9ee507f8ceb891605080

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 ## [Unreleased]
 
+## [0.5.1] - 2021-08-12
+
+- Requiring byebug in bin file. 😮
+
+## [0.5.0] - 2021-08-011
+
+- Job retries! The jobs now have ids, so you can follow the job and it's retries from the log. 
+- Quite a lot has changed internally, so if you were not using the Belated::Queue class to enqueue your jobs, you will need to update your code.
+
+## [0.4.4] - 2021-08-07
+
+- Now if you pass something with a syntax error in it as a job, it should not bring down the whole app! 
+
+## [0.4.3] - 2021-08-06
+
+- Client now starts the banker thread to execute jobs that were enqueued when there was no connection to Belated only if necessary. 
 ## [0.4.2] - 2021-08-05
 
 - Client also handles no connection, now it saves jobs to a bank and adds them to the queue once it has a connection.  

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    belated (0.4.2)
+    belated (0.5.1)
       drb
       dry-configurable
 

data/README.md

@@ -2,24 +2,31 @@
 
 [![CodeFactor](https://www.codefactor.io/repository/github/sampokuokkanen/belated/badge)](https://www.codefactor.io/repository/github/sampokuokkanen/belated) [![Gem Version](https://badge.fury.io/rb/belated.svg)](https://badge.fury.io/rb/belated)
 
-This is Belated, a new Ruby backend job library! It supports running procs and classes in the background. To deal with restarts, it uses YAML to load the queue into a file, which it then calls at startup to find the previous jobs.
+This is Belated, a new Ruby backend job library! It supports running procs, lambdas and classes in the background. To deal with restarts, it uses YAML to load the queue into a file, which it then calls at startup to find the previous jobs. There is no way in Ruby to save procs or lambdas to a file, so they are discarded when the process restarts.
+
+Belated uses the Ruby Queue class, so it's First In, First Out (FIFO). 
 
 Note that Belated used to be called HardWorker. That name was already in use in Sidekiq documentation and a bit too generic anyway. 
 
 It uses dRuby to do the communication! Which is absolute great. No need for Redis or PostgreSQL, just Ruby standard libraries.
 
+Note that currently the timezone is hardcoded to UTC. 
+
 Can be used with or without Rails. 
 
 TODO LIST:
 
+- Don't use class instance variables.
+- Make port configurable.
+- Don't hardcode timezone.
 - Add some checks to the client for proper jobs.
-- Don't crash on errors (Partially done)
 - Have multiple queues?
 - Maybe support ActiveJob?
-- Have a web UI
-- Do some performance testing
+- Have a web UI.
+- Have a job history
+- Do some performance testing.
 - Deploy a Rails app to production that is using Belated
-  and mention it in the readme. 
+  and mention it in the readme. (Capistrano support?)
 - Add a section telling people to use Sidekiq if they can
 
 ## Installation
@@ -48,15 +55,6 @@ Then, in another program, connect to Belated and give it a job to do.
 Sample below:
 
 ```ruby
-class DummyWorker
-  attr_accessor :queue
-
-  def initialize
-    server_uri = Belated::URI
-    self.queue = DRbObject.new_with_uri(server_uri)
-  end
-end
-
 class DumDum
   # classes need to have a perform method
   def perform
@@ -64,11 +62,14 @@ class DumDum
   end
 end
 
-# Need to start dRuby on the client side
-DRb.start_service
-dummy = DummyWorker.new
-dummy.queue.push(proc { 2 / 1 })
-dummy.queue.push(DumDum.new)
+client = Belated::Client.new
+client.perform_belated(proc { 2 / 1 })
+client.perform_belated(DumDum.new)
+# client.perform, client.perform_later are also good
+# if you want to do something later:
+client.perform_belated(DumDum.new, at: Time.now + 5 * 60)
+# max retries:
+client.perform_belated(DumDum.new, max_retries: 3) # default 5
 ```
 
 Belated runs on localhost, port 8788. 
@@ -88,6 +89,7 @@ and you can use the client!
 Call
 
 ```ruby
+job = proc { 2 / 1 }
 client.perform_belated(job)
 ```
 
@@ -99,6 +101,8 @@ If you don't want the job to run right away, you can also pass it a keyword para
 client.perform_belated(job, Time.now + 1.month)
 ```
 
+Note that you probably want to memoize the client, as it always creates a 'banker thread' now if you have no connection and there is the overhead of connecting to dRuby. Maybe even use it as a global!(`$client`)
+
 # Settings
 
 Configuring Belated:

data/bin/belated

@@ -5,7 +5,6 @@ $TESTING = false
 $LOAD_PATH.unshift File.expand_path("#{File.dirname(__FILE__)}/../lib")
 require 'belated'
 require 'optparse'
-require 'byebug'
 
 OptionParser.new { |opts|
   opts.banner = 'Usage: belated [options]'

data/lib/belated.rb

@@ -1,15 +1,16 @@
 # frozen_string_literal: true
 
+require_relative 'belated/logging'
 require_relative 'belated/version'
 require_relative 'belated/worker'
-require_relative 'belated/logging'
+require 'belated/client'
+require 'belated/job_wrapper'
+require 'belated/queue'
 require 'drb'
-require 'yaml'
-require 'singleton'
 require 'dry-configurable'
-require 'belated/client'
 require 'logger'
-require 'belated/queue'
+require 'singleton'
+require 'yaml'
 
 # Belated is a pure Ruby job backend.
 # It has limited functionality, as it only accepts
@@ -64,7 +65,11 @@ class Belated
           @@queue.push(:shutdown)
         end
         Thread.new { stop_workers }
-        sleep 0.1 until @@queue.empty? || $TESTING
+        # Max 30 seconds to shutdown
+        timeout = 0
+        until (timeout += 0.1) >= 30 || @@queue.empty? || $TESTING
+          sleep 0.1
+        end
         exit
       end
     end
@@ -87,9 +92,9 @@ class Belated
     log 'starting future jobs thread'
     loop do
       @@queue.future_jobs.each_with_index do |job, i|
-        if job[:at] <= Time.now.utc
-          log @@queue.future_jobs.delete_at(i)
-          @@queue.push(job[:klass])
+        if job.at <= Time.now.utc
+          log "Deleting #{@@queue.future_jobs.delete_at(i)} from future jobs"
+          @@queue.push(job)
         end
       end
       sleep 0.01
@@ -151,12 +156,16 @@ class Belated
     @@queue.clear
   end
 
+  def self.fetch_job
+    @@queue.pop
+  end
+
   def job_list
     @@queue
   end
 
-  def self.fetch_job
-    @@queue.pop
+  def self.job_list
+    @@queue
   end
 
   class Error < StandardError; end

data/lib/belated/client.rb

@@ -1,3 +1,4 @@
+require 'belated/job_wrapper'
 class Belated
   # The client class is responsible for managing the connection to the
   # DRb server. If it has no connection, it adds the jobs to a bank queue.
@@ -13,16 +14,21 @@ class Belated
     # @return [void]
     def initialize
       server_uri = Belated::URI
-      # @bank =
       DRb.start_service
       self.bank = Thread::Queue.new
       self.queue = DRbObject.new_with_uri(server_uri)
+    end
+
+    # Thread in charge of handling the bank queue.
+    # You probably want to memoize the client in order to avoid
+    # having many threads in the sleep state.
+    # @return [void]
+    def start_banker_thread
       self.banker_thread = Thread.new do
         loop do
-          sleep 0.05
-          next unless (job, at = bank.pop)
+          job = bank.pop
 
-          perform(job, at: at)
+          perform(job)
         end
       end
     end
@@ -31,11 +37,20 @@ class Belated
     # If there is no connection, it pushes the job to the bank.
     # @param job [Object] - The the job to be pushed.
     # @param at [Date] - The time at which the job should be executed.
-    # @return [Object] - The job that was pushed.
-    def perform(job, at: nil)
-      queue.push(job, at: at)
+    # @param max_retries [Integer] - Times the job should be retried if it fails.
+    # @return [JobWrapper] - The job wrapper for the queue.
+    def perform(job, at: nil, max_retries: 5)
+      job_wrapper = if job.is_a?(JobWrapper)
+                      job
+                    else
+                      JobWrapper.new(job: job, at: at, max_retries: max_retries)
+                    end
+      pp queue.push(job_wrapper)
+      job_wrapper
     rescue DRb::DRbConnError
-      bank.push([job, at])
+      bank.push(job_wrapper)
+      start_banker_thread if banker_thread.nil?
+      banker_thread.wakeup if banker_thread.status == 'sleep'
     end
     alias perform_belated perform
     alias perform_later perform

data/lib/belated/job_wrapper.rb

@@ -0,0 +1,44 @@
+require 'securerandom'
+require_relative 'logging'
+
+class Belated
+  class JobWrapper
+    include Logging
+    attr_accessor :retries, :max_retries, :id, :job, :at
+
+    def initialize(job:, max_retries: 5, at: nil)
+      self.retries = 0
+      self.max_retries = max_retries
+      self.id = SecureRandom.uuid
+      self.job = job
+      self.at = at
+    end
+
+    # rubocop:disable Lint/RescueException
+    def perform
+      if job.respond_to?(:call)
+        job.call
+      else
+        job.perform
+      end
+    rescue Exception => e
+      case e.class
+      when Interrupt, SignalException
+        raise e
+      else
+        retry_job
+        "Error while executing job, #{e.inspect}. Retry #{retries} of #{max_retries}"
+      end
+    end
+    # rubocop:enable Lint/RescueException
+
+    def retry_job
+      self.retries += 1
+      return if retries > max_retries
+
+      self.at = Time.now.utc + (retries.next**4)
+      log "Job #{id} failed, retrying at #{at}"
+      Belated.job_list.push(self)
+    end
+  end
+end

data/lib/belated/queue.rb

@@ -2,6 +2,7 @@
 
 require 'belated/job'
 require 'belated/logging'
+require 'belated/job_wrapper'
 class Belated
   class Queue
     include Logging
@@ -14,11 +15,11 @@ class Belated
       self.future_jobs = future_jobs
     end
 
-    def push(job, at: nil)
-      if at.nil?
+    def push(job)
+      if job.at.nil? || job.at <= Time.now.utc
         @queue.push(job)
       else
-        @future_jobs << Job.new(job, at)
+        @future_jobs << job
       end
     end
 
@@ -45,7 +46,7 @@ class Belated
 
       jobs = YAML.load(File.binread(FILE_NAME))
       jobs.each do |job|
-        if job.is_a?(Job)
+        if job.at && job.at > Time.now.utc
           future_jobs.push(job)
         else
           @queue.push(job)
@@ -73,7 +74,7 @@ class Belated
     private
 
     def proc_or_shutdown?(job)
-      job.instance_of?(Proc) || job == :shutdown
+      job.job.instance_of?(Proc) || job == :shutdown
     end
   end
 end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.4.2'
+  VERSION = '0.5.1'
 end

data/lib/belated/worker.rb

@@ -18,18 +18,9 @@ class Belated
 
         break if job == :shutdown
 
-        log call_job(job)
-      end
-    end
-
-    def call_job(job)
-      if job.respond_to?(:call)
-        job.call
-      else
+        log "Worker #{@number} got job: #{job.inspect}"
         job.perform
       end
-    rescue StandardError => e
-      e.inspect
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.1
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-05 00:00:00.000000000 Z
+date: 2021-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -81,6 +81,7 @@ files:
 - lib/belated.rb
 - lib/belated/client.rb
 - lib/belated/job.rb
+- lib/belated/job_wrapper.rb
 - lib/belated/logging.rb
 - lib/belated/queue.rb
 - lib/belated/rails.rb