belated 0.5.4 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 918d89e2f82daa2abdd57976ca18edbf994dcd4c92b4480bf139983f2a51e26d
-  data.tar.gz: 1c684c38e41ba507d048463c4e287802fed0b067f9263702ea9fb3bbc4bfb22c
+  metadata.gz: 4ca7f7d0a3a78ca41e6ba3afed7630e1ac72d4a88c4fdea08604066518c843a6
+  data.tar.gz: 2fb05c90501681f9f45ab4476cbdc4c1ac88de9addafaa1c54da0a16731426df
 SHA512:
-  metadata.gz: 7db6cbabae254f0152bd4a4367de72ad18085c58923dcb0420e20b94dbab0fb713fc00c54031bbe6788363a626d0a2ad315e6690b6cba0849aade14a406cb2f8
-  data.tar.gz: '0578bcaadeac30fa0d4cea61862de5927ca8624364fcb954a5a2f7f57c3cc1871c4fc99cb0730505aeb7e1f81ee0eb24327f5f48ef5aae4bf85fc5d5647baef3'
+  metadata.gz: 5182420208c866b37511bda56cc5a2e8fddac8d87a23274923318f576b66f3d3848b33c6633296bbfed088e6bd339a458e92ccc057a63c8a815dfaf12aa3fdf7
+  data.tar.gz: 78ef5824e2afdd45910dbe81e79e54986dcc2c7ccf346515228c978a55c5d2aa0d0f29c4d894e219667ee6d6b735fbef43c94b61bd36d645a3e9ca1152bf6a86

data/.github/workflows/main.yml

@@ -10,7 +10,8 @@ jobs:
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 3.0.1
+        ruby-version: 3.0.2
         bundler-cache: true
+        cache-version: 1
     - name: Run the default task
       run: bundle exec rake

data/.rubocop.yml

@@ -1,7 +1,12 @@
-require: rubocop-performance
 AllCops:
   TargetRubyVersion: 2.7
   NewCops: enable
+  Exclude:
+    - 'dummy/**/*'
+    - 'vendor/**/*'
+
+Lint/HashCompareByIdentity:
+  Enabled: false
 
 Style/StringLiterals:
   Enabled: true
@@ -23,9 +28,6 @@ Style/AsciiComments:
 Style/ClassVars:
   Enabled: false
 
-Style/Documentation:
-  Enabled: false
-
 Style/GlobalVars:
   Enabled: false
 

data/CHANGELOG.md

@@ -1,6 +1,23 @@
 ## [Unreleased]
 
-## [0.5.4] - 2021-08-14
+## [0.6.0] - 2021-08-19
+
+- Only need to keep references on the client side for procs. Not needed for classes, as they are pass-by-value. However, you can only pass procs by reference, so need to keep track of them. They're removed from the client side when they're completed though. 
+- The client is now a singleton. This is because it had some overhead when pushing the jobs to dRuby, so I took the approach of also doing that in a background thread. You however do not want more than one client to be running at the same time, so making it a singleton is the best option. Call the `.instance` method to get the singleton and then `.start` to get it started. 
+
+## [0.5.7] - 2021-08-18
+
+- Got errors under heavy load and restarting. Hopefully fixed by rescuing the DRb connection error.
+## [0.5.6] - 2021-08-17
+
+- Now the client has a hash table that holds references to the objects you push through it. This is to get by GC, otherwise the objects are collected on clientside, but are still referenced on server side. This means that you do not want to use two instances of Client at the same time. Also, might need to write a way to close the client...
+- Should work a bit more safely now! Performance testing 0.5.5 online using a Rails project, it performed horribly, so it should be a bit better now. 
+## [0.5.5] - 2021-08-15
+
+- Use SortedSet for future jobs, to avoid having to go through the whole list every few seconds. 
+
+
+## [0.5.4] - 2021-08-13
 
 - Client was using 100% CPU when it had no connection. (on $5 Digital Ocean droplet) Should be fixed now. 
 

data/Gemfile

@@ -13,6 +13,6 @@ gem 'database_cleaner-active_record'
 gem 'rails', '>= 6.1.3'
 gem 'rspec-rails'
 gem 'rubocop', '~> 1.7'
-gem 'rubocop-performance', require: false
+gem 'rubocop-discourse'
 gem 'sqlite3'
 gem 'stackprof'

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    belated (0.5.4)
+    belated (0.6.0)
       drb
       dry-configurable
+      sorted_set
 
 GEM
   remote: https://rubygems.org/
@@ -135,6 +136,7 @@ GEM
       thor (~> 1.0)
     rainbow (3.0.0)
     rake (13.0.6)
+    rbtree (0.4.4)
     regexp_parser (2.1.1)
     rexml (3.2.5)
     rspec (3.10.0)
@@ -158,21 +160,28 @@ GEM
       rspec-mocks (~> 3.10)
       rspec-support (~> 3.10)
     rspec-support (3.10.2)
-    rubocop (1.18.3)
+    rubocop (1.19.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 1.9.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.7.0)
+    rubocop-ast (1.10.0)
       parser (>= 3.0.1.1)
-    rubocop-performance (1.11.4)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
+    rubocop-discourse (2.4.2)
+      rubocop (>= 1.1.0)
+      rubocop-rspec (>= 2.0.0)
+    rubocop-rspec (2.4.0)
+      rubocop (~> 1.0)
+      rubocop-ast (>= 1.1.0)
     ruby-progressbar (1.11.0)
+    set (1.0.1)
+    sorted_set (1.0.3)
+      rbtree
+      set (~> 1.0)
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -203,7 +212,7 @@ DEPENDENCIES
   rspec (~> 3.0)
   rspec-rails
   rubocop (~> 1.7)
-  rubocop-performance
+  rubocop-discourse
   sqlite3
   stackprof
 

data/README.md

@@ -16,8 +16,10 @@ Can be used with or without Rails.
 
 TODO LIST:
 
+- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. 
+- Rescue `DRb::DRbRemoteError` when shutting down, might not need to if using GDBM?
 - Don't use class instance variables.
-- Make port configurable.
+- Make DRb port configurable.
 - Don't hardcode timezone.
 - Add some checks to the client for proper jobs.
 - Have multiple queues?
@@ -27,6 +29,7 @@ TODO LIST:
 - Do some performance testing.
 - Deploy a Rails app to production that is using Belated
   and mention it in the readme. (Capistrano support?)
+  ([Wasurechatta](https://wasurechatta.com/))
 - Add a section telling people to use Sidekiq if they can
 
 ## Installation
@@ -82,10 +85,17 @@ First, start up Belated.
 Then,
 
 ```ruby
-client = Belated::Client.new
+# Get the client
+client = Belated::Client.instance
+# Start the client, only need to do this once
+client.start unless client.started?
 ```
 
 and you can use the client!
+Note that the client is a singleton. 
+This means that you can only have one client running at a time, 
+but it also means you only have one connection to dRuby, and that the number of threads in charge of queuing the jobs is only one.
+
 Call
 
 ```ruby
@@ -101,7 +111,7 @@ If you don't want the job to run right away, you can also pass it a keyword para
 client.perform_belated(job, Time.now + 1.month)
 ```
 
-Note that you probably want to memoize the client, as it always creates a 'banker thread' now if you have no connection and there is the overhead of connecting to dRuby. Maybe even use it as a global!(`$client`)
+The client also holds references to the jobs that are instances of `Proc` that have been pushed so that they are not collected by GC. This is because procs are passed by reference, and the client needs to keep them alive. They are removed from the list when the job is done.
 
 # Settings
 

data/belated.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   # Uncomment to register a new dependency of your gem
   spec.add_dependency 'drb'
   spec.add_dependency 'dry-configurable'
+  spec.add_dependency 'sorted_set'
   spec.add_development_dependency 'byebug'
 
   # For more information and examples about making a new gem, checkout our

data/lib/belated.rb

@@ -65,9 +65,9 @@ class Belated
           @@queue.push(:shutdown)
         end
         Thread.new { stop_workers }
-        # Max 30 seconds to shutdown
+        # Max 40 seconds to shutdown
         timeout = 0
-        until (timeout += 0.1) >= 30 || @@queue.empty? || $TESTING
+        until (timeout += 0.1) >= 40 || @@queue.empty? || $TESTING
           sleep 0.1
         end
         exit
@@ -89,19 +89,20 @@ class Belated
   end
 
   def enqueue_future_jobs
-    log 'starting future jobs thread'
     loop do
       sleep 0.1
-      if @@queue.future_jobs.empty?
-        sleep 1
+      job = @@queue.future_jobs.min
+      if job.nil?
+        sleep 5
         next
       end
-      @@queue.future_jobs.each_with_index do |job, i|
-        if job.at <= Time.now.utc
-          log "Deleting #{@@queue.future_jobs.delete_at(i)} from future jobs"
-          @@queue.push(job)
-        end
+      if job.at <= Time.now.utc
+        log "Deleting #{@@queue.future_jobs.delete(job)} from future jobs"
+        @@queue.push(job)
       end
+    rescue DRb::DRbConnError
+      log 'DRb connection error!!!!!!'
+      log stats
     end
   end
 
@@ -112,7 +113,8 @@ class Belated
 
   def stop_workers
     @worker_list&.each do |worker|
-      sleep 0.1 if worker.alive?
+      i = 0
+      sleep 0.1 while worker.alive? || (i + 0.1) < 10
       Thread.kill(worker)
     end
     @@queue.save_jobs
@@ -138,7 +140,7 @@ class Belated
 
   def banner_and_info
     log banner
-    log "Currently running Belated version #{Belated::VERSION}"
+    log "Currently running Belated version #{Belated::VERSION} in #{Belated.config.environment}"
     log %(Belated running #{@worker_list&.length.to_i} workers on #{URI}...)
   end
 

data/lib/belated/client.rb

@@ -1,4 +1,5 @@
 require 'belated/job_wrapper'
+require 'singleton'
 class Belated
   # The client class is responsible for managing the connection to the
   # DRb server. If it has no connection, it adds the jobs to a bank queue.
@@ -7,16 +8,25 @@ class Belated
   #   client = Belated::Client.new
   #   client.enqueue(JubJub.new, at: Time.now + 5.seconds)
   class Client
-    attr_accessor :queue, :bank, :banker_thread
+    include Singleton unless $TESTING
+
+    attr_accessor :queue, :bank, :banker_thread, :proc_table
 
     # Starts up the client.
     # Connects to the queue through DRb.
     # @return [void]
-    def initialize
+    def start
       server_uri = Belated::URI
       DRb.start_service
+      self.proc_table = {}
       self.bank = Thread::Queue.new
       self.queue = DRbObject.new_with_uri(server_uri)
+      @started = true
+    end
+    alias initialize start
+
+    def started?
+      @started
     end
 
     # Thread in charge of handling the bank queue.
@@ -26,16 +36,25 @@ class Belated
     def start_banker_thread
       self.banker_thread = Thread.new do
         loop do
-          sleep 0.01
-          unless drb_connected?
-            sleep(10)
+          delete_from_table
+          if bank.empty?
+            sleep 10
             next
           end
+          begin
+            queue.push(wrapper = bank.pop)
+          rescue DRb::DRbConnError
+            bank.push(wrapper)
+          end
+        end
+      end
+    end
 
-          job = bank.pop
+    def delete_from_table
+      return if proc_table.empty?
 
-          perform(job)
-        end
+      proc_table.select { |_k, v| v.completed }.each do |key, _value|
+        proc_table.delete(key)
       end
     end
 
@@ -46,16 +65,17 @@ class Belated
     # @param max_retries [Integer] - Times the job should be retried if it fails.
     # @return [JobWrapper] - The job wrapper for the queue.
     def perform(job, at: nil, max_retries: 5)
+      log 'Passing a proc and at time is deprecated and will be removed in 0.6' if job.instance_of?(Proc) && !at.nil?
+
       job_wrapper = if job.is_a?(JobWrapper)
                       job
                     else
                       JobWrapper.new(job: job, at: at, max_retries: max_retries)
                     end
-      pp queue.push(job_wrapper)
-      job_wrapper
-    rescue DRb::DRbConnError
       bank.push(job_wrapper)
+      proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
       start_banker_thread if banker_thread.nil?
+      job_wrapper
     end
     alias perform_belated perform
     alias perform_later perform

data/lib/belated/job_wrapper.rb

@@ -2,9 +2,18 @@ require 'securerandom'
 require_relative 'logging'
 
 class Belated
+  # JobWrapper is a wrapper for a job. It is responsible for
+  # - logging
+  # - error handling
+  # - job execution
+  # - job result handling
+  # - job result logging
+  # - job retries
+  # - job retry delay
   class JobWrapper
+    include Comparable
     include Logging
-    attr_accessor :retries, :max_retries, :id, :job, :at
+    attr_accessor :retries, :max_retries, :id, :job, :at, :completed, :proc_klass
 
     def initialize(job:, max_retries: 5, at: nil)
       self.retries = 0
@@ -12,15 +21,23 @@ class Belated
       self.id = SecureRandom.uuid
       self.job = job
       self.at = at
+      self.completed = false
+      self.proc_klass = job.instance_of?(Proc)
+    end
+
+    def <=>(other)
+      at <=> other.at
     end
 
     # rubocop:disable Lint/RescueException
     def perform
-      if job.respond_to?(:call)
-        job.call
-      else
-        job.perform
-      end
+      resp = if job.respond_to?(:call)
+               job.call
+             else
+               job.perform
+             end
+      self.completed = true
+      resp
     rescue Exception => e
       case e.class
       when Interrupt, SignalException

data/lib/belated/logging.rb

@@ -1,4 +1,6 @@
 class Belated
+  # Logger for Belated.
+  # Include this module in your class to get a logger.
   module Logging
     extend self
 

data/lib/belated/queue.rb

@@ -3,20 +3,27 @@
 require 'belated/job'
 require 'belated/logging'
 require 'belated/job_wrapper'
+require 'sorted_set'
+
 class Belated
+  # Job queues that Belated uses.
+  # queue is the jobs that are currenly
+  # waiting for a worker to start working on them.
+  # future_jobs is a SortedSet of jobs that are going
+  # to be added to queue at some point in the future.
   class Queue
     include Logging
     attr_accessor :future_jobs
 
     FILE_NAME = 'belated_dump'
 
-    def initialize(queue: Thread::Queue.new, future_jobs: [])
+    def initialize(queue: Thread::Queue.new, future_jobs: SortedSet.new)
       @queue = queue
       self.future_jobs = future_jobs
     end
 
     def push(job)
-      if job == :shutdown || job.at.nil? ||
+      if job.is_a?(Symbol) || job.at.nil? ||
          job.at <= Time.now.utc
         @queue.push(job)
       else
@@ -79,7 +86,7 @@ class Belated
     private
 
     def proc_or_shutdown?(job)
-      job.job.instance_of?(Proc) || job == :shutdown
+      job.is_a?(Symbol) || job.job.instance_of?(Proc)
     end
   end
 end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.5.4'
+  VERSION = '0.6.0'
 end

data/lib/belated/worker.rb

@@ -16,10 +16,12 @@ class Belated
         log "Worker #{@number} fetching jobs!"
         next unless (job = Belated.fetch_job)
 
-        break if job == :shutdown
+        break if job.is_a?(Symbol)
 
         log "Worker #{@number} got job: #{job.inspect}"
-        job.perform
+        log job.perform
+      rescue Errno::ECONNREFUSED, RangeError => e
+        log e
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.0
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-13 00:00:00.000000000 Z
+date: 2021-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sorted_set
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement