belated 0.5.7 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c1e102693d57c373c84b2f6a4fb5624770334b0fb15bc3cffa05541cf7bbe29
-  data.tar.gz: bb18635e1ce9a04d83618cbd41fbdb91912c9b2f5c2d5f9dc6715154ef9254dc
+  metadata.gz: ae5f827cc80ba5fba59161a50ae5797ef774be28feeb478ea641dda6bca79b3a
+  data.tar.gz: 01a61d9a44ae2b76ad27827e58dd8ba81e2040753925dc7172458c04495f010a
 SHA512:
-  metadata.gz: 3465242fd5fe5a55ccd288cbc7d034ad6538e4bd75d8eeaa60c65b73e2f2a2c545ce63885172b85a8ec1e5c5a320dd51eaaf2038542bcc0f95da7a381973a217
-  data.tar.gz: f889516ac99765cfb5aa744589c614e7d6c59b7488223214e1f8a8eb6da879fef0a0b66556609dd625a04dcba454300f1915b1e45ff01a68e2bc212ea4ed4857
+  metadata.gz: 73a7fe76eacd1334186ef70628a78f193a20a72373d4faaa6bc9b8688ff0d9786156a69b44c3dfb9405837f0f5d980d1317277ad87c33d239d5d8c13b73207a4
+  data.tar.gz: bcc797637c63a712e81b77189ecd93c325a67fb67f91b392a86d8231b685394341447539c12012280fee392a52a71a6fb20713fdb0ae0be32201c2b2670f5a33

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 ## [Unreleased]
 
+## [0.6.3] - 2021-08-21
+
+- Needed to have the hash inside the mutex when going over it; otherwise you still the get can't add key into hash during iteration error. Of course.
+
+## [0.6.2] - 2021-08-20
+
+- Use a mutex for the proc_table used by the client. Fixes
+`RuntimeError: can't add a new key into hash during iteration (Most recent call first)`, so starting improving thread safety with this fix. 
+
+## [0.6.1] - 2021-08-20
+
+When the client closes and worker has a reference to a proc, a `DRb::ConnError` is raised. Rescueing it and ignoring it.
+## [0.6.0] - 2021-08-19
+
+- Only need to keep references on the client side for procs. Not needed for classes, as they are pass-by-value. However, you can only pass procs by reference, so need to keep track of them. They're removed from the client side when they're completed though. 
+- The client is now a singleton. This is because it had some overhead when pushing the jobs to dRuby, so I took the approach of also doing that in a background thread. You however do not want more than one client to be running at the same time, so making it a singleton is the best option. Call the `.instance` method to get the singleton and then `.start` to get it started. 
+
 ## [0.5.7] - 2021-08-18
 
 - Got errors under heavy load and restarting. Hopefully fixed by rescuing the DRb connection error.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    belated (0.5.7)
+    belated (0.6.3)
       drb
       dry-configurable
       sorted_set

data/README.md

@@ -14,15 +14,15 @@ Note that currently the timezone is hardcoded to UTC.
 
 Can be used with or without Rails. 
 
+Can be used if you're on a normal instance such as EC2 or Digital Ocean drop. Not if you're on a Heroku or Docker, or anything with ephemeral storage. 
+
 TODO LIST:
 
-- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. 
-- Rescue `DRb::DRbRemoteError` when shutting down, might not need to if using GDBM?
-- Don't use class instance variables.
+- Improve thread safety. 
+- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. Or RocksDB? 
 - Make DRb port configurable.
-- Don't hardcode timezone.
+- Don't hardcode timezone to UTC.
 - Add some checks to the client for proper jobs.
-- Have multiple queues?
 - Maybe support ActiveJob?
 - Have a web UI.
 - Have a job history
@@ -85,10 +85,17 @@ First, start up Belated.
 Then,
 
 ```ruby
-client = Belated::Client.new
+# Get the client
+client = Belated::Client.instance
+# Start the client, only need to do this once
+client.start unless client.started?
 ```
 
 and you can use the client!
+Note that the client is a singleton. 
+This means that you can only have one client running at a time, 
+but it also means you only have one connection to dRuby, and that the number of threads in charge of queuing the jobs is only one.
+
 Call
 
 ```ruby
@@ -104,9 +111,7 @@ If you don't want the job to run right away, you can also pass it a keyword para
 client.perform_belated(job, Time.now + 1.month)
 ```
 
-Note that you probably want to memoize the client, as it always creates a 'banker thread' and there is the overhead of connecting to dRuby. Maybe even use it as a global!(`$client`).
-
-The client also holds references to the jobs that have been pushed so that they are not collected by GC.
+The client also holds references to the jobs that are instances of `Proc` that have been pushed so that they are not collected by GC. This is because procs are passed by reference, and the client needs to keep them alive. They are removed from the list when the job is done.
 
 # Settings
 

data/lib/belated.rb

@@ -113,7 +113,8 @@ class Belated
 
   def stop_workers
     @worker_list&.each do |worker|
-      sleep 0.1 if worker.alive?
+      i = 0
+      sleep 0.1 while worker.alive? || (i + 0.1) < 10
       Thread.kill(worker)
     end
     @@queue.save_jobs

data/lib/belated/client.rb

@@ -1,4 +1,5 @@
 require 'belated/job_wrapper'
+require 'singleton'
 class Belated
   # The client class is responsible for managing the connection to the
   # DRb server. If it has no connection, it adds the jobs to a bank queue.
@@ -7,18 +8,26 @@ class Belated
   #   client = Belated::Client.new
   #   client.enqueue(JubJub.new, at: Time.now + 5.seconds)
   class Client
-    attr_accessor :queue, :bank, :banker_thread, :table
+    include Singleton unless $TESTING
+
+    attr_accessor :queue, :bank, :banker_thread, :proc_table
 
     # Starts up the client.
     # Connects to the queue through DRb.
     # @return [void]
-    def initialize
+    def start
       server_uri = Belated::URI
       DRb.start_service
-      self.table = {}
+      self.proc_table = {}
       self.bank = Thread::Queue.new
       self.queue = DRbObject.new_with_uri(server_uri)
-      start_banker_thread
+      @started = true
+      @mutex = Mutex.new
+    end
+    alias initialize start
+
+    def started?
+      @started
     end
 
     # Thread in charge of handling the bank queue.
@@ -28,24 +37,30 @@ class Belated
     def start_banker_thread
       self.banker_thread = Thread.new do
         loop do
-          sleep 0.01
-
-          delete_from_table
-
+          delete_from_table if proc_table.length > 20
           if bank.empty?
             sleep 10
-          else
-            perform(bank.pop)
+            next
+          end
+          until bank.empty?
+            begin
+              queue.push(wrapper = bank.pop)
+            rescue DRb::DRbConnError
+              bank.push(wrapper)
+              sleep 5
+            end
           end
         end
       end
     end
 
     def delete_from_table
-      return if table.empty?
+      return if proc_table.empty?
 
-      table.select { |_k, v| v.completed }.each do |key, _value|
-        table.delete(key)
+      @mutex.synchronize do
+        proc_table.select { |_k, v| v.completed }.each do |key, _value|
+          proc_table.delete(key)
+        end
       end
     end
 
@@ -56,23 +71,25 @@ class Belated
     # @param max_retries [Integer] - Times the job should be retried if it fails.
     # @return [JobWrapper] - The job wrapper for the queue.
     def perform(job, at: nil, max_retries: 5)
-      log 'Passing a proc and at time is deprecated and will be removed in 0.6' if job.instance_of?(Proc) && !at.nil?
-
-      job_wrapper = if job.is_a?(JobWrapper)
-                      job
-                    else
-                      JobWrapper.new(job: job, at: at, max_retries: max_retries)
-                    end
-      queue.push(job_wrapper)
-      table[job_wrapper.object_id] = job_wrapper
-    rescue DRb::DRbConnError
+      job_wrapper = wrap_job(job, at: at, max_retries: max_retries)
       bank.push(job_wrapper)
+      @mutex.synchronize do
+        proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
+      end
+      start_banker_thread if banker_thread.nil?
+      job_wrapper
     end
     alias perform_belated perform
     alias perform_later perform
 
     private
 
+    def wrap_job(job, at:, max_retries:)
+      return job if job.is_a?(JobWrapper)
+
+      JobWrapper.new(job: job, at: at, max_retries: max_retries)
+    end
+
     def drb_connected?
       queue.connected?
     rescue StandardError

data/lib/belated/job_wrapper.rb

@@ -13,7 +13,7 @@ class Belated
   class JobWrapper
     include Comparable
     include Logging
-    attr_accessor :retries, :max_retries, :id, :job, :at, :completed
+    attr_accessor :retries, :max_retries, :id, :job, :at, :completed, :proc_klass
 
     def initialize(job:, max_retries: 5, at: nil)
       self.retries = 0
@@ -22,6 +22,7 @@ class Belated
       self.job = job
       self.at = at
       self.completed = false
+      self.proc_klass = job.instance_of?(Proc)
     end
 
     def <=>(other)

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.5.7'
+  VERSION = '0.6.3'
 end

data/lib/belated/worker.rb

@@ -20,7 +20,7 @@ class Belated
 
         log "Worker #{@number} got job: #{job.inspect}"
         log job.perform
-      rescue Errno::ECONNREFUSED, RangeError => e
+      rescue DRb::DRbConnError, Errno::ECONNREFUSED, RangeError => e
         log e
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.5.7
+  version: 0.6.3
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-18 00:00:00.000000000 Z
+date: 2021-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb