belated 0.6.0 → 0.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ca7f7d0a3a78ca41e6ba3afed7630e1ac72d4a88c4fdea08604066518c843a6
-  data.tar.gz: 2fb05c90501681f9f45ab4476cbdc4c1ac88de9addafaa1c54da0a16731426df
+  metadata.gz: 535f6d7f0af3565867c6f621fb328966d35ff524e38ffc22813e4410fd66eb2f
+  data.tar.gz: ace8cc58b9519d07e7bce2f12c5b1e45eba570f75a0a5f34b0496a5acfac679d
 SHA512:
-  metadata.gz: 5182420208c866b37511bda56cc5a2e8fddac8d87a23274923318f576b66f3d3848b33c6633296bbfed088e6bd339a458e92ccc057a63c8a815dfaf12aa3fdf7
-  data.tar.gz: 78ef5824e2afdd45910dbe81e79e54986dcc2c7ccf346515228c978a55c5d2aa0d0f29c4d894e219667ee6d6b735fbef43c94b61bd36d645a3e9ca1152bf6a86
+  metadata.gz: '058ed5f47971a9f39d7ec39997f23bdfeb54221b516ee9162330e9a0c783dc9ec6d7633ad029b7d96792e08bd317d5efced10ca2ea97b0e8322507a6e5f9a045'
+  data.tar.gz: c5ef2e3ad64ecda5e1fa2cb0feab2dda1b50388a523e571fe3be829af76e864973429bb95d0433353dd5a58bf97f2f366d19bd19a7ae0d6d578030646156f542

data/.rubocop.yml

@@ -37,6 +37,9 @@ Style/BlockDelimiters:
 Style/ModuleFunction:
   EnforcedStyle: extend_self
 
+Metrics/AbcSize:
+  Max: 17
+
 Metrics/BlockLength:
   Exclude:
     - 'spec/**/*.rb'

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 ## [Unreleased]
 
+## [0.6.4] - 2021-08-22
+- Inline jobs for testing!
+```ruby
+`belated/testing`
+Belated::Testing.inline!
+```
+- Very much inspired by how Sidekiq is doing this. 
+- Read more in the testing part of README.md
+
+## [0.6.3] - 2021-08-21
+
+- Needed to have the hash inside the mutex when going over it; otherwise you still the get can't add key into hash during iteration error. Of course.
+
+## [0.6.2] - 2021-08-20
+
+- Use a mutex for the proc_table used by the client. Fixes
+`RuntimeError: can't add a new key into hash during iteration (Most recent call first)`, so starting improving thread safety with this fix. 
+
+## [0.6.1] - 2021-08-20
+
+When the client closes and worker has a reference to a proc, a `DRb::ConnError` is raised. Rescueing it and ignoring it.
 ## [0.6.0] - 2021-08-19
 
 - Only need to keep references on the client side for procs. Not needed for classes, as they are pass-by-value. However, you can only pass procs by reference, so need to keep track of them. They're removed from the client side when they're completed though. 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    belated (0.6.0)
+    belated (0.6.4)
       drb
       dry-configurable
       sorted_set

data/README.md

@@ -14,15 +14,14 @@ Note that currently the timezone is hardcoded to UTC.
 
 Can be used with or without Rails. 
 
+Can be used if you're on a normal instance such as EC2 or Digital Ocean drop. Not if you're on a Heroku or Docker, or anything with ephemeral storage. 
+
 TODO LIST:
 
-- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. 
-- Rescue `DRb::DRbRemoteError` when shutting down, might not need to if using GDBM?
-- Don't use class instance variables.
+- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. Or RocksDB? Would need to be configurable if you don't have something installed.
 - Make DRb port configurable.
-- Don't hardcode timezone.
+- Don't hardcode timezone to UTC.
 - Add some checks to the client for proper jobs.
-- Have multiple queues?
 - Maybe support ActiveJob?
 - Have a web UI.
 - Have a job history
@@ -140,6 +139,22 @@ Path to Rails project.
 
 Number of workers.
 
+## Testing
+
+When testing, you can require `belated/testing` and then call `Belated::Testing.inline!` to make your jobs perform inline.
+
+```ruby
+`belated/testing`
+c = Belated::Client.instance
+c.start
+c.perform(proc { 2/ 1}) # Tries to push the job to the drb backend
+# <Belated::JobWrapper:0x00005654bc2db1f0 @at=nil, @completed=false, @id="95e4dc6a-1876-4adf-ae0f-5ae902f5f024", @job=#<Proc:0x00005654bc2db330 (irb):3>, @max_retries=5, @proc_klass=true, @retries=0>
+Belated::Testing.inline! # Sidekiq-inspired, now jobs run inline
+c.perform(proc { 2/ 1}) # Returns 2 right away
+# 2
+Belated::Client.test_mode_off! # Turn off inline job processing
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/belated.rb

@@ -29,7 +29,7 @@ class Belated
   setting :rails_path, '.'
   setting :workers, 1
   setting :connect, true
-  setting :environment, 'development'
+  setting :environment, 'development', reader: true
   setting :logger, Logger.new($stdout), reader: true
   setting :log_level, :info, reader: true
 

data/lib/belated/client.rb

@@ -16,12 +16,15 @@ class Belated
     # Connects to the queue through DRb.
     # @return [void]
     def start
+      return if started?
+
       server_uri = Belated::URI
       DRb.start_service
       self.proc_table = {}
       self.bank = Thread::Queue.new
       self.queue = DRbObject.new_with_uri(server_uri)
       @started = true
+      @mutex = Mutex.new
     end
     alias initialize start
 
@@ -29,32 +32,41 @@ class Belated
       @started
     end
 
+    # Makes it possible to reset the client
+    def turn_off
+      @started = false
+      banker_thread&.kill
+    end
+
     # Thread in charge of handling the bank queue.
     # You probably want to memoize the client in order to avoid
     # having many threads in the sleep state.
     # @return [void]
     def start_banker_thread
-      self.banker_thread = Thread.new do
+      Thread.new do
         loop do
           delete_from_table
-          if bank.empty?
-            sleep 10
-            next
-          end
-          begin
-            queue.push(wrapper = bank.pop)
-          rescue DRb::DRbConnError
-            bank.push(wrapper)
+          sleep 10 and next if bank.empty?
+
+          until bank.empty?
+            begin
+              queue.push(wrapper = bank.pop)
+            rescue DRb::DRbConnError
+              bank.push(wrapper)
+              sleep 5
+            end
           end
         end
       end
     end
 
     def delete_from_table
-      return if proc_table.empty?
+      return if proc_table.length < 25
 
-      proc_table.select { |_k, v| v.completed }.each do |key, _value|
-        proc_table.delete(key)
+      @mutex.synchronize do
+        proc_table.select { |_k, v| v.completed }.each do |key, _value|
+          proc_table.delete(key)
+        end
       end
     end
 
@@ -65,16 +77,14 @@ class Belated
     # @param max_retries [Integer] - Times the job should be retried if it fails.
     # @return [JobWrapper] - The job wrapper for the queue.
     def perform(job, at: nil, max_retries: 5)
-      log 'Passing a proc and at time is deprecated and will be removed in 0.6' if job.instance_of?(Proc) && !at.nil?
+      log 'Call .start on the client instance first!' unless started?
 
-      job_wrapper = if job.is_a?(JobWrapper)
-                      job
-                    else
-                      JobWrapper.new(job: job, at: at, max_retries: max_retries)
-                    end
+      job_wrapper = wrap_job(job, at: at, max_retries: max_retries)
       bank.push(job_wrapper)
-      proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
-      start_banker_thread if banker_thread.nil?
+      @mutex.synchronize do
+        proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
+      end
+      self.banker_thread = start_banker_thread if banker_thread.nil?
       job_wrapper
     end
     alias perform_belated perform
@@ -82,6 +92,12 @@ class Belated
 
     private
 
+    def wrap_job(job, at:, max_retries:)
+      return job if job.is_a?(JobWrapper)
+
+      JobWrapper.new(job: job, at: at, max_retries: max_retries)
+    end
+
     def drb_connected?
       queue.connected?
     rescue StandardError

data/lib/belated/testing.rb

@@ -0,0 +1,37 @@
+class Belated
+  # Testing helpers
+  # Enable or disable testing
+  class Testing
+    @@testing = false
+
+    def self.inline?
+      @@testing == true
+    end
+
+    def self.inline!
+      @@testing = true
+    end
+
+    def self.test_mode_off!
+      @@testing = false
+    end
+  end
+end
+
+class Belated
+  # A client that can perform jobs inline
+  class Client
+    alias old_perform perform
+    def perform(job, at: nil, max_retries: 5)
+      if Belated::Testing.inline?
+        if job.respond_to?(:call)
+          job.call
+        else
+          job.perform
+        end
+      else
+        old_perform(job, at: at, max_retries: max_retries)
+      end
+    end
+  end
+end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.6.0'
+  VERSION = '0.6.4'
 end

data/lib/belated/worker.rb

@@ -20,7 +20,7 @@ class Belated
 
         log "Worker #{@number} got job: #{job.inspect}"
         log job.perform
-      rescue Errno::ECONNREFUSED, RangeError => e
+      rescue DRb::DRbConnError, Errno::ECONNREFUSED, RangeError => e
         log e
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.4
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-19 00:00:00.000000000 Z
+date: 2021-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -99,6 +99,7 @@ files:
 - lib/belated/logging.rb
 - lib/belated/queue.rb
 - lib/belated/rails.rb
+- lib/belated/testing.rb
 - lib/belated/version.rb
 - lib/belated/worker.rb
 homepage: https://github.com/sampokuokkanen/belated