belated 0.6.1 → 0.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c9e0e522a8de63f70d61dfeb7d02e8382c957f9e39858a5ce76fc063d4b2beb
-  data.tar.gz: 5cd83e71ab30ce2ceb5bc357bf53963399ed6b8aae5beae6930eff98a0f79b10
+  metadata.gz: b76db91c6812b59cfd2ede6c6e8d5664cde8547de40b0d05345f624e9cba4729
+  data.tar.gz: 99e886f91004d6d467db2aa7103e59c0da5bedaa5d2359fd9e91bd69b17a424d
 SHA512:
-  metadata.gz: 3cc2687dd4cdbbae82dc205a5c2ea851dd8b6a7f24471b424e8e1aae547a3fb01be063adbf765f92de10fcf886cf5eb10ac23b2c1144fa7c2d0c5cff59acb803
-  data.tar.gz: 748fd28b0eed08dfb16f221e3519c11c41235cdfd7527fe0ce68c6cdd5b996614c7e61ca6b48f31b66c32f3e20d94dce681be307abbcc616f8df39f6752df3b3
+  metadata.gz: a1c095daefe085f5913c363609a3a5b038a0025e133ed0ab16ad131d09a26c4c4d69489b87e5a4939dd6da105b428f6ea9ab186ef38a53bb2ec3605bdbb8086d
+  data.tar.gz: 79532774d5169e6781828792b94ef1dd38c0cedbf68b838b95e2c2d64d01d922e0cb9ac093e76524f3c784f9b346bb0e87bda319fb50e1bb1ac8ecd9abd81fbc

data/.rubocop.yml

@@ -37,6 +37,9 @@ Style/BlockDelimiters:
 Style/ModuleFunction:
   EnforcedStyle: extend_self
 
+Metrics/AbcSize:
+  Max: 17
+
 Metrics/BlockLength:
   Exclude:
     - 'spec/**/*.rb'

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 ## [Unreleased]
 
+## [0.6.5] - 2021-08-23
+
+- Timezone used inside Belated is all using the server time now, so it's up to the user to take care of that(using `Time.now` instead of `Time.now.utc`)
+- Possible to configure host and port.
+- No need to call `.start` on the client anymore. 
+- Logging some error and warn messages now too, instead of it all being info
+
+## [0.6.4] - 2021-08-22
+- Inline jobs for testing!
+```ruby
+`belated/testing`
+Belated::Testing.inline!
+```
+- Very much inspired by how Sidekiq is doing this. 
+- Read more in the testing part of README.md
+
+## [0.6.3] - 2021-08-21
+
+- Needed to have the hash inside the mutex when going over it; otherwise you still the get can't add key into hash during iteration error. Of course.
+
+## [0.6.2] - 2021-08-20
+
+- Use a mutex for the proc_table used by the client. Fixes
+`RuntimeError: can't add a new key into hash during iteration (Most recent call first)`, so starting improving thread safety with this fix. 
+
 ## [0.6.1] - 2021-08-20
 
 When the client closes and worker has a reference to a proc, a `DRb::ConnError` is raised. Rescueing it and ignoring it.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    belated (0.6.1)
+    belated (0.6.5)
       drb
       dry-configurable
       sorted_set

data/README.md

@@ -18,11 +18,7 @@ Can be used if you're on a normal instance such as EC2 or Digital Ocean drop. No
 
 TODO LIST:
 
-- Improve thread safety. 
-- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. Or RocksDB? 
-- Make DRb port configurable.
-- Don't hardcode timezone to UTC.
-- Add some checks to the client for proper jobs.
+- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. Or RocksDB? Would need to be configurable if you don't have something installed.
 - Maybe support ActiveJob?
 - Have a web UI.
 - Have a job history
@@ -87,8 +83,6 @@ Then,
 ```ruby
 # Get the client
 client = Belated::Client.instance
-# Start the client, only need to do this once
-client.start unless client.started?
 ```
 
 and you can use the client!
@@ -138,8 +132,31 @@ Path to Rails project.
 
     $ bundle exec belated --workers=10
 
+Other available settings:
+
+    $ bundle exec belated --host=1.1.1.1 --port=1234 
+    # druby://1.1.1.1:1234
+    $ bundle exec belated --env 
+    # environment
+
+
 Number of workers.
 
+## Testing
+
+When testing, you can require `belated/testing` and then call `Belated::Testing.inline!` to make your jobs perform inline.
+
+```ruby
+`belated/testing`
+c = Belated::Client.instance
+c.perform(proc { 2/ 1}) # Tries to push the job to the drb backend
+# <Belated::JobWrapper:0x00005654bc2db1f0 @at=nil, @completed=false, @id="95e4dc6a-1876-4adf-ae0f-5ae902f5f024", @job=#<Proc:0x00005654bc2db330 (irb):3>, @max_retries=5, @proc_klass=true, @retries=0>
+Belated::Testing.inline! # Sidekiq-inspired, now jobs run inline
+c.perform(proc { 2/ 1}) # Returns 2 right away
+# 2
+Belated::Client.test_mode_off! # Turn off inline job processing
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/bin/belated

@@ -28,9 +28,17 @@ OptionParser.new { |opts|
     Belated.config.env = env
   end
 
-  opts.on('-c=CONNECT', '--connect=CONNECT', 'Start dRuby connection, default true') do |connect|
+  opts.on('-c=CONNECT', '--connect=CONNECT', 'Start dRuby connection, default true, use for testing only') do |connect|
     Belated.config.connect = connect == 'true'
   end
+
+  opts.on('-h=HOST', '--host=HOST', 'dRuby host') do |host|
+    Belated.config.host = host
+  end
+
+  opts.on('-p=PORT', '--port=PORT', 'dRuby port') do |port|
+    Belated.config.port = port
+  end
 }.parse!
 
 instance = Belated.instance

data/lib/belated.rb

@@ -22,16 +22,18 @@ class Belated
   extend Dry::Configurable
   include Logging
   include Singleton unless $TESTING
-  URI = 'druby://localhost:8788'
   @@queue = Belated::Queue.new
 
   setting :rails, true
   setting :rails_path, '.'
   setting :workers, 1
   setting :connect, true
-  setting :environment, 'development'
+  setting :environment, 'development', reader: true
   setting :logger, Logger.new($stdout), reader: true
   setting :log_level, :info, reader: true
+  setting :host, 'localhost', reader: true
+  setting :port, '8788', reader: true
+  URI = "druby://#{Belated.host}:#{Belated.port}"
 
   # Since it's running as a singleton, we need something to start it up.
   # Aliased for testing purposes.
@@ -55,7 +57,7 @@ class Belated
   def connect!
     DRb.start_service(URI, @@queue, verbose: true)
   rescue DRb::DRbConnError, Errno::EADDRINUSE
-    Belated.logger.error 'Could not connect to DRb server.'
+    error 'Could not connect to DRb server.'
   end
 
   def trap_signals
@@ -96,12 +98,12 @@ class Belated
         sleep 5
         next
       end
-      if job.at <= Time.now.utc
+      if job.at <= Time.now
         log "Deleting #{@@queue.future_jobs.delete(job)} from future jobs"
         @@queue.push(job)
       end
     rescue DRb::DRbConnError
-      log 'DRb connection error!!!!!!'
+      error 'DRb connection error!!!!!!'
       log stats
     end
   end
@@ -173,8 +175,6 @@ class Belated
   def self.job_list
     @@queue
   end
-
-  class Error < StandardError; end
 end
 
 require 'belated/rails' if defined?(::Rails::Engine)

data/lib/belated/client.rb

@@ -1,4 +1,5 @@
 require 'belated/job_wrapper'
+require 'belated/exceptions'
 require 'singleton'
 class Belated
   # The client class is responsible for managing the connection to the
@@ -16,12 +17,15 @@ class Belated
     # Connects to the queue through DRb.
     # @return [void]
     def start
+      return if started?
+
       server_uri = Belated::URI
       DRb.start_service
       self.proc_table = {}
       self.bank = Thread::Queue.new
       self.queue = DRbObject.new_with_uri(server_uri)
       @started = true
+      @mutex = Mutex.new
     end
     alias initialize start
 
@@ -29,32 +33,41 @@ class Belated
       @started
     end
 
+    # Makes it possible to reset the client
+    def turn_off
+      @started = false
+      banker_thread&.kill
+    end
+
     # Thread in charge of handling the bank queue.
     # You probably want to memoize the client in order to avoid
     # having many threads in the sleep state.
     # @return [void]
     def start_banker_thread
-      self.banker_thread = Thread.new do
+      Thread.new do
         loop do
           delete_from_table
-          if bank.empty?
-            sleep 10
-            next
-          end
-          begin
-            queue.push(wrapper = bank.pop)
-          rescue DRb::DRbConnError
-            bank.push(wrapper)
+          sleep 10 and next if bank.empty?
+
+          until bank.empty?
+            begin
+              queue.push(wrapper = bank.pop)
+            rescue DRb::DRbConnError
+              bank.push(wrapper)
+              sleep 5
+            end
           end
         end
       end
     end
 
     def delete_from_table
-      return if proc_table.empty?
+      return if proc_table.length < 25
 
-      proc_table.select { |_k, v| v.completed }.each do |key, _value|
-        proc_table.delete(key)
+      @mutex.synchronize do
+        proc_table.select { |_k, v| v.completed }.each do |key, _value|
+          proc_table.delete(key)
+        end
       end
     end
 
@@ -65,16 +78,14 @@ class Belated
     # @param max_retries [Integer] - Times the job should be retried if it fails.
     # @return [JobWrapper] - The job wrapper for the queue.
     def perform(job, at: nil, max_retries: 5)
-      log 'Passing a proc and at time is deprecated and will be removed in 0.6' if job.instance_of?(Proc) && !at.nil?
-
-      job_wrapper = if job.is_a?(JobWrapper)
-                      job
-                    else
-                      JobWrapper.new(job: job, at: at, max_retries: max_retries)
-                    end
+      start unless started?
+      check_if_proper_job!(job)
+      job_wrapper = wrap_job(job, at: at, max_retries: max_retries)
       bank.push(job_wrapper)
-      proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
-      start_banker_thread if banker_thread.nil?
+      @mutex.synchronize do
+        proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
+      end
+      self.banker_thread = start_banker_thread if banker_thread.nil?
       job_wrapper
     end
     alias perform_belated perform
@@ -82,6 +93,18 @@ class Belated
 
     private
 
+    def check_if_proper_job!(job)
+      return if job.respond_to?(:call) || job.respond_to?(:perform)
+
+      raise JobError, 'job does not implement .call nor .perform!'
+    end
+
+    def wrap_job(job, at:, max_retries:)
+      return job if job.is_a?(JobWrapper)
+
+      JobWrapper.new(job: job, at: at, max_retries: max_retries)
+    end
+
     def drb_connected?
       queue.connected?
     rescue StandardError

data/lib/belated/exceptions.rb

@@ -0,0 +1,3 @@
+class Belated
+  class JobError < StandardError; end
+end

data/lib/belated/job_wrapper.rb

@@ -53,7 +53,7 @@ class Belated
       self.retries += 1
       return if retries > max_retries
 
-      self.at = Time.now.utc + (retries.next**4)
+      self.at = Time.now + (retries.next**4)
       log "Job #{id} failed, retrying at #{at}"
       Belated.job_list.push(self)
     end

data/lib/belated/logging.rb

@@ -12,6 +12,14 @@ class Belated
       logger.__send__(Belated.log_level, message)
     end
 
+    def warn(message)
+      logger.warn(message)
+    end
+
+    def error(message)
+      logger.error(message)
+    end
+
     def logger=(logger)
       @logger = logger
     end

data/lib/belated/testing.rb

@@ -0,0 +1,37 @@
+class Belated
+  # Testing helpers
+  # Enable or disable testing
+  class Testing
+    @@testing = false
+
+    def self.inline?
+      @@testing == true
+    end
+
+    def self.inline!
+      @@testing = true
+    end
+
+    def self.test_mode_off!
+      @@testing = false
+    end
+  end
+end
+
+class Belated
+  # A client that can perform jobs inline
+  class Client
+    alias old_perform perform
+    def perform(job, at: nil, max_retries: 5)
+      if Belated::Testing.inline?
+        if job.respond_to?(:call)
+          job.call
+        else
+          job.perform
+        end
+      else
+        old_perform(job, at: at, max_retries: max_retries)
+      end
+    end
+  end
+end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.6.1'
+  VERSION = '0.6.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.5
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-20 00:00:00.000000000 Z
+date: 2021-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -94,11 +94,13 @@ files:
 - bin/setup
 - lib/belated.rb
 - lib/belated/client.rb
+- lib/belated/exceptions.rb
 - lib/belated/job.rb
 - lib/belated/job_wrapper.rb
 - lib/belated/logging.rb
 - lib/belated/queue.rb
 - lib/belated/rails.rb
+- lib/belated/testing.rb
 - lib/belated/version.rb
 - lib/belated/worker.rb
 homepage: https://github.com/sampokuokkanen/belated