belated 0.6.3 → 0.6.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae5f827cc80ba5fba59161a50ae5797ef774be28feeb478ea641dda6bca79b3a
-  data.tar.gz: 01a61d9a44ae2b76ad27827e58dd8ba81e2040753925dc7172458c04495f010a
+  metadata.gz: e0c28df2d6dffdff23e48d5fba469ca7a67378bebdf6538067f748ebc28d8a67
+  data.tar.gz: 1805135d97859dd9d399b3e46d636141b014e102630eefa1e7d0a9dc4a2a19bc
 SHA512:
-  metadata.gz: 73a7fe76eacd1334186ef70628a78f193a20a72373d4faaa6bc9b8688ff0d9786156a69b44c3dfb9405837f0f5d980d1317277ad87c33d239d5d8c13b73207a4
-  data.tar.gz: bcc797637c63a712e81b77189ecd93c325a67fb67f91b392a86d8231b685394341447539c12012280fee392a52a71a6fb20713fdb0ae0be32201c2b2670f5a33
+  metadata.gz: '086a0dbc146fe3386a24bef59523aeca27dcb72506e83189f39553f63e20f867f6a094faab2ce07fdaadeb0f159bd740fde9bb8bc7f6bf8b2e4f04ea551f5e2e'
+  data.tar.gz: dea44b6d05c0eda6e5cb5d58c3b317c150d1d5b4399fcac55d80c8e044faecbfd1a21d6f96394950ab271e9db2eb346057c99de4cc02df4d25e8929f25487ff3

data/.github/workflows/main.yml

@@ -4,14 +4,19 @@ on: [push,pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
+        ruby: [2.6, 2.7, '3.0']
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 3.0.2
-        bundler-cache: true
-        cache-version: 1
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
     - name: Run the default task
       run: bundle exec rake

data/.rubocop.yml

@@ -1,9 +1,10 @@
 AllCops:
-  TargetRubyVersion: 2.7
+  TargetRubyVersion: 2.6
   NewCops: enable
   Exclude:
     - 'dummy/**/*'
     - 'vendor/**/*'
+    - '*gemspec'
 
 Lint/HashCompareByIdentity:
   Enabled: false
@@ -37,6 +38,9 @@ Style/BlockDelimiters:
 Style/ModuleFunction:
   EnforcedStyle: extend_self
 
+Metrics/AbcSize:
+  Max: 17
+
 Metrics/BlockLength:
   Exclude:
     - 'spec/**/*.rb'

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 ## [Unreleased]
 
+## [0.6.7] - 2021-08-25
+
+- A bug fix for bad jobs bringing down client side. 
+- Heartbeat option for server side. It only affects jobs in the wait list, it's used to determine how often to check the future jobs queue. 
+## [0.6.6] - 2021-08-25
+
+- Tests now run agains Ruby 2.6, so relaxing the version constraint. 
+- Add client_heartbeat option, so you can define how often you want jobs sent to the Belated server. 
+## [0.6.5] - 2021-08-23
+
+- Timezone used inside Belated is all using the server time now, so it's up to the user to take care of that(using `Time.now` instead of `Time.now.utc`)
+- Possible to configure host and port.
+- No need to call `.start` on the client anymore. 
+- Logging some error and warn messages now too, instead of it all being info
+
+## [0.6.4] - 2021-08-22
+- Inline jobs for testing!
+```ruby
+`belated/testing`
+Belated::Testing.inline!
+```
+- Very much inspired by how Sidekiq is doing this. 
+- Read more in the testing part of README.md
+
 ## [0.6.3] - 2021-08-21
 
 - Needed to have the hash inside the mutex when going over it; otherwise you still the get can't add key into hash during iteration error. Of course.

data/Gemfile

@@ -15,4 +15,4 @@ gem 'rspec-rails'
 gem 'rubocop', '~> 1.7'
 gem 'rubocop-discourse'
 gem 'sqlite3'
-gem 'stackprof'
+# gem 'stackprof'

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    belated (0.6.3)
+    belated (0.6.7)
       drb
       dry-configurable
+      ruby2_keywords
       sorted_set
 
 GEM
@@ -97,9 +98,15 @@ GEM
     marcel (1.0.1)
     method_source (1.0.0)
     mini_mime (1.1.0)
+    mini_portile2 (2.6.1)
     minitest (5.14.4)
     nio4r (2.5.7)
-    nokogiri (1.11.7-x86_64-linux)
+    nokogiri (1.12.3)
+      mini_portile2 (~> 2.6.1)
+      racc (~> 1.4)
+    nokogiri (1.12.3-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.12.3-x86_64-linux)
       racc (~> 1.4)
     parallel (1.20.1)
     parser (3.0.2.0)
@@ -178,6 +185,7 @@ GEM
       rubocop (~> 1.0)
       rubocop-ast (>= 1.1.0)
     ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
     set (1.0.1)
     sorted_set (1.0.3)
       rbtree
@@ -190,7 +198,6 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
-    stackprof (0.2.16)
     thor (1.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
@@ -201,6 +208,8 @@ GEM
     zeitwerk (2.4.2)
 
 PLATFORMS
+  ruby
+  x86_64-darwin-19
   x86_64-linux
 
 DEPENDENCIES
@@ -214,7 +223,6 @@ DEPENDENCIES
   rubocop (~> 1.7)
   rubocop-discourse
   sqlite3
-  stackprof
 
 BUNDLED WITH
    2.2.22

data/README.md

@@ -10,19 +10,13 @@ Note that Belated used to be called HardWorker. That name was already in use in
 
 It uses dRuby to do the communication! Which is absolute great. No need for Redis or PostgreSQL, just Ruby standard libraries.
 
-Note that currently the timezone is hardcoded to UTC. 
-
 Can be used with or without Rails. 
 
 Can be used if you're on a normal instance such as EC2 or Digital Ocean drop. Not if you're on a Heroku or Docker, or anything with ephemeral storage. 
 
 TODO LIST:
 
-- Improve thread safety. 
-- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. Or RocksDB? 
-- Make DRb port configurable.
-- Don't hardcode timezone to UTC.
-- Add some checks to the client for proper jobs.
+- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. Or RocksDB? Would need to be configurable if you don't have something installed.
 - Maybe support ActiveJob?
 - Have a web UI.
 - Have a job history
@@ -87,8 +81,6 @@ Then,
 ```ruby
 # Get the client
 client = Belated::Client.instance
-# Start the client, only need to do this once
-client.start unless client.started?
 ```
 
 and you can use the client!
@@ -138,8 +130,33 @@ Path to Rails project.
 
     $ bundle exec belated --workers=10
 
+Other available settings:
+
+    $ bundle exec belated --host=1.1.1.1 --port=1234 
+    # druby://1.1.1.1:1234
+    $ bundle exec belated --env=test
+    # environment
+    $ bundle exec belated --client_heartbeat=10
+    # how often client sends jobs to server, default is once every 5 seconds
+
+
 Number of workers.
 
+## Testing
+
+When testing, you can require `belated/testing` and then call `Belated::Testing.inline!` to make your jobs perform inline.
+
+```ruby
+`belated/testing`
+c = Belated::Client.instance
+c.perform(proc { 2/ 1}) # Tries to push the job to the drb backend
+# <Belated::JobWrapper:0x00005654bc2db1f0 @at=nil, @completed=false, @id="95e4dc6a-1876-4adf-ae0f-5ae902f5f024", @job=#<Proc:0x00005654bc2db330 (irb):3>, @max_retries=5, @proc_klass=true, @retries=0>
+Belated::Testing.inline! # Sidekiq-inspired, now jobs run inline
+c.perform(proc { 2/ 1}) # Returns 2 right away
+# 2
+Belated::Client.test_mode_off! # Turn off inline job processing
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/belated.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
   ).gsub(/\s+/, ' ').strip
   spec.homepage      = 'https://github.com/sampokuokkanen/belated'
   spec.license       = 'MIT'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.7.0')
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   # Uncomment to register a new dependency of your gem
   spec.add_dependency 'drb'
   spec.add_dependency 'dry-configurable'
+  spec.add_dependency 'ruby2_keywords'
   spec.add_dependency 'sorted_set'
   spec.add_development_dependency 'byebug'
 

data/bin/belated

@@ -28,9 +28,17 @@ OptionParser.new { |opts|
     Belated.config.env = env
   end
 
-  opts.on('-c=CONNECT', '--connect=CONNECT', 'Start dRuby connection, default true') do |connect|
+  opts.on('-c=CONNECT', '--connect=CONNECT', 'Start dRuby connection, default true, use for testing only') do |connect|
     Belated.config.connect = connect == 'true'
   end
+
+  opts.on('-h=HOST', '--host=HOST', 'dRuby host') do |host|
+    Belated.config.host = host
+  end
+
+  opts.on('-p=PORT', '--port=PORT', 'dRuby port') do |port|
+    Belated.config.port = port
+  end
 }.parse!
 
 instance = Belated.instance

data/lib/belated/client.rb

@@ -1,4 +1,5 @@
 require 'belated/job_wrapper'
+require 'belated/exceptions'
 require 'singleton'
 class Belated
   # The client class is responsible for managing the connection to the
@@ -16,6 +17,8 @@ class Belated
     # Connects to the queue through DRb.
     # @return [void]
     def start
+      return if started?
+
       server_uri = Belated::URI
       DRb.start_service
       self.proc_table = {}
@@ -30,32 +33,33 @@ class Belated
       @started
     end
 
+    # Makes it possible to reset the client
+    def turn_off
+      @started = false
+      banker_thread&.kill
+    end
+
     # Thread in charge of handling the bank queue.
     # You probably want to memoize the client in order to avoid
     # having many threads in the sleep state.
     # @return [void]
     def start_banker_thread
-      self.banker_thread = Thread.new do
+      Thread.new do
         loop do
-          delete_from_table if proc_table.length > 20
-          if bank.empty?
-            sleep 10
-            next
-          end
-          until bank.empty?
-            begin
-              queue.push(wrapper = bank.pop)
-            rescue DRb::DRbConnError
-              bank.push(wrapper)
-              sleep 5
-            end
+          delete_from_table
+          sleep Belated.client_heartbeat and next if bank.empty?
+
+          bank.length.times do
+            queue.push(wrapper = bank.pop)
+          rescue DRb::DRbConnError
+            bank.push(wrapper)
           end
         end
       end
     end
 
     def delete_from_table
-      return if proc_table.empty?
+      return if proc_table.length < 25
 
       @mutex.synchronize do
         proc_table.select { |_k, v| v.completed }.each do |key, _value|
@@ -71,12 +75,15 @@ class Belated
     # @param max_retries [Integer] - Times the job should be retried if it fails.
     # @return [JobWrapper] - The job wrapper for the queue.
     def perform(job, at: nil, max_retries: 5)
+      start unless started?
+      return unless proper_job?(job)
+
       job_wrapper = wrap_job(job, at: at, max_retries: max_retries)
       bank.push(job_wrapper)
       @mutex.synchronize do
         proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
       end
-      start_banker_thread if banker_thread.nil?
+      self.banker_thread = start_banker_thread if banker_thread.nil?
       job_wrapper
     end
     alias perform_belated perform
@@ -84,6 +91,13 @@ class Belated
 
     private
 
+    def proper_job?(job)
+      return true if job.respond_to?(:call) || job.respond_to?(:perform)
+
+      warn 'job does not implement .call nor .perform!'
+      false
+    end
+
     def wrap_job(job, at:, max_retries:)
       return job if job.is_a?(JobWrapper)
 

data/lib/belated/exceptions.rb

@@ -0,0 +1,3 @@
+class Belated
+  class JobError < StandardError; end
+end

data/lib/belated/job_wrapper.rb

@@ -53,7 +53,7 @@ class Belated
       self.retries += 1
       return if retries > max_retries
 
-      self.at = Time.now.utc + (retries.next**4)
+      self.at = Time.now + (retries.next**4)
       log "Job #{id} failed, retrying at #{at}"
       Belated.job_list.push(self)
     end

data/lib/belated/logging.rb

@@ -12,6 +12,14 @@ class Belated
       logger.__send__(Belated.log_level, message)
     end
 
+    def warn(message)
+      logger.warn(message)
+    end
+
+    def error(message)
+      logger.error(message)
+    end
+
     def logger=(logger)
       @logger = logger
     end

data/lib/belated/queue.rb

@@ -19,6 +19,7 @@ class Belated
 
     def initialize(queue: Thread::Queue.new, future_jobs: SortedSet.new)
       @queue = queue
+      @mutex = Mutex.new
       self.future_jobs = future_jobs
     end
 
@@ -27,7 +28,9 @@ class Belated
          job.at <= Time.now.utc
         @queue.push(job)
       else
-        @future_jobs << job
+        @mutex.synchronize do
+          @future_jobs << job
+        end
       end
     end
 

data/lib/belated/testing.rb

@@ -0,0 +1,37 @@
+class Belated
+  # Testing helpers
+  # Enable or disable testing
+  class Testing
+    @@testing = false
+
+    def self.inline?
+      @@testing == true
+    end
+
+    def self.inline!
+      @@testing = true
+    end
+
+    def self.test_mode_off!
+      @@testing = false
+    end
+  end
+end
+
+class Belated
+  # A client that can perform jobs inline
+  class Client
+    alias old_perform perform
+    def perform(job, at: nil, max_retries: 5)
+      if Belated::Testing.inline?
+        if job.respond_to?(:call)
+          job.call
+        else
+          job.perform
+        end
+      else
+        old_perform(job, at: at, max_retries: max_retries)
+      end
+    end
+  end
+end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.6.3'
+  VERSION = '0.6.7'
 end

data/lib/belated.rb

@@ -22,16 +22,20 @@ class Belated
   extend Dry::Configurable
   include Logging
   include Singleton unless $TESTING
-  URI = 'druby://localhost:8788'
   @@queue = Belated::Queue.new
 
   setting :rails, true
   setting :rails_path, '.'
   setting :workers, 1
   setting :connect, true
-  setting :environment, 'development'
+  setting :environment, 'development', reader: true
   setting :logger, Logger.new($stdout), reader: true
   setting :log_level, :info, reader: true
+  setting :host, 'localhost', reader: true
+  setting :port, '8788', reader: true
+  setting :heartbeat, 1, reader: true
+  setting :client_heartbeat, 5, reader: true
+  URI = "druby://#{Belated.host}:#{Belated.port}"
 
   # Since it's running as a singleton, we need something to start it up.
   # Aliased for testing purposes.
@@ -53,9 +57,11 @@ class Belated
 
   # Handles connection to DRb server.
   def connect!
+    i = 0
     DRb.start_service(URI, @@queue, verbose: true)
   rescue DRb::DRbConnError, Errno::EADDRINUSE
-    Belated.logger.error 'Could not connect to DRb server.'
+    sleep 0.1 and retry if (i += 1) < 5
+    error 'Could not connect to DRb server.'
   end
 
   def trap_signals
@@ -90,18 +96,17 @@ class Belated
 
   def enqueue_future_jobs
     loop do
-      sleep 0.1
       job = @@queue.future_jobs.min
       if job.nil?
-        sleep 5
+        sleep Belated.heartbeat
         next
       end
-      if job.at <= Time.now.utc
+      if job.at <= Time.now
         log "Deleting #{@@queue.future_jobs.delete(job)} from future jobs"
         @@queue.push(job)
       end
     rescue DRb::DRbConnError
-      log 'DRb connection error!!!!!!'
+      error 'DRb connection error!!!!!!'
       log stats
     end
   end
@@ -173,8 +178,6 @@ class Belated
   def self.job_list
     @@queue
   end
-
-  class Error < StandardError; end
 end
 
 require 'belated/rails' if defined?(::Rails::Engine)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.7
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-21 00:00:00.000000000 Z
+date: 2021-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby2_keywords
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sorted_set
   requirement: !ruby/object:Gem::Requirement
@@ -94,11 +108,13 @@ files:
 - bin/setup
 - lib/belated.rb
 - lib/belated/client.rb
+- lib/belated/exceptions.rb
 - lib/belated/job.rb
 - lib/belated/job_wrapper.rb
 - lib/belated/logging.rb
 - lib/belated/queue.rb
 - lib/belated/rails.rb
+- lib/belated/testing.rb
 - lib/belated/version.rb
 - lib/belated/worker.rb
 homepage: https://github.com/sampokuokkanen/belated
@@ -116,7 +132,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="