belated 0.6.2 → 0.6.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 422e1b6b1f3973315fb01e1e47b7a976d0ac1a62897c0d32880fd21a7dec8c92
-  data.tar.gz: 3daed6a19c67909b50bb93c8b39ba729b43d33dbbd3a6966ed425249c6eec89f
+  metadata.gz: 628f04309c824751bd90f72163d29738f5600f7ed5b759b0106d09b9eccbf23e
+  data.tar.gz: 1318b353fb2047ae6e185f0af720a987a3d8c784a483df04a79601fb595ff28f
 SHA512:
-  metadata.gz: 489ee717aaf5a1cbdf26f134e4341894dd088d4ba067f9b0f16c9c64a8f4692252e98b4dfa21d0e45fe45ba8c8cbfda77118483758bc0649b031bf8d9049c785
-  data.tar.gz: 811c5bc89bdbafaee6869a8f267e0c748ef5ca98ccc10edd65780e1a548205471ad881c879e12dc8e6162331e7065d8afb4bd765c96c694de8748600854f6feb
+  metadata.gz: de9793daf6f3987be47b798916dbdfa16e7be92211f46395ed4d63eabf7ae127420952c047df749de584404cd18e80a002f5fadb40896421583e80f3ebe9bf2a
+  data.tar.gz: 3a09388e73a833d3230787f6191cd02a86cc4ebf3e58ba950d8631eba31a67fe4cf319bfaa448bc471b5fe1a933d9564c7d846d3416d91832a7b5331c78e62b9

data/.github/workflows/main.yml

@@ -4,14 +4,19 @@ on: [push,pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
+        ruby: [2.6, 2.7, '3.0']
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 3.0.2
-        bundler-cache: true
-        cache-version: 1
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
     - name: Run the default task
       run: bundle exec rake

data/.rubocop.yml

@@ -1,9 +1,10 @@
 AllCops:
-  TargetRubyVersion: 2.7
+  TargetRubyVersion: 2.6
   NewCops: enable
   Exclude:
     - 'dummy/**/*'
     - 'vendor/**/*'
+    - '*gemspec'
 
 Lint/HashCompareByIdentity:
   Enabled: false
@@ -37,6 +38,9 @@ Style/BlockDelimiters:
 Style/ModuleFunction:
   EnforcedStyle: extend_self
 
+Metrics/AbcSize:
+  Max: 17
+
 Metrics/BlockLength:
   Exclude:
     - 'spec/**/*.rb'

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 ## [Unreleased]
 
+## [0.6.6] - 2021-08-25
+
+- Tests now run agains Ruby 2.6, so relaxing the version constraint. 
+- Add client_heartbeat option, so you can define how often you want jobs sent to the Belated server. 
+## [0.6.5] - 2021-08-23
+
+- Timezone used inside Belated is all using the server time now, so it's up to the user to take care of that(using `Time.now` instead of `Time.now.utc`)
+- Possible to configure host and port.
+- No need to call `.start` on the client anymore. 
+- Logging some error and warn messages now too, instead of it all being info
+
+## [0.6.4] - 2021-08-22
+- Inline jobs for testing!
+```ruby
+`belated/testing`
+Belated::Testing.inline!
+```
+- Very much inspired by how Sidekiq is doing this. 
+- Read more in the testing part of README.md
+
+## [0.6.3] - 2021-08-21
+
+- Needed to have the hash inside the mutex when going over it; otherwise you still the get can't add key into hash during iteration error. Of course.
+
 ## [0.6.2] - 2021-08-20
 
 - Use a mutex for the proc_table used by the client. Fixes

data/Gemfile

@@ -15,4 +15,4 @@ gem 'rspec-rails'
 gem 'rubocop', '~> 1.7'
 gem 'rubocop-discourse'
 gem 'sqlite3'
-gem 'stackprof'
+# gem 'stackprof'

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    belated (0.6.2)
+    belated (0.6.6)
       drb
       dry-configurable
+      ruby2_keywords
       sorted_set
 
 GEM
@@ -97,9 +98,15 @@ GEM
     marcel (1.0.1)
     method_source (1.0.0)
     mini_mime (1.1.0)
+    mini_portile2 (2.6.1)
     minitest (5.14.4)
     nio4r (2.5.7)
-    nokogiri (1.11.7-x86_64-linux)
+    nokogiri (1.12.3)
+      mini_portile2 (~> 2.6.1)
+      racc (~> 1.4)
+    nokogiri (1.12.3-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.12.3-x86_64-linux)
       racc (~> 1.4)
     parallel (1.20.1)
     parser (3.0.2.0)
@@ -178,6 +185,7 @@ GEM
       rubocop (~> 1.0)
       rubocop-ast (>= 1.1.0)
     ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
     set (1.0.1)
     sorted_set (1.0.3)
       rbtree
@@ -190,7 +198,6 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
-    stackprof (0.2.16)
     thor (1.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
@@ -201,6 +208,8 @@ GEM
     zeitwerk (2.4.2)
 
 PLATFORMS
+  ruby
+  x86_64-darwin-19
   x86_64-linux
 
 DEPENDENCIES
@@ -214,7 +223,6 @@ DEPENDENCIES
   rubocop (~> 1.7)
   rubocop-discourse
   sqlite3
-  stackprof
 
 BUNDLED WITH
    2.2.22

data/README.md

@@ -10,19 +10,13 @@ Note that Belated used to be called HardWorker. That name was already in use in
 
 It uses dRuby to do the communication! Which is absolute great. No need for Redis or PostgreSQL, just Ruby standard libraries.
 
-Note that currently the timezone is hardcoded to UTC. 
-
 Can be used with or without Rails. 
 
 Can be used if you're on a normal instance such as EC2 or Digital Ocean drop. Not if you're on a Heroku or Docker, or anything with ephemeral storage. 
 
 TODO LIST:
 
-- Improve thread safety. 
-- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. Or RocksDB? 
-- Make DRb port configurable.
-- Don't hardcode timezone to UTC.
-- Add some checks to the client for proper jobs.
+- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. Or RocksDB? Would need to be configurable if you don't have something installed.
 - Maybe support ActiveJob?
 - Have a web UI.
 - Have a job history
@@ -87,8 +81,6 @@ Then,
 ```ruby
 # Get the client
 client = Belated::Client.instance
-# Start the client, only need to do this once
-client.start unless client.started?
 ```
 
 and you can use the client!
@@ -138,8 +130,33 @@ Path to Rails project.
 
     $ bundle exec belated --workers=10
 
+Other available settings:
+
+    $ bundle exec belated --host=1.1.1.1 --port=1234 
+    # druby://1.1.1.1:1234
+    $ bundle exec belated --env=test
+    # environment
+    $ bundle exec belated --client_heartbeat=10
+    # how often client sends jobs to server, default is once every 5 seconds
+
+
 Number of workers.
 
+## Testing
+
+When testing, you can require `belated/testing` and then call `Belated::Testing.inline!` to make your jobs perform inline.
+
+```ruby
+`belated/testing`
+c = Belated::Client.instance
+c.perform(proc { 2/ 1}) # Tries to push the job to the drb backend
+# <Belated::JobWrapper:0x00005654bc2db1f0 @at=nil, @completed=false, @id="95e4dc6a-1876-4adf-ae0f-5ae902f5f024", @job=#<Proc:0x00005654bc2db330 (irb):3>, @max_retries=5, @proc_klass=true, @retries=0>
+Belated::Testing.inline! # Sidekiq-inspired, now jobs run inline
+c.perform(proc { 2/ 1}) # Returns 2 right away
+# 2
+Belated::Client.test_mode_off! # Turn off inline job processing
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/belated.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
   ).gsub(/\s+/, ' ').strip
   spec.homepage      = 'https://github.com/sampokuokkanen/belated'
   spec.license       = 'MIT'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.7.0')
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   # Uncomment to register a new dependency of your gem
   spec.add_dependency 'drb'
   spec.add_dependency 'dry-configurable'
+  spec.add_dependency 'ruby2_keywords'
   spec.add_dependency 'sorted_set'
   spec.add_development_dependency 'byebug'
 

data/bin/belated

@@ -28,9 +28,17 @@ OptionParser.new { |opts|
     Belated.config.env = env
   end
 
-  opts.on('-c=CONNECT', '--connect=CONNECT', 'Start dRuby connection, default true') do |connect|
+  opts.on('-c=CONNECT', '--connect=CONNECT', 'Start dRuby connection, default true, use for testing only') do |connect|
     Belated.config.connect = connect == 'true'
   end
+
+  opts.on('-h=HOST', '--host=HOST', 'dRuby host') do |host|
+    Belated.config.host = host
+  end
+
+  opts.on('-p=PORT', '--port=PORT', 'dRuby port') do |port|
+    Belated.config.port = port
+  end
 }.parse!
 
 instance = Belated.instance

data/lib/belated/client.rb

@@ -1,4 +1,5 @@
 require 'belated/job_wrapper'
+require 'belated/exceptions'
 require 'singleton'
 class Belated
   # The client class is responsible for managing the connection to the
@@ -16,6 +17,8 @@ class Belated
     # Connects to the queue through DRb.
     # @return [void]
     def start
+      return if started?
+
       server_uri = Belated::URI
       DRb.start_service
       self.proc_table = {}
@@ -30,19 +33,23 @@ class Belated
       @started
     end
 
+    # Makes it possible to reset the client
+    def turn_off
+      @started = false
+      banker_thread&.kill
+    end
+
     # Thread in charge of handling the bank queue.
     # You probably want to memoize the client in order to avoid
     # having many threads in the sleep state.
     # @return [void]
     def start_banker_thread
-      self.banker_thread = Thread.new do
+      Thread.new do
         loop do
           delete_from_table
-          if bank.empty?
-            sleep 10
-            next
-          end
-          begin
+          sleep Belated.client_heartbeat and next if bank.empty?
+
+          bank.length.times do
             queue.push(wrapper = bank.pop)
           rescue DRb::DRbConnError
             bank.push(wrapper)
@@ -52,10 +59,10 @@ class Belated
     end
 
     def delete_from_table
-      return if proc_table.empty?
+      return if proc_table.length < 25
 
-      proc_table.select { |_k, v| v.completed }.each do |key, _value|
-        @mutex.synchronize do
+      @mutex.synchronize do
+        proc_table.select { |_k, v| v.completed }.each do |key, _value|
           proc_table.delete(key)
         end
       end
@@ -68,12 +75,14 @@ class Belated
     # @param max_retries [Integer] - Times the job should be retried if it fails.
     # @return [JobWrapper] - The job wrapper for the queue.
     def perform(job, at: nil, max_retries: 5)
+      start unless started?
+      check_if_proper_job!(job)
       job_wrapper = wrap_job(job, at: at, max_retries: max_retries)
       bank.push(job_wrapper)
       @mutex.synchronize do
         proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
       end
-      start_banker_thread if banker_thread.nil?
+      self.banker_thread = start_banker_thread if banker_thread.nil?
       job_wrapper
     end
     alias perform_belated perform
@@ -81,6 +90,12 @@ class Belated
 
     private
 
+    def check_if_proper_job!(job)
+      return if job.respond_to?(:call) || job.respond_to?(:perform)
+
+      raise JobError, 'job does not implement .call nor .perform!'
+    end
+
     def wrap_job(job, at:, max_retries:)
       return job if job.is_a?(JobWrapper)
 

data/lib/belated/exceptions.rb

@@ -0,0 +1,3 @@
+class Belated
+  class JobError < StandardError; end
+end

data/lib/belated/job_wrapper.rb

@@ -53,7 +53,7 @@ class Belated
       self.retries += 1
       return if retries > max_retries
 
-      self.at = Time.now.utc + (retries.next**4)
+      self.at = Time.now + (retries.next**4)
       log "Job #{id} failed, retrying at #{at}"
       Belated.job_list.push(self)
     end

data/lib/belated/logging.rb

@@ -12,6 +12,14 @@ class Belated
       logger.__send__(Belated.log_level, message)
     end
 
+    def warn(message)
+      logger.warn(message)
+    end
+
+    def error(message)
+      logger.error(message)
+    end
+
     def logger=(logger)
       @logger = logger
     end

data/lib/belated/testing.rb

@@ -0,0 +1,37 @@
+class Belated
+  # Testing helpers
+  # Enable or disable testing
+  class Testing
+    @@testing = false
+
+    def self.inline?
+      @@testing == true
+    end
+
+    def self.inline!
+      @@testing = true
+    end
+
+    def self.test_mode_off!
+      @@testing = false
+    end
+  end
+end
+
+class Belated
+  # A client that can perform jobs inline
+  class Client
+    alias old_perform perform
+    def perform(job, at: nil, max_retries: 5)
+      if Belated::Testing.inline?
+        if job.respond_to?(:call)
+          job.call
+        else
+          job.perform
+        end
+      else
+        old_perform(job, at: at, max_retries: max_retries)
+      end
+    end
+  end
+end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.6.2'
+  VERSION = '0.6.6'
 end

data/lib/belated.rb

@@ -22,16 +22,19 @@ class Belated
   extend Dry::Configurable
   include Logging
   include Singleton unless $TESTING
-  URI = 'druby://localhost:8788'
   @@queue = Belated::Queue.new
 
   setting :rails, true
   setting :rails_path, '.'
   setting :workers, 1
   setting :connect, true
-  setting :environment, 'development'
+  setting :environment, 'development', reader: true
   setting :logger, Logger.new($stdout), reader: true
   setting :log_level, :info, reader: true
+  setting :host, 'localhost', reader: true
+  setting :port, '8788', reader: true
+  setting :client_heartbeat, 5, reader: true
+  URI = "druby://#{Belated.host}:#{Belated.port}"
 
   # Since it's running as a singleton, we need something to start it up.
   # Aliased for testing purposes.
@@ -55,7 +58,7 @@ class Belated
   def connect!
     DRb.start_service(URI, @@queue, verbose: true)
   rescue DRb::DRbConnError, Errno::EADDRINUSE
-    Belated.logger.error 'Could not connect to DRb server.'
+    error 'Could not connect to DRb server.'
   end
 
   def trap_signals
@@ -96,12 +99,12 @@ class Belated
         sleep 5
         next
       end
-      if job.at <= Time.now.utc
+      if job.at <= Time.now
         log "Deleting #{@@queue.future_jobs.delete(job)} from future jobs"
         @@queue.push(job)
       end
     rescue DRb::DRbConnError
-      log 'DRb connection error!!!!!!'
+      error 'DRb connection error!!!!!!'
       log stats
     end
   end
@@ -173,8 +176,6 @@ class Belated
   def self.job_list
     @@queue
   end
-
-  class Error < StandardError; end
 end
 
 require 'belated/rails' if defined?(::Rails::Engine)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.6
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-20 00:00:00.000000000 Z
+date: 2021-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby2_keywords
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sorted_set
   requirement: !ruby/object:Gem::Requirement
@@ -94,11 +108,13 @@ files:
 - bin/setup
 - lib/belated.rb
 - lib/belated/client.rb
+- lib/belated/exceptions.rb
 - lib/belated/job.rb
 - lib/belated/job_wrapper.rb
 - lib/belated/logging.rb
 - lib/belated/queue.rb
 - lib/belated/rails.rb
+- lib/belated/testing.rb
 - lib/belated/version.rb
 - lib/belated/worker.rb
 homepage: https://github.com/sampokuokkanen/belated
@@ -116,7 +132,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="