belated 0.6.4 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 535f6d7f0af3565867c6f621fb328966d35ff524e38ffc22813e4410fd66eb2f
-  data.tar.gz: ace8cc58b9519d07e7bce2f12c5b1e45eba570f75a0a5f34b0496a5acfac679d
+  metadata.gz: 625adf33d85f1e94d4d7416db02a50e11fe66bfeb5fa265fa4de2c9b6f1c0750
+  data.tar.gz: 663fa5c0fcb7903b66324723a8cd19a771d596a88a396974d80060bf8c0bfd4a
 SHA512:
-  metadata.gz: '058ed5f47971a9f39d7ec39997f23bdfeb54221b516ee9162330e9a0c783dc9ec6d7633ad029b7d96792e08bd317d5efced10ca2ea97b0e8322507a6e5f9a045'
-  data.tar.gz: c5ef2e3ad64ecda5e1fa2cb0feab2dda1b50388a523e571fe3be829af76e864973429bb95d0433353dd5a58bf97f2f366d19bd19a7ae0d6d578030646156f542
+  metadata.gz: e14413f88843ed9c80391987c5891f821ce0eb43e2759d5be034bb17a9cda78f4de7a67b97f44af933013dd4aef39e0d23d91df9ecfbbf3dbead38427a925450
+  data.tar.gz: aef2baada5a40a7da090e71e1c490d6f901e5178bd5be05001fb53c2863e9b03b86f9b997a53ced594d40aad8eb8700e25419b1378eba13b444b3b62f48d1b65

data/.github/workflows/main.yml

@@ -4,14 +4,19 @@ on: [push,pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
+        ruby: [2.7, '3.0']
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 3.0.2
-        bundler-cache: true
-        cache-version: 1
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
     - name: Run the default task
       run: bundle exec rake

data/.rubocop.yml

@@ -1,9 +1,10 @@
 AllCops:
-  TargetRubyVersion: 2.7
+  TargetRubyVersion: 2.6
   NewCops: enable
   Exclude:
     - 'dummy/**/*'
     - 'vendor/**/*'
+    - '*gemspec'
 
 Lint/HashCompareByIdentity:
   Enabled: false

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 ## [Unreleased]
 
+## [0.7] - 2021-09-04
+- ActiveJob support! Retries, exception rescuing should work as expected. 
+- Second Moderna jab took me out for a while... sorry for the long wait. 
+## [0.6.7] - 2021-08-25
+
+- A bug fix for bad jobs bringing down client side. 
+- Heartbeat option for server side. It only affects jobs in the wait list, it's used to determine how often to check the future jobs queue. 
+## [0.6.6] - 2021-08-25
+
+- Tests now run agains Ruby 2.6, so relaxing the version constraint. 
+- Add client_heartbeat option, so you can define how often you want jobs sent to the Belated server. 
+## [0.6.5] - 2021-08-23
+
+- Timezone used inside Belated is all using the server time now, so it's up to the user to take care of that(using `Time.now` instead of `Time.now.utc`)
+- Possible to configure host and port.
+- No need to call `.start` on the client anymore. 
+- Logging some error and warn messages now too, instead of it all being info
+
 ## [0.6.4] - 2021-08-22
 - Inline jobs for testing!
 ```ruby

data/Gemfile

@@ -15,4 +15,4 @@ gem 'rspec-rails'
 gem 'rubocop', '~> 1.7'
 gem 'rubocop-discourse'
 gem 'sqlite3'
-gem 'stackprof'
+# gem 'stackprof'

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    belated (0.6.4)
+    belated (0.7.0)
       drb
       dry-configurable
+      ruby2_keywords
       sorted_set
 
 GEM
@@ -97,9 +98,15 @@ GEM
     marcel (1.0.1)
     method_source (1.0.0)
     mini_mime (1.1.0)
+    mini_portile2 (2.6.1)
     minitest (5.14.4)
     nio4r (2.5.7)
-    nokogiri (1.11.7-x86_64-linux)
+    nokogiri (1.12.3)
+      mini_portile2 (~> 2.6.1)
+      racc (~> 1.4)
+    nokogiri (1.12.3-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.12.3-x86_64-linux)
       racc (~> 1.4)
     parallel (1.20.1)
     parser (3.0.2.0)
@@ -178,6 +185,7 @@ GEM
       rubocop (~> 1.0)
       rubocop-ast (>= 1.1.0)
     ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
     set (1.0.1)
     sorted_set (1.0.3)
       rbtree
@@ -190,7 +198,6 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
-    stackprof (0.2.16)
     thor (1.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
@@ -201,6 +208,8 @@ GEM
     zeitwerk (2.4.2)
 
 PLATFORMS
+  ruby
+  x86_64-darwin-19
   x86_64-linux
 
 DEPENDENCIES
@@ -214,7 +223,6 @@ DEPENDENCIES
   rubocop (~> 1.7)
   rubocop-discourse
   sqlite3
-  stackprof
 
 BUNDLED WITH
    2.2.22

data/README.md

@@ -10,8 +10,6 @@ Note that Belated used to be called HardWorker. That name was already in use in
 
 It uses dRuby to do the communication! Which is absolute great. No need for Redis or PostgreSQL, just Ruby standard libraries.
 
-Note that currently the timezone is hardcoded to UTC. 
-
 Can be used with or without Rails. 
 
 Can be used if you're on a normal instance such as EC2 or Digital Ocean drop. Not if you're on a Heroku or Docker, or anything with ephemeral storage. 
@@ -19,10 +17,6 @@ Can be used if you're on a normal instance such as EC2 or Digital Ocean drop. No
 TODO LIST:
 
 - Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup. Or RocksDB? Would need to be configurable if you don't have something installed.
-- Make DRb port configurable.
-- Don't hardcode timezone to UTC.
-- Add some checks to the client for proper jobs.
-- Maybe support ActiveJob?
 - Have a web UI.
 - Have a job history
 - Do some performance testing.
@@ -53,7 +47,15 @@ Start up Belated!
 
     $ belated
 
-Then, in another program, connect to Belated and give it a job to do.
+If you're using Rails, just set Belated to be the ActiveJob adapter like below:
+
+```ruby
+config.active_job.adapter = :belated
+```
+
+And you're good to go!
+
+If not, in your non-ActiveJob using program, connect to Belated and give it a job to do.
 Sample below:
 
 ```ruby
@@ -86,8 +88,6 @@ Then,
 ```ruby
 # Get the client
 client = Belated::Client.instance
-# Start the client, only need to do this once
-client.start unless client.started?
 ```
 
 and you can use the client!
@@ -137,6 +137,16 @@ Path to Rails project.
 
     $ bundle exec belated --workers=10
 
+Other available settings:
+
+    $ bundle exec belated --host=1.1.1.1 --port=1234 
+    # druby://1.1.1.1:1234
+    $ bundle exec belated --env=test
+    # environment
+    $ bundle exec belated --client_heartbeat=10
+    # how often client sends jobs to server, default is once every 5 seconds
+
+
 Number of workers.
 
 ## Testing
@@ -146,7 +156,6 @@ When testing, you can require `belated/testing` and then call `Belated::Testing.
 ```ruby
 `belated/testing`
 c = Belated::Client.instance
-c.start
 c.perform(proc { 2/ 1}) # Tries to push the job to the drb backend
 # <Belated::JobWrapper:0x00005654bc2db1f0 @at=nil, @completed=false, @id="95e4dc6a-1876-4adf-ae0f-5ae902f5f024", @job=#<Proc:0x00005654bc2db330 (irb):3>, @max_retries=5, @proc_klass=true, @retries=0>
 Belated::Testing.inline! # Sidekiq-inspired, now jobs run inline

data/belated.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
   ).gsub(/\s+/, ' ').strip
   spec.homepage      = 'https://github.com/sampokuokkanen/belated'
   spec.license       = 'MIT'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.7.0')
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   # Uncomment to register a new dependency of your gem
   spec.add_dependency 'drb'
   spec.add_dependency 'dry-configurable'
+  spec.add_dependency 'ruby2_keywords'
   spec.add_dependency 'sorted_set'
   spec.add_development_dependency 'byebug'
 

data/bin/belated

@@ -28,9 +28,17 @@ OptionParser.new { |opts|
     Belated.config.env = env
   end
 
-  opts.on('-c=CONNECT', '--connect=CONNECT', 'Start dRuby connection, default true') do |connect|
+  opts.on('-c=CONNECT', '--connect=CONNECT', 'Start dRuby connection, default true, use for testing only') do |connect|
     Belated.config.connect = connect == 'true'
   end
+
+  opts.on('-h=HOST', '--host=HOST', 'dRuby host') do |host|
+    Belated.config.host = host
+  end
+
+  opts.on('-p=PORT', '--port=PORT', 'dRuby port') do |port|
+    Belated.config.port = port
+  end
 }.parse!
 
 instance = Belated.instance

data/lib/active_job/queue_adapters/belated_adapter.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'belated'
+
+module ActiveJob # :nodoc:
+  module QueueAdapters # :nodoc:
+    # The adapter in charge of handling ActiveJob integration.
+    # WIP
+    class BelatedAdapter
+      def instance
+        @instance ||= Belated::Client.instance
+      rescue StandardError
+        @instance = Belated::Client.new
+      end
+
+      def enqueue(job) # :nodoc:
+        Rails.logger.info "Belated got job #{job}"
+        instance.perform(job, active_job: true)
+      end
+
+      def enqueue_at(job, timestamp) # :nodoc:
+        Rails.logger.info "Belated got job #{job} to be performed at #{Time.at(timestamp)}"
+        instance.perform_belated(job, at: timestamp, active_job: true)
+      end
+    end
+  end
+end

data/lib/belated/client.rb

@@ -1,4 +1,5 @@
 require 'belated/job_wrapper'
+require 'belated/exceptions'
 require 'singleton'
 class Belated
   # The client class is responsible for managing the connection to the
@@ -46,15 +47,12 @@ class Belated
       Thread.new do
         loop do
           delete_from_table
-          sleep 10 and next if bank.empty?
-
-          until bank.empty?
-            begin
-              queue.push(wrapper = bank.pop)
-            rescue DRb::DRbConnError
-              bank.push(wrapper)
-              sleep 5
-            end
+          sleep Belated.client_heartbeat and next if bank.empty?
+
+          bank.length.times do
+            queue.push(wrapper = bank.pop)
+          rescue DRb::DRbConnError
+            bank.push(wrapper)
           end
         end
       end
@@ -76,10 +74,11 @@ class Belated
     # @param at [Date] - The time at which the job should be executed.
     # @param max_retries [Integer] - Times the job should be retried if it fails.
     # @return [JobWrapper] - The job wrapper for the queue.
-    def perform(job, at: nil, max_retries: 5)
-      log 'Call .start on the client instance first!' unless started?
+    def perform(job, at: nil, max_retries: 5, active_job: false)
+      start unless started?
+      return unless proper_job?(job)
 
-      job_wrapper = wrap_job(job, at: at, max_retries: max_retries)
+      job_wrapper = wrap_job(job, at: at.to_f, max_retries: max_retries, active_job: active_job)
       bank.push(job_wrapper)
       @mutex.synchronize do
         proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
@@ -92,10 +91,17 @@ class Belated
 
     private
 
-    def wrap_job(job, at:, max_retries:)
+    def proper_job?(job)
+      return true if job.respond_to?(:call) || job.respond_to?(:perform)
+
+      warn 'job does not implement .call nor .perform!'
+      false
+    end
+
+    def wrap_job(job, at:, max_retries:, active_job:)
       return job if job.is_a?(JobWrapper)
 
-      JobWrapper.new(job: job, at: at, max_retries: max_retries)
+      JobWrapper.new(job: job, at: at, max_retries: max_retries, active_job: active_job)
     end
 
     def drb_connected?

data/lib/belated/exceptions.rb

@@ -0,0 +1,3 @@
+class Belated
+  class JobError < StandardError; end
+end

data/lib/belated/job_wrapper.rb

@@ -13,16 +13,17 @@ class Belated
   class JobWrapper
     include Comparable
     include Logging
-    attr_accessor :retries, :max_retries, :id, :job, :at, :completed, :proc_klass
+    attr_accessor :retries, :max_retries, :id, :job, :at, :completed, :proc_klass, :error, :active_job
 
-    def initialize(job:, max_retries: 5, at: nil)
+    def initialize(job:, max_retries: 5, at: nil, active_job: false)
       self.retries = 0
       self.max_retries = max_retries
-      self.id = SecureRandom.uuid
+      self.id = job.respond_to?(:job_id) ? job.job_id : SecureRandom.uuid
       self.job = job
       self.at = at
       self.completed = false
       self.proc_klass = job.instance_of?(Proc)
+      self.active_job = active_job
     end
 
     def <=>(other)
@@ -31,11 +32,7 @@ class Belated
 
     # rubocop:disable Lint/RescueException
     def perform
-      resp = if job.respond_to?(:call)
-               job.call
-             else
-               job.perform
-             end
+      resp = execute
       self.completed = true
       resp
     rescue Exception => e
@@ -43,17 +40,33 @@ class Belated
       when Interrupt, SignalException
         raise e
       else
-        retry_job
-        "Error while executing job, #{e.inspect}. Retry #{retries} of #{max_retries}"
+        retry_job(e)
+        "Error while executing job #{job.inspect}, #{e.inspect}. Retry #{retries} of #{max_retries}"
       end
     end
+
     # rubocop:enable Lint/RescueException
+    def execute
+      if active_job
+        ActiveJob::Base.execute job.serialize
+      elsif job.respond_to?(:call)
+        job.call
+      elsif job.respond_to?(:arguments)
+        job.perform(*job.arguments)
+      else
+        job.perform
+      end
+    end
 
-    def retry_job
+    def retry_job(error)
       self.retries += 1
-      return if retries > max_retries
+      if retries > max_retries
+        self.error = error
+        return
+      end
 
-      self.at = Time.now.utc + (retries.next**4)
+      seconds_to_retry = $TESTING ? 0.05 : retries.next**4
+      self.at = (Time.now + seconds_to_retry).to_f
       log "Job #{id} failed, retrying at #{at}"
       Belated.job_list.push(self)
     end

data/lib/belated/logging.rb

@@ -12,6 +12,14 @@ class Belated
       logger.__send__(Belated.log_level, message)
     end
 
+    def warn(message)
+      logger.warn(message)
+    end
+
+    def error(message)
+      logger.error(message)
+    end
+
     def logger=(logger)
       @logger = logger
     end

data/lib/belated/queue.rb

@@ -19,15 +19,18 @@ class Belated
 
     def initialize(queue: Thread::Queue.new, future_jobs: SortedSet.new)
       @queue = queue
+      @mutex = Mutex.new
       self.future_jobs = future_jobs
     end
 
     def push(job)
       if job.is_a?(Symbol) || job.at.nil? ||
-         job.at <= Time.now.utc
+         job.at <= Time.now.to_f
         @queue.push(job)
       else
-        @future_jobs << job
+        @mutex.synchronize do
+          @future_jobs << job
+        end
       end
     end
 
@@ -54,7 +57,7 @@ class Belated
 
       jobs = YAML.load(File.binread(FILE_NAME))
       jobs.each do |job|
-        if job.at && job.at > Time.now.utc
+        if job.at && job.at > Time.now.to_f
           future_jobs.push(job)
         else
           @queue.push(job)

data/lib/belated/testing.rb

@@ -22,7 +22,7 @@ class Belated
   # A client that can perform jobs inline
   class Client
     alias old_perform perform
-    def perform(job, at: nil, max_retries: 5)
+    def perform(job, at: nil, max_retries: 5, active_job: false)
       if Belated::Testing.inline?
         if job.respond_to?(:call)
           job.call
@@ -30,7 +30,7 @@ class Belated
           job.perform
         end
       else
-        old_perform(job, at: at, max_retries: max_retries)
+        old_perform(job, at: at, max_retries: max_retries, active_job: active_job)
       end
     end
   end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.6.4'
+  VERSION = '0.7.0'
 end

data/lib/belated.rb

@@ -22,7 +22,6 @@ class Belated
   extend Dry::Configurable
   include Logging
   include Singleton unless $TESTING
-  URI = 'druby://localhost:8788'
   @@queue = Belated::Queue.new
 
   setting :rails, true
@@ -32,6 +31,11 @@ class Belated
   setting :environment, 'development', reader: true
   setting :logger, Logger.new($stdout), reader: true
   setting :log_level, :info, reader: true
+  setting :host, 'localhost', reader: true
+  setting :port, '8788', reader: true
+  setting :heartbeat, 1, reader: true
+  setting :client_heartbeat, 5, reader: true
+  URI = "druby://#{Belated.host}:#{Belated.port}"
 
   # Since it's running as a singleton, we need something to start it up.
   # Aliased for testing purposes.
@@ -53,12 +57,15 @@ class Belated
 
   # Handles connection to DRb server.
   def connect!
+    i = 0
     DRb.start_service(URI, @@queue, verbose: true)
   rescue DRb::DRbConnError, Errno::EADDRINUSE
-    Belated.logger.error 'Could not connect to DRb server.'
+    sleep 0.1 and retry if (i += 1) < 5
+    error 'Could not connect to DRb server.'
   end
 
   def trap_signals
+    pp 'trap'
     %w[INT TERM].each do |signal|
       Signal.trap(signal) do
         @worker_list.length.times do
@@ -90,18 +97,17 @@ class Belated
 
   def enqueue_future_jobs
     loop do
-      sleep 0.1
       job = @@queue.future_jobs.min
       if job.nil?
-        sleep 5
+        sleep Belated.heartbeat
         next
       end
-      if job.at <= Time.now.utc
+      if job.at <= Time.now.to_f
         log "Deleting #{@@queue.future_jobs.delete(job)} from future jobs"
         @@queue.push(job)
       end
     rescue DRb::DRbConnError
-      log 'DRb connection error!!!!!!'
+      error 'DRb connection error!!!!!!'
       log stats
     end
   end
@@ -151,19 +157,25 @@ class Belated
     }
   end
 
-  def self.kill_and_clear_queue!
-    @worker_list&.each do |worker|
-      Thread.kill(worker)
+  class << self
+    def find(job_id)
+      @@queue.future_jobs.find { |job| job.id == job_id }
     end
-    clear_queue!
-  end
 
-  def self.clear_queue!
-    @@queue.clear
-  end
+    def kill_and_clear_queue!
+      @worker_list&.each do |worker|
+        Thread.kill(worker)
+      end
+      clear_queue!
+    end
 
-  def self.fetch_job
-    @@queue.pop
+    def clear_queue!
+      @@queue.clear
+    end
+
+    def fetch_job
+      @@queue.pop
+    end
   end
 
   def job_list
@@ -173,8 +185,5 @@ class Belated
   def self.job_list
     @@queue
   end
-
-  class Error < StandardError; end
 end
-
-require 'belated/rails' if defined?(::Rails::Engine)
+require 'active_job/queue_adapters/belated_adapter' if defined?(::Rails)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.7.0
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-22 00:00:00.000000000 Z
+date: 2021-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby2_keywords
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sorted_set
   requirement: !ruby/object:Gem::Requirement
@@ -92,8 +106,10 @@ files:
 - bin/bundle
 - bin/console
 - bin/setup
+- lib/active_job/queue_adapters/belated_adapter.rb
 - lib/belated.rb
 - lib/belated/client.rb
+- lib/belated/exceptions.rb
 - lib/belated/job.rb
 - lib/belated/job_wrapper.rb
 - lib/belated/logging.rb
@@ -117,7 +133,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="