belated 0.3.3 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 639e202c5feb0ecb2af5bcd7f22815128f670f36c2aa793ce5bc0c107477a92f
-  data.tar.gz: 65864414d040a9b1cd3e92897a97b5f596683c42cb6941c5cdbaee731502a622
+  metadata.gz: 598cfda479ae3ec79564557a4f111ff5600c255bdf3cc8b3e03194b0b0b6ce86
+  data.tar.gz: f6b07150c218bf9754f5d40625e7e5d26673f87bf4d8a1baa4ef87814edbbc09
 SHA512:
-  metadata.gz: bda697be3405328560b19afbf9a90cc8cac0a8487e627ba0d6ee02ef2bbd6bf933cf3235fa312a1f817224ade6837da390d6794742a5110b255ca6052cafb838
-  data.tar.gz: da4c2c0b49901ca28aed07eff2d279293c28a3121031500cc4846627d01619fbd24c52b0c382c8c4091d46238df53d827449c7092a85286072a1b74600402313
+  metadata.gz: 43ab001fcc721b6750ec2c7d3ee00f32ab58ff9909268bb9dda92d7096a5cca85782fbd8f3502295cc07086ac6ef18c056124f55ef4a706b607d2aab3c667ef8
+  data.tar.gz: c83cec5063743c8ed84a28a1b5995b7b3d7deeade1f6f352d81d1dca5b1531231dc46678f14e1ad6795662d8105dfc957ca81f7868996b310bc8b4128132e0c9

data/.gitignore

@@ -14,3 +14,4 @@
 /dummy/log/
 /dummy/log/test.log
 belated_dump
+stackprof*

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 ## [Unreleased]
 
+## [0.4.3] - 2021-08-06
+
+- Client now starts the banker thread to execute jobs that were enqueued when there was no connection to Belated only if necessary. 
+## [0.4.2] - 2021-08-05
+
+- Client also handles no connection, now it saves jobs to a bank and adds them to the queue once it has a connection.  
+## [0.4.1] - 2021-08-05
+
+- Now handles saving future jobs too! So if you have a job enqueued for tomorrow, and restart Belated, it should still be enqueued. 
+
+## [0.4.0] - 2021-08-03
+
+- Now you can enqueue jobs to be done at a later time. Just pass an `at:` keyword param to the client. 
+- Does not save the jobs when you quit. 
+
 ## [0.3.3] - 2021-08-01
 
 - Shutdown trapped signal thread, make sure :shutdown is not recorded as a job. 

data/Gemfile

@@ -11,5 +11,8 @@ gem 'rspec', '~> 3.0'
 
 gem 'rubocop', '~> 1.7'
 
+gem 'database_cleaner-active_record'
 gem 'rails', '>= 6.1.3'
+gem 'rspec-rails'
 gem 'sqlite3'
+gem 'stackprof'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    belated (0.3.3)
+    belated (0.4.3)
       drb
       dry-configurable
 
@@ -72,6 +72,10 @@ GEM
     byebug (11.1.3)
     concurrent-ruby (1.1.9)
     crass (1.0.6)
+    database_cleaner-active_record (2.0.1)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
     diff-lcs (1.4.4)
     drb (2.0.4)
     dry-configurable (0.12.1)
@@ -145,6 +149,14 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
+    rspec-rails (5.0.1)
+      actionpack (>= 5.2)
+      activesupport (>= 5.2)
+      railties (>= 5.2)
+      rspec-core (~> 3.10)
+      rspec-expectations (~> 3.10)
+      rspec-mocks (~> 3.10)
+      rspec-support (~> 3.10)
     rspec-support (3.10.2)
     rubocop (1.18.3)
       parallel (~> 1.10)
@@ -166,6 +178,7 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
+    stackprof (0.2.16)
     thor (1.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
@@ -181,11 +194,14 @@ PLATFORMS
 DEPENDENCIES
   belated!
   byebug
+  database_cleaner-active_record
   rails (>= 6.1.3)
   rake (~> 13.0)
   rspec (~> 3.0)
+  rspec-rails
   rubocop (~> 1.7)
   sqlite3
+  stackprof
 
 BUNDLED WITH
    2.2.17

data/README.md

@@ -4,29 +4,26 @@
 
 This is Belated, a new Ruby backend job library! It supports running procs and classes in the background. To deal with restarts, it uses YAML to load the queue into a file, which it then calls at startup to find the previous jobs.
 
+Belated uses the Ruby Queue class, so it's First In, First Out (FIFO). 
+
 Note that Belated used to be called HardWorker. That name was already in use in Sidekiq documentation and a bit too generic anyway. 
 
 It uses dRuby to do the communication! Which is absolute great. No need for Redis or PostgreSQL, just Ruby standard libraries.
 
+Can be used with or without Rails. 
+
 TODO LIST:
 
-- Catch SIGTERM and friends
-  - Now supports it, partly. 
+- Add some checks to the client for proper jobs.
 - Don't crash on errors (Partially done)
-- Make it possible to schedule jobs
+- Have multiple queues?
 - Maybe support ActiveJob?
 - Have a web UI
 - Do some performance testing
+- Deploy a Rails app to production that is using Belated
+  and mention it in the readme. 
 - Add a section telling people to use Sidekiq if they can
 
-DONE
-
-- ~~Marshal the job queue into a file so you don't lose all progress~~
-  (Ended up using YAML)
-- ~~Add a logger~~
-- ~~Support Rails~~ (Supported!)
-- ~~Parse options from command line, eg. `--workers 10`~~(Done!)
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -98,6 +95,14 @@ client.perform_belated(job)
 
 If you want to pass a job to Belated.
 
+If you don't want the job to run right away, you can also pass it a keyword param `at:` like so:
+
+```ruby
+client.perform_belated(job, Time.now + 1.month)
+```
+
+Note that you probably want to memoize the client, as it always creates a 'banker thread' now if you have no connection. Maybe even use it as a global!(`$client`)
+
 # Settings
 
 Configuring Belated:
@@ -113,15 +118,15 @@ end
 
 From command line:
 
-$ bundle exec belated --rails=true
+    $ bundle exec belated --rails=true
 
 Use Rails or not.
 
-$ bundle exec belated --rails_path=/my_rails_project
+    $ bundle exec belated --rails_path=/my_rails_project
 
 Path to Rails project.
 
-$ bundle exec belated --workers=10
+    $ bundle exec belated --workers=10
 
 Number of workers.
 

data/lib/belated.rb

@@ -9,6 +9,7 @@ require 'singleton'
 require 'dry-configurable'
 require 'belated/client'
 require 'logger'
+require 'belated/queue'
 
 # Belated is a pure Ruby job backend.
 # It has limited functionality, as it only accepts
@@ -21,8 +22,7 @@ class Belated
   include Logging
   include Singleton unless $TESTING
   URI = 'druby://localhost:8788'
-  FILE_NAME = 'belated_dump'
-  @@queue = Queue.new
+  @@queue = Belated::Queue.new
 
   setting :rails, true
   setting :rails_path, '.'
@@ -36,17 +36,17 @@ class Belated
   # Aliased for testing purposes.
   # This is only run from the bin file.
   def start
-    boot_app && load_jobs
+    boot_app && @@queue.load_jobs
     @worker_list = []
-    Belated.config.workers.times do |_i|
-      @worker_list << Thread.new { Worker.new }
+    Belated.config.workers.times do |i|
+      @worker_list << Thread.new { Worker.new(number: i.next) }
     end
     return unless Belated.config.connect
 
     connect!
     banner_and_info
     trap_signals
-    DRb.thread.join
+    enqueue_future_jobs
   end
   alias initialize start
 
@@ -55,7 +55,6 @@ class Belated
     DRb.start_service(URI, @@queue, verbose: true)
   rescue DRb::DRbConnError, Errno::EADDRINUSE
     Belated.logger.error 'Could not connect to DRb server.'
-    retry
   end
 
   def trap_signals
@@ -65,7 +64,7 @@ class Belated
           @@queue.push(:shutdown)
         end
         Thread.new { stop_workers }
-        sleep 0.1 until @@queue.empty?
+        sleep 0.1 until @@queue.empty? || $TESTING
         exit
       end
     end
@@ -84,20 +83,22 @@ class Belated
     Belated.config.rails
   end
 
-  def load_jobs
-    log "reloading... if file exists #{File.exist?(Belated::FILE_NAME)}"
-    return unless File.exist?(Belated::FILE_NAME)
-
-    jobs = YAML.load(File.binread(FILE_NAME))
-    jobs.each do |job|
-      @@queue.push(job)
+  def enqueue_future_jobs
+    log 'starting future jobs thread'
+    loop do
+      @@queue.future_jobs.each_with_index do |job, i|
+        if job[:at] <= Time.now.utc
+          log @@queue.future_jobs.delete_at(i)
+          @@queue.push(job[:klass])
+        end
+      end
+      sleep 0.01
     end
-    File.delete(Belated::FILE_NAME)
   end
 
   def reload
     log 'reloading...'
-    load_jobs
+    @@queue.load_jobs
   end
 
   def stop_workers
@@ -105,13 +106,7 @@ class Belated
       sleep 0.1 if worker.alive?
       Thread.kill(worker)
     end
-    class_array = []
-    @@queue.size.times do |_i|
-      unless (klass = @@queue.pop).instance_of?(Proc) || klass == :shutdown
-        class_array << klass
-      end
-    end
-    pp File.open(FILE_NAME, 'wb') { |f| f.write(YAML.dump(class_array)) }
+    @@queue.save_jobs
     exit unless $TESTING
   end
 

data/lib/belated/client.rb

@@ -1,16 +1,48 @@
 class Belated
+  # The client class is responsible for managing the connection to the
+  # DRb server. If it has no connection, it adds the jobs to a bank queue.
+  # You can enqueue jobs to be processed by the server.
+  # Example:
+  #   client = Belated::Client.new
+  #   client.enqueue(JubJub.new, at: Time.now + 5.seconds)
   class Client
-    attr_accessor :queue
+    attr_accessor :queue, :bank, :banker_thread
 
+    # Starts up the client.
+    # Connects to the queue through DRb.
+    # @return [void]
     def initialize
       server_uri = Belated::URI
-      # @bank =
       DRb.start_service
+      self.bank = Thread::Queue.new
       self.queue = DRbObject.new_with_uri(server_uri)
     end
 
-    def perform(job)
-      queue.push(job)
+    # Thread in charge of handling the bank queue.
+    # You probably want to memoize the client in order to avoid
+    # having many threads in the sleep state.
+    # @return [void]
+    def start_banker_thread
+      self.banker_thread = Thread.new do
+        loop do
+          job, at = bank.pop
+
+          perform(job, at: at)
+        end
+      end
+    end
+
+    # The method that pushes the jobs to the queue.
+    # If there is no connection, it pushes the job to the bank.
+    # @param job [Object] - The the job to be pushed.
+    # @param at [Date] - The time at which the job should be executed.
+    # @return [Object] - The job that was pushed.
+    def perform(job, at: nil)
+      queue.push(job, at: at)
+    rescue DRb::DRbConnError
+      bank.push([job, at])
+      start_banker_thread if banker_thread.nil?
+      # banker_thread.wakeup if banker_thread.status == 'sleep'
     end
     alias perform_belated perform
     alias perform_later perform

data/lib/belated/job.rb

@@ -0,0 +1 @@
+Job = Struct.new(:klass, :at)

data/lib/belated/queue.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'belated/job'
+require 'belated/logging'
+class Belated
+  class Queue
+    include Logging
+    attr_accessor :future_jobs
+
+    FILE_NAME = 'belated_dump'
+
+    def initialize(queue: Thread::Queue.new, future_jobs: [])
+      @queue = queue
+      self.future_jobs = future_jobs
+    end
+
+    def push(job, at: nil)
+      if at.nil?
+        @queue.push(job)
+      else
+        @future_jobs << Job.new(job, at)
+      end
+    end
+
+    def pop
+      @queue.pop
+    end
+
+    def clear
+      @queue.clear
+      self.future_jobs = []
+    end
+
+    def empty?
+      @queue.empty?
+    end
+
+    def length
+      @queue.length
+    end
+
+    def load_jobs
+      log "reloading... if file exists #{File.exist?(FILE_NAME)}"
+      return unless File.exist?(FILE_NAME)
+
+      jobs = YAML.load(File.binread(FILE_NAME))
+      jobs.each do |job|
+        if job.is_a?(Job)
+          future_jobs.push(job)
+        else
+          @queue.push(job)
+        end
+      end
+      File.delete(FILE_NAME)
+    end
+
+    def save_jobs
+      class_array = []
+      @queue.length.times do |_i|
+        unless proc_or_shutdown?(klass = @queue.pop)
+          class_array << klass
+        end
+      end
+      future_jobs.each do |_job|
+        unless proc_or_shutdown?(klass = future_jobs.pop)
+          class_array << klass
+        end
+      end
+
+      pp File.open(FILE_NAME, 'wb') { |f| f.write(YAML.dump(class_array)) }
+    end
+
+    private
+
+    def proc_or_shutdown?(job)
+      job.instance_of?(Proc) || job == :shutdown
+    end
+  end
+end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.3.3'
+  VERSION = '0.4.3'
 end

data/lib/belated/worker.rb

@@ -1,22 +1,24 @@
 require_relative 'logging'
 class Belated
   # The worker class that actually gets the jobs from the queue
-  # and calls them. Expects the jobs to be procs.
+  # and calls them. Expects the jobs to be procs or
+  # classes that have a perform method.
   class Worker
     include Logging
 
-    def initialize
+    def initialize(number: 1)
+      @number = number
       start_working
     end
 
     def start_working
       loop do
+        log "Worker #{@number} fetching jobs!"
         next unless (job = Belated.fetch_job)
 
         break if job == :shutdown
 
         log call_job(job)
-        log 'fetching jobs...'
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.3
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-01 00:00:00.000000000 Z
+date: 2021-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -80,7 +80,9 @@ files:
 - bin/setup
 - lib/belated.rb
 - lib/belated/client.rb
+- lib/belated/job.rb
 - lib/belated/logging.rb
+- lib/belated/queue.rb
 - lib/belated/rails.rb
 - lib/belated/version.rb
 - lib/belated/worker.rb