belated 0.3.2 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 995560f345bb7644aa19035b27a68d43a1210ee36c21d13e21cc430b76d9fded
-  data.tar.gz: 103441a7e60e79319c65bacbedfb4dc337224c41c0e9560225f3b2421e219473
+  metadata.gz: 33b9abbd449108f850f65e0afaa92929c780b03856e9e151efb6cc33b84021d5
+  data.tar.gz: eb9a7142602bae2572454c227d057adf9f38397c1638dcc6f66f2cd16019b766
 SHA512:
-  metadata.gz: ecc27c208d2fdb26c93ee39ebba7ada11236e9f564565d69258dfa6381a407c924e076365329f088ca4071955fd0eaeb8e9e1a1119f77d824473987590fee670
-  data.tar.gz: 57db27785351cbba33b6c704f877eacf4343d95a02267411e79bf63f1efadb49a67430342d145898a8571318bc61eafdd756bc2618388603122a0733bad7a3ed
+  metadata.gz: c50c1dfe80dceee431785ec150a0a5b6a7c220e18fc8c6a05069637d474d848719c5bfd05f0c0f4125e379348296ebb92823e71b985e76cccffe9e3080349822
+  data.tar.gz: 357053481de792ea44b832eade91895d5257074e3381333ddb58aaa48c20d08b6d02d6e84b57664218108ae2927d8247db7229f1a1fa91483a745a3202031a0d

data/.gitignore

@@ -14,3 +14,4 @@
 /dummy/log/
 /dummy/log/test.log
 belated_dump
+stackprof*

data/.rubocop.yml

@@ -41,5 +41,8 @@ Metrics/BlockLength:
 Metrics/ClassLength:
   Max: 200
 
+Metrics/MethodLength:
+  Max: 15
+
 Security/YAMLLoad:
   Enabled: false

data/CHANGELOG.md

@@ -1,6 +1,20 @@
 ## [Unreleased]
 
+## [0.4.2] - 2021-08-05
 
+- Client also handles no connection, now it saves jobs to a bank and adds them to the queue once it has a connection.  
+## [0.4.1] - 2021-08-05
+
+- Now handles saving future jobs too! So if you have a job enqueued for tomorrow, and restart Belated, it should still be enqueued. 
+
+## [0.4.0] - 2021-08-03
+
+- Now you can enqueue jobs to be done at a later time. Just pass an `at:` keyword param to the client. 
+- Does not save the jobs when you quit. 
+
+## [0.3.3] - 2021-08-01
+
+- Shutdown trapped signal thread, make sure :shutdown is not recorded as a job. 
 
 ## [0.3.2] - 2021-07-31
 

data/Gemfile

@@ -11,5 +11,8 @@ gem 'rspec', '~> 3.0'
 
 gem 'rubocop', '~> 1.7'
 
+gem 'database_cleaner-active_record'
 gem 'rails', '>= 6.1.3'
+gem 'rspec-rails'
 gem 'sqlite3'
+gem 'stackprof'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    belated (0.3.2)
+    belated (0.4.2)
       drb
       dry-configurable
 
@@ -72,6 +72,10 @@ GEM
     byebug (11.1.3)
     concurrent-ruby (1.1.9)
     crass (1.0.6)
+    database_cleaner-active_record (2.0.1)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
     diff-lcs (1.4.4)
     drb (2.0.4)
     dry-configurable (0.12.1)
@@ -145,6 +149,14 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
+    rspec-rails (5.0.1)
+      actionpack (>= 5.2)
+      activesupport (>= 5.2)
+      railties (>= 5.2)
+      rspec-core (~> 3.10)
+      rspec-expectations (~> 3.10)
+      rspec-mocks (~> 3.10)
+      rspec-support (~> 3.10)
     rspec-support (3.10.2)
     rubocop (1.18.3)
       parallel (~> 1.10)
@@ -166,6 +178,7 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
+    stackprof (0.2.16)
     thor (1.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
@@ -181,11 +194,14 @@ PLATFORMS
 DEPENDENCIES
   belated!
   byebug
+  database_cleaner-active_record
   rails (>= 6.1.3)
   rake (~> 13.0)
   rspec (~> 3.0)
+  rspec-rails
   rubocop (~> 1.7)
   sqlite3
+  stackprof
 
 BUNDLED WITH
    2.2.17

data/README.md

@@ -8,25 +8,20 @@ Note that Belated used to be called HardWorker. That name was already in use in
 
 It uses dRuby to do the communication! Which is absolute great. No need for Redis or PostgreSQL, just Ruby standard libraries.
 
+Can be used with or without Rails. 
+
 TODO LIST:
 
-- Catch SIGTERM and friends
-  - Now supports it, partly. 
+- Add some checks to the client for proper jobs.
 - Don't crash on errors (Partially done)
-- Make it possible to schedule jobs
+- Have multiple queues?
 - Maybe support ActiveJob?
 - Have a web UI
 - Do some performance testing
+- Deploy a Rails app to production that is using Belated
+  and mention it in the readme. 
 - Add a section telling people to use Sidekiq if they can
 
-DONE
-
-- ~~Marshal the job queue into a file so you don't lose all progress~~
-  (Ended up using YAML)
-- ~~Add a logger~~
-- ~~Support Rails~~ (Supported!)
-- ~~Parse options from command line, eg. `--workers 10`~~(Done!)
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -98,6 +93,12 @@ client.perform_belated(job)
 
 If you want to pass a job to Belated.
 
+If you don't want the job to run right away, you can also pass it a keyword param `at:` like so:
+
+```ruby
+client.perform_belated(job, Time.now + 1.month)
+```
+
 # Settings
 
 Configuring Belated:
@@ -113,15 +114,15 @@ end
 
 From command line:
 
-$ bundle exec belated --rails=true
+    $ bundle exec belated --rails=true
 
 Use Rails or not.
 
-$ bundle exec belated --rails_path=/my_rails_project
+    $ bundle exec belated --rails_path=/my_rails_project
 
 Path to Rails project.
 
-$ bundle exec belated --workers=10
+    $ bundle exec belated --workers=10
 
 Number of workers.
 

data/lib/belated.rb

@@ -9,6 +9,7 @@ require 'singleton'
 require 'dry-configurable'
 require 'belated/client'
 require 'logger'
+require 'belated/queue'
 
 # Belated is a pure Ruby job backend.
 # It has limited functionality, as it only accepts
@@ -21,8 +22,7 @@ class Belated
   include Logging
   include Singleton unless $TESTING
   URI = 'druby://localhost:8788'
-  FILE_NAME = 'belated_dump'
-  @@queue = Queue.new
+  @@queue = Belated::Queue.new
 
   setting :rails, true
   setting :rails_path, '.'
@@ -36,17 +36,17 @@ class Belated
   # Aliased for testing purposes.
   # This is only run from the bin file.
   def start
-    boot_app && load_jobs
+    boot_app && @@queue.load_jobs
     @worker_list = []
-    Belated.config.workers.times do |_i|
-      @worker_list << Thread.new { Worker.new }
+    Belated.config.workers.times do |i|
+      @worker_list << Thread.new { Worker.new(number: i.next) }
     end
     return unless Belated.config.connect
 
     connect!
     banner_and_info
     trap_signals
-    DRb.thread.join
+    enqueue_future_jobs
   end
   alias initialize start
 
@@ -55,7 +55,6 @@ class Belated
     DRb.start_service(URI, @@queue, verbose: true)
   rescue DRb::DRbConnError, Errno::EADDRINUSE
     Belated.logger.error 'Could not connect to DRb server.'
-    retry
   end
 
   def trap_signals
@@ -65,7 +64,7 @@ class Belated
           @@queue.push(:shutdown)
         end
         Thread.new { stop_workers }
-        sleep 0.1 until @@queue.empty?
+        sleep 0.1 until @@queue.empty? || $TESTING
         exit
       end
     end
@@ -84,20 +83,22 @@ class Belated
     Belated.config.rails
   end
 
-  def load_jobs
-    log "reloading... if file exists #{File.exist?(Belated::FILE_NAME)}"
-    return unless File.exist?(Belated::FILE_NAME)
-
-    jobs = YAML.load(File.binread(FILE_NAME))
-    jobs.each do |job|
-      @@queue.push(job)
+  def enqueue_future_jobs
+    log 'starting future jobs thread'
+    loop do
+      @@queue.future_jobs.each_with_index do |job, i|
+        if job[:at] <= Time.now.utc
+          log @@queue.future_jobs.delete_at(i)
+          @@queue.push(job[:klass])
+        end
+      end
+      sleep 0.01
     end
-    File.delete(Belated::FILE_NAME)
   end
 
   def reload
     log 'reloading...'
-    load_jobs
+    @@queue.load_jobs
   end
 
   def stop_workers
@@ -105,13 +106,8 @@ class Belated
       sleep 0.1 if worker.alive?
       Thread.kill(worker)
     end
-    class_array = []
-    @@queue.size.times do |_i|
-      next if (klass = @@queue.pop).instance_of?(Proc)
-
-      class_array << klass
-    end
-    pp File.open(FILE_NAME, 'wb') { |f| f.write(YAML.dump(class_array)) }
+    @@queue.save_jobs
+    exit unless $TESTING
   end
 
   def banner

data/lib/belated/client.rb

@@ -1,16 +1,41 @@
 class Belated
+  # The client class is responsible for managing the connection to the
+  # DRb server. If it has no connection, it adds the jobs to a bank queue.
+  # You can enqueue jobs to be processed by the server.
+  # Example:
+  #   client = Belated::Client.new
+  #   client.enqueue(JubJub.new, at: Time.now + 5.seconds)
   class Client
-    attr_accessor :queue
+    attr_accessor :queue, :bank, :banker_thread
 
+    # Starts up the client.
+    # Connects to the queue through DRb.
+    # @return [void]
     def initialize
       server_uri = Belated::URI
       # @bank =
       DRb.start_service
+      self.bank = Thread::Queue.new
       self.queue = DRbObject.new_with_uri(server_uri)
+      self.banker_thread = Thread.new do
+        loop do
+          sleep 0.05
+          next unless (job, at = bank.pop)
+
+          perform(job, at: at)
+        end
+      end
     end
 
-    def perform(job)
-      queue.push(job)
+    # The method that pushes the jobs to the queue.
+    # If there is no connection, it pushes the job to the bank.
+    # @param job [Object] - The the job to be pushed.
+    # @param at [Date] - The time at which the job should be executed.
+    # @return [Object] - The job that was pushed.
+    def perform(job, at: nil)
+      queue.push(job, at: at)
+    rescue DRb::DRbConnError
+      bank.push([job, at])
     end
     alias perform_belated perform
     alias perform_later perform

data/lib/belated/job.rb

@@ -0,0 +1 @@
+Job = Struct.new(:klass, :at)

data/lib/belated/queue.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'belated/job'
+require 'belated/logging'
+class Belated
+  class Queue
+    include Logging
+    attr_accessor :future_jobs
+
+    FILE_NAME = 'belated_dump'
+
+    def initialize(queue: Thread::Queue.new, future_jobs: [])
+      @queue = queue
+      self.future_jobs = future_jobs
+    end
+
+    def push(job, at: nil)
+      if at.nil?
+        @queue.push(job)
+      else
+        @future_jobs << Job.new(job, at)
+      end
+    end
+
+    def pop
+      @queue.pop
+    end
+
+    def clear
+      @queue.clear
+      self.future_jobs = []
+    end
+
+    def empty?
+      @queue.empty?
+    end
+
+    def length
+      @queue.length
+    end
+
+    def load_jobs
+      log "reloading... if file exists #{File.exist?(FILE_NAME)}"
+      return unless File.exist?(FILE_NAME)
+
+      jobs = YAML.load(File.binread(FILE_NAME))
+      jobs.each do |job|
+        if job.is_a?(Job)
+          future_jobs.push(job)
+        else
+          @queue.push(job)
+        end
+      end
+      File.delete(FILE_NAME)
+    end
+
+    def save_jobs
+      class_array = []
+      @queue.length.times do |_i|
+        unless proc_or_shutdown?(klass = @queue.pop)
+          class_array << klass
+        end
+      end
+      future_jobs.each do |_job|
+        unless proc_or_shutdown?(klass = future_jobs.pop)
+          class_array << klass
+        end
+      end
+
+      pp File.open(FILE_NAME, 'wb') { |f| f.write(YAML.dump(class_array)) }
+    end
+
+    private
+
+    def proc_or_shutdown?(job)
+      job.instance_of?(Proc) || job == :shutdown
+    end
+  end
+end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.3.2'
+  VERSION = '0.4.2'
 end

data/lib/belated/worker.rb

@@ -1,22 +1,24 @@
 require_relative 'logging'
 class Belated
   # The worker class that actually gets the jobs from the queue
-  # and calls them. Expects the jobs to be procs.
+  # and calls them. Expects the jobs to be procs or
+  # classes that have a perform method.
   class Worker
     include Logging
 
-    def initialize
+    def initialize(number: 1)
+      @number = number
       start_working
     end
 
     def start_working
       loop do
+        log "Worker #{@number} fetching jobs!"
         next unless (job = Belated.fetch_job)
 
         break if job == :shutdown
 
         log call_job(job)
-        log 'fetching jobs...'
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.2
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-31 00:00:00.000000000 Z
+date: 2021-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -80,7 +80,9 @@ files:
 - bin/setup
 - lib/belated.rb
 - lib/belated/client.rb
+- lib/belated/job.rb
 - lib/belated/logging.rb
+- lib/belated/queue.rb
 - lib/belated/rails.rb
 - lib/belated/version.rb
 - lib/belated/worker.rb