belated 0.3.1 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99491dbba348e9e991f00436b70faa69d7ee3646961172014499a09276e9cb1e
-  data.tar.gz: ebb40194db4cf9893edb0b06f6b940a4076e5ec328ea00ba2ab40bd839ebe7e6
+  metadata.gz: ed06672798a42b07c69c9dc84f32d1ba12a58a7b95028f0c4a2f5d1a5f0b04c4
+  data.tar.gz: 0ecf13b8abf076c5fbe8a0037017840e55f32c674f4a40092a6e35845503f4e9
 SHA512:
-  metadata.gz: 0b40900814f6cae2396025fd6ebda46b6fedc11ae7bca92ffcd534cdd6a782e0cca581a264c6f0b3bac05c1c925de8d32590270370445dab62a8c50f9749df90
-  data.tar.gz: 397925608f6083f3722e69661e7731d9f56557ee44d8b79e811b655a7d25dd2f58755f607ad00137296607dd6f1f3fe88045237152134769bb4b5b09aea5cb73
+  metadata.gz: 1e16e639db3c05d9112a4b31232d0cb78dbe6fe93ed05cd33f1d9a7856ded7c40319c145c20cddf1dd5a8bafb2894d8f96d46fc423a6718180e4d5e4584fdc41
+  data.tar.gz: 86793a533285a0ae5761b3eb11d4f16fe44793cc83d2a24c6f7aebae6a4d27ff7795b3ef2e69ce7e2b12f5c86fb9e108d5252f2aa863c645262f880f0ad56cf7

data/.gitignore

@@ -12,4 +12,6 @@
 .byebug_history
 /dummy/tmp/cache/
 /dummy/log/
-belated_dump
+/dummy/log/test.log
+belated_dump
+stackprof*

data/.rubocop.yml

@@ -41,5 +41,8 @@ Metrics/BlockLength:
 Metrics/ClassLength:
   Max: 200
 
+Metrics/MethodLength:
+  Max: 15
+
 Security/YAMLLoad:
   Enabled: false

data/CHANGELOG.md

@@ -1,6 +1,21 @@
 ## [Unreleased]
 
+## [0.4.1] - 2021-08-05
 
+- Now handles saving future jobs too! So if you have a job enqueued for tomorrow, and restart Belated, it should still be enqueued. 
+
+## [0.4.0] - 2021-08-03
+
+- Now you can enqueue jobs to be done at a later time. Just pass an `at:` keyword param to the client. 
+- Does not save the jobs when you quit. 
+
+## [0.3.3] - 2021-08-01
+
+- Shutdown trapped signal thread, make sure :shutdown is not recorded as a job. 
+
+## [0.3.2] - 2021-07-31
+
+- Trap INT and TERM, so now the shutdown is a little bit more graceful. 
 
 ## [0.3.1] - 2021-07-29
 

data/Gemfile

@@ -11,5 +11,8 @@ gem 'rspec', '~> 3.0'
 
 gem 'rubocop', '~> 1.7'
 
+gem 'database_cleaner-active_record'
 gem 'rails', '>= 6.1.3'
+gem 'rspec-rails'
 gem 'sqlite3'
+gem 'stackprof'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    belated (0.3.1)
+    belated (0.4.1)
       drb
       dry-configurable
 
@@ -72,6 +72,10 @@ GEM
     byebug (11.1.3)
     concurrent-ruby (1.1.9)
     crass (1.0.6)
+    database_cleaner-active_record (2.0.1)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
     diff-lcs (1.4.4)
     drb (2.0.4)
     dry-configurable (0.12.1)
@@ -145,6 +149,14 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
+    rspec-rails (5.0.1)
+      actionpack (>= 5.2)
+      activesupport (>= 5.2)
+      railties (>= 5.2)
+      rspec-core (~> 3.10)
+      rspec-expectations (~> 3.10)
+      rspec-mocks (~> 3.10)
+      rspec-support (~> 3.10)
     rspec-support (3.10.2)
     rubocop (1.18.3)
       parallel (~> 1.10)
@@ -166,6 +178,7 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
+    stackprof (0.2.16)
     thor (1.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
@@ -181,11 +194,14 @@ PLATFORMS
 DEPENDENCIES
   belated!
   byebug
+  database_cleaner-active_record
   rails (>= 6.1.3)
   rake (~> 13.0)
   rspec (~> 3.0)
+  rspec-rails
   rubocop (~> 1.7)
   sqlite3
+  stackprof
 
 BUNDLED WITH
    2.2.17

data/README.md

@@ -8,21 +8,28 @@ Note that Belated used to be called HardWorker. That name was already in use in
 
 It uses dRuby to do the communication! Which is absolute great. No need for Redis or PostgreSQL, just Ruby standard libraries.
 
+Can be used with or without Rails. 
+
 TODO LIST:
 
-- ~~Marshal the job queue into a file so you don't lose all progress~~
-  (Ended up using YAML)
 - Catch SIGTERM and friends
-- ~~Support Rails~~ (Supported!)
-- ~~Parse options from command line, eg. `--workers 10`~~(Done!)
+  - Now supports it, partly. 
 - Don't crash on errors (Partially done)
-- ~~Add a logger~~
-- Make it possible to schedule jobs
+- Have multiple queues?
 - Maybe support ActiveJob?
 - Have a web UI
 - Do some performance testing
 - Add a section telling people to use Sidekiq if they can
 
+DONE
+
+- ~~Make it possible to schedule jobs~~
+- ~~Marshal the job queue into a file so you don't lose all progress~~
+  (Ended up using YAML)
+- ~~Add a logger~~
+- ~~Support Rails~~ (Supported!)
+- ~~Parse options from command line, eg. `--workers 10`~~(Done!)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -94,6 +101,12 @@ client.perform_belated(job)
 
 If you want to pass a job to Belated.
 
+If you don't want the job to run right away, you can also pass it a keyword param `at:` like so:
+
+```ruby
+client.perform_belated(job, Time.now + 1.month)
+```
+
 # Settings
 
 Configuring Belated:
@@ -109,15 +122,15 @@ end
 
 From command line:
 
-$ bundle exec belated --rails=true
+    $ bundle exec belated --rails=true
 
 Use Rails or not.
 
-$ bundle exec belated --rails_path=/my_rails_project
+    $ bundle exec belated --rails_path=/my_rails_project
 
 Path to Rails project.
 
-$ bundle exec belated --workers=10
+    $ bundle exec belated --workers=10
 
 Number of workers.
 

data/lib/belated.rb

@@ -9,6 +9,7 @@ require 'singleton'
 require 'dry-configurable'
 require 'belated/client'
 require 'logger'
+require 'belated/queue'
 
 # Belated is a pure Ruby job backend.
 # It has limited functionality, as it only accepts
@@ -21,8 +22,7 @@ class Belated
   include Logging
   include Singleton unless $TESTING
   URI = 'druby://localhost:8788'
-  FILE_NAME = 'belated_dump'
-  @@queue = Queue.new
+  @@queue = Belated::Queue.new
 
   setting :rails, true
   setting :rails_path, '.'
@@ -36,17 +36,17 @@ class Belated
   # Aliased for testing purposes.
   # This is only run from the bin file.
   def start
-    boot_app
-    load_jobs
+    boot_app && @@queue.load_jobs
     @worker_list = []
-    Belated.config.workers.times do |_i|
-      @worker_list << Thread.new { Worker.new }
+    Belated.config.workers.times do |i|
+      @worker_list << Thread.new { Worker.new(number: i.next) }
     end
     return unless Belated.config.connect
 
     connect!
     banner_and_info
-    DRb.thread.join
+    trap_signals
+    enqueue_future_jobs
   end
   alias initialize start
 
@@ -55,10 +55,19 @@ class Belated
     DRb.start_service(URI, @@queue, verbose: true)
   rescue DRb::DRbConnError, Errno::EADDRINUSE
     Belated.logger.error 'Could not connect to DRb server.'
-    uri = "druby://localhost:#{Array.new(4) { rand(10) }.join}"
-    self.class.send(:remove_const, 'URI')
-    self.class.const_set('URI', uri)
-    retry
+  end
+
+  def trap_signals
+    %w[INT TERM].each do |signal|
+      Signal.trap(signal) do
+        @worker_list.length.times do
+          @@queue.push(:shutdown)
+        end
+        Thread.new { stop_workers }
+        sleep 0.1 until @@queue.empty? || $TESTING
+        exit
+      end
+    end
   end
 
   def boot_app
@@ -74,33 +83,31 @@ class Belated
     Belated.config.rails
   end
 
-  def load_jobs
-    log "reloading... if file exists #{File.exist?(Belated::FILE_NAME)}"
-    return unless File.exist?(Belated::FILE_NAME)
-
-    jobs = YAML.load(File.binread(FILE_NAME))
-    jobs.each do |job|
-      @@queue.push(job)
+  def enqueue_future_jobs
+    log 'starting future jobs thread'
+    loop do
+      @@queue.future_jobs.each_with_index do |job, i|
+        if job[:at] <= Time.now.utc
+          log @@queue.future_jobs.delete_at(i)
+          @@queue.push(job[:klass])
+        end
+      end
+      sleep 0.01
     end
-    File.delete(Belated::FILE_NAME)
   end
 
   def reload
     log 'reloading...'
-    load_jobs
+    @@queue.load_jobs
   end
 
   def stop_workers
     @worker_list&.each do |worker|
+      sleep 0.1 if worker.alive?
       Thread.kill(worker)
     end
-    class_array = []
-    @@queue.size.times do |_i|
-      next if (klass = @@queue.pop).instance_of?(Proc)
-
-      class_array << klass
-    end
-    pp File.open(FILE_NAME, 'wb') { |f| f.write(YAML.dump(class_array)) }
+    @@queue.save_jobs
+    exit unless $TESTING
   end
 
   def banner

data/lib/belated/client.rb

@@ -1,7 +1,16 @@
 class Belated
+  # The client class is responsible for managing the connection to the
+  # DRb server.
+  # You can enqueue jobs to be processed by the server.
+  # Example:
+  #   client = Belated::Client.new
+  #   client.enqueue(JubJub.new, at: Time.now + 5.seconds)
   class Client
     attr_accessor :queue
 
+    # Starts up the client.
+    # Connects to the queue through DRb.
+    # @return [void]
     def initialize
       server_uri = Belated::URI
       # @bank =
@@ -9,8 +18,12 @@ class Belated
       self.queue = DRbObject.new_with_uri(server_uri)
     end
 
-    def perform(job)
-      queue.push(job)
+    # The method that pushes the jobs to the queue.
+    # @param job [Object] - The the job to be pushed.
+    # @param at [Date] - The time at which the job should be executed.
+    # @return [Object] - The job that was pushed.
+    def perform(job, at: nil)
+      queue.push(job, at: at)
     end
     alias perform_belated perform
     alias perform_later perform

data/lib/belated/job.rb

@@ -0,0 +1 @@
+Job = Struct.new(:klass, :at)

data/lib/belated/queue.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'belated/job'
+require 'belated/logging'
+class Belated
+  class Queue
+    include Logging
+    attr_accessor :future_jobs
+
+    FILE_NAME = 'belated_dump'
+
+    def initialize(queue: Thread::Queue.new, future_jobs: [])
+      @queue = queue
+      self.future_jobs = future_jobs
+    end
+
+    def push(job, at: nil)
+      if at.nil?
+        @queue.push(job)
+      else
+        @future_jobs << Job.new(job, at)
+      end
+    end
+
+    def pop
+      @queue.pop
+    end
+
+    def clear
+      @queue.clear
+      self.future_jobs = []
+    end
+
+    def empty?
+      @queue.empty?
+    end
+
+    def length
+      @queue.length
+    end
+
+    def load_jobs
+      log "reloading... if file exists #{File.exist?(FILE_NAME)}"
+      return unless File.exist?(FILE_NAME)
+
+      jobs = YAML.load(File.binread(FILE_NAME))
+      jobs.each do |job|
+        if job.is_a?(Job)
+          future_jobs.push(job)
+        else
+          @queue.push(job)
+        end
+      end
+      File.delete(FILE_NAME)
+    end
+
+    def save_jobs
+      class_array = []
+      @queue.length.times do |_i|
+        unless proc_or_shutdown?(klass = @queue.pop)
+          class_array << klass
+        end
+      end
+      future_jobs.each do |_job|
+        unless proc_or_shutdown?(klass = future_jobs.pop)
+          class_array << klass
+        end
+      end
+
+      pp File.open(FILE_NAME, 'wb') { |f| f.write(YAML.dump(class_array)) }
+    end
+
+    private
+
+    def proc_or_shutdown?(job)
+      job.instance_of?(Proc) || job == :shutdown
+    end
+  end
+end

data/lib/belated/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Belated
-  VERSION = '0.3.1'
+  VERSION = '0.4.1'
 end

data/lib/belated/worker.rb

@@ -1,21 +1,24 @@
 require_relative 'logging'
 class Belated
   # The worker class that actually gets the jobs from the queue
-  # and calls them. Expects the jobs to be procs.
+  # and calls them. Expects the jobs to be procs or
+  # classes that have a perform method.
   class Worker
     include Logging
 
-    def initialize
+    def initialize(number: 1)
+      @number = number
       start_working
     end
 
     def start_working
       loop do
-        job = Belated.fetch_job
-        next unless job
+        log "Worker #{@number} fetching jobs!"
+        next unless (job = Belated.fetch_job)
+
+        break if job == :shutdown
 
         log call_job(job)
-        log 'fetching jobs...'
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: belated
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.1
 platform: ruby
 authors:
 - Sampo Kuokkanen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2021-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: drb
@@ -78,10 +78,11 @@ files:
 - bin/bundle
 - bin/console
 - bin/setup
-- hard_worker_dump
 - lib/belated.rb
 - lib/belated/client.rb
+- lib/belated/job.rb
 - lib/belated/logging.rb
+- lib/belated/queue.rb
 - lib/belated/rails.rb
 - lib/belated/version.rb
 - lib/belated/worker.rb

data/hard_worker_dump

@@ -1 +0,0 @@
---- []