belated 0.6.5 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +9 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +16 -0
- data/Gemfile +1 -1
- data/Gemfile.lock +14 -4
- data/README.md +20 -15
- data/belated.gemspec +3 -1
- data/lib/active_job/queue_adapters/belated_adapter.rb +34 -0
- data/lib/belated/client.rb +21 -17
- data/lib/belated/job_wrapper.rb +29 -16
- data/lib/belated/queue.rb +45 -16
- data/lib/belated/testing.rb +2 -2
- data/lib/belated/version.rb +1 -1
- data/lib/belated/worker.rb +20 -0
- data/lib/belated.rb +23 -31
- metadata +32 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9bd1b76847288931abea65941cf12792557ce92d930f07132d3d78f40cd736e0
|
4
|
+
data.tar.gz: 264e80282918062e70948fcc6f9357b56e49170eabab1c963ecf6ab4767401a2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5f96e1e24d51fea623952ac7d0dba6333bd7f2ec114fe271a7dc56a37b768f3bad58f0bcec1323038d0cf1fd0cb5104a0e66e93757ad7e9d4efe579596c95fac
|
7
|
+
data.tar.gz: ceab192601a13e0e8413f6205e5d82e1ebca1a483f4201cabeda670f8f18d0cf7225fc1442eec96474d7cafc561e4c4808e14ebbffae436b144a3d4805f2770f
|
data/.github/workflows/main.yml
CHANGED
@@ -4,14 +4,19 @@ on: [push,pull_request]
|
|
4
4
|
|
5
5
|
jobs:
|
6
6
|
build:
|
7
|
-
|
7
|
+
strategy:
|
8
|
+
fail-fast: false
|
9
|
+
matrix:
|
10
|
+
os: [ubuntu-latest]
|
11
|
+
# Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
|
12
|
+
ruby: [2.6, 2.7, '3.0']
|
13
|
+
runs-on: ${{ matrix.os }}
|
8
14
|
steps:
|
9
15
|
- uses: actions/checkout@v2
|
10
16
|
- name: Set up Ruby
|
11
17
|
uses: ruby/setup-ruby@v1
|
12
18
|
with:
|
13
|
-
ruby-version:
|
14
|
-
bundler-cache: true
|
15
|
-
cache-version: 1
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
16
21
|
- name: Run the default task
|
17
22
|
run: bundle exec rake
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,21 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
|
3
|
+
## [0.8] - 2021-09-08
|
4
|
+
- Using PStore for future jobs backup and job history. Job history is rotated daily, future jobs cannot at the moment.
|
5
|
+
- PStore has ultrasafe mode, should test whether that makes things very slow. If not, might be worth using it for the peace of mind. Or at least have it as an option.
|
6
|
+
- ActiveJob support is a bit more natural now code-wise.
|
7
|
+
|
8
|
+
## [0.7] - 2021-09-04
|
9
|
+
- ActiveJob support! Retries, exception rescuing should work as expected.
|
10
|
+
- Second Moderna jab took me out for a while... sorry for the long wait.
|
11
|
+
## [0.6.7] - 2021-08-25
|
12
|
+
|
13
|
+
- A bug fix for bad jobs bringing down client side.
|
14
|
+
- Heartbeat option for server side. It only affects jobs in the wait list, it's used to determine how often to check the future jobs queue.
|
15
|
+
## [0.6.6] - 2021-08-25
|
16
|
+
|
17
|
+
- Tests now run agains Ruby 2.6, so relaxing the version constraint.
|
18
|
+
- Add client_heartbeat option, so you can define how often you want jobs sent to the Belated server.
|
3
19
|
## [0.6.5] - 2021-08-23
|
4
20
|
|
5
21
|
- Timezone used inside Belated is all using the server time now, so it's up to the user to take care of that(using `Time.now` instead of `Time.now.utc`)
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
belated (0.
|
4
|
+
belated (0.8.0)
|
5
5
|
drb
|
6
6
|
dry-configurable
|
7
|
+
pstore
|
8
|
+
ruby2_keywords
|
7
9
|
sorted_set
|
8
10
|
|
9
11
|
GEM
|
@@ -97,13 +99,20 @@ GEM
|
|
97
99
|
marcel (1.0.1)
|
98
100
|
method_source (1.0.0)
|
99
101
|
mini_mime (1.1.0)
|
102
|
+
mini_portile2 (2.6.1)
|
100
103
|
minitest (5.14.4)
|
101
104
|
nio4r (2.5.7)
|
102
|
-
nokogiri (1.
|
105
|
+
nokogiri (1.12.3)
|
106
|
+
mini_portile2 (~> 2.6.1)
|
107
|
+
racc (~> 1.4)
|
108
|
+
nokogiri (1.12.3-x86_64-darwin)
|
109
|
+
racc (~> 1.4)
|
110
|
+
nokogiri (1.12.3-x86_64-linux)
|
103
111
|
racc (~> 1.4)
|
104
112
|
parallel (1.20.1)
|
105
113
|
parser (3.0.2.0)
|
106
114
|
ast (~> 2.4.1)
|
115
|
+
pstore (0.1.1)
|
107
116
|
racc (1.5.2)
|
108
117
|
rack (2.2.3)
|
109
118
|
rack-test (1.1.0)
|
@@ -178,6 +187,7 @@ GEM
|
|
178
187
|
rubocop (~> 1.0)
|
179
188
|
rubocop-ast (>= 1.1.0)
|
180
189
|
ruby-progressbar (1.11.0)
|
190
|
+
ruby2_keywords (0.0.5)
|
181
191
|
set (1.0.1)
|
182
192
|
sorted_set (1.0.3)
|
183
193
|
rbtree
|
@@ -190,7 +200,6 @@ GEM
|
|
190
200
|
activesupport (>= 4.0)
|
191
201
|
sprockets (>= 3.0.0)
|
192
202
|
sqlite3 (1.4.2)
|
193
|
-
stackprof (0.2.16)
|
194
203
|
thor (1.1.0)
|
195
204
|
tzinfo (2.0.4)
|
196
205
|
concurrent-ruby (~> 1.0)
|
@@ -201,6 +210,8 @@ GEM
|
|
201
210
|
zeitwerk (2.4.2)
|
202
211
|
|
203
212
|
PLATFORMS
|
213
|
+
ruby
|
214
|
+
x86_64-darwin-19
|
204
215
|
x86_64-linux
|
205
216
|
|
206
217
|
DEPENDENCIES
|
@@ -214,7 +225,6 @@ DEPENDENCIES
|
|
214
225
|
rubocop (~> 1.7)
|
215
226
|
rubocop-discourse
|
216
227
|
sqlite3
|
217
|
-
stackprof
|
218
228
|
|
219
229
|
BUNDLED WITH
|
220
230
|
2.2.22
|
data/README.md
CHANGED
@@ -2,31 +2,26 @@
|
|
2
2
|
|
3
3
|
[![CodeFactor](https://www.codefactor.io/repository/github/sampokuokkanen/belated/badge)](https://www.codefactor.io/repository/github/sampokuokkanen/belated) [![Gem Version](https://badge.fury.io/rb/belated.svg)](https://badge.fury.io/rb/belated)
|
4
4
|
|
5
|
-
This is Belated, a new Ruby backend job library! It supports running procs, lambdas and classes in the background. To deal with restarts, it uses YAML to load the
|
5
|
+
This is Belated, a new Ruby backend job library! It supports running procs, lambdas and classes in the background. To deal with restarts, it uses YAML for the current jobs in the queue waiting to be processed and PStore for the future jobs to load the queues into a file, which it then calls at startup to find the previous jobs. There is no way in Ruby to save procs or lambdas to a file, so they are discarded when the process restarts.
|
6
6
|
|
7
|
-
Belated uses the Ruby Queue class, so it's First In, First Out (FIFO).
|
8
|
-
|
9
|
-
Note that Belated used to be called HardWorker. That name was already in use in Sidekiq documentation and a bit too generic anyway.
|
7
|
+
Belated uses the Ruby Queue class, so it's First In, First Out (FIFO), unless of course you want to run the job in the future. In that case the order is decided by the time the job is scheduled to be executed.
|
10
8
|
|
11
9
|
It uses dRuby to do the communication! Which is absolute great. No need for Redis or PostgreSQL, just Ruby standard libraries.
|
12
10
|
|
13
|
-
Note that currently the timezone is hardcoded to UTC.
|
14
|
-
|
15
11
|
Can be used with or without Rails.
|
16
12
|
|
17
13
|
Can be used if you're on a normal instance such as EC2 or Digital Ocean drop. Not if you're on a Heroku or Docker, or anything with ephemeral storage.
|
18
14
|
|
19
15
|
TODO LIST:
|
20
16
|
|
21
|
-
-
|
22
|
-
- Maybe support ActiveJob?
|
23
|
-
- Have a web UI.
|
24
|
-
- Have a job history
|
25
|
-
- Do some performance testing.
|
17
|
+
- Have a web UI with job history.
|
26
18
|
- Deploy a Rails app to production that is using Belated
|
27
19
|
and mention it in the readme. (Capistrano support?)
|
28
|
-
([Wasurechatta](https://wasurechatta.com/))
|
29
|
-
|
20
|
+
([Wasurechatta](https://wasurechatta.com/) deployed, still need to setup Capistrano)
|
21
|
+
|
22
|
+
# Why not Sidekiq?
|
23
|
+
|
24
|
+
If you can, definitely use Sidekiq!!! Belated is supposed to be used if you can't get anything else to work. Like if you want to use SQLite in a Rails app and don't want to have Redis running. Or maybe you just want to run procs in the background?
|
30
25
|
|
31
26
|
## Installation
|
32
27
|
|
@@ -50,7 +45,15 @@ Start up Belated!
|
|
50
45
|
|
51
46
|
$ belated
|
52
47
|
|
53
|
-
|
48
|
+
If you're using Rails, just set Belated to be the ActiveJob adapter like below:
|
49
|
+
|
50
|
+
```ruby
|
51
|
+
config.active_job.adapter = :belated
|
52
|
+
```
|
53
|
+
|
54
|
+
And you're good to go!
|
55
|
+
|
56
|
+
If not, in your non-ActiveJob using program, connect to Belated and give it a job to do.
|
54
57
|
Sample below:
|
55
58
|
|
56
59
|
```ruby
|
@@ -136,8 +139,10 @@ Other available settings:
|
|
136
139
|
|
137
140
|
$ bundle exec belated --host=1.1.1.1 --port=1234
|
138
141
|
# druby://1.1.1.1:1234
|
139
|
-
$ bundle exec belated --env
|
142
|
+
$ bundle exec belated --env=test
|
140
143
|
# environment
|
144
|
+
$ bundle exec belated --client_heartbeat=10
|
145
|
+
# how often client sends jobs to server, default is once every 5 seconds
|
141
146
|
|
142
147
|
|
143
148
|
Number of workers.
|
data/belated.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
|
|
15
15
|
).gsub(/\s+/, ' ').strip
|
16
16
|
spec.homepage = 'https://github.com/sampokuokkanen/belated'
|
17
17
|
spec.license = 'MIT'
|
18
|
-
spec.required_ruby_version = Gem::Requirement.new('>= 2.
|
18
|
+
spec.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
|
19
19
|
|
20
20
|
spec.metadata['homepage_uri'] = spec.homepage
|
21
21
|
spec.metadata['source_code_uri'] = spec.homepage
|
@@ -33,6 +33,8 @@ Gem::Specification.new do |spec|
|
|
33
33
|
# Uncomment to register a new dependency of your gem
|
34
34
|
spec.add_dependency 'drb'
|
35
35
|
spec.add_dependency 'dry-configurable'
|
36
|
+
spec.add_dependency 'ruby2_keywords'
|
37
|
+
spec.add_dependency 'pstore'
|
36
38
|
spec.add_dependency 'sorted_set'
|
37
39
|
spec.add_development_dependency 'byebug'
|
38
40
|
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'belated'
|
4
|
+
|
5
|
+
module ActiveJob # :nodoc:
|
6
|
+
module QueueAdapters # :nodoc:
|
7
|
+
# The adapter in charge of handling ActiveJob integration.
|
8
|
+
# WIP
|
9
|
+
class BelatedAdapter
|
10
|
+
def instance
|
11
|
+
@instance ||= Belated::Client.instance
|
12
|
+
rescue StandardError
|
13
|
+
@instance = Belated::Client.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def enqueue(job) # :nodoc:
|
17
|
+
Rails.logger.info "Belated got job #{job}"
|
18
|
+
instance.perform(job, active_job: true)
|
19
|
+
end
|
20
|
+
|
21
|
+
def enqueue_at(job, timestamp) # :nodoc:
|
22
|
+
Rails.logger.info "Belated got job #{job} to be performed at #{Time.at(timestamp)}"
|
23
|
+
instance.perform_belated(job, at: timestamp, active_job: true)
|
24
|
+
end
|
25
|
+
|
26
|
+
# JobWrapper that overwrites perform for ActiveJob
|
27
|
+
class JobWrapper < Belated::JobWrapper
|
28
|
+
def perform
|
29
|
+
Base.execute job.serialize
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
data/lib/belated/client.rb
CHANGED
@@ -47,15 +47,12 @@ class Belated
|
|
47
47
|
Thread.new do
|
48
48
|
loop do
|
49
49
|
delete_from_table
|
50
|
-
sleep
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
bank.push(wrapper)
|
57
|
-
sleep 5
|
58
|
-
end
|
50
|
+
sleep Belated.client_heartbeat and next if bank.empty?
|
51
|
+
|
52
|
+
bank.length.times do
|
53
|
+
queue.push(wrapper = bank.pop)
|
54
|
+
rescue DRb::DRbConnError
|
55
|
+
bank.push(wrapper)
|
59
56
|
end
|
60
57
|
end
|
61
58
|
end
|
@@ -77,10 +74,11 @@ class Belated
|
|
77
74
|
# @param at [Date] - The time at which the job should be executed.
|
78
75
|
# @param max_retries [Integer] - Times the job should be retried if it fails.
|
79
76
|
# @return [JobWrapper] - The job wrapper for the queue.
|
80
|
-
def perform(job, at: nil, max_retries: 5)
|
77
|
+
def perform(job, at: nil, max_retries: 5, active_job: false)
|
81
78
|
start unless started?
|
82
|
-
|
83
|
-
|
79
|
+
return unless proper_job?(job)
|
80
|
+
|
81
|
+
job_wrapper = wrap_job(job, at: at.to_f, max_retries: max_retries, active_job: active_job)
|
84
82
|
bank.push(job_wrapper)
|
85
83
|
@mutex.synchronize do
|
86
84
|
proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
|
@@ -93,16 +91,22 @@ class Belated
|
|
93
91
|
|
94
92
|
private
|
95
93
|
|
96
|
-
def
|
97
|
-
return if job.respond_to?(:call) || job.respond_to?(:perform)
|
94
|
+
def proper_job?(job)
|
95
|
+
return true if job.respond_to?(:call) || job.respond_to?(:perform)
|
98
96
|
|
99
|
-
|
97
|
+
warn 'job does not implement .call nor .perform!'
|
98
|
+
false
|
100
99
|
end
|
101
100
|
|
102
|
-
def wrap_job(job, at:, max_retries:)
|
101
|
+
def wrap_job(job, at:, max_retries:, active_job:)
|
103
102
|
return job if job.is_a?(JobWrapper)
|
104
103
|
|
105
|
-
|
104
|
+
wrapper = if active_job
|
105
|
+
ActiveJob::QueueAdapters::BelatedAdapter::JobWrapper
|
106
|
+
else
|
107
|
+
JobWrapper
|
108
|
+
end
|
109
|
+
wrapper.new(job: job, at: at, max_retries: max_retries, active_job: active_job)
|
106
110
|
end
|
107
111
|
|
108
112
|
def drb_connected?
|
data/lib/belated/job_wrapper.rb
CHANGED
@@ -13,47 +13,60 @@ class Belated
|
|
13
13
|
class JobWrapper
|
14
14
|
include Comparable
|
15
15
|
include Logging
|
16
|
-
attr_accessor :retries, :max_retries, :id, :job, :at,
|
16
|
+
attr_accessor :retries, :max_retries, :id, :job, :at,
|
17
|
+
:completed, :proc_klass, :error, :active_job
|
18
|
+
|
19
|
+
def initialize(job:, max_retries: 5, at: nil, active_job: false)
|
20
|
+
raise 'JobError' unless job.respond_to?(:call) || job.respond_to?(:perform)
|
17
21
|
|
18
|
-
def initialize(job:, max_retries: 5, at: nil)
|
19
22
|
self.retries = 0
|
20
23
|
self.max_retries = max_retries
|
21
|
-
self.id = SecureRandom.uuid
|
24
|
+
self.id = job.respond_to?(:job_id) ? job.job_id : SecureRandom.uuid
|
22
25
|
self.job = job
|
23
26
|
self.at = at
|
24
|
-
self.completed = false
|
25
27
|
self.proc_klass = job.instance_of?(Proc)
|
28
|
+
self.active_job = active_job
|
26
29
|
end
|
27
30
|
|
28
31
|
def <=>(other)
|
29
|
-
at <=> other
|
32
|
+
at <=> (other&.at || other&.scheduled_at)
|
30
33
|
end
|
31
34
|
|
32
35
|
# rubocop:disable Lint/RescueException
|
33
36
|
def perform
|
34
|
-
resp =
|
35
|
-
job.call
|
36
|
-
else
|
37
|
-
job.perform
|
38
|
-
end
|
37
|
+
resp = execute
|
39
38
|
self.completed = true
|
40
39
|
resp
|
41
40
|
rescue Exception => e
|
42
41
|
case e.class
|
43
|
-
when Interrupt, SignalException
|
42
|
+
when Interrupt, SignalException, NoMethodError
|
44
43
|
raise e
|
45
44
|
else
|
46
|
-
retry_job
|
47
|
-
"Error while executing job, #{e.inspect}. Retry #{retries} of #{max_retries}"
|
45
|
+
retry_job(e)
|
46
|
+
"Error while executing job #{job.inspect}, #{e.inspect}. Retry #{retries} of #{max_retries}"
|
48
47
|
end
|
49
48
|
end
|
49
|
+
|
50
50
|
# rubocop:enable Lint/RescueException
|
51
|
+
def execute
|
52
|
+
if job.respond_to?(:call)
|
53
|
+
job.call
|
54
|
+
elsif job.respond_to?(:arguments)
|
55
|
+
job.perform(*job.arguments)
|
56
|
+
else
|
57
|
+
job.perform
|
58
|
+
end
|
59
|
+
end
|
51
60
|
|
52
|
-
def retry_job
|
61
|
+
def retry_job(error)
|
53
62
|
self.retries += 1
|
54
|
-
|
63
|
+
if retries > max_retries
|
64
|
+
self.error = error
|
65
|
+
return
|
66
|
+
end
|
55
67
|
|
56
|
-
|
68
|
+
seconds_to_retry = $TESTING ? 0.05 : retries.next**4
|
69
|
+
self.at = (Time.now + seconds_to_retry).to_f
|
57
70
|
log "Job #{id} failed, retrying at #{at}"
|
58
71
|
Belated.job_list.push(self)
|
59
72
|
end
|
data/lib/belated/queue.rb
CHANGED
@@ -4,7 +4,7 @@ require 'belated/job'
|
|
4
4
|
require 'belated/logging'
|
5
5
|
require 'belated/job_wrapper'
|
6
6
|
require 'sorted_set'
|
7
|
-
|
7
|
+
require 'pstore'
|
8
8
|
class Belated
|
9
9
|
# Job queues that Belated uses.
|
10
10
|
# queue is the jobs that are currenly
|
@@ -13,21 +13,43 @@ class Belated
|
|
13
13
|
# to be added to queue at some point in the future.
|
14
14
|
class Queue
|
15
15
|
include Logging
|
16
|
-
attr_accessor :future_jobs
|
16
|
+
attr_accessor :future_jobs, :future_jobs_db
|
17
17
|
|
18
18
|
FILE_NAME = 'belated_dump'
|
19
19
|
|
20
20
|
def initialize(queue: Thread::Queue.new, future_jobs: SortedSet.new)
|
21
21
|
@queue = queue
|
22
|
+
@mutex = Mutex.new
|
22
23
|
self.future_jobs = future_jobs
|
24
|
+
self.future_jobs_db = PStore.new('future_jobs.pstore', true) # pass true for thread safety
|
25
|
+
end
|
26
|
+
|
27
|
+
def enqueue_future_jobs
|
28
|
+
loop do
|
29
|
+
job = future_jobs.min
|
30
|
+
if job.nil?
|
31
|
+
sleep Belated.heartbeat
|
32
|
+
next
|
33
|
+
end
|
34
|
+
if job.at <= Time.now.to_f
|
35
|
+
delete_job(job)
|
36
|
+
push(job)
|
37
|
+
end
|
38
|
+
rescue DRb::DRbConnError
|
39
|
+
error 'DRb connection error!!!!!!'
|
40
|
+
log stats
|
41
|
+
end
|
23
42
|
end
|
24
43
|
|
25
44
|
def push(job)
|
26
45
|
if job.is_a?(Symbol) || job.at.nil? ||
|
27
|
-
job.at <= Time.now.
|
46
|
+
job.at <= Time.now.to_f
|
28
47
|
@queue.push(job)
|
29
48
|
else
|
30
|
-
@
|
49
|
+
@mutex.synchronize do
|
50
|
+
@future_jobs << job
|
51
|
+
insert_into_future_jobs_db(job) unless job.proc_klass
|
52
|
+
end
|
31
53
|
end
|
32
54
|
end
|
33
55
|
|
@@ -49,16 +71,16 @@ class Belated
|
|
49
71
|
end
|
50
72
|
|
51
73
|
def load_jobs
|
52
|
-
|
74
|
+
future_jobs_db.transaction(true) do
|
75
|
+
future_jobs_db.roots.each do |id|
|
76
|
+
future_jobs << future_jobs_db[id]
|
77
|
+
end
|
78
|
+
end
|
53
79
|
return unless File.exist?(FILE_NAME)
|
54
80
|
|
55
81
|
jobs = YAML.load(File.binread(FILE_NAME))
|
56
82
|
jobs.each do |job|
|
57
|
-
|
58
|
-
future_jobs.push(job)
|
59
|
-
else
|
60
|
-
@queue.push(job)
|
61
|
-
end
|
83
|
+
@queue.push(job)
|
62
84
|
end
|
63
85
|
File.delete(FILE_NAME)
|
64
86
|
end
|
@@ -70,12 +92,6 @@ class Belated
|
|
70
92
|
class_array << klass
|
71
93
|
end
|
72
94
|
end
|
73
|
-
future_jobs.each do |_job|
|
74
|
-
unless proc_or_shutdown?(klass = future_jobs.pop)
|
75
|
-
class_array << klass
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
95
|
pp File.open(FILE_NAME, 'wb') { |f| f.write(YAML.dump(class_array)) }
|
80
96
|
end
|
81
97
|
|
@@ -88,5 +104,18 @@ class Belated
|
|
88
104
|
def proc_or_shutdown?(job)
|
89
105
|
job.is_a?(Symbol) || job.job.instance_of?(Proc)
|
90
106
|
end
|
107
|
+
|
108
|
+
def delete_job(job)
|
109
|
+
log "Deleting #{future_jobs.delete(job)} from future jobs"
|
110
|
+
future_jobs_db.transaction do
|
111
|
+
future_jobs_db.delete(job.id)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def insert_into_future_jobs_db(job)
|
116
|
+
future_jobs_db.transaction do
|
117
|
+
future_jobs_db[job.id] = job
|
118
|
+
end
|
119
|
+
end
|
91
120
|
end
|
92
121
|
end
|
data/lib/belated/testing.rb
CHANGED
@@ -22,7 +22,7 @@ class Belated
|
|
22
22
|
# A client that can perform jobs inline
|
23
23
|
class Client
|
24
24
|
alias old_perform perform
|
25
|
-
def perform(job, at: nil, max_retries: 5)
|
25
|
+
def perform(job, at: nil, max_retries: 5, active_job: false)
|
26
26
|
if Belated::Testing.inline?
|
27
27
|
if job.respond_to?(:call)
|
28
28
|
job.call
|
@@ -30,7 +30,7 @@ class Belated
|
|
30
30
|
job.perform
|
31
31
|
end
|
32
32
|
else
|
33
|
-
old_perform(job, at: at, max_retries: max_retries)
|
33
|
+
old_perform(job, at: at, max_retries: max_retries, active_job: active_job)
|
34
34
|
end
|
35
35
|
end
|
36
36
|
end
|
data/lib/belated/version.rb
CHANGED
data/lib/belated/worker.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
require_relative 'logging'
|
2
|
+
require 'pstore'
|
3
|
+
|
2
4
|
class Belated
|
3
5
|
# The worker class that actually gets the jobs from the queue
|
4
6
|
# and calls them. Expects the jobs to be procs or
|
@@ -20,9 +22,27 @@ class Belated
|
|
20
22
|
|
21
23
|
log "Worker #{@number} got job: #{job.inspect}"
|
22
24
|
log job.perform
|
25
|
+
history_insert(job) unless job.proc_klass || !job.completed
|
23
26
|
rescue DRb::DRbConnError, Errno::ECONNREFUSED, RangeError => e
|
24
27
|
log e
|
25
28
|
end
|
26
29
|
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def history_insert(job)
|
34
|
+
store.transaction do
|
35
|
+
store[job.id] = job
|
36
|
+
end
|
37
|
+
rescue StandardError => e
|
38
|
+
error e
|
39
|
+
end
|
40
|
+
|
41
|
+
def store
|
42
|
+
today = Time.now.strftime('%F')
|
43
|
+
return @store if @store&.path&.include?(today)
|
44
|
+
|
45
|
+
@store = PStore.new("history-#{today}.pstore", true)
|
46
|
+
end
|
27
47
|
end
|
28
48
|
end
|
data/lib/belated.rb
CHANGED
@@ -33,6 +33,8 @@ class Belated
|
|
33
33
|
setting :log_level, :info, reader: true
|
34
34
|
setting :host, 'localhost', reader: true
|
35
35
|
setting :port, '8788', reader: true
|
36
|
+
setting :heartbeat, 1, reader: true
|
37
|
+
setting :client_heartbeat, 5, reader: true
|
36
38
|
URI = "druby://#{Belated.host}:#{Belated.port}"
|
37
39
|
|
38
40
|
# Since it's running as a singleton, we need something to start it up.
|
@@ -49,18 +51,21 @@ class Belated
|
|
49
51
|
connect!
|
50
52
|
banner_and_info
|
51
53
|
trap_signals
|
52
|
-
enqueue_future_jobs
|
54
|
+
@@queue.enqueue_future_jobs
|
53
55
|
end
|
54
56
|
alias initialize start
|
55
57
|
|
56
58
|
# Handles connection to DRb server.
|
57
59
|
def connect!
|
60
|
+
i = 0
|
58
61
|
DRb.start_service(URI, @@queue, verbose: true)
|
59
62
|
rescue DRb::DRbConnError, Errno::EADDRINUSE
|
63
|
+
sleep 0.1 and retry if (i += 1) < 5
|
60
64
|
error 'Could not connect to DRb server.'
|
61
65
|
end
|
62
66
|
|
63
67
|
def trap_signals
|
68
|
+
pp 'trap'
|
64
69
|
%w[INT TERM].each do |signal|
|
65
70
|
Signal.trap(signal) do
|
66
71
|
@worker_list.length.times do
|
@@ -90,24 +95,6 @@ class Belated
|
|
90
95
|
Belated.config.rails
|
91
96
|
end
|
92
97
|
|
93
|
-
def enqueue_future_jobs
|
94
|
-
loop do
|
95
|
-
sleep 0.1
|
96
|
-
job = @@queue.future_jobs.min
|
97
|
-
if job.nil?
|
98
|
-
sleep 5
|
99
|
-
next
|
100
|
-
end
|
101
|
-
if job.at <= Time.now
|
102
|
-
log "Deleting #{@@queue.future_jobs.delete(job)} from future jobs"
|
103
|
-
@@queue.push(job)
|
104
|
-
end
|
105
|
-
rescue DRb::DRbConnError
|
106
|
-
error 'DRb connection error!!!!!!'
|
107
|
-
log stats
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
98
|
def reload
|
112
99
|
log 'reloading...'
|
113
100
|
@@queue.load_jobs
|
@@ -153,19 +140,25 @@ class Belated
|
|
153
140
|
}
|
154
141
|
end
|
155
142
|
|
156
|
-
|
157
|
-
|
158
|
-
|
143
|
+
class << self
|
144
|
+
def find(job_id)
|
145
|
+
@@queue.future_jobs.find { |job| job.id == job_id }
|
159
146
|
end
|
160
|
-
clear_queue!
|
161
|
-
end
|
162
147
|
|
163
|
-
|
164
|
-
|
165
|
-
|
148
|
+
def kill_and_clear_queue!
|
149
|
+
@worker_list&.each do |worker|
|
150
|
+
Thread.kill(worker)
|
151
|
+
end
|
152
|
+
clear_queue!
|
153
|
+
end
|
166
154
|
|
167
|
-
|
168
|
-
|
155
|
+
def clear_queue!
|
156
|
+
@@queue.clear
|
157
|
+
end
|
158
|
+
|
159
|
+
def fetch_job
|
160
|
+
@@queue.pop
|
161
|
+
end
|
169
162
|
end
|
170
163
|
|
171
164
|
def job_list
|
@@ -176,5 +169,4 @@ class Belated
|
|
176
169
|
@@queue
|
177
170
|
end
|
178
171
|
end
|
179
|
-
|
180
|
-
require 'belated/rails' if defined?(::Rails::Engine)
|
172
|
+
require 'active_job/queue_adapters/belated_adapter' if defined?(::Rails)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: belated
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sampo Kuokkanen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-08
|
11
|
+
date: 2021-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: drb
|
@@ -38,6 +38,34 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: ruby2_keywords
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: pstore
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
41
69
|
- !ruby/object:Gem::Dependency
|
42
70
|
name: sorted_set
|
43
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -92,6 +120,7 @@ files:
|
|
92
120
|
- bin/bundle
|
93
121
|
- bin/console
|
94
122
|
- bin/setup
|
123
|
+
- lib/active_job/queue_adapters/belated_adapter.rb
|
95
124
|
- lib/belated.rb
|
96
125
|
- lib/belated/client.rb
|
97
126
|
- lib/belated/exceptions.rb
|
@@ -118,7 +147,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
118
147
|
requirements:
|
119
148
|
- - ">="
|
120
149
|
- !ruby/object:Gem::Version
|
121
|
-
version: 2.
|
150
|
+
version: 2.6.0
|
122
151
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
123
152
|
requirements:
|
124
153
|
- - ">="
|