belated 0.5.4 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +2 -1
- data/.rubocop.yml +6 -4
- data/CHANGELOG.md +18 -1
- data/Gemfile +1 -1
- data/Gemfile.lock +17 -8
- data/README.md +13 -3
- data/belated.gemspec +1 -0
- data/lib/belated.rb +14 -12
- data/lib/belated/client.rb +31 -11
- data/lib/belated/job_wrapper.rb +23 -6
- data/lib/belated/logging.rb +2 -0
- data/lib/belated/queue.rb +10 -3
- data/lib/belated/version.rb +1 -1
- data/lib/belated/worker.rb +4 -2
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4ca7f7d0a3a78ca41e6ba3afed7630e1ac72d4a88c4fdea08604066518c843a6
|
4
|
+
data.tar.gz: 2fb05c90501681f9f45ab4476cbdc4c1ac88de9addafaa1c54da0a16731426df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5182420208c866b37511bda56cc5a2e8fddac8d87a23274923318f576b66f3d3848b33c6633296bbfed088e6bd339a458e92ccc057a63c8a815dfaf12aa3fdf7
|
7
|
+
data.tar.gz: 78ef5824e2afdd45910dbe81e79e54986dcc2c7ccf346515228c978a55c5d2aa0d0f29c4d894e219667ee6d6b735fbef43c94b61bd36d645a3e9ca1152bf6a86
|
data/.github/workflows/main.yml
CHANGED
data/.rubocop.yml
CHANGED
@@ -1,7 +1,12 @@
|
|
1
|
-
require: rubocop-performance
|
2
1
|
AllCops:
|
3
2
|
TargetRubyVersion: 2.7
|
4
3
|
NewCops: enable
|
4
|
+
Exclude:
|
5
|
+
- 'dummy/**/*'
|
6
|
+
- 'vendor/**/*'
|
7
|
+
|
8
|
+
Lint/HashCompareByIdentity:
|
9
|
+
Enabled: false
|
5
10
|
|
6
11
|
Style/StringLiterals:
|
7
12
|
Enabled: true
|
@@ -23,9 +28,6 @@ Style/AsciiComments:
|
|
23
28
|
Style/ClassVars:
|
24
29
|
Enabled: false
|
25
30
|
|
26
|
-
Style/Documentation:
|
27
|
-
Enabled: false
|
28
|
-
|
29
31
|
Style/GlobalVars:
|
30
32
|
Enabled: false
|
31
33
|
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,23 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
|
3
|
-
## [0.
|
3
|
+
## [0.6.0] - 2021-08-19
|
4
|
+
|
5
|
+
- Only need to keep references on the client side for procs. Not needed for classes, as they are pass-by-value. However, you can only pass procs by reference, so need to keep track of them. They're removed from the client side when they're completed though.
|
6
|
+
- The client is now a singleton. This is because it had some overhead when pushing the jobs to dRuby, so I took the approach of also doing that in a background thread. You however do not want more than one client to be running at the same time, so making it a singleton is the best option. Call the `.instance` method to get the singleton and then `.start` to get it started.
|
7
|
+
|
8
|
+
## [0.5.7] - 2021-08-18
|
9
|
+
|
10
|
+
- Got errors under heavy load and restarting. Hopefully fixed by rescuing the DRb connection error.
|
11
|
+
## [0.5.6] - 2021-08-17
|
12
|
+
|
13
|
+
- Now the client has a hash table that holds references to the objects you push through it. This is to get by GC, otherwise the objects are collected on clientside, but are still referenced on server side. This means that you do not want to use two instances of Client at the same time. Also, might need to write a way to close the client...
|
14
|
+
- Should work a bit more safely now! Performance testing 0.5.5 online using a Rails project, it performed horribly, so it should be a bit better now.
|
15
|
+
## [0.5.5] - 2021-08-15
|
16
|
+
|
17
|
+
- Use SortedSet for future jobs, to avoid having to go through the whole list every few seconds.
|
18
|
+
|
19
|
+
|
20
|
+
## [0.5.4] - 2021-08-13
|
4
21
|
|
5
22
|
- Client was using 100% CPU when it had no connection. (on $5 Digital Ocean droplet) Should be fixed now.
|
6
23
|
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
belated (0.
|
4
|
+
belated (0.6.0)
|
5
5
|
drb
|
6
6
|
dry-configurable
|
7
|
+
sorted_set
|
7
8
|
|
8
9
|
GEM
|
9
10
|
remote: https://rubygems.org/
|
@@ -135,6 +136,7 @@ GEM
|
|
135
136
|
thor (~> 1.0)
|
136
137
|
rainbow (3.0.0)
|
137
138
|
rake (13.0.6)
|
139
|
+
rbtree (0.4.4)
|
138
140
|
regexp_parser (2.1.1)
|
139
141
|
rexml (3.2.5)
|
140
142
|
rspec (3.10.0)
|
@@ -158,21 +160,28 @@ GEM
|
|
158
160
|
rspec-mocks (~> 3.10)
|
159
161
|
rspec-support (~> 3.10)
|
160
162
|
rspec-support (3.10.2)
|
161
|
-
rubocop (1.
|
163
|
+
rubocop (1.19.0)
|
162
164
|
parallel (~> 1.10)
|
163
165
|
parser (>= 3.0.0.0)
|
164
166
|
rainbow (>= 2.2.2, < 4.0)
|
165
167
|
regexp_parser (>= 1.8, < 3.0)
|
166
168
|
rexml
|
167
|
-
rubocop-ast (>= 1.
|
169
|
+
rubocop-ast (>= 1.9.1, < 2.0)
|
168
170
|
ruby-progressbar (~> 1.7)
|
169
171
|
unicode-display_width (>= 1.4.0, < 3.0)
|
170
|
-
rubocop-ast (1.
|
172
|
+
rubocop-ast (1.10.0)
|
171
173
|
parser (>= 3.0.1.1)
|
172
|
-
rubocop-
|
173
|
-
rubocop (>= 1.
|
174
|
-
rubocop-
|
174
|
+
rubocop-discourse (2.4.2)
|
175
|
+
rubocop (>= 1.1.0)
|
176
|
+
rubocop-rspec (>= 2.0.0)
|
177
|
+
rubocop-rspec (2.4.0)
|
178
|
+
rubocop (~> 1.0)
|
179
|
+
rubocop-ast (>= 1.1.0)
|
175
180
|
ruby-progressbar (1.11.0)
|
181
|
+
set (1.0.1)
|
182
|
+
sorted_set (1.0.3)
|
183
|
+
rbtree
|
184
|
+
set (~> 1.0)
|
176
185
|
sprockets (4.0.2)
|
177
186
|
concurrent-ruby (~> 1.0)
|
178
187
|
rack (> 1, < 3)
|
@@ -203,7 +212,7 @@ DEPENDENCIES
|
|
203
212
|
rspec (~> 3.0)
|
204
213
|
rspec-rails
|
205
214
|
rubocop (~> 1.7)
|
206
|
-
rubocop-
|
215
|
+
rubocop-discourse
|
207
216
|
sqlite3
|
208
217
|
stackprof
|
209
218
|
|
data/README.md
CHANGED
@@ -16,8 +16,10 @@ Can be used with or without Rails.
|
|
16
16
|
|
17
17
|
TODO LIST:
|
18
18
|
|
19
|
+
- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup.
|
20
|
+
- Rescue `DRb::DRbRemoteError` when shutting down, might not need to if using GDBM?
|
19
21
|
- Don't use class instance variables.
|
20
|
-
- Make port configurable.
|
22
|
+
- Make DRb port configurable.
|
21
23
|
- Don't hardcode timezone.
|
22
24
|
- Add some checks to the client for proper jobs.
|
23
25
|
- Have multiple queues?
|
@@ -27,6 +29,7 @@ TODO LIST:
|
|
27
29
|
- Do some performance testing.
|
28
30
|
- Deploy a Rails app to production that is using Belated
|
29
31
|
and mention it in the readme. (Capistrano support?)
|
32
|
+
([Wasurechatta](https://wasurechatta.com/))
|
30
33
|
- Add a section telling people to use Sidekiq if they can
|
31
34
|
|
32
35
|
## Installation
|
@@ -82,10 +85,17 @@ First, start up Belated.
|
|
82
85
|
Then,
|
83
86
|
|
84
87
|
```ruby
|
85
|
-
|
88
|
+
# Get the client
|
89
|
+
client = Belated::Client.instance
|
90
|
+
# Start the client, only need to do this once
|
91
|
+
client.start unless client.started?
|
86
92
|
```
|
87
93
|
|
88
94
|
and you can use the client!
|
95
|
+
Note that the client is a singleton.
|
96
|
+
This means that you can only have one client running at a time,
|
97
|
+
but it also means you only have one connection to dRuby, and that the number of threads in charge of queuing the jobs is only one.
|
98
|
+
|
89
99
|
Call
|
90
100
|
|
91
101
|
```ruby
|
@@ -101,7 +111,7 @@ If you don't want the job to run right away, you can also pass it a keyword para
|
|
101
111
|
client.perform_belated(job, Time.now + 1.month)
|
102
112
|
```
|
103
113
|
|
104
|
-
|
114
|
+
The client also holds references to the jobs that are instances of `Proc` that have been pushed so that they are not collected by GC. This is because procs are passed by reference, and the client needs to keep them alive. They are removed from the list when the job is done.
|
105
115
|
|
106
116
|
# Settings
|
107
117
|
|
data/belated.gemspec
CHANGED
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
|
|
33
33
|
# Uncomment to register a new dependency of your gem
|
34
34
|
spec.add_dependency 'drb'
|
35
35
|
spec.add_dependency 'dry-configurable'
|
36
|
+
spec.add_dependency 'sorted_set'
|
36
37
|
spec.add_development_dependency 'byebug'
|
37
38
|
|
38
39
|
# For more information and examples about making a new gem, checkout our
|
data/lib/belated.rb
CHANGED
@@ -65,9 +65,9 @@ class Belated
|
|
65
65
|
@@queue.push(:shutdown)
|
66
66
|
end
|
67
67
|
Thread.new { stop_workers }
|
68
|
-
# Max
|
68
|
+
# Max 40 seconds to shutdown
|
69
69
|
timeout = 0
|
70
|
-
until (timeout += 0.1) >=
|
70
|
+
until (timeout += 0.1) >= 40 || @@queue.empty? || $TESTING
|
71
71
|
sleep 0.1
|
72
72
|
end
|
73
73
|
exit
|
@@ -89,19 +89,20 @@ class Belated
|
|
89
89
|
end
|
90
90
|
|
91
91
|
def enqueue_future_jobs
|
92
|
-
log 'starting future jobs thread'
|
93
92
|
loop do
|
94
93
|
sleep 0.1
|
95
|
-
|
96
|
-
|
94
|
+
job = @@queue.future_jobs.min
|
95
|
+
if job.nil?
|
96
|
+
sleep 5
|
97
97
|
next
|
98
98
|
end
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
@@queue.push(job)
|
103
|
-
end
|
99
|
+
if job.at <= Time.now.utc
|
100
|
+
log "Deleting #{@@queue.future_jobs.delete(job)} from future jobs"
|
101
|
+
@@queue.push(job)
|
104
102
|
end
|
103
|
+
rescue DRb::DRbConnError
|
104
|
+
log 'DRb connection error!!!!!!'
|
105
|
+
log stats
|
105
106
|
end
|
106
107
|
end
|
107
108
|
|
@@ -112,7 +113,8 @@ class Belated
|
|
112
113
|
|
113
114
|
def stop_workers
|
114
115
|
@worker_list&.each do |worker|
|
115
|
-
|
116
|
+
i = 0
|
117
|
+
sleep 0.1 while worker.alive? || (i + 0.1) < 10
|
116
118
|
Thread.kill(worker)
|
117
119
|
end
|
118
120
|
@@queue.save_jobs
|
@@ -138,7 +140,7 @@ class Belated
|
|
138
140
|
|
139
141
|
def banner_and_info
|
140
142
|
log banner
|
141
|
-
log "Currently running Belated version #{Belated::VERSION}"
|
143
|
+
log "Currently running Belated version #{Belated::VERSION} in #{Belated.config.environment}"
|
142
144
|
log %(Belated running #{@worker_list&.length.to_i} workers on #{URI}...)
|
143
145
|
end
|
144
146
|
|
data/lib/belated/client.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'belated/job_wrapper'
|
2
|
+
require 'singleton'
|
2
3
|
class Belated
|
3
4
|
# The client class is responsible for managing the connection to the
|
4
5
|
# DRb server. If it has no connection, it adds the jobs to a bank queue.
|
@@ -7,16 +8,25 @@ class Belated
|
|
7
8
|
# client = Belated::Client.new
|
8
9
|
# client.enqueue(JubJub.new, at: Time.now + 5.seconds)
|
9
10
|
class Client
|
10
|
-
|
11
|
+
include Singleton unless $TESTING
|
12
|
+
|
13
|
+
attr_accessor :queue, :bank, :banker_thread, :proc_table
|
11
14
|
|
12
15
|
# Starts up the client.
|
13
16
|
# Connects to the queue through DRb.
|
14
17
|
# @return [void]
|
15
|
-
def
|
18
|
+
def start
|
16
19
|
server_uri = Belated::URI
|
17
20
|
DRb.start_service
|
21
|
+
self.proc_table = {}
|
18
22
|
self.bank = Thread::Queue.new
|
19
23
|
self.queue = DRbObject.new_with_uri(server_uri)
|
24
|
+
@started = true
|
25
|
+
end
|
26
|
+
alias initialize start
|
27
|
+
|
28
|
+
def started?
|
29
|
+
@started
|
20
30
|
end
|
21
31
|
|
22
32
|
# Thread in charge of handling the bank queue.
|
@@ -26,16 +36,25 @@ class Belated
|
|
26
36
|
def start_banker_thread
|
27
37
|
self.banker_thread = Thread.new do
|
28
38
|
loop do
|
29
|
-
|
30
|
-
|
31
|
-
sleep
|
39
|
+
delete_from_table
|
40
|
+
if bank.empty?
|
41
|
+
sleep 10
|
32
42
|
next
|
33
43
|
end
|
44
|
+
begin
|
45
|
+
queue.push(wrapper = bank.pop)
|
46
|
+
rescue DRb::DRbConnError
|
47
|
+
bank.push(wrapper)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
34
52
|
|
35
|
-
|
53
|
+
def delete_from_table
|
54
|
+
return if proc_table.empty?
|
36
55
|
|
37
|
-
|
38
|
-
|
56
|
+
proc_table.select { |_k, v| v.completed }.each do |key, _value|
|
57
|
+
proc_table.delete(key)
|
39
58
|
end
|
40
59
|
end
|
41
60
|
|
@@ -46,16 +65,17 @@ class Belated
|
|
46
65
|
# @param max_retries [Integer] - Times the job should be retried if it fails.
|
47
66
|
# @return [JobWrapper] - The job wrapper for the queue.
|
48
67
|
def perform(job, at: nil, max_retries: 5)
|
68
|
+
log 'Passing a proc and at time is deprecated and will be removed in 0.6' if job.instance_of?(Proc) && !at.nil?
|
69
|
+
|
49
70
|
job_wrapper = if job.is_a?(JobWrapper)
|
50
71
|
job
|
51
72
|
else
|
52
73
|
JobWrapper.new(job: job, at: at, max_retries: max_retries)
|
53
74
|
end
|
54
|
-
pp queue.push(job_wrapper)
|
55
|
-
job_wrapper
|
56
|
-
rescue DRb::DRbConnError
|
57
75
|
bank.push(job_wrapper)
|
76
|
+
proc_table[job_wrapper.object_id] = job_wrapper if job_wrapper.proc_klass
|
58
77
|
start_banker_thread if banker_thread.nil?
|
78
|
+
job_wrapper
|
59
79
|
end
|
60
80
|
alias perform_belated perform
|
61
81
|
alias perform_later perform
|
data/lib/belated/job_wrapper.rb
CHANGED
@@ -2,9 +2,18 @@ require 'securerandom'
|
|
2
2
|
require_relative 'logging'
|
3
3
|
|
4
4
|
class Belated
|
5
|
+
# JobWrapper is a wrapper for a job. It is responsible for
|
6
|
+
# - logging
|
7
|
+
# - error handling
|
8
|
+
# - job execution
|
9
|
+
# - job result handling
|
10
|
+
# - job result logging
|
11
|
+
# - job retries
|
12
|
+
# - job retry delay
|
5
13
|
class JobWrapper
|
14
|
+
include Comparable
|
6
15
|
include Logging
|
7
|
-
attr_accessor :retries, :max_retries, :id, :job, :at
|
16
|
+
attr_accessor :retries, :max_retries, :id, :job, :at, :completed, :proc_klass
|
8
17
|
|
9
18
|
def initialize(job:, max_retries: 5, at: nil)
|
10
19
|
self.retries = 0
|
@@ -12,15 +21,23 @@ class Belated
|
|
12
21
|
self.id = SecureRandom.uuid
|
13
22
|
self.job = job
|
14
23
|
self.at = at
|
24
|
+
self.completed = false
|
25
|
+
self.proc_klass = job.instance_of?(Proc)
|
26
|
+
end
|
27
|
+
|
28
|
+
def <=>(other)
|
29
|
+
at <=> other.at
|
15
30
|
end
|
16
31
|
|
17
32
|
# rubocop:disable Lint/RescueException
|
18
33
|
def perform
|
19
|
-
if job.respond_to?(:call)
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
34
|
+
resp = if job.respond_to?(:call)
|
35
|
+
job.call
|
36
|
+
else
|
37
|
+
job.perform
|
38
|
+
end
|
39
|
+
self.completed = true
|
40
|
+
resp
|
24
41
|
rescue Exception => e
|
25
42
|
case e.class
|
26
43
|
when Interrupt, SignalException
|
data/lib/belated/logging.rb
CHANGED
data/lib/belated/queue.rb
CHANGED
@@ -3,20 +3,27 @@
|
|
3
3
|
require 'belated/job'
|
4
4
|
require 'belated/logging'
|
5
5
|
require 'belated/job_wrapper'
|
6
|
+
require 'sorted_set'
|
7
|
+
|
6
8
|
class Belated
|
9
|
+
# Job queues that Belated uses.
|
10
|
+
# queue is the jobs that are currenly
|
11
|
+
# waiting for a worker to start working on them.
|
12
|
+
# future_jobs is a SortedSet of jobs that are going
|
13
|
+
# to be added to queue at some point in the future.
|
7
14
|
class Queue
|
8
15
|
include Logging
|
9
16
|
attr_accessor :future_jobs
|
10
17
|
|
11
18
|
FILE_NAME = 'belated_dump'
|
12
19
|
|
13
|
-
def initialize(queue: Thread::Queue.new, future_jobs:
|
20
|
+
def initialize(queue: Thread::Queue.new, future_jobs: SortedSet.new)
|
14
21
|
@queue = queue
|
15
22
|
self.future_jobs = future_jobs
|
16
23
|
end
|
17
24
|
|
18
25
|
def push(job)
|
19
|
-
if job
|
26
|
+
if job.is_a?(Symbol) || job.at.nil? ||
|
20
27
|
job.at <= Time.now.utc
|
21
28
|
@queue.push(job)
|
22
29
|
else
|
@@ -79,7 +86,7 @@ class Belated
|
|
79
86
|
private
|
80
87
|
|
81
88
|
def proc_or_shutdown?(job)
|
82
|
-
job.job.instance_of?(Proc)
|
89
|
+
job.is_a?(Symbol) || job.job.instance_of?(Proc)
|
83
90
|
end
|
84
91
|
end
|
85
92
|
end
|
data/lib/belated/version.rb
CHANGED
data/lib/belated/worker.rb
CHANGED
@@ -16,10 +16,12 @@ class Belated
|
|
16
16
|
log "Worker #{@number} fetching jobs!"
|
17
17
|
next unless (job = Belated.fetch_job)
|
18
18
|
|
19
|
-
break if job
|
19
|
+
break if job.is_a?(Symbol)
|
20
20
|
|
21
21
|
log "Worker #{@number} got job: #{job.inspect}"
|
22
|
-
job.perform
|
22
|
+
log job.perform
|
23
|
+
rescue Errno::ECONNREFUSED, RangeError => e
|
24
|
+
log e
|
23
25
|
end
|
24
26
|
end
|
25
27
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: belated
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sampo Kuokkanen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-08-
|
11
|
+
date: 2021-08-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: drb
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: sorted_set
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: byebug
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|