belated 0.5.2 → 0.5.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +2 -1
- data/.rubocop.yml +6 -3
- data/CHANGELOG.md +18 -1
- data/Gemfile +2 -2
- data/Gemfile.lock +17 -4
- data/README.md +14 -3
- data/belated.gemspec +1 -0
- data/lib/belated.rb +10 -6
- data/lib/belated/client.rb +31 -7
- data/lib/belated/job_wrapper.rb +22 -6
- data/lib/belated/logging.rb +2 -0
- data/lib/belated/queue.rb +14 -3
- data/lib/belated/version.rb +1 -1
- data/lib/belated/worker.rb +4 -2
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0ae6d21b68590fe6edb5ed817c1dd0806dadf5ee7320955cc3f4f652969f6501
|
4
|
+
data.tar.gz: 744ab565ddd144e5562ba39f675416e3dd17154b4ab70c916443d2711c148311
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b1baddc3b1a949af6420e6925e6e3b1565d006684d59333d89334f2e298c3f3eccdf0347402035417ee4faa300e643b4fc6824049cebabf8dab53fb2a49d0cf6
|
7
|
+
data.tar.gz: 065cd54c037f481a5d91a2da1800a721d40aceba6881315d261022b77d29898e6b738e72c885ef3876c6e8552fd2303a1b08270361fc624162c11c3b3b720ac2
|
data/.github/workflows/main.yml
CHANGED
data/.rubocop.yml
CHANGED
@@ -1,6 +1,12 @@
|
|
1
1
|
AllCops:
|
2
2
|
TargetRubyVersion: 2.7
|
3
3
|
NewCops: enable
|
4
|
+
Exclude:
|
5
|
+
- 'dummy/**/*'
|
6
|
+
- 'vendor/**/*'
|
7
|
+
|
8
|
+
Lint/HashCompareByIdentity:
|
9
|
+
Enabled: false
|
4
10
|
|
5
11
|
Style/StringLiterals:
|
6
12
|
Enabled: true
|
@@ -22,9 +28,6 @@ Style/AsciiComments:
|
|
22
28
|
Style/ClassVars:
|
23
29
|
Enabled: false
|
24
30
|
|
25
|
-
Style/Documentation:
|
26
|
-
Enabled: false
|
27
|
-
|
28
31
|
Style/GlobalVars:
|
29
32
|
Enabled: false
|
30
33
|
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,23 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
|
3
|
-
|
3
|
+
|
4
|
+
## [0.5.6] - 2021-08-17
|
5
|
+
|
6
|
+
- Now the client has a hash table that holds references to the objects you push through it. This is to get by GC, otherwise the objects are collected on clientside, but are still referenced on server side. This means that you do not want to use two instances of Client at the same time. Also, might need to write a way to close the client...
|
7
|
+
- Should work a bit more safely now! Performance testing 0.5.5 online using a Rails project, it performed horribly, so it should be a bit better now.
|
8
|
+
## [0.5.5] - 2021-08-15
|
9
|
+
|
10
|
+
- Use SortedSet for future jobs, to avoid having to go through the whole list every few seconds.
|
11
|
+
|
12
|
+
|
13
|
+
## [0.5.4] - 2021-08-13
|
14
|
+
|
15
|
+
- Client was using 100% CPU when it had no connection. (on $5 Digital Ocean droplet) Should be fixed now.
|
16
|
+
|
17
|
+
## [0.5.3] - 2021-08-13
|
18
|
+
|
19
|
+
- A bit less looping - better performance.
|
20
|
+
## [0.5.2] - 2021-08-13
|
4
21
|
|
5
22
|
- An error with shutdown handling was fixed.
|
6
23
|
|
data/Gemfile
CHANGED
@@ -9,10 +9,10 @@ gem 'rake', '~> 13.0'
|
|
9
9
|
|
10
10
|
gem 'rspec', '~> 3.0'
|
11
11
|
|
12
|
-
gem 'rubocop', '~> 1.7'
|
13
|
-
|
14
12
|
gem 'database_cleaner-active_record'
|
15
13
|
gem 'rails', '>= 6.1.3'
|
16
14
|
gem 'rspec-rails'
|
15
|
+
gem 'rubocop', '~> 1.7'
|
16
|
+
gem 'rubocop-discourse'
|
17
17
|
gem 'sqlite3'
|
18
18
|
gem 'stackprof'
|
data/Gemfile.lock
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
belated (0.5.
|
4
|
+
belated (0.5.6)
|
5
5
|
drb
|
6
6
|
dry-configurable
|
7
|
+
sorted_set
|
7
8
|
|
8
9
|
GEM
|
9
10
|
remote: https://rubygems.org/
|
@@ -135,6 +136,7 @@ GEM
|
|
135
136
|
thor (~> 1.0)
|
136
137
|
rainbow (3.0.0)
|
137
138
|
rake (13.0.6)
|
139
|
+
rbtree (0.4.4)
|
138
140
|
regexp_parser (2.1.1)
|
139
141
|
rexml (3.2.5)
|
140
142
|
rspec (3.10.0)
|
@@ -158,18 +160,28 @@ GEM
|
|
158
160
|
rspec-mocks (~> 3.10)
|
159
161
|
rspec-support (~> 3.10)
|
160
162
|
rspec-support (3.10.2)
|
161
|
-
rubocop (1.
|
163
|
+
rubocop (1.19.0)
|
162
164
|
parallel (~> 1.10)
|
163
165
|
parser (>= 3.0.0.0)
|
164
166
|
rainbow (>= 2.2.2, < 4.0)
|
165
167
|
regexp_parser (>= 1.8, < 3.0)
|
166
168
|
rexml
|
167
|
-
rubocop-ast (>= 1.
|
169
|
+
rubocop-ast (>= 1.9.1, < 2.0)
|
168
170
|
ruby-progressbar (~> 1.7)
|
169
171
|
unicode-display_width (>= 1.4.0, < 3.0)
|
170
|
-
rubocop-ast (1.
|
172
|
+
rubocop-ast (1.10.0)
|
171
173
|
parser (>= 3.0.1.1)
|
174
|
+
rubocop-discourse (2.4.2)
|
175
|
+
rubocop (>= 1.1.0)
|
176
|
+
rubocop-rspec (>= 2.0.0)
|
177
|
+
rubocop-rspec (2.4.0)
|
178
|
+
rubocop (~> 1.0)
|
179
|
+
rubocop-ast (>= 1.1.0)
|
172
180
|
ruby-progressbar (1.11.0)
|
181
|
+
set (1.0.1)
|
182
|
+
sorted_set (1.0.3)
|
183
|
+
rbtree
|
184
|
+
set (~> 1.0)
|
173
185
|
sprockets (4.0.2)
|
174
186
|
concurrent-ruby (~> 1.0)
|
175
187
|
rack (> 1, < 3)
|
@@ -200,6 +212,7 @@ DEPENDENCIES
|
|
200
212
|
rspec (~> 3.0)
|
201
213
|
rspec-rails
|
202
214
|
rubocop (~> 1.7)
|
215
|
+
rubocop-discourse
|
203
216
|
sqlite3
|
204
217
|
stackprof
|
205
218
|
|
data/README.md
CHANGED
@@ -16,8 +16,10 @@ Can be used with or without Rails.
|
|
16
16
|
|
17
17
|
TODO LIST:
|
18
18
|
|
19
|
+
- Use GDBM for queue storage? That way could maybe get rid of YAML dumping and make things a bit safer. Not ordered though, so maybe keep a list of the jobs as YAML and update it sometimes? Just as backup.
|
20
|
+
- Rescue `DRb::DRbRemoteError` when shutting down, might not need to if using GDBM?
|
19
21
|
- Don't use class instance variables.
|
20
|
-
- Make port configurable.
|
22
|
+
- Make DRb port configurable.
|
21
23
|
- Don't hardcode timezone.
|
22
24
|
- Add some checks to the client for proper jobs.
|
23
25
|
- Have multiple queues?
|
@@ -27,6 +29,7 @@ TODO LIST:
|
|
27
29
|
- Do some performance testing.
|
28
30
|
- Deploy a Rails app to production that is using Belated
|
29
31
|
and mention it in the readme. (Capistrano support?)
|
32
|
+
([Wasurechatta](https://wasurechatta.com/))
|
30
33
|
- Add a section telling people to use Sidekiq if they can
|
31
34
|
|
32
35
|
## Installation
|
@@ -101,7 +104,9 @@ If you don't want the job to run right away, you can also pass it a keyword para
|
|
101
104
|
client.perform_belated(job, Time.now + 1.month)
|
102
105
|
```
|
103
106
|
|
104
|
-
Note that you probably want to memoize the client, as it always creates a 'banker thread'
|
107
|
+
Note that you probably want to memoize the client, as it always creates a 'banker thread' and there is the overhead of connecting to dRuby. Maybe even use it as a global!(`$client`).
|
108
|
+
|
109
|
+
The client also holds references to the jobs that have been pushed so that they are not collected by GC.
|
105
110
|
|
106
111
|
# Settings
|
107
112
|
|
@@ -136,9 +141,15 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
|
|
136
141
|
|
137
142
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
138
143
|
|
144
|
+
|
145
|
+
# Possible problems
|
146
|
+
|
147
|
+
If you have the port 8788 already in use, you can check the ports in use in Linux with the following command:
|
148
|
+
|
149
|
+
$ sudo lsof -i -P -n | grep LISTEN
|
139
150
|
## Contributing
|
140
151
|
|
141
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/sampokuokkanen/belated. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/sampokuokkanen/belated/blob/master/CODE_OF_CONDUCT.md).
|
152
|
+
Bug reports, questions and pull requests are welcome on GitHub at https://github.com/sampokuokkanen/belated. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/sampokuokkanen/belated/blob/master/CODE_OF_CONDUCT.md).
|
142
153
|
|
143
154
|
## License
|
144
155
|
|
data/belated.gemspec
CHANGED
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
|
|
33
33
|
# Uncomment to register a new dependency of your gem
|
34
34
|
spec.add_dependency 'drb'
|
35
35
|
spec.add_dependency 'dry-configurable'
|
36
|
+
spec.add_dependency 'sorted_set'
|
36
37
|
spec.add_development_dependency 'byebug'
|
37
38
|
|
38
39
|
# For more information and examples about making a new gem, checkout our
|
data/lib/belated.rb
CHANGED
@@ -91,13 +91,17 @@ class Belated
|
|
91
91
|
def enqueue_future_jobs
|
92
92
|
log 'starting future jobs thread'
|
93
93
|
loop do
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
94
|
+
sleep 0.1
|
95
|
+
job = @@queue.future_jobs.min
|
96
|
+
if job.nil?
|
97
|
+
sleep 5
|
98
|
+
next
|
99
|
+
end
|
100
|
+
|
101
|
+
if job.at <= Time.now.utc
|
102
|
+
log "Deleting #{@@queue.future_jobs.delete(job)} from future jobs"
|
103
|
+
@@queue.push(job)
|
99
104
|
end
|
100
|
-
sleep 0.01
|
101
105
|
end
|
102
106
|
end
|
103
107
|
|
data/lib/belated/client.rb
CHANGED
@@ -7,7 +7,7 @@ class Belated
|
|
7
7
|
# client = Belated::Client.new
|
8
8
|
# client.enqueue(JubJub.new, at: Time.now + 5.seconds)
|
9
9
|
class Client
|
10
|
-
attr_accessor :queue, :bank, :banker_thread
|
10
|
+
attr_accessor :queue, :bank, :banker_thread, :table
|
11
11
|
|
12
12
|
# Starts up the client.
|
13
13
|
# Connects to the queue through DRb.
|
@@ -15,8 +15,10 @@ class Belated
|
|
15
15
|
def initialize
|
16
16
|
server_uri = Belated::URI
|
17
17
|
DRb.start_service
|
18
|
+
self.table = {}
|
18
19
|
self.bank = Thread::Queue.new
|
19
20
|
self.queue = DRbObject.new_with_uri(server_uri)
|
21
|
+
start_banker_thread
|
20
22
|
end
|
21
23
|
|
22
24
|
# Thread in charge of handling the bank queue.
|
@@ -26,13 +28,27 @@ class Belated
|
|
26
28
|
def start_banker_thread
|
27
29
|
self.banker_thread = Thread.new do
|
28
30
|
loop do
|
29
|
-
|
31
|
+
sleep 0.01
|
30
32
|
|
31
|
-
|
33
|
+
delete_from_table
|
34
|
+
|
35
|
+
if bank.empty?
|
36
|
+
sleep 10
|
37
|
+
else
|
38
|
+
perform(bank.pop)
|
39
|
+
end
|
32
40
|
end
|
33
41
|
end
|
34
42
|
end
|
35
43
|
|
44
|
+
def delete_from_table
|
45
|
+
return if table.empty?
|
46
|
+
|
47
|
+
table.select { |_k, v| v.completed }.each do |key, _value|
|
48
|
+
table.delete(key)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
36
52
|
# The method that pushes the jobs to the queue.
|
37
53
|
# If there is no connection, it pushes the job to the bank.
|
38
54
|
# @param job [Object] - The the job to be pushed.
|
@@ -40,19 +56,27 @@ class Belated
|
|
40
56
|
# @param max_retries [Integer] - Times the job should be retried if it fails.
|
41
57
|
# @return [JobWrapper] - The job wrapper for the queue.
|
42
58
|
def perform(job, at: nil, max_retries: 5)
|
59
|
+
log 'Passing a proc and at time is deprecated and will be removed in 0.6' if job.instance_of?(Proc) && !at.nil?
|
60
|
+
|
43
61
|
job_wrapper = if job.is_a?(JobWrapper)
|
44
62
|
job
|
45
63
|
else
|
46
64
|
JobWrapper.new(job: job, at: at, max_retries: max_retries)
|
47
65
|
end
|
48
|
-
|
49
|
-
job_wrapper
|
66
|
+
queue.push(job_wrapper)
|
67
|
+
table[job_wrapper.object_id] = job_wrapper
|
50
68
|
rescue DRb::DRbConnError
|
51
69
|
bank.push(job_wrapper)
|
52
|
-
start_banker_thread if banker_thread.nil?
|
53
|
-
banker_thread.wakeup if banker_thread.status == 'sleep'
|
54
70
|
end
|
55
71
|
alias perform_belated perform
|
56
72
|
alias perform_later perform
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def drb_connected?
|
77
|
+
queue.connected?
|
78
|
+
rescue StandardError
|
79
|
+
false
|
80
|
+
end
|
57
81
|
end
|
58
82
|
end
|
data/lib/belated/job_wrapper.rb
CHANGED
@@ -2,9 +2,18 @@ require 'securerandom'
|
|
2
2
|
require_relative 'logging'
|
3
3
|
|
4
4
|
class Belated
|
5
|
+
# JobWrapper is a wrapper for a job. It is responsible for
|
6
|
+
# - logging
|
7
|
+
# - error handling
|
8
|
+
# - job execution
|
9
|
+
# - job result handling
|
10
|
+
# - job result logging
|
11
|
+
# - job retries
|
12
|
+
# - job retry delay
|
5
13
|
class JobWrapper
|
14
|
+
include Comparable
|
6
15
|
include Logging
|
7
|
-
attr_accessor :retries, :max_retries, :id, :job, :at
|
16
|
+
attr_accessor :retries, :max_retries, :id, :job, :at, :completed
|
8
17
|
|
9
18
|
def initialize(job:, max_retries: 5, at: nil)
|
10
19
|
self.retries = 0
|
@@ -12,15 +21,22 @@ class Belated
|
|
12
21
|
self.id = SecureRandom.uuid
|
13
22
|
self.job = job
|
14
23
|
self.at = at
|
24
|
+
self.completed = false
|
25
|
+
end
|
26
|
+
|
27
|
+
def <=>(other)
|
28
|
+
at <=> other.at
|
15
29
|
end
|
16
30
|
|
17
31
|
# rubocop:disable Lint/RescueException
|
18
32
|
def perform
|
19
|
-
if job.respond_to?(:call)
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
33
|
+
resp = if job.respond_to?(:call)
|
34
|
+
job.call
|
35
|
+
else
|
36
|
+
job.perform
|
37
|
+
end
|
38
|
+
self.completed = true
|
39
|
+
resp
|
24
40
|
rescue Exception => e
|
25
41
|
case e.class
|
26
42
|
when Interrupt, SignalException
|
data/lib/belated/logging.rb
CHANGED
data/lib/belated/queue.rb
CHANGED
@@ -3,20 +3,27 @@
|
|
3
3
|
require 'belated/job'
|
4
4
|
require 'belated/logging'
|
5
5
|
require 'belated/job_wrapper'
|
6
|
+
require 'sorted_set'
|
7
|
+
|
6
8
|
class Belated
|
9
|
+
# Job queues that Belated uses.
|
10
|
+
# queue is the jobs that are currenly
|
11
|
+
# waiting for a worker to start working on them.
|
12
|
+
# future_jobs is a SortedSet of jobs that are going
|
13
|
+
# to be added to queue at some point in the future.
|
7
14
|
class Queue
|
8
15
|
include Logging
|
9
16
|
attr_accessor :future_jobs
|
10
17
|
|
11
18
|
FILE_NAME = 'belated_dump'
|
12
19
|
|
13
|
-
def initialize(queue: Thread::Queue.new, future_jobs:
|
20
|
+
def initialize(queue: Thread::Queue.new, future_jobs: SortedSet.new)
|
14
21
|
@queue = queue
|
15
22
|
self.future_jobs = future_jobs
|
16
23
|
end
|
17
24
|
|
18
25
|
def push(job)
|
19
|
-
if job
|
26
|
+
if job.is_a?(Symbol) || job.at.nil? ||
|
20
27
|
job.at <= Time.now.utc
|
21
28
|
@queue.push(job)
|
22
29
|
else
|
@@ -72,10 +79,14 @@ class Belated
|
|
72
79
|
pp File.open(FILE_NAME, 'wb') { |f| f.write(YAML.dump(class_array)) }
|
73
80
|
end
|
74
81
|
|
82
|
+
def connected?
|
83
|
+
true
|
84
|
+
end
|
85
|
+
|
75
86
|
private
|
76
87
|
|
77
88
|
def proc_or_shutdown?(job)
|
78
|
-
job.job.instance_of?(Proc)
|
89
|
+
job.is_a?(Symbol) || job.job.instance_of?(Proc)
|
79
90
|
end
|
80
91
|
end
|
81
92
|
end
|
data/lib/belated/version.rb
CHANGED
data/lib/belated/worker.rb
CHANGED
@@ -16,10 +16,12 @@ class Belated
|
|
16
16
|
log "Worker #{@number} fetching jobs!"
|
17
17
|
next unless (job = Belated.fetch_job)
|
18
18
|
|
19
|
-
break if job
|
19
|
+
break if job.is_a?(Symbol)
|
20
20
|
|
21
21
|
log "Worker #{@number} got job: #{job.inspect}"
|
22
|
-
job.perform
|
22
|
+
log job.perform
|
23
|
+
rescue RangeError => e
|
24
|
+
log e
|
23
25
|
end
|
24
26
|
end
|
25
27
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: belated
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sampo Kuokkanen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-08-
|
11
|
+
date: 2021-08-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: drb
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: sorted_set
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: byebug
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|