workhorse 1.0.0 → 1.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/CHANGELOG.md +44 -1
- data/README.md +4 -4
- data/VERSION +1 -1
- data/lib/active_job/queue_adapters/workhorse_adapter.rb +2 -2
- data/lib/generators/workhorse/templates/create_table_jobs.rb +2 -0
- data/lib/workhorse.rb +3 -2
- data/lib/workhorse/daemon.rb +3 -1
- data/lib/workhorse/daemon/shell_handler.rb +2 -2
- data/lib/workhorse/db_job.rb +1 -1
- data/lib/workhorse/enqueuer.rb +11 -4
- data/lib/workhorse/jobs/detect_stale_jobs_job.rb +2 -2
- data/lib/workhorse/performer.rb +1 -0
- data/lib/workhorse/poller.rb +3 -3
- data/lib/workhorse/worker.rb +2 -2
- data/test/lib/db_schema.rb +2 -0
- data/test/workhorse/enqueuer_test.rb +9 -0
- data/test/workhorse/poller_test.rb +3 -1
- data/workhorse.gemspec +3 -3
- metadata +9 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e3c7e4fe252910423baf366bed04b4dab80fc529860d95aa4f858954d9735f14
|
4
|
+
data.tar.gz: 4aa288f74516cc5b097c79a6a492d15433bd7e5c0d4e90e5818818714ecf37d9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9aa54bffce19e617fc34ba1d900e30dbae109fcda66163c47e3a1bc1a1bb7b3c8ba17c15582577a5b72c24ddccaac55ca92528dd2c7f18f90c7756eedfd72153
|
7
|
+
data.tar.gz: 58af82fc7c8dff5375aeb31fa71084c7718998bf242736e056ae6b20787f591d4d4e26484153874b16badbffb1aa846ca73d76ea5efbd4f21a049367537704e7
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,8 +1,51 @@
|
|
1
1
|
# Workhorse Changelog
|
2
2
|
|
3
|
+
## 1.2.1 - 2021-01-27
|
4
|
+
|
5
|
+
* Log job description (if given) when performing job
|
6
|
+
|
7
|
+
## 1.2.0 - 2021-01-27
|
8
|
+
|
9
|
+
* `Workhorse.enqueue_active_job`
|
10
|
+
* Change `perform_at` to a keyword arument
|
11
|
+
* Allow passing `:description` and `:queue`
|
12
|
+
|
13
|
+
Note that when upgrading, change:
|
14
|
+
|
15
|
+
```ruby
|
16
|
+
# From
|
17
|
+
Workhorse.enqueue_active_job MyJob, 5.minutes.from_now
|
18
|
+
|
19
|
+
# To
|
20
|
+
Workhorse.enqueue_active_job MyJob, perform_at: 5.minutes.from_now
|
21
|
+
```
|
22
|
+
|
23
|
+
## 1.1.1 - 2021-01-19
|
24
|
+
|
25
|
+
* Remove deprecation warnings with Ruby 2.7
|
26
|
+
|
27
|
+
## 1.1.0 - 2020-12-24
|
28
|
+
|
29
|
+
* Add `description` column to `DbJob`.
|
30
|
+
|
31
|
+
If you're upgrading from a previous version, add the `description` column
|
32
|
+
to your `DbJob` table, e.g. with such a migration:
|
33
|
+
|
34
|
+
```ruby
|
35
|
+
class AddDescriptionToWorkhorseDbJobs < ActiveRecord::Migration[6.0]
|
36
|
+
def change
|
37
|
+
add_column :jobs, :description, :string, after: :perform_at, null: true
|
38
|
+
end
|
39
|
+
end
|
40
|
+
```
|
41
|
+
|
42
|
+
## 1.0.1 - 2020-12-15
|
43
|
+
|
44
|
+
* Fix handling of empty pid files
|
45
|
+
|
3
46
|
## 1.0.0 - 2020-09-21
|
4
47
|
|
5
|
-
* Stable release,
|
48
|
+
* Stable release, identical to 1.0.0.beta2 but now extensively battle-tested
|
6
49
|
|
7
50
|
## 1.0.0.beta2 - 2020-08-27
|
8
51
|
|
data/README.md
CHANGED
@@ -81,7 +81,7 @@ GRANT execute ON DBMS_LOCK TO <schema-name>;
|
|
81
81
|
|
82
82
|
Workhorse can handle any jobs that support the `perform` method and are
|
83
83
|
serializable. To queue a basic job, use the static method `Workhorse.enqueue`.
|
84
|
-
You can optionally pass a queue name and a
|
84
|
+
You can optionally pass a queue name, a priority and a description (as a string).
|
85
85
|
|
86
86
|
```ruby
|
87
87
|
class MyJob
|
@@ -94,7 +94,7 @@ class MyJob
|
|
94
94
|
end
|
95
95
|
end
|
96
96
|
|
97
|
-
Workhorse.enqueue MyJob.new('John'), queue: :test, priority: 2
|
97
|
+
Workhorse.enqueue MyJob.new('John'), queue: :test, priority: 2, description: 'Basic Job'
|
98
98
|
```
|
99
99
|
|
100
100
|
### RailsOps operations
|
@@ -381,7 +381,7 @@ configuration or else using `self.queue_adapter` in a job class inheriting from
|
|
381
381
|
Per default, jobs remain in the database, no matter in which state. This can
|
382
382
|
eventually lead to a very large jobs database. You are advised to clean your
|
383
383
|
jobs database on a regular interval. Workhorse provides the job
|
384
|
-
`
|
384
|
+
`Workhorse::Jobs::CleanupSucceededJobs` for this purpose that cleans up all
|
385
385
|
succeeded jobs. You can run this using your scheduler in a specific interval.
|
386
386
|
|
387
387
|
## Caveats
|
@@ -421,4 +421,4 @@ Please consult the [FAQ](FAQ.md).
|
|
421
421
|
|
422
422
|
## Copyright
|
423
423
|
|
424
|
-
Copyright ©
|
424
|
+
Copyright © 2021 Sitrox. See `LICENSE` for further details.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.2.1
|
@@ -14,8 +14,8 @@ module ActiveJob
|
|
14
14
|
Workhorse.enqueue_active_job(job)
|
15
15
|
end
|
16
16
|
|
17
|
-
def enqueue_at(job, timestamp) #:nodoc:
|
18
|
-
Workhorse.enqueue_active_job(job, timestamp)
|
17
|
+
def enqueue_at(job, timestamp = Time.now) #:nodoc:
|
18
|
+
Workhorse.enqueue_active_job(job, perform_at: timestamp)
|
19
19
|
end
|
20
20
|
end
|
21
21
|
end
|
data/lib/workhorse.rb
CHANGED
data/lib/workhorse/daemon.rb
CHANGED
@@ -177,7 +177,9 @@ module Workhorse
|
|
177
177
|
file = pid_file_for(worker)
|
178
178
|
|
179
179
|
if File.exist?(file)
|
180
|
-
|
180
|
+
raw_pid = IO.read(file)
|
181
|
+
return nil, nil if raw_pid.blank?
|
182
|
+
pid = Integer(raw_pid)
|
181
183
|
return file, process?(pid) ? pid : nil
|
182
184
|
else
|
183
185
|
return nil, nil
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module Workhorse
|
2
2
|
class Daemon::ShellHandler
|
3
|
-
def self.run(options
|
3
|
+
def self.run(**options, &block)
|
4
4
|
unless ARGV.count == 1
|
5
5
|
usage
|
6
6
|
exit 99
|
@@ -10,7 +10,7 @@ module Workhorse
|
|
10
10
|
lockfile = File.open(lockfile_path, 'a')
|
11
11
|
lockfile.flock(File::LOCK_EX || File::LOCK_NB)
|
12
12
|
|
13
|
-
daemon = Workhorse::Daemon.new(options, &block)
|
13
|
+
daemon = Workhorse::Daemon.new(**options, &block)
|
14
14
|
|
15
15
|
begin
|
16
16
|
case ARGV.first
|
data/lib/workhorse/db_job.rb
CHANGED
data/lib/workhorse/enqueuer.rb
CHANGED
@@ -1,20 +1,27 @@
|
|
1
1
|
module Workhorse
|
2
2
|
module Enqueuer
|
3
3
|
# Enqueue any object that is serializable and has a `perform` method
|
4
|
-
def enqueue(job, queue: nil, priority: 0, perform_at: Time.now)
|
4
|
+
def enqueue(job, queue: nil, priority: 0, perform_at: Time.now, description: nil)
|
5
5
|
return DbJob.create!(
|
6
6
|
queue: queue,
|
7
7
|
priority: priority,
|
8
8
|
perform_at: perform_at,
|
9
|
+
description: description,
|
9
10
|
handler: Marshal.dump(job)
|
10
11
|
)
|
11
12
|
end
|
12
13
|
|
13
14
|
# Enqueue an ActiveJob job
|
14
|
-
def enqueue_active_job(job, perform_at
|
15
|
+
def enqueue_active_job(job, perform_at: Time.now, queue: nil, description: nil)
|
15
16
|
wrapper_job = Jobs::RunActiveJob.new(job.serialize)
|
16
|
-
queue
|
17
|
-
db_job = enqueue(
|
17
|
+
queue ||= job.queue_name if job.queue_name.present?
|
18
|
+
db_job = enqueue(
|
19
|
+
wrapper_job,
|
20
|
+
queue: queue,
|
21
|
+
priority: job.priority || 0,
|
22
|
+
perform_at: Time.at(perform_at),
|
23
|
+
description: description
|
24
|
+
)
|
18
25
|
job.provider_job_id = db_job.id
|
19
26
|
return db_job
|
20
27
|
end
|
@@ -22,7 +22,7 @@ module Workhorse::Jobs
|
|
22
22
|
rel = rel.where('locked_at < ?', @locked_to_started_threshold.seconds.ago)
|
23
23
|
ids = rel.pluck(:id)
|
24
24
|
|
25
|
-
|
25
|
+
unless ids.empty?
|
26
26
|
messages << "Detected #{ids.size} jobs that were locked more than "\
|
27
27
|
"#{@locked_to_started_threshold}s ago and might be stale: #{ids.inspect}."
|
28
28
|
end
|
@@ -34,7 +34,7 @@ module Workhorse::Jobs
|
|
34
34
|
rel = rel.where('started_at < ?', @run_time_threshold.seconds.ago)
|
35
35
|
ids = rel.pluck(:id)
|
36
36
|
|
37
|
-
|
37
|
+
unless ids.empty?
|
38
38
|
messages << "Detected #{ids.size} jobs that are running for longer than "\
|
39
39
|
"#{@run_time_threshold}s ago and might be stale: #{ids.inspect}."
|
40
40
|
end
|
data/lib/workhorse/performer.rb
CHANGED
@@ -59,6 +59,7 @@ module Workhorse
|
|
59
59
|
# Deserialize and perform job
|
60
60
|
# ---------------------------------------------------------------
|
61
61
|
log 'Performing', :info
|
62
|
+
log "Description: #{@db_job.description}", :info unless @db_job.description.blank?
|
62
63
|
|
63
64
|
if Workhorse.perform_jobs_in_tx
|
64
65
|
Workhorse.tx_callback.call do
|
data/lib/workhorse/poller.rb
CHANGED
@@ -3,8 +3,8 @@ module Workhorse
|
|
3
3
|
MIN_LOCK_TIMEOUT = 0.1 # In seconds
|
4
4
|
MAX_LOCK_TIMEOUT = 1.0 # In seconds
|
5
5
|
|
6
|
-
ORACLE_LOCK_MODE
|
7
|
-
ORACLE_LOCK_HANDLE =
|
6
|
+
ORACLE_LOCK_MODE = 6 # X_MODE (exclusive)
|
7
|
+
ORACLE_LOCK_HANDLE = 478_564_848 # Randomly chosen number
|
8
8
|
|
9
9
|
attr_reader :worker
|
10
10
|
attr_reader :table
|
@@ -72,7 +72,7 @@ module Workhorse
|
|
72
72
|
end
|
73
73
|
end
|
74
74
|
|
75
|
-
def with_global_lock(name: :workhorse, timeout: 2, &
|
75
|
+
def with_global_lock(name: :workhorse, timeout: 2, &_block)
|
76
76
|
if @is_oracle
|
77
77
|
result = Workhorse::DbJob.connection.select_all(
|
78
78
|
"SELECT DBMS_LOCK.REQUEST(#{ORACLE_LOCK_HANDLE}, #{ORACLE_LOCK_MODE}, #{timeout}) FROM DUAL"
|
data/lib/workhorse/worker.rb
CHANGED
@@ -13,8 +13,8 @@ module Workhorse
|
|
13
13
|
|
14
14
|
# Instantiates and starts a new worker with the given arguments and then
|
15
15
|
# waits for its completion (i.e. an interrupt).
|
16
|
-
def self.start_and_wait(
|
17
|
-
worker = new(
|
16
|
+
def self.start_and_wait(**args)
|
17
|
+
worker = new(**args)
|
18
18
|
worker.start
|
19
19
|
worker.wait
|
20
20
|
end
|
data/test/lib/db_schema.rb
CHANGED
@@ -33,6 +33,15 @@ class Workhorse::EnqueuerTest < WorkhorseTest
|
|
33
33
|
assert_equal 1, Workhorse::DbJob.first.priority
|
34
34
|
end
|
35
35
|
|
36
|
+
def test_with_description
|
37
|
+
assert_equal 0, Workhorse::DbJob.all.count
|
38
|
+
Workhorse.enqueue BasicJob.new, description: 'Lorem ipsum'
|
39
|
+
assert_equal 1, Workhorse::DbJob.all.count
|
40
|
+
|
41
|
+
db_job = Workhorse::DbJob.first
|
42
|
+
assert_equal 'Lorem ipsum', db_job.description
|
43
|
+
end
|
44
|
+
|
36
45
|
def test_op
|
37
46
|
Workhorse.enqueue_op DummyRailsOpsOp, { queue: :q1 }, foo: :bar
|
38
47
|
|
@@ -48,7 +48,7 @@ class Workhorse::PollerTest < WorkhorseTest
|
|
48
48
|
assert_equal %w[q1 q2], w.poller.send(:valid_queues)
|
49
49
|
end
|
50
50
|
|
51
|
-
def
|
51
|
+
def test_valid_queues_2
|
52
52
|
w = Workhorse::Worker.new(polling_interval: 60)
|
53
53
|
|
54
54
|
assert_equal [], w.poller.send(:valid_queues)
|
@@ -148,6 +148,7 @@ class Workhorse::PollerTest < WorkhorseTest
|
|
148
148
|
assert_equal 25, used_workers
|
149
149
|
end
|
150
150
|
|
151
|
+
# rubocop: disable Style/GlobalVars
|
151
152
|
def test_connection_loss
|
152
153
|
$thread_conn = nil
|
153
154
|
|
@@ -174,6 +175,7 @@ class Workhorse::PollerTest < WorkhorseTest
|
|
174
175
|
|
175
176
|
assert_equal 1, Workhorse::DbJob.succeeded.count
|
176
177
|
end
|
178
|
+
# rubocop: enable Style/GlobalVars
|
177
179
|
|
178
180
|
private
|
179
181
|
|
data/workhorse.gemspec
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
|
-
# stub: workhorse 1.
|
2
|
+
# stub: workhorse 1.2.1 ruby lib
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "workhorse".freeze
|
6
|
-
s.version = "1.
|
6
|
+
s.version = "1.2.1"
|
7
7
|
|
8
8
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
9
9
|
s.require_paths = ["lib".freeze]
|
10
10
|
s.authors = ["Sitrox".freeze]
|
11
|
-
s.date = "
|
11
|
+
s.date = "2021-01-27"
|
12
12
|
s.files = [".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, ".travis.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
|
13
13
|
s.rubygems_version = "3.0.3".freeze
|
14
14
|
s.summary = "Multi-threaded job backend with database queuing for ruby.".freeze
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: workhorse
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sitrox
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -192,8 +192,8 @@ dependencies:
|
|
192
192
|
- - ">="
|
193
193
|
- !ruby/object:Gem::Version
|
194
194
|
version: '0'
|
195
|
-
description:
|
196
|
-
email:
|
195
|
+
description:
|
196
|
+
email:
|
197
197
|
executables: []
|
198
198
|
extensions: []
|
199
199
|
extra_rdoc_files: []
|
@@ -241,10 +241,10 @@ files:
|
|
241
241
|
- test/workhorse/pool_test.rb
|
242
242
|
- test/workhorse/worker_test.rb
|
243
243
|
- workhorse.gemspec
|
244
|
-
homepage:
|
244
|
+
homepage:
|
245
245
|
licenses: []
|
246
246
|
metadata: {}
|
247
|
-
post_install_message:
|
247
|
+
post_install_message:
|
248
248
|
rdoc_options: []
|
249
249
|
require_paths:
|
250
250
|
- lib
|
@@ -259,8 +259,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
259
259
|
- !ruby/object:Gem::Version
|
260
260
|
version: '0'
|
261
261
|
requirements: []
|
262
|
-
rubygems_version: 3.1.
|
263
|
-
signing_key:
|
262
|
+
rubygems_version: 3.1.4
|
263
|
+
signing_key:
|
264
264
|
specification_version: 4
|
265
265
|
summary: Multi-threaded job backend with database queuing for ruby.
|
266
266
|
test_files:
|