parallel_minion 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +6 -4
- data/Rakefile +18 -15
- data/lib/parallel_minion.rb +1 -1
- data/lib/parallel_minion/minion.rb +198 -127
- data/lib/parallel_minion/railtie.rb +8 -0
- data/lib/parallel_minion/version.rb +1 -1
- data/test/minion_scope_test.rb +10 -19
- data/test/minion_test.rb +61 -44
- data/test/test_db.sqlite3 +0 -0
- data/test/test_helper.rb +2 -8
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c4ec6fafacfe5cf2b3e090a1e1ed733fd6c3bc33
|
4
|
+
data.tar.gz: a054a763962d6ee10fb8bc75587ec4d1c5f3d456
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 13909e2ad5221826dd9ac6cbbd15364566e4cf6b76fc33bc77d7cc8c35acc6ddb06cc3b53845bb6df053b031a13fb20808ce10c7e2f7bbb141912e2a86a03aff
|
7
|
+
data.tar.gz: 01b7b72f77160109e4bfa9ab995c6f98be128a7e837bc284d3932f399cdae9389b866d83a534d5ec618425c201d3607d5ff19b06bf803a3054da4edc409bcaa5
|
data/README.md
CHANGED
@@ -15,9 +15,11 @@ existing code to use minions.
|
|
15
15
|
## Example
|
16
16
|
|
17
17
|
```ruby
|
18
|
-
minion = ParallelMinion::Minion.new(
|
19
|
-
|
20
|
-
|
18
|
+
minion = ParallelMinion::Minion.new(
|
19
|
+
10.days.ago,
|
20
|
+
description: 'Doing something else in parallel',
|
21
|
+
timeout: 1000
|
22
|
+
) do |date|
|
21
23
|
MyTable.where('created_at <= ?', date).count
|
22
24
|
end
|
23
25
|
|
@@ -59,7 +61,7 @@ This project uses [Semantic Versioning](http://semver.org/).
|
|
59
61
|
|
60
62
|
## License
|
61
63
|
|
62
|
-
Copyright 2013, 2014,
|
64
|
+
Copyright 2013, 2014, 2015, 2016, 2017 Reid Morrison
|
63
65
|
|
64
66
|
Licensed under the Apache License, Version 2.0 (the "License");
|
65
67
|
you may not use this file except in compliance with the License.
|
data/Rakefile
CHANGED
@@ -1,28 +1,31 @@
|
|
1
|
-
|
2
|
-
require '
|
1
|
+
# Setup bundler to avoid having to run bundle exec all the time.
|
2
|
+
require 'rubygems'
|
3
|
+
require 'bundler/setup'
|
3
4
|
|
4
|
-
|
5
|
-
|
5
|
+
require 'rake/testtask'
|
6
|
+
require_relative 'lib/parallel_minion/version'
|
6
7
|
|
7
8
|
task :gem do
|
8
|
-
system
|
9
|
+
system 'gem build parallel_minion.gemspec'
|
9
10
|
end
|
10
11
|
|
11
12
|
task :publish => :gem do
|
12
13
|
system "git tag -a v#{ParallelMinion::VERSION} -m 'Tagging #{ParallelMinion::VERSION}'"
|
13
|
-
system
|
14
|
+
system 'git push --tags'
|
14
15
|
system "gem push parallel_minion-#{ParallelMinion::VERSION}.gem"
|
15
16
|
system "rm parallel_minion-#{ParallelMinion::VERSION}.gem"
|
16
17
|
end
|
17
18
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
t.verbose = true
|
23
|
-
end
|
24
|
-
|
25
|
-
Rake::Task['functional'].invoke
|
19
|
+
Rake::TestTask.new(:test) do |t|
|
20
|
+
t.pattern = 'test/**/*_test.rb'
|
21
|
+
t.verbose = true
|
22
|
+
t.warning = false
|
26
23
|
end
|
27
24
|
|
28
|
-
|
25
|
+
# By default run tests against all appraisals
|
26
|
+
if !ENV["APPRAISAL_INITIALIZED"] && !ENV["TRAVIS"]
|
27
|
+
require 'appraisal'
|
28
|
+
task default: :appraisal
|
29
|
+
else
|
30
|
+
task default: :test
|
31
|
+
end
|
data/lib/parallel_minion.rb
CHANGED
@@ -19,26 +19,34 @@ module ParallelMinion
|
|
19
19
|
# Returns nil if the minion is still running
|
20
20
|
attr_reader :duration
|
21
21
|
|
22
|
+
# Metrics [String]
|
23
|
+
attr_reader :metric, :wait_metric
|
24
|
+
|
25
|
+
attr_reader :on_timeout, :log_exception, :start_time
|
26
|
+
|
22
27
|
# Give an infinite amount of time to wait for a Minion to complete a task
|
23
28
|
INFINITE = 0
|
24
29
|
|
25
|
-
# Sets whether
|
26
|
-
#
|
27
|
-
# By Setting _enabled_ to false all Minions that have not yet been started
|
28
|
-
# will run in the thread from which it is created and not on its own thread
|
30
|
+
# Sets whether Minions should run in a separate thread.
|
29
31
|
#
|
30
|
-
#
|
31
|
-
#
|
32
|
-
# -
|
33
|
-
#
|
34
|
-
#
|
32
|
+
# By Setting _enabled_ to false all Minions that have not yet been created
|
33
|
+
# will run in the thread in which it is created.
|
34
|
+
# - Development:
|
35
|
+
# Use a debugger, since the code will run in the current thread.
|
36
|
+
# - Test:
|
37
|
+
# Keep test execution in the current thread.
|
38
|
+
# Supports rolling back database changes after each test, since all changes are
|
39
|
+
# performed on the same database connection.
|
40
|
+
# - Production:
|
41
|
+
# Batch processing in Rocket Job where throughput is more important than latency.
|
42
|
+
# http://rocketjob.io
|
35
43
|
def self.enabled=(enabled)
|
36
|
-
|
44
|
+
@enabled = enabled
|
37
45
|
end
|
38
46
|
|
39
47
|
# Returns whether minions are enabled to run in their own threads
|
40
48
|
def self.enabled?
|
41
|
-
|
49
|
+
@enabled
|
42
50
|
end
|
43
51
|
|
44
52
|
# The list of classes for which the current scope must be copied into the
|
@@ -47,25 +55,85 @@ module ParallelMinion
|
|
47
55
|
# Example:
|
48
56
|
# ...
|
49
57
|
def self.scoped_classes
|
50
|
-
|
58
|
+
@scoped_classes
|
51
59
|
end
|
52
60
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
#
|
58
|
-
#
|
61
|
+
def self.scoped_classes=(scoped_classes)
|
62
|
+
@scoped_classes = scoped_classes.dup
|
63
|
+
end
|
64
|
+
|
65
|
+
# Change the log level for the Started log message.
|
66
|
+
#
|
67
|
+
# Default: :info
|
68
|
+
#
|
69
|
+
# Valid levels:
|
70
|
+
# :trace, :debug, :info, :warn, :error, :fatal
|
71
|
+
def self.started_log_level=(level)
|
72
|
+
raise(ArgumentError, "Invalid log level: #{level}") unless SemanticLogger::LEVELS.include?(level)
|
73
|
+
@started_log_level = level
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.started_log_level
|
77
|
+
@started_log_level
|
78
|
+
end
|
79
|
+
|
80
|
+
# Change the log level for the Completed log message.
|
81
|
+
#
|
82
|
+
# Default: :info
|
83
|
+
#
|
84
|
+
# Valid levels:
|
85
|
+
# :trace, :debug, :info, :warn, :error, :fatal
|
86
|
+
def self.completed_log_level=(level)
|
87
|
+
raise(ArgumentError, "Invalid log level: #{level}") unless SemanticLogger::LEVELS.include?(level)
|
88
|
+
@completed_log_level = level
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.completed_log_level
|
92
|
+
@completed_log_level
|
93
|
+
end
|
94
|
+
|
95
|
+
self.started_log_level = :info
|
96
|
+
self.completed_log_level = :info
|
97
|
+
self.enabled = true
|
98
|
+
self.scoped_classes = []
|
99
|
+
logger.name = 'Minion'
|
100
|
+
|
101
|
+
# Create a new Minion
|
102
|
+
#
|
103
|
+
# Creates a new thread and logs the time for the supplied block to complete processing.
|
104
|
+
# The exception without stack trace is logged whenever an exception is thrown in the thread.
|
105
|
+
#
|
106
|
+
# Re-raises any unhandled exception in the calling thread when `#result` is called.
|
107
|
+
# Copies the logging tags and specified ActiveRecord scopes to the new thread.
|
59
108
|
#
|
60
109
|
# Parameters
|
110
|
+
# *arguments
|
111
|
+
# Any number of arguments can be supplied that are passed into the block
|
112
|
+
# in the order they are listed.
|
113
|
+
#
|
114
|
+
# Note:
|
115
|
+
# All arguments must be supplied by copy and not by reference.
|
116
|
+
# For example, use `#dup` to create copies of passed data.
|
117
|
+
# Pass by copy is critical to prevent concurrency issues when multiple threads
|
118
|
+
# attempt to update the same object at the same time.
|
119
|
+
#
|
120
|
+
# Proc / lambda
|
121
|
+
# A block of code must be supplied that the Minion will execute.
|
122
|
+
#
|
123
|
+
# Note:
|
124
|
+
# This block will be executed within the scope of the created minion instance
|
125
|
+
# and _not_ within the scope of where the Proc/lambda was originally created.
|
126
|
+
# This is done to force all parameters to be passed in explicitly
|
127
|
+
# and should be read-only or duplicates of the original data.
|
128
|
+
#
|
61
129
|
# :description [String]
|
62
|
-
# Description for this task that the Minion is performing
|
63
|
-
#
|
130
|
+
# Description for this task that the Minion is performing.
|
131
|
+
# Written to the log file along with the time take to complete the task.
|
64
132
|
#
|
65
133
|
# :timeout [Integer]
|
66
134
|
# Maximum amount of time in milli-seconds that the task may take to complete
|
67
|
-
# before #result times out
|
68
|
-
# Set to 0 to give the thread an infinite amount of time to complete
|
135
|
+
# before #result times out.
|
136
|
+
# Set to 0 to give the thread an infinite amount of time to complete.
|
69
137
|
# Default: 0 ( Wait forever )
|
70
138
|
#
|
71
139
|
# Notes:
|
@@ -76,11 +144,6 @@ module ParallelMinion
|
|
76
144
|
# then :timeout is ignored and assumed to be Minion::INFINITE
|
77
145
|
# since the code is run in the calling thread when the Minion is created
|
78
146
|
#
|
79
|
-
# :enabled [Boolean]
|
80
|
-
# Whether the minion should run in a separate thread
|
81
|
-
# Not recommended in Production, but is useful for debugging purposes
|
82
|
-
# Default: ParallelMinion::Minion.enabled?
|
83
|
-
#
|
84
147
|
# :on_timeout [Exception]
|
85
148
|
# The class to raise on the minion when the minion times out.
|
86
149
|
# By raising the exception on the running thread it ensures that the thread
|
@@ -92,21 +155,30 @@ module ParallelMinion
|
|
92
155
|
#
|
93
156
|
# Note: :on_timeout has no effect if not #enabled?
|
94
157
|
#
|
95
|
-
#
|
96
|
-
#
|
97
|
-
#
|
98
|
-
# It is recommended to duplicate and/or freeze objects passed as arguments
|
99
|
-
# so that they are not modified at the same time by multiple threads
|
100
|
-
# These arguments are accessible while and after the minion is running
|
101
|
-
# by calling #arguments
|
158
|
+
# :metric [String]
|
159
|
+
# Name of the metric to forward to Semantic Logger when measuring the minion execution time.
|
160
|
+
# Example: inquiry/address_cleansing
|
102
161
|
#
|
103
|
-
#
|
104
|
-
#
|
105
|
-
#
|
106
|
-
#
|
107
|
-
#
|
108
|
-
#
|
109
|
-
#
|
162
|
+
# When a metric is supplied the following metrics will also be generated:
|
163
|
+
# - wait
|
164
|
+
# Duration waiting for a minion to complete.
|
165
|
+
#
|
166
|
+
# The additional metrics are added to the supplied metric name. For example:
|
167
|
+
# - inquiry/address_cleansing/wait
|
168
|
+
#
|
169
|
+
# :log_exception [Symbol]
|
170
|
+
# Control whether or how an exception thrown in the block is
|
171
|
+
# reported by Semantic Logger. Values:
|
172
|
+
# :full
|
173
|
+
# Log the exception class, message, and backtrace
|
174
|
+
# :partial
|
175
|
+
# Log the exception class and message. The backtrace will not be logged
|
176
|
+
# :off
|
177
|
+
# Any unhandled exception raised in the block will not be logged
|
178
|
+
# Default: :partial
|
179
|
+
#
|
180
|
+
# :enabled [Boolean]
|
181
|
+
# Override the global setting: `ParallelMinion::Minion.enabled?` for this minion instance.
|
110
182
|
#
|
111
183
|
# The overhead for moving the task to a Minion (separate thread) vs running it
|
112
184
|
# sequentially is about 0.3 ms if performing other tasks in-between starting
|
@@ -116,86 +188,32 @@ module ParallelMinion
|
|
116
188
|
# code in-line:
|
117
189
|
# ParallelMinion::Minion.new(description: 'Count', timeout: 5) { 1 }.result
|
118
190
|
#
|
119
|
-
#
|
120
|
-
# On JRuby it is
|
121
|
-
#
|
191
|
+
# Note:
|
192
|
+
# On JRuby it is recommended to add the following setting to .jrubyrc
|
193
|
+
# thread.pool.enabled=true
|
122
194
|
#
|
123
195
|
# Example:
|
124
196
|
# ParallelMinion::Minion.new(10.days.ago, description: 'Doing something else in parallel', timeout: 1000) do |date|
|
125
197
|
# MyTable.where('created_at <= ?', date).count
|
126
198
|
# end
|
127
|
-
def initialize(*
|
128
|
-
raise
|
199
|
+
def initialize(*arguments, description: 'Minion', metric: nil, log_exception: nil, enabled: self.class.enabled?, timeout: INFINITE, on_timeout: nil, wait_metric: nil, &block)
|
200
|
+
raise 'Missing mandatory block that Minion must perform' unless block
|
129
201
|
@start_time = Time.now
|
130
202
|
@exception = nil
|
131
|
-
@arguments =
|
132
|
-
|
133
|
-
@
|
134
|
-
@
|
135
|
-
@
|
136
|
-
@
|
137
|
-
@
|
138
|
-
@enabled = self.class.enabled? if @enabled.nil?
|
139
|
-
@on_timeout = options.delete(:on_timeout)
|
140
|
-
|
141
|
-
# Warn about any unknown options.
|
142
|
-
options.each_pair do | key, val |
|
143
|
-
logger.warn "Ignoring unknown option: #{key.inspect} => #{val.inspect}"
|
144
|
-
warn "ParallelMinion::Minion Ignoring unknown option: #{key.inspect} => #{val.inspect}"
|
145
|
-
end
|
203
|
+
@arguments = arguments
|
204
|
+
@timeout = timeout.to_f
|
205
|
+
@description = description.to_s
|
206
|
+
@metric = metric
|
207
|
+
@log_exception = log_exception
|
208
|
+
@enabled = enabled
|
209
|
+
@on_timeout = on_timeout
|
146
210
|
|
147
|
-
|
148
|
-
# debugging purposes
|
149
|
-
if @enabled == false
|
150
|
-
begin
|
151
|
-
logger.info("Started in the current thread: #{@description}")
|
152
|
-
logger.benchmark_info("Completed in the current thread: #{@description}", log_exception: @log_exception, metric: @metric) do
|
153
|
-
@result = instance_exec(*@arguments, &block)
|
154
|
-
end
|
155
|
-
rescue Exception => exc
|
156
|
-
@exception = exc
|
157
|
-
ensure
|
158
|
-
@duration = Time.now - @start_time
|
159
|
-
end
|
160
|
-
return
|
161
|
-
end
|
211
|
+
@wait_metric = (wait_metric || "#{metric}/wait") if @metric
|
162
212
|
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
@thread = Thread.new(*@arguments) do
|
169
|
-
# Copy logging tags from parent thread
|
170
|
-
logger.tagged(*tags) do
|
171
|
-
# Set the current thread name to the description for this Minion
|
172
|
-
# so that all log entries in this thread use this thread name
|
173
|
-
Thread.current.name = "#{@description}-#{Thread.current.object_id}"
|
174
|
-
logger.info("Started #{@description}")
|
175
|
-
|
176
|
-
begin
|
177
|
-
logger.benchmark_info("Completed #{@description}", log_exception: @log_exception, metric: @metric) do
|
178
|
-
# Use the current scope for the duration of the task execution
|
179
|
-
if scopes.nil? || (scopes.size == 0)
|
180
|
-
@result = instance_exec(*@arguments, &block)
|
181
|
-
else
|
182
|
-
# Each Class to scope requires passing a block to .scoping
|
183
|
-
proc = Proc.new { instance_exec(*@arguments, &block) }
|
184
|
-
first = scopes.shift
|
185
|
-
scopes.each {|scope| proc = Proc.new { scope.scoping(&proc) } }
|
186
|
-
@result = first.scoping(&proc)
|
187
|
-
end
|
188
|
-
end
|
189
|
-
rescue Exception => exc
|
190
|
-
@exception = exc
|
191
|
-
nil
|
192
|
-
ensure
|
193
|
-
# Return any database connections used by this thread back to the pool
|
194
|
-
ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
|
195
|
-
@duration = Time.now - @start_time
|
196
|
-
end
|
197
|
-
end
|
198
|
-
end
|
213
|
+
# When minion is disabled it is obvious in the logs since the name will now be 'Inline' instead of 'Minion'
|
214
|
+
self.logger = SemanticLogger['Inline'] unless @enabled
|
215
|
+
|
216
|
+
@enabled ? run(&block) : run_inline(&block)
|
199
217
|
end
|
200
218
|
|
201
219
|
# Returns the result when the thread completes
|
@@ -207,10 +225,10 @@ module ParallelMinion
|
|
207
225
|
# Return nil if Minion is still working and has time left to finish
|
208
226
|
if working?
|
209
227
|
ms = time_left
|
210
|
-
logger.
|
211
|
-
if @thread.join(ms.nil? ? nil: ms / 1000).nil?
|
212
|
-
@thread.raise(@on_timeout.new("Minion: #{
|
213
|
-
logger.warn("Timed out waiting for
|
228
|
+
logger.measure(self.class.completed_log_level, "Waited for Minion to complete: #{description}", min_duration: 0.01, metric: wait_metric) do
|
229
|
+
if @thread.join(ms.nil? ? nil : ms / 1000).nil?
|
230
|
+
@thread.raise(@on_timeout.new("Minion: #{description} timed out")) if @on_timeout
|
231
|
+
logger.warn("Timed out waiting for: #{description}")
|
214
232
|
return
|
215
233
|
end
|
216
234
|
end
|
@@ -239,8 +257,8 @@ module ParallelMinion
|
|
239
257
|
# Returns 0 if no time is left
|
240
258
|
# Returns nil if their is no time limit. I.e. :timeout was set to Minion::INFINITE (infinite time left)
|
241
259
|
def time_left
|
242
|
-
return nil if (
|
243
|
-
duration =
|
260
|
+
return nil if (timeout == 0) || (timeout == -1)
|
261
|
+
duration = timeout - (Time.now - start_time) * 1000
|
244
262
|
duration <= 0 ? 0 : duration
|
245
263
|
end
|
246
264
|
|
@@ -252,25 +270,78 @@ module ParallelMinion
|
|
252
270
|
# Returns the current scopes for each of the models for which scopes will be
|
253
271
|
# copied to the Minions
|
254
272
|
if defined?(ActiveRecord)
|
255
|
-
if
|
273
|
+
if ActiveRecord::VERSION::MAJOR >= 4
|
256
274
|
def self.current_scopes
|
257
|
-
|
275
|
+
scoped_classes.collect { |klass| klass.all }
|
258
276
|
end
|
259
277
|
else
|
260
278
|
def self.current_scopes
|
261
|
-
|
279
|
+
scoped_classes.collect { |klass| klass.scoped }
|
262
280
|
end
|
263
281
|
end
|
264
282
|
end
|
265
283
|
|
266
|
-
|
284
|
+
private
|
267
285
|
|
268
|
-
|
269
|
-
|
286
|
+
# Run the supplied block of code in the current thread.
|
287
|
+
# Useful for debugging, testing, and when running in batch environments.
|
288
|
+
def run_inline(&block)
|
289
|
+
begin
|
290
|
+
logger.public_send(self.class.started_log_level, "Started #{@description}")
|
291
|
+
logger.measure(self.class.completed_log_level, "Completed #{@description}", log_exception: @log_exception, metric: metric) do
|
292
|
+
@result = instance_exec(*arguments, &block)
|
293
|
+
end
|
294
|
+
rescue Exception => exc
|
295
|
+
@exception = exc
|
296
|
+
ensure
|
297
|
+
@duration = Time.now - start_time
|
298
|
+
end
|
299
|
+
end
|
300
|
+
|
301
|
+
def run(&block)
|
302
|
+
# Capture tags from current thread
|
303
|
+
tags = SemanticLogger.tags
|
304
|
+
tags = tags.nil? || tags.empty? ? nil : tags.dup
|
305
|
+
|
306
|
+
named_tags = SemanticLogger.named_tags
|
307
|
+
named_tags = named_tags.nil? || named_tags.empty? ? nil : named_tags.dup
|
308
|
+
|
309
|
+
# Captures scopes from current thread. Only applicable for AR models
|
310
|
+
scopes = self.class.current_scopes if defined?(ActiveRecord::Base)
|
311
|
+
|
312
|
+
@thread = Thread.new(*arguments) do
|
313
|
+
Thread.current.name = "#{description}-#{Thread.current.object_id}"
|
314
|
+
|
315
|
+
# Copy logging tags from parent thread, if any
|
316
|
+
proc = Proc.new { run_in_scope(scopes, &block) }
|
317
|
+
proc2 = tags ? Proc.new { SemanticLogger.tagged(*tags, &proc) } : proc
|
318
|
+
proc3 = named_tags ? Proc.new { SemanticLogger.named_tagged(named_tags, &proc2) } : proc2
|
319
|
+
|
320
|
+
logger.public_send(self.class.started_log_level, "Started #{description}")
|
321
|
+
begin
|
322
|
+
logger.measure(self.class.completed_log_level, "Completed #{description}", log_exception: log_exception, metric: metric, &proc3)
|
323
|
+
rescue Exception => exc
|
324
|
+
@exception = exc
|
325
|
+
nil
|
326
|
+
ensure
|
327
|
+
@duration = Time.now - start_time
|
328
|
+
# Return any database connections used by this thread back to the pool
|
329
|
+
ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
270
333
|
|
271
|
-
|
272
|
-
|
273
|
-
|
334
|
+
def run_in_scope(scopes, &block)
|
335
|
+
if scopes.nil? || scopes.empty?
|
336
|
+
@result = instance_exec(*@arguments, &block)
|
337
|
+
else
|
338
|
+
# Use the captured scope when running the block.
|
339
|
+
# Each Class to scope requires passing a block to .scoping.
|
340
|
+
proc = Proc.new { instance_exec(*@arguments, &block) }
|
341
|
+
first = scopes.shift
|
342
|
+
scopes.each { |scope| proc = Proc.new { scope.scoping(&proc) } }
|
343
|
+
@result = first.scoping(&proc)
|
344
|
+
end
|
274
345
|
end
|
275
346
|
|
276
347
|
end
|
@@ -11,6 +11,14 @@ module ParallelMinion #:nodoc:
|
|
11
11
|
# # Run Minions in the current thread to make debugging easier
|
12
12
|
# config.parallel_minion.enabled = false
|
13
13
|
#
|
14
|
+
# # Change the log level for the started log messages to :debug,
|
15
|
+
# # so that they do not show up in production when log level is :info.
|
16
|
+
# config.parallel_minion.started_log_level = :debug
|
17
|
+
#
|
18
|
+
# # Change the log level for the completed log messages to :debug,
|
19
|
+
# # so that they do not show up in production when log level is :info.
|
20
|
+
# config.parallel_minion.completed_log_level = :debug
|
21
|
+
#
|
14
22
|
# # Add a model so that its current scope is copied to the Minion
|
15
23
|
# config.after_initialize do
|
16
24
|
# # Perform in an after_initialize so that the model has been loaded
|
data/test/minion_scope_test.rb
CHANGED
@@ -1,18 +1,10 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'rubygems'
|
1
|
+
require_relative './test_helper'
|
4
2
|
require 'erb'
|
5
|
-
#require 'test/unit'
|
6
|
-
# Since we want both the AR and Mongoid extensions loaded we need to require them first
|
7
3
|
require 'active_record'
|
8
|
-
require 'active_record/relation'
|
9
|
-
# Should redefines Proc#bind so must include after Rails
|
10
|
-
#require 'shoulda'
|
11
|
-
#require 'parallel_minion'
|
12
4
|
|
13
|
-
ActiveRecord::Base.logger
|
5
|
+
ActiveRecord::Base.logger = SemanticLogger[ActiveRecord]
|
14
6
|
ActiveRecord::Base.configurations = YAML::load(ERB.new(IO.read('test/config/database.yml')).result)
|
15
|
-
ActiveRecord::Base.establish_connection(
|
7
|
+
ActiveRecord::Base.establish_connection(:test)
|
16
8
|
|
17
9
|
ActiveRecord::Schema.define :version => 0 do
|
18
10
|
create_table :people, :force => true do |t|
|
@@ -26,15 +18,14 @@ class Person < ActiveRecord::Base
|
|
26
18
|
end
|
27
19
|
|
28
20
|
class MinionScopeTest < Minitest::Test
|
29
|
-
|
30
|
-
context ParallelMinion::Minion do
|
21
|
+
describe ParallelMinion::Minion do
|
31
22
|
[false, true].each do |enabled|
|
32
|
-
|
33
|
-
|
23
|
+
describe ".new with enabled: #{enabled.inspect}" do
|
24
|
+
before do
|
34
25
|
Person.create(name: 'Jack', state: 'FL', zip_code: 38729)
|
35
26
|
Person.create(name: 'John', state: 'FL', zip_code: 35363)
|
36
27
|
Person.create(name: 'Jill', state: 'FL', zip_code: 73534)
|
37
|
-
Person.create(name: 'Joe',
|
28
|
+
Person.create(name: 'Joe', state: 'NY', zip_code: 45325)
|
38
29
|
Person.create(name: 'Jane', state: 'NY', zip_code: 45325)
|
39
30
|
Person.create(name: 'James', state: 'CA', zip_code: 123123)
|
40
31
|
# Instruct Minions to adhere to any dynamic scopes for Person model
|
@@ -42,13 +33,13 @@ class MinionScopeTest < Minitest::Test
|
|
42
33
|
ParallelMinion::Minion.enabled = enabled
|
43
34
|
end
|
44
35
|
|
45
|
-
|
36
|
+
after do
|
46
37
|
Person.destroy_all
|
47
38
|
ParallelMinion::Minion.scoped_classes.clear
|
48
39
|
SemanticLogger.flush
|
49
40
|
end
|
50
41
|
|
51
|
-
|
42
|
+
it 'copy across model scope' do
|
52
43
|
assert_equal 6, Person.count
|
53
44
|
|
54
45
|
Person.unscoped.where(state: 'FL').scoping { Person.count }
|
@@ -65,4 +56,4 @@ class MinionScopeTest < Minitest::Test
|
|
65
56
|
end
|
66
57
|
end
|
67
58
|
end
|
68
|
-
end
|
59
|
+
end
|
data/test/minion_test.rb
CHANGED
@@ -1,47 +1,53 @@
|
|
1
|
-
|
1
|
+
require_relative './test_helper'
|
2
2
|
|
3
3
|
# Test ParallelMinion standalone without Rails
|
4
4
|
# Run this test standalone to verify it has no Rails dependencies
|
5
5
|
class MinionTest < Minitest::Test
|
6
6
|
include SemanticLogger::Loggable
|
7
7
|
|
8
|
-
|
8
|
+
describe ParallelMinion::Minion do
|
9
9
|
|
10
10
|
[false, true].each do |enabled|
|
11
|
-
|
12
|
-
|
11
|
+
describe enabled ? 'enabled' : 'disabled' do
|
12
|
+
before do
|
13
13
|
ParallelMinion::Minion.enabled = enabled
|
14
|
-
$
|
14
|
+
$log_structs = []
|
15
15
|
end
|
16
16
|
|
17
|
-
|
17
|
+
it 'without parameters' do
|
18
18
|
minion = ParallelMinion::Minion.new { 196 }
|
19
19
|
assert_equal 196, minion.result
|
20
20
|
end
|
21
21
|
|
22
|
-
|
22
|
+
it 'with a description' do
|
23
23
|
minion = ParallelMinion::Minion.new(description: 'Test') { 197 }
|
24
24
|
assert_equal 197, minion.result
|
25
25
|
end
|
26
26
|
|
27
|
-
|
28
|
-
p1
|
27
|
+
it 'with an argument' do
|
28
|
+
p1 = {name: 198}
|
29
29
|
minion = ParallelMinion::Minion.new(p1, description: 'Test') do |v|
|
30
30
|
v[:name]
|
31
31
|
end
|
32
32
|
assert_equal 198, minion.result
|
33
33
|
end
|
34
34
|
|
35
|
-
|
35
|
+
it 'raise exception' do
|
36
36
|
minion = ParallelMinion::Minion.new(description: 'Test') { raise "An exception" }
|
37
37
|
assert_raises RuntimeError do
|
38
38
|
minion.result
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
+
it 'has correct logger name' do
|
43
|
+
minion = ParallelMinion::Minion.new { 196 }
|
44
|
+
name = enabled ? 'Minion' : 'Inline'
|
45
|
+
assert_equal name, minion.logger.name
|
46
|
+
end
|
47
|
+
|
42
48
|
# TODO Blocks still have access to their original scope if variables cannot be
|
43
49
|
# resolved first by the parameters, then by the values in Minion itself
|
44
|
-
#
|
50
|
+
# it 'not have access to local variables' do
|
45
51
|
# name = 'Jack'
|
46
52
|
# minion = ParallelMinion::Minion.new(description: 'Test') { puts name }
|
47
53
|
# assert_raises NameError do
|
@@ -49,11 +55,11 @@ class MinionTest < Minitest::Test
|
|
49
55
|
# end
|
50
56
|
# end
|
51
57
|
|
52
|
-
|
53
|
-
hash
|
54
|
-
value
|
58
|
+
it 'run minion' do
|
59
|
+
hash = {value: 23}
|
60
|
+
value = 47
|
55
61
|
minion = ParallelMinion::Minion.new(hash, description: 'Test') do |h|
|
56
|
-
value
|
62
|
+
value = 321
|
57
63
|
h[:value] = 123
|
58
64
|
456
|
59
65
|
end
|
@@ -62,10 +68,10 @@ class MinionTest < Minitest::Test
|
|
62
68
|
assert_equal 321, value
|
63
69
|
end
|
64
70
|
|
65
|
-
|
71
|
+
it 'copy across logging tags' do
|
66
72
|
minion = nil
|
67
|
-
|
68
|
-
assert_equal 'TAG',
|
73
|
+
SemanticLogger.tagged('TAG') do
|
74
|
+
assert_equal 'TAG', SemanticLogger.tags.last
|
69
75
|
minion = ParallelMinion::Minion.new(description: 'Tag Test') do
|
70
76
|
logger.tags.last
|
71
77
|
end
|
@@ -73,27 +79,38 @@ class MinionTest < Minitest::Test
|
|
73
79
|
assert_equal 'TAG', minion.result
|
74
80
|
end
|
75
81
|
|
76
|
-
|
77
|
-
metric_name = '/
|
78
|
-
hash
|
79
|
-
value
|
80
|
-
minion
|
81
|
-
value
|
82
|
+
it 'include metric' do
|
83
|
+
metric_name = 'model/method'
|
84
|
+
hash = {value: 23}
|
85
|
+
value = 47
|
86
|
+
minion = ParallelMinion::Minion.new(hash, description: 'Test', metric: metric_name) do |h|
|
87
|
+
value = 321
|
82
88
|
h[:value] = 123
|
89
|
+
sleep 1
|
83
90
|
456
|
84
91
|
end
|
85
92
|
assert_equal 456, minion.result
|
86
93
|
assert_equal 123, hash[:value]
|
87
94
|
assert_equal 321, value
|
88
95
|
SemanticLogger.flush
|
89
|
-
|
96
|
+
assert log = $log_structs.first, -> { $log_structs.ai }
|
97
|
+
if enabled
|
98
|
+
# Completed log message
|
99
|
+
assert_equal metric_name, log.metric, -> { $log_structs.ai }
|
100
|
+
# Wait log message
|
101
|
+
assert log = $log_structs.last, -> { $log_structs.ai }
|
102
|
+
assert_equal "#{metric_name}/wait", log.metric, -> { $log_structs.ai }
|
103
|
+
else
|
104
|
+
# Timeout and wait has no effect when run inline
|
105
|
+
assert_equal metric_name, log.metric, -> { $log_structs.ai }
|
106
|
+
end
|
90
107
|
end
|
91
108
|
|
92
|
-
|
109
|
+
it 'handle multiple minions concurrently' do
|
93
110
|
# Start 10 minions
|
94
111
|
minions = 10.times.collect do |i|
|
95
112
|
# Each Minion returns its index in the collection
|
96
|
-
ParallelMinion::Minion.new(i, description: "Minion:#{i}") {|counter| counter }
|
113
|
+
ParallelMinion::Minion.new(i, description: "Minion:#{i}") { |counter| counter }
|
97
114
|
end
|
98
115
|
assert_equal 10, minions.count
|
99
116
|
# Fetch the result from each Minion
|
@@ -102,58 +119,58 @@ class MinionTest < Minitest::Test
|
|
102
119
|
end
|
103
120
|
end
|
104
121
|
|
105
|
-
|
106
|
-
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 100) { sleep 1 }
|
122
|
+
it 'timeout' do
|
107
123
|
if enabled
|
108
|
-
|
124
|
+
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 100) { sleep 1 }
|
125
|
+
assert_nil minion.result
|
109
126
|
end
|
110
127
|
end
|
111
128
|
|
112
|
-
|
113
|
-
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 100, on_timeout: Timeout::Error) { sleep 1 }
|
129
|
+
it 'timeout and terminate thread with Exception' do
|
114
130
|
if enabled
|
115
|
-
|
131
|
+
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 100, on_timeout: Timeout::Error) { sleep 1 }
|
132
|
+
assert_nil minion.result
|
116
133
|
# Give time for thread to terminate
|
117
134
|
sleep 0.1
|
118
135
|
assert_equal Timeout::Error, minion.exception.class
|
119
136
|
assert_equal false, minion.working?
|
120
|
-
assert_equal true,
|
121
|
-
assert_equal true,
|
122
|
-
assert_equal 0,
|
137
|
+
assert_equal true, minion.completed?
|
138
|
+
assert_equal true, minion.failed?
|
139
|
+
assert_equal 0, minion.time_left
|
123
140
|
end
|
124
141
|
end
|
125
142
|
|
126
|
-
|
143
|
+
it 'make description instance variable available' do
|
127
144
|
minion = ParallelMinion::Minion.new(description: 'Test') do
|
128
145
|
description
|
129
146
|
end
|
130
147
|
assert_equal 'Test', minion.result
|
131
148
|
end
|
132
149
|
|
133
|
-
|
134
|
-
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 1000
|
150
|
+
it 'make timeout instance variable available' do
|
151
|
+
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 1000) do
|
135
152
|
timeout
|
136
153
|
end
|
137
154
|
assert_equal 1000, minion.result
|
138
155
|
end
|
139
156
|
|
140
|
-
|
157
|
+
it 'make enabled? method available' do
|
141
158
|
minion = ParallelMinion::Minion.new(description: 'Test') do
|
142
159
|
enabled?
|
143
160
|
end
|
144
161
|
assert_equal enabled, minion.result
|
145
162
|
end
|
146
163
|
|
147
|
-
|
148
|
-
minion = ParallelMinion::Minion.new(1, 'data', 14.1, description: 'Test') do |
|
164
|
+
it 'keep the original arguments' do
|
165
|
+
minion = ParallelMinion::Minion.new(1, 'data', 14.1, description: 'Test') do |num, str, float|
|
149
166
|
num + float
|
150
167
|
end
|
151
168
|
assert_equal 15.1, minion.result
|
152
|
-
assert_equal [
|
169
|
+
assert_equal [1, 'data', 14.1], minion.arguments
|
153
170
|
end
|
154
171
|
end
|
155
172
|
|
156
173
|
end
|
157
174
|
|
158
175
|
end
|
159
|
-
end
|
176
|
+
end
|
data/test/test_db.sqlite3
CHANGED
Binary file
|
data/test/test_helper.rb
CHANGED
@@ -1,19 +1,13 @@
|
|
1
1
|
$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
-
require 'minitest/reporters'
|
5
|
-
require 'minitest/stub_any_instance'
|
6
|
-
require 'shoulda/context'
|
7
4
|
require 'parallel_minion'
|
8
5
|
require 'semantic_logger'
|
9
6
|
|
10
|
-
Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
|
11
|
-
|
12
|
-
# Register an appender if one is not already registered
|
13
7
|
SemanticLogger.default_level = :trace
|
14
|
-
SemanticLogger.add_appender('test.log',
|
8
|
+
SemanticLogger.add_appender(file_name: 'test.log', formatter: :color)
|
15
9
|
|
16
10
|
# Setup global callback for metric so that it can be tested below
|
17
11
|
SemanticLogger.on_metric do |log_struct|
|
18
|
-
$
|
12
|
+
($log_structs ||= []) << log_struct.dup
|
19
13
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel_minion
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: semantic_logger
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0'
|
19
|
+
version: '4.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0'
|
26
|
+
version: '4.0'
|
27
27
|
description: Parallel Minion allows you to take existing blocks of code and wrap them
|
28
28
|
in a minion so that they can run asynchronously in a separate thread
|
29
29
|
email:
|
@@ -56,7 +56,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
56
56
|
requirements:
|
57
57
|
- - ">="
|
58
58
|
- !ruby/object:Gem::Version
|
59
|
-
version: '
|
59
|
+
version: '2.1'
|
60
60
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
61
61
|
requirements:
|
62
62
|
- - ">="
|
@@ -64,7 +64,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
64
64
|
version: '0'
|
65
65
|
requirements: []
|
66
66
|
rubyforge_project:
|
67
|
-
rubygems_version: 2.
|
67
|
+
rubygems_version: 2.6.11
|
68
68
|
signing_key:
|
69
69
|
specification_version: 4
|
70
70
|
summary: Pragmatic approach to parallel and asynchronous processing in Ruby
|