parallel_minion 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +6 -4
- data/Rakefile +18 -15
- data/lib/parallel_minion.rb +1 -1
- data/lib/parallel_minion/minion.rb +198 -127
- data/lib/parallel_minion/railtie.rb +8 -0
- data/lib/parallel_minion/version.rb +1 -1
- data/test/minion_scope_test.rb +10 -19
- data/test/minion_test.rb +61 -44
- data/test/test_db.sqlite3 +0 -0
- data/test/test_helper.rb +2 -8
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c4ec6fafacfe5cf2b3e090a1e1ed733fd6c3bc33
|
4
|
+
data.tar.gz: a054a763962d6ee10fb8bc75587ec4d1c5f3d456
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 13909e2ad5221826dd9ac6cbbd15364566e4cf6b76fc33bc77d7cc8c35acc6ddb06cc3b53845bb6df053b031a13fb20808ce10c7e2f7bbb141912e2a86a03aff
|
7
|
+
data.tar.gz: 01b7b72f77160109e4bfa9ab995c6f98be128a7e837bc284d3932f399cdae9389b866d83a534d5ec618425c201d3607d5ff19b06bf803a3054da4edc409bcaa5
|
data/README.md
CHANGED
@@ -15,9 +15,11 @@ existing code to use minions.
|
|
15
15
|
## Example
|
16
16
|
|
17
17
|
```ruby
|
18
|
-
minion = ParallelMinion::Minion.new(
|
19
|
-
|
20
|
-
|
18
|
+
minion = ParallelMinion::Minion.new(
|
19
|
+
10.days.ago,
|
20
|
+
description: 'Doing something else in parallel',
|
21
|
+
timeout: 1000
|
22
|
+
) do |date|
|
21
23
|
MyTable.where('created_at <= ?', date).count
|
22
24
|
end
|
23
25
|
|
@@ -59,7 +61,7 @@ This project uses [Semantic Versioning](http://semver.org/).
|
|
59
61
|
|
60
62
|
## License
|
61
63
|
|
62
|
-
Copyright 2013, 2014,
|
64
|
+
Copyright 2013, 2014, 2015, 2016, 2017 Reid Morrison
|
63
65
|
|
64
66
|
Licensed under the Apache License, Version 2.0 (the "License");
|
65
67
|
you may not use this file except in compliance with the License.
|
data/Rakefile
CHANGED
@@ -1,28 +1,31 @@
|
|
1
|
-
|
2
|
-
require '
|
1
|
+
# Setup bundler to avoid having to run bundle exec all the time.
|
2
|
+
require 'rubygems'
|
3
|
+
require 'bundler/setup'
|
3
4
|
|
4
|
-
|
5
|
-
|
5
|
+
require 'rake/testtask'
|
6
|
+
require_relative 'lib/parallel_minion/version'
|
6
7
|
|
7
8
|
task :gem do
|
8
|
-
system
|
9
|
+
system 'gem build parallel_minion.gemspec'
|
9
10
|
end
|
10
11
|
|
11
12
|
task :publish => :gem do
|
12
13
|
system "git tag -a v#{ParallelMinion::VERSION} -m 'Tagging #{ParallelMinion::VERSION}'"
|
13
|
-
system
|
14
|
+
system 'git push --tags'
|
14
15
|
system "gem push parallel_minion-#{ParallelMinion::VERSION}.gem"
|
15
16
|
system "rm parallel_minion-#{ParallelMinion::VERSION}.gem"
|
16
17
|
end
|
17
18
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
t.verbose = true
|
23
|
-
end
|
24
|
-
|
25
|
-
Rake::Task['functional'].invoke
|
19
|
+
Rake::TestTask.new(:test) do |t|
|
20
|
+
t.pattern = 'test/**/*_test.rb'
|
21
|
+
t.verbose = true
|
22
|
+
t.warning = false
|
26
23
|
end
|
27
24
|
|
28
|
-
|
25
|
+
# By default run tests against all appraisals
|
26
|
+
if !ENV["APPRAISAL_INITIALIZED"] && !ENV["TRAVIS"]
|
27
|
+
require 'appraisal'
|
28
|
+
task default: :appraisal
|
29
|
+
else
|
30
|
+
task default: :test
|
31
|
+
end
|
data/lib/parallel_minion.rb
CHANGED
@@ -19,26 +19,34 @@ module ParallelMinion
|
|
19
19
|
# Returns nil if the minion is still running
|
20
20
|
attr_reader :duration
|
21
21
|
|
22
|
+
# Metrics [String]
|
23
|
+
attr_reader :metric, :wait_metric
|
24
|
+
|
25
|
+
attr_reader :on_timeout, :log_exception, :start_time
|
26
|
+
|
22
27
|
# Give an infinite amount of time to wait for a Minion to complete a task
|
23
28
|
INFINITE = 0
|
24
29
|
|
25
|
-
# Sets whether
|
26
|
-
#
|
27
|
-
# By Setting _enabled_ to false all Minions that have not yet been started
|
28
|
-
# will run in the thread from which it is created and not on its own thread
|
30
|
+
# Sets whether Minions should run in a separate thread.
|
29
31
|
#
|
30
|
-
#
|
31
|
-
#
|
32
|
-
# -
|
33
|
-
#
|
34
|
-
#
|
32
|
+
# By Setting _enabled_ to false all Minions that have not yet been created
|
33
|
+
# will run in the thread in which it is created.
|
34
|
+
# - Development:
|
35
|
+
# Use a debugger, since the code will run in the current thread.
|
36
|
+
# - Test:
|
37
|
+
# Keep test execution in the current thread.
|
38
|
+
# Supports rolling back database changes after each test, since all changes are
|
39
|
+
# performed on the same database connection.
|
40
|
+
# - Production:
|
41
|
+
# Batch processing in Rocket Job where throughput is more important than latency.
|
42
|
+
# http://rocketjob.io
|
35
43
|
def self.enabled=(enabled)
|
36
|
-
|
44
|
+
@enabled = enabled
|
37
45
|
end
|
38
46
|
|
39
47
|
# Returns whether minions are enabled to run in their own threads
|
40
48
|
def self.enabled?
|
41
|
-
|
49
|
+
@enabled
|
42
50
|
end
|
43
51
|
|
44
52
|
# The list of classes for which the current scope must be copied into the
|
@@ -47,25 +55,85 @@ module ParallelMinion
|
|
47
55
|
# Example:
|
48
56
|
# ...
|
49
57
|
def self.scoped_classes
|
50
|
-
|
58
|
+
@scoped_classes
|
51
59
|
end
|
52
60
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
#
|
58
|
-
#
|
61
|
+
def self.scoped_classes=(scoped_classes)
|
62
|
+
@scoped_classes = scoped_classes.dup
|
63
|
+
end
|
64
|
+
|
65
|
+
# Change the log level for the Started log message.
|
66
|
+
#
|
67
|
+
# Default: :info
|
68
|
+
#
|
69
|
+
# Valid levels:
|
70
|
+
# :trace, :debug, :info, :warn, :error, :fatal
|
71
|
+
def self.started_log_level=(level)
|
72
|
+
raise(ArgumentError, "Invalid log level: #{level}") unless SemanticLogger::LEVELS.include?(level)
|
73
|
+
@started_log_level = level
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.started_log_level
|
77
|
+
@started_log_level
|
78
|
+
end
|
79
|
+
|
80
|
+
# Change the log level for the Completed log message.
|
81
|
+
#
|
82
|
+
# Default: :info
|
83
|
+
#
|
84
|
+
# Valid levels:
|
85
|
+
# :trace, :debug, :info, :warn, :error, :fatal
|
86
|
+
def self.completed_log_level=(level)
|
87
|
+
raise(ArgumentError, "Invalid log level: #{level}") unless SemanticLogger::LEVELS.include?(level)
|
88
|
+
@completed_log_level = level
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.completed_log_level
|
92
|
+
@completed_log_level
|
93
|
+
end
|
94
|
+
|
95
|
+
self.started_log_level = :info
|
96
|
+
self.completed_log_level = :info
|
97
|
+
self.enabled = true
|
98
|
+
self.scoped_classes = []
|
99
|
+
logger.name = 'Minion'
|
100
|
+
|
101
|
+
# Create a new Minion
|
102
|
+
#
|
103
|
+
# Creates a new thread and logs the time for the supplied block to complete processing.
|
104
|
+
# The exception without stack trace is logged whenever an exception is thrown in the thread.
|
105
|
+
#
|
106
|
+
# Re-raises any unhandled exception in the calling thread when `#result` is called.
|
107
|
+
# Copies the logging tags and specified ActiveRecord scopes to the new thread.
|
59
108
|
#
|
60
109
|
# Parameters
|
110
|
+
# *arguments
|
111
|
+
# Any number of arguments can be supplied that are passed into the block
|
112
|
+
# in the order they are listed.
|
113
|
+
#
|
114
|
+
# Note:
|
115
|
+
# All arguments must be supplied by copy and not by reference.
|
116
|
+
# For example, use `#dup` to create copies of passed data.
|
117
|
+
# Pass by copy is critical to prevent concurrency issues when multiple threads
|
118
|
+
# attempt to update the same object at the same time.
|
119
|
+
#
|
120
|
+
# Proc / lambda
|
121
|
+
# A block of code must be supplied that the Minion will execute.
|
122
|
+
#
|
123
|
+
# Note:
|
124
|
+
# This block will be executed within the scope of the created minion instance
|
125
|
+
# and _not_ within the scope of where the Proc/lambda was originally created.
|
126
|
+
# This is done to force all parameters to be passed in explicitly
|
127
|
+
# and should be read-only or duplicates of the original data.
|
128
|
+
#
|
61
129
|
# :description [String]
|
62
|
-
# Description for this task that the Minion is performing
|
63
|
-
#
|
130
|
+
# Description for this task that the Minion is performing.
|
131
|
+
# Written to the log file along with the time take to complete the task.
|
64
132
|
#
|
65
133
|
# :timeout [Integer]
|
66
134
|
# Maximum amount of time in milli-seconds that the task may take to complete
|
67
|
-
# before #result times out
|
68
|
-
# Set to 0 to give the thread an infinite amount of time to complete
|
135
|
+
# before #result times out.
|
136
|
+
# Set to 0 to give the thread an infinite amount of time to complete.
|
69
137
|
# Default: 0 ( Wait forever )
|
70
138
|
#
|
71
139
|
# Notes:
|
@@ -76,11 +144,6 @@ module ParallelMinion
|
|
76
144
|
# then :timeout is ignored and assumed to be Minion::INFINITE
|
77
145
|
# since the code is run in the calling thread when the Minion is created
|
78
146
|
#
|
79
|
-
# :enabled [Boolean]
|
80
|
-
# Whether the minion should run in a separate thread
|
81
|
-
# Not recommended in Production, but is useful for debugging purposes
|
82
|
-
# Default: ParallelMinion::Minion.enabled?
|
83
|
-
#
|
84
147
|
# :on_timeout [Exception]
|
85
148
|
# The class to raise on the minion when the minion times out.
|
86
149
|
# By raising the exception on the running thread it ensures that the thread
|
@@ -92,21 +155,30 @@ module ParallelMinion
|
|
92
155
|
#
|
93
156
|
# Note: :on_timeout has no effect if not #enabled?
|
94
157
|
#
|
95
|
-
#
|
96
|
-
#
|
97
|
-
#
|
98
|
-
# It is recommended to duplicate and/or freeze objects passed as arguments
|
99
|
-
# so that they are not modified at the same time by multiple threads
|
100
|
-
# These arguments are accessible while and after the minion is running
|
101
|
-
# by calling #arguments
|
158
|
+
# :metric [String]
|
159
|
+
# Name of the metric to forward to Semantic Logger when measuring the minion execution time.
|
160
|
+
# Example: inquiry/address_cleansing
|
102
161
|
#
|
103
|
-
#
|
104
|
-
#
|
105
|
-
#
|
106
|
-
#
|
107
|
-
#
|
108
|
-
#
|
109
|
-
#
|
162
|
+
# When a metric is supplied the following metrics will also be generated:
|
163
|
+
# - wait
|
164
|
+
# Duration waiting for a minion to complete.
|
165
|
+
#
|
166
|
+
# The additional metrics are added to the supplied metric name. For example:
|
167
|
+
# - inquiry/address_cleansing/wait
|
168
|
+
#
|
169
|
+
# :log_exception [Symbol]
|
170
|
+
# Control whether or how an exception thrown in the block is
|
171
|
+
# reported by Semantic Logger. Values:
|
172
|
+
# :full
|
173
|
+
# Log the exception class, message, and backtrace
|
174
|
+
# :partial
|
175
|
+
# Log the exception class and message. The backtrace will not be logged
|
176
|
+
# :off
|
177
|
+
# Any unhandled exception raised in the block will not be logged
|
178
|
+
# Default: :partial
|
179
|
+
#
|
180
|
+
# :enabled [Boolean]
|
181
|
+
# Override the global setting: `ParallelMinion::Minion.enabled?` for this minion instance.
|
110
182
|
#
|
111
183
|
# The overhead for moving the task to a Minion (separate thread) vs running it
|
112
184
|
# sequentially is about 0.3 ms if performing other tasks in-between starting
|
@@ -116,86 +188,32 @@ module ParallelMinion
|
|
116
188
|
# code in-line:
|
117
189
|
# ParallelMinion::Minion.new(description: 'Count', timeout: 5) { 1 }.result
|
118
190
|
#
|
119
|
-
#
|
120
|
-
# On JRuby it is
|
121
|
-
#
|
191
|
+
# Note:
|
192
|
+
# On JRuby it is recommended to add the following setting to .jrubyrc
|
193
|
+
# thread.pool.enabled=true
|
122
194
|
#
|
123
195
|
# Example:
|
124
196
|
# ParallelMinion::Minion.new(10.days.ago, description: 'Doing something else in parallel', timeout: 1000) do |date|
|
125
197
|
# MyTable.where('created_at <= ?', date).count
|
126
198
|
# end
|
127
|
-
def initialize(*
|
128
|
-
raise
|
199
|
+
def initialize(*arguments, description: 'Minion', metric: nil, log_exception: nil, enabled: self.class.enabled?, timeout: INFINITE, on_timeout: nil, wait_metric: nil, &block)
|
200
|
+
raise 'Missing mandatory block that Minion must perform' unless block
|
129
201
|
@start_time = Time.now
|
130
202
|
@exception = nil
|
131
|
-
@arguments =
|
132
|
-
|
133
|
-
@
|
134
|
-
@
|
135
|
-
@
|
136
|
-
@
|
137
|
-
@
|
138
|
-
@enabled = self.class.enabled? if @enabled.nil?
|
139
|
-
@on_timeout = options.delete(:on_timeout)
|
140
|
-
|
141
|
-
# Warn about any unknown options.
|
142
|
-
options.each_pair do | key, val |
|
143
|
-
logger.warn "Ignoring unknown option: #{key.inspect} => #{val.inspect}"
|
144
|
-
warn "ParallelMinion::Minion Ignoring unknown option: #{key.inspect} => #{val.inspect}"
|
145
|
-
end
|
203
|
+
@arguments = arguments
|
204
|
+
@timeout = timeout.to_f
|
205
|
+
@description = description.to_s
|
206
|
+
@metric = metric
|
207
|
+
@log_exception = log_exception
|
208
|
+
@enabled = enabled
|
209
|
+
@on_timeout = on_timeout
|
146
210
|
|
147
|
-
|
148
|
-
# debugging purposes
|
149
|
-
if @enabled == false
|
150
|
-
begin
|
151
|
-
logger.info("Started in the current thread: #{@description}")
|
152
|
-
logger.benchmark_info("Completed in the current thread: #{@description}", log_exception: @log_exception, metric: @metric) do
|
153
|
-
@result = instance_exec(*@arguments, &block)
|
154
|
-
end
|
155
|
-
rescue Exception => exc
|
156
|
-
@exception = exc
|
157
|
-
ensure
|
158
|
-
@duration = Time.now - @start_time
|
159
|
-
end
|
160
|
-
return
|
161
|
-
end
|
211
|
+
@wait_metric = (wait_metric || "#{metric}/wait") if @metric
|
162
212
|
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
@thread = Thread.new(*@arguments) do
|
169
|
-
# Copy logging tags from parent thread
|
170
|
-
logger.tagged(*tags) do
|
171
|
-
# Set the current thread name to the description for this Minion
|
172
|
-
# so that all log entries in this thread use this thread name
|
173
|
-
Thread.current.name = "#{@description}-#{Thread.current.object_id}"
|
174
|
-
logger.info("Started #{@description}")
|
175
|
-
|
176
|
-
begin
|
177
|
-
logger.benchmark_info("Completed #{@description}", log_exception: @log_exception, metric: @metric) do
|
178
|
-
# Use the current scope for the duration of the task execution
|
179
|
-
if scopes.nil? || (scopes.size == 0)
|
180
|
-
@result = instance_exec(*@arguments, &block)
|
181
|
-
else
|
182
|
-
# Each Class to scope requires passing a block to .scoping
|
183
|
-
proc = Proc.new { instance_exec(*@arguments, &block) }
|
184
|
-
first = scopes.shift
|
185
|
-
scopes.each {|scope| proc = Proc.new { scope.scoping(&proc) } }
|
186
|
-
@result = first.scoping(&proc)
|
187
|
-
end
|
188
|
-
end
|
189
|
-
rescue Exception => exc
|
190
|
-
@exception = exc
|
191
|
-
nil
|
192
|
-
ensure
|
193
|
-
# Return any database connections used by this thread back to the pool
|
194
|
-
ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
|
195
|
-
@duration = Time.now - @start_time
|
196
|
-
end
|
197
|
-
end
|
198
|
-
end
|
213
|
+
# When minion is disabled it is obvious in the logs since the name will now be 'Inline' instead of 'Minion'
|
214
|
+
self.logger = SemanticLogger['Inline'] unless @enabled
|
215
|
+
|
216
|
+
@enabled ? run(&block) : run_inline(&block)
|
199
217
|
end
|
200
218
|
|
201
219
|
# Returns the result when the thread completes
|
@@ -207,10 +225,10 @@ module ParallelMinion
|
|
207
225
|
# Return nil if Minion is still working and has time left to finish
|
208
226
|
if working?
|
209
227
|
ms = time_left
|
210
|
-
logger.
|
211
|
-
if @thread.join(ms.nil? ? nil: ms / 1000).nil?
|
212
|
-
@thread.raise(@on_timeout.new("Minion: #{
|
213
|
-
logger.warn("Timed out waiting for
|
228
|
+
logger.measure(self.class.completed_log_level, "Waited for Minion to complete: #{description}", min_duration: 0.01, metric: wait_metric) do
|
229
|
+
if @thread.join(ms.nil? ? nil : ms / 1000).nil?
|
230
|
+
@thread.raise(@on_timeout.new("Minion: #{description} timed out")) if @on_timeout
|
231
|
+
logger.warn("Timed out waiting for: #{description}")
|
214
232
|
return
|
215
233
|
end
|
216
234
|
end
|
@@ -239,8 +257,8 @@ module ParallelMinion
|
|
239
257
|
# Returns 0 if no time is left
|
240
258
|
# Returns nil if their is no time limit. I.e. :timeout was set to Minion::INFINITE (infinite time left)
|
241
259
|
def time_left
|
242
|
-
return nil if (
|
243
|
-
duration =
|
260
|
+
return nil if (timeout == 0) || (timeout == -1)
|
261
|
+
duration = timeout - (Time.now - start_time) * 1000
|
244
262
|
duration <= 0 ? 0 : duration
|
245
263
|
end
|
246
264
|
|
@@ -252,25 +270,78 @@ module ParallelMinion
|
|
252
270
|
# Returns the current scopes for each of the models for which scopes will be
|
253
271
|
# copied to the Minions
|
254
272
|
if defined?(ActiveRecord)
|
255
|
-
if
|
273
|
+
if ActiveRecord::VERSION::MAJOR >= 4
|
256
274
|
def self.current_scopes
|
257
|
-
|
275
|
+
scoped_classes.collect { |klass| klass.all }
|
258
276
|
end
|
259
277
|
else
|
260
278
|
def self.current_scopes
|
261
|
-
|
279
|
+
scoped_classes.collect { |klass| klass.scoped }
|
262
280
|
end
|
263
281
|
end
|
264
282
|
end
|
265
283
|
|
266
|
-
|
284
|
+
private
|
267
285
|
|
268
|
-
|
269
|
-
|
286
|
+
# Run the supplied block of code in the current thread.
|
287
|
+
# Useful for debugging, testing, and when running in batch environments.
|
288
|
+
def run_inline(&block)
|
289
|
+
begin
|
290
|
+
logger.public_send(self.class.started_log_level, "Started #{@description}")
|
291
|
+
logger.measure(self.class.completed_log_level, "Completed #{@description}", log_exception: @log_exception, metric: metric) do
|
292
|
+
@result = instance_exec(*arguments, &block)
|
293
|
+
end
|
294
|
+
rescue Exception => exc
|
295
|
+
@exception = exc
|
296
|
+
ensure
|
297
|
+
@duration = Time.now - start_time
|
298
|
+
end
|
299
|
+
end
|
300
|
+
|
301
|
+
def run(&block)
|
302
|
+
# Capture tags from current thread
|
303
|
+
tags = SemanticLogger.tags
|
304
|
+
tags = tags.nil? || tags.empty? ? nil : tags.dup
|
305
|
+
|
306
|
+
named_tags = SemanticLogger.named_tags
|
307
|
+
named_tags = named_tags.nil? || named_tags.empty? ? nil : named_tags.dup
|
308
|
+
|
309
|
+
# Captures scopes from current thread. Only applicable for AR models
|
310
|
+
scopes = self.class.current_scopes if defined?(ActiveRecord::Base)
|
311
|
+
|
312
|
+
@thread = Thread.new(*arguments) do
|
313
|
+
Thread.current.name = "#{description}-#{Thread.current.object_id}"
|
314
|
+
|
315
|
+
# Copy logging tags from parent thread, if any
|
316
|
+
proc = Proc.new { run_in_scope(scopes, &block) }
|
317
|
+
proc2 = tags ? Proc.new { SemanticLogger.tagged(*tags, &proc) } : proc
|
318
|
+
proc3 = named_tags ? Proc.new { SemanticLogger.named_tagged(named_tags, &proc2) } : proc2
|
319
|
+
|
320
|
+
logger.public_send(self.class.started_log_level, "Started #{description}")
|
321
|
+
begin
|
322
|
+
logger.measure(self.class.completed_log_level, "Completed #{description}", log_exception: log_exception, metric: metric, &proc3)
|
323
|
+
rescue Exception => exc
|
324
|
+
@exception = exc
|
325
|
+
nil
|
326
|
+
ensure
|
327
|
+
@duration = Time.now - start_time
|
328
|
+
# Return any database connections used by this thread back to the pool
|
329
|
+
ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
270
333
|
|
271
|
-
|
272
|
-
|
273
|
-
|
334
|
+
def run_in_scope(scopes, &block)
|
335
|
+
if scopes.nil? || scopes.empty?
|
336
|
+
@result = instance_exec(*@arguments, &block)
|
337
|
+
else
|
338
|
+
# Use the captured scope when running the block.
|
339
|
+
# Each Class to scope requires passing a block to .scoping.
|
340
|
+
proc = Proc.new { instance_exec(*@arguments, &block) }
|
341
|
+
first = scopes.shift
|
342
|
+
scopes.each { |scope| proc = Proc.new { scope.scoping(&proc) } }
|
343
|
+
@result = first.scoping(&proc)
|
344
|
+
end
|
274
345
|
end
|
275
346
|
|
276
347
|
end
|
@@ -11,6 +11,14 @@ module ParallelMinion #:nodoc:
|
|
11
11
|
# # Run Minions in the current thread to make debugging easier
|
12
12
|
# config.parallel_minion.enabled = false
|
13
13
|
#
|
14
|
+
# # Change the log level for the started log messages to :debug,
|
15
|
+
# # so that they do not show up in production when log level is :info.
|
16
|
+
# config.parallel_minion.started_log_level = :debug
|
17
|
+
#
|
18
|
+
# # Change the log level for the completed log messages to :debug,
|
19
|
+
# # so that they do not show up in production when log level is :info.
|
20
|
+
# config.parallel_minion.completed_log_level = :debug
|
21
|
+
#
|
14
22
|
# # Add a model so that its current scope is copied to the Minion
|
15
23
|
# config.after_initialize do
|
16
24
|
# # Perform in an after_initialize so that the model has been loaded
|
data/test/minion_scope_test.rb
CHANGED
@@ -1,18 +1,10 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'rubygems'
|
1
|
+
require_relative './test_helper'
|
4
2
|
require 'erb'
|
5
|
-
#require 'test/unit'
|
6
|
-
# Since we want both the AR and Mongoid extensions loaded we need to require them first
|
7
3
|
require 'active_record'
|
8
|
-
require 'active_record/relation'
|
9
|
-
# Should redefines Proc#bind so must include after Rails
|
10
|
-
#require 'shoulda'
|
11
|
-
#require 'parallel_minion'
|
12
4
|
|
13
|
-
ActiveRecord::Base.logger
|
5
|
+
ActiveRecord::Base.logger = SemanticLogger[ActiveRecord]
|
14
6
|
ActiveRecord::Base.configurations = YAML::load(ERB.new(IO.read('test/config/database.yml')).result)
|
15
|
-
ActiveRecord::Base.establish_connection(
|
7
|
+
ActiveRecord::Base.establish_connection(:test)
|
16
8
|
|
17
9
|
ActiveRecord::Schema.define :version => 0 do
|
18
10
|
create_table :people, :force => true do |t|
|
@@ -26,15 +18,14 @@ class Person < ActiveRecord::Base
|
|
26
18
|
end
|
27
19
|
|
28
20
|
class MinionScopeTest < Minitest::Test
|
29
|
-
|
30
|
-
context ParallelMinion::Minion do
|
21
|
+
describe ParallelMinion::Minion do
|
31
22
|
[false, true].each do |enabled|
|
32
|
-
|
33
|
-
|
23
|
+
describe ".new with enabled: #{enabled.inspect}" do
|
24
|
+
before do
|
34
25
|
Person.create(name: 'Jack', state: 'FL', zip_code: 38729)
|
35
26
|
Person.create(name: 'John', state: 'FL', zip_code: 35363)
|
36
27
|
Person.create(name: 'Jill', state: 'FL', zip_code: 73534)
|
37
|
-
Person.create(name: 'Joe',
|
28
|
+
Person.create(name: 'Joe', state: 'NY', zip_code: 45325)
|
38
29
|
Person.create(name: 'Jane', state: 'NY', zip_code: 45325)
|
39
30
|
Person.create(name: 'James', state: 'CA', zip_code: 123123)
|
40
31
|
# Instruct Minions to adhere to any dynamic scopes for Person model
|
@@ -42,13 +33,13 @@ class MinionScopeTest < Minitest::Test
|
|
42
33
|
ParallelMinion::Minion.enabled = enabled
|
43
34
|
end
|
44
35
|
|
45
|
-
|
36
|
+
after do
|
46
37
|
Person.destroy_all
|
47
38
|
ParallelMinion::Minion.scoped_classes.clear
|
48
39
|
SemanticLogger.flush
|
49
40
|
end
|
50
41
|
|
51
|
-
|
42
|
+
it 'copy across model scope' do
|
52
43
|
assert_equal 6, Person.count
|
53
44
|
|
54
45
|
Person.unscoped.where(state: 'FL').scoping { Person.count }
|
@@ -65,4 +56,4 @@ class MinionScopeTest < Minitest::Test
|
|
65
56
|
end
|
66
57
|
end
|
67
58
|
end
|
68
|
-
end
|
59
|
+
end
|
data/test/minion_test.rb
CHANGED
@@ -1,47 +1,53 @@
|
|
1
|
-
|
1
|
+
require_relative './test_helper'
|
2
2
|
|
3
3
|
# Test ParallelMinion standalone without Rails
|
4
4
|
# Run this test standalone to verify it has no Rails dependencies
|
5
5
|
class MinionTest < Minitest::Test
|
6
6
|
include SemanticLogger::Loggable
|
7
7
|
|
8
|
-
|
8
|
+
describe ParallelMinion::Minion do
|
9
9
|
|
10
10
|
[false, true].each do |enabled|
|
11
|
-
|
12
|
-
|
11
|
+
describe enabled ? 'enabled' : 'disabled' do
|
12
|
+
before do
|
13
13
|
ParallelMinion::Minion.enabled = enabled
|
14
|
-
$
|
14
|
+
$log_structs = []
|
15
15
|
end
|
16
16
|
|
17
|
-
|
17
|
+
it 'without parameters' do
|
18
18
|
minion = ParallelMinion::Minion.new { 196 }
|
19
19
|
assert_equal 196, minion.result
|
20
20
|
end
|
21
21
|
|
22
|
-
|
22
|
+
it 'with a description' do
|
23
23
|
minion = ParallelMinion::Minion.new(description: 'Test') { 197 }
|
24
24
|
assert_equal 197, minion.result
|
25
25
|
end
|
26
26
|
|
27
|
-
|
28
|
-
p1
|
27
|
+
it 'with an argument' do
|
28
|
+
p1 = {name: 198}
|
29
29
|
minion = ParallelMinion::Minion.new(p1, description: 'Test') do |v|
|
30
30
|
v[:name]
|
31
31
|
end
|
32
32
|
assert_equal 198, minion.result
|
33
33
|
end
|
34
34
|
|
35
|
-
|
35
|
+
it 'raise exception' do
|
36
36
|
minion = ParallelMinion::Minion.new(description: 'Test') { raise "An exception" }
|
37
37
|
assert_raises RuntimeError do
|
38
38
|
minion.result
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
+
it 'has correct logger name' do
|
43
|
+
minion = ParallelMinion::Minion.new { 196 }
|
44
|
+
name = enabled ? 'Minion' : 'Inline'
|
45
|
+
assert_equal name, minion.logger.name
|
46
|
+
end
|
47
|
+
|
42
48
|
# TODO Blocks still have access to their original scope if variables cannot be
|
43
49
|
# resolved first by the parameters, then by the values in Minion itself
|
44
|
-
#
|
50
|
+
# it 'not have access to local variables' do
|
45
51
|
# name = 'Jack'
|
46
52
|
# minion = ParallelMinion::Minion.new(description: 'Test') { puts name }
|
47
53
|
# assert_raises NameError do
|
@@ -49,11 +55,11 @@ class MinionTest < Minitest::Test
|
|
49
55
|
# end
|
50
56
|
# end
|
51
57
|
|
52
|
-
|
53
|
-
hash
|
54
|
-
value
|
58
|
+
it 'run minion' do
|
59
|
+
hash = {value: 23}
|
60
|
+
value = 47
|
55
61
|
minion = ParallelMinion::Minion.new(hash, description: 'Test') do |h|
|
56
|
-
value
|
62
|
+
value = 321
|
57
63
|
h[:value] = 123
|
58
64
|
456
|
59
65
|
end
|
@@ -62,10 +68,10 @@ class MinionTest < Minitest::Test
|
|
62
68
|
assert_equal 321, value
|
63
69
|
end
|
64
70
|
|
65
|
-
|
71
|
+
it 'copy across logging tags' do
|
66
72
|
minion = nil
|
67
|
-
|
68
|
-
assert_equal 'TAG',
|
73
|
+
SemanticLogger.tagged('TAG') do
|
74
|
+
assert_equal 'TAG', SemanticLogger.tags.last
|
69
75
|
minion = ParallelMinion::Minion.new(description: 'Tag Test') do
|
70
76
|
logger.tags.last
|
71
77
|
end
|
@@ -73,27 +79,38 @@ class MinionTest < Minitest::Test
|
|
73
79
|
assert_equal 'TAG', minion.result
|
74
80
|
end
|
75
81
|
|
76
|
-
|
77
|
-
metric_name = '/
|
78
|
-
hash
|
79
|
-
value
|
80
|
-
minion
|
81
|
-
value
|
82
|
+
it 'include metric' do
|
83
|
+
metric_name = 'model/method'
|
84
|
+
hash = {value: 23}
|
85
|
+
value = 47
|
86
|
+
minion = ParallelMinion::Minion.new(hash, description: 'Test', metric: metric_name) do |h|
|
87
|
+
value = 321
|
82
88
|
h[:value] = 123
|
89
|
+
sleep 1
|
83
90
|
456
|
84
91
|
end
|
85
92
|
assert_equal 456, minion.result
|
86
93
|
assert_equal 123, hash[:value]
|
87
94
|
assert_equal 321, value
|
88
95
|
SemanticLogger.flush
|
89
|
-
|
96
|
+
assert log = $log_structs.first, -> { $log_structs.ai }
|
97
|
+
if enabled
|
98
|
+
# Completed log message
|
99
|
+
assert_equal metric_name, log.metric, -> { $log_structs.ai }
|
100
|
+
# Wait log message
|
101
|
+
assert log = $log_structs.last, -> { $log_structs.ai }
|
102
|
+
assert_equal "#{metric_name}/wait", log.metric, -> { $log_structs.ai }
|
103
|
+
else
|
104
|
+
# Timeout and wait has no effect when run inline
|
105
|
+
assert_equal metric_name, log.metric, -> { $log_structs.ai }
|
106
|
+
end
|
90
107
|
end
|
91
108
|
|
92
|
-
|
109
|
+
it 'handle multiple minions concurrently' do
|
93
110
|
# Start 10 minions
|
94
111
|
minions = 10.times.collect do |i|
|
95
112
|
# Each Minion returns its index in the collection
|
96
|
-
ParallelMinion::Minion.new(i, description: "Minion:#{i}") {|counter| counter }
|
113
|
+
ParallelMinion::Minion.new(i, description: "Minion:#{i}") { |counter| counter }
|
97
114
|
end
|
98
115
|
assert_equal 10, minions.count
|
99
116
|
# Fetch the result from each Minion
|
@@ -102,58 +119,58 @@ class MinionTest < Minitest::Test
|
|
102
119
|
end
|
103
120
|
end
|
104
121
|
|
105
|
-
|
106
|
-
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 100) { sleep 1 }
|
122
|
+
it 'timeout' do
|
107
123
|
if enabled
|
108
|
-
|
124
|
+
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 100) { sleep 1 }
|
125
|
+
assert_nil minion.result
|
109
126
|
end
|
110
127
|
end
|
111
128
|
|
112
|
-
|
113
|
-
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 100, on_timeout: Timeout::Error) { sleep 1 }
|
129
|
+
it 'timeout and terminate thread with Exception' do
|
114
130
|
if enabled
|
115
|
-
|
131
|
+
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 100, on_timeout: Timeout::Error) { sleep 1 }
|
132
|
+
assert_nil minion.result
|
116
133
|
# Give time for thread to terminate
|
117
134
|
sleep 0.1
|
118
135
|
assert_equal Timeout::Error, minion.exception.class
|
119
136
|
assert_equal false, minion.working?
|
120
|
-
assert_equal true,
|
121
|
-
assert_equal true,
|
122
|
-
assert_equal 0,
|
137
|
+
assert_equal true, minion.completed?
|
138
|
+
assert_equal true, minion.failed?
|
139
|
+
assert_equal 0, minion.time_left
|
123
140
|
end
|
124
141
|
end
|
125
142
|
|
126
|
-
|
143
|
+
it 'make description instance variable available' do
|
127
144
|
minion = ParallelMinion::Minion.new(description: 'Test') do
|
128
145
|
description
|
129
146
|
end
|
130
147
|
assert_equal 'Test', minion.result
|
131
148
|
end
|
132
149
|
|
133
|
-
|
134
|
-
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 1000
|
150
|
+
it 'make timeout instance variable available' do
|
151
|
+
minion = ParallelMinion::Minion.new(description: 'Test', timeout: 1000) do
|
135
152
|
timeout
|
136
153
|
end
|
137
154
|
assert_equal 1000, minion.result
|
138
155
|
end
|
139
156
|
|
140
|
-
|
157
|
+
it 'make enabled? method available' do
|
141
158
|
minion = ParallelMinion::Minion.new(description: 'Test') do
|
142
159
|
enabled?
|
143
160
|
end
|
144
161
|
assert_equal enabled, minion.result
|
145
162
|
end
|
146
163
|
|
147
|
-
|
148
|
-
minion = ParallelMinion::Minion.new(1, 'data', 14.1, description: 'Test') do |
|
164
|
+
it 'keep the original arguments' do
|
165
|
+
minion = ParallelMinion::Minion.new(1, 'data', 14.1, description: 'Test') do |num, str, float|
|
149
166
|
num + float
|
150
167
|
end
|
151
168
|
assert_equal 15.1, minion.result
|
152
|
-
assert_equal [
|
169
|
+
assert_equal [1, 'data', 14.1], minion.arguments
|
153
170
|
end
|
154
171
|
end
|
155
172
|
|
156
173
|
end
|
157
174
|
|
158
175
|
end
|
159
|
-
end
|
176
|
+
end
|
data/test/test_db.sqlite3
CHANGED
Binary file
|
data/test/test_helper.rb
CHANGED
@@ -1,19 +1,13 @@
|
|
1
1
|
$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
-
require 'minitest/reporters'
|
5
|
-
require 'minitest/stub_any_instance'
|
6
|
-
require 'shoulda/context'
|
7
4
|
require 'parallel_minion'
|
8
5
|
require 'semantic_logger'
|
9
6
|
|
10
|
-
Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
|
11
|
-
|
12
|
-
# Register an appender if one is not already registered
|
13
7
|
SemanticLogger.default_level = :trace
|
14
|
-
SemanticLogger.add_appender('test.log',
|
8
|
+
SemanticLogger.add_appender(file_name: 'test.log', formatter: :color)
|
15
9
|
|
16
10
|
# Setup global callback for metric so that it can be tested below
|
17
11
|
SemanticLogger.on_metric do |log_struct|
|
18
|
-
$
|
12
|
+
($log_structs ||= []) << log_struct.dup
|
19
13
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel_minion
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: semantic_logger
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0'
|
19
|
+
version: '4.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0'
|
26
|
+
version: '4.0'
|
27
27
|
description: Parallel Minion allows you to take existing blocks of code and wrap them
|
28
28
|
in a minion so that they can run asynchronously in a separate thread
|
29
29
|
email:
|
@@ -56,7 +56,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
56
56
|
requirements:
|
57
57
|
- - ">="
|
58
58
|
- !ruby/object:Gem::Version
|
59
|
-
version: '
|
59
|
+
version: '2.1'
|
60
60
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
61
61
|
requirements:
|
62
62
|
- - ">="
|
@@ -64,7 +64,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
64
64
|
version: '0'
|
65
65
|
requirements: []
|
66
66
|
rubyforge_project:
|
67
|
-
rubygems_version: 2.
|
67
|
+
rubygems_version: 2.6.11
|
68
68
|
signing_key:
|
69
69
|
specification_version: 4
|
70
70
|
summary: Pragmatic approach to parallel and asynchronous processing in Ruby
|