parallel_minion 0.4.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE.txt +202 -0
- data/README.md +26 -231
- data/Rakefile +13 -8
- data/lib/parallel_minion/minion.rb +14 -10
- data/lib/parallel_minion/version.rb +1 -1
- data/test/minion_test.rb +7 -0
- data/test/test_db.sqlite3 +0 -0
- metadata +6 -7
- data/lib/parallel_minion/horde.rb +0 -41
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3b02090a750670c6d1b845c4c88daf19451f591c
|
|
4
|
+
data.tar.gz: ebff60fff44c11157e93ca57fa682fd19ee226ef
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 57033a4c8856bd4befea01eb8af8f974b15507ab9cfabbf65a767b05cd045fbca0b1a18ca979ebda8bdf1a301d385694aa9730b0658e75a2dbfacf2d125172a5
|
|
7
|
+
data.tar.gz: ef9594b1ae35312037a02e4dae1c3100d042964f178ea7787e46f3ef3273c5e0c1e4784ec53192c0369e321f1db22329094ddfb16fd6dfb6ac95215decfb6429
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "{}"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright 2013, 2014 Reid Morrison
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
|
202
|
+
|
data/README.md
CHANGED
|
@@ -1,52 +1,23 @@
|
|
|
1
|
-
parallel_minion
|
|
1
|
+
parallel_minion [](http://travis-ci.org/reidmorrison/parallel_minion)
|
|
2
2
|
===============
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
that would normally be performed sequentially can easily be executed in parallel.
|
|
6
|
-
This allows Ruby and Rails applications to very easily do many tasks at the same
|
|
7
|
-
time so that results are returned more quickly.
|
|
4
|
+
Pragmatic approach to parallel and asynchronous processing in Ruby
|
|
8
5
|
|
|
9
|
-
|
|
10
|
-
be useful to run some of the steps in fulfilling a single request in parallel.
|
|
6
|
+
## Description
|
|
11
7
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
- Makes exception handling simple with a drop-in replacement for existing code
|
|
19
|
-
- Avoids having to implement more complex actors and supervisors required
|
|
20
|
-
by some concurrency frameworks
|
|
21
|
-
|
|
22
|
-
Timeouts
|
|
23
|
-
|
|
24
|
-
- Timeout when a minion does not return within a specified time
|
|
25
|
-
- Timeouts are a useful feature when one of the minions fails to respond in a
|
|
26
|
-
reasonable amount of time. For example when a call to a remote service hangs
|
|
27
|
-
we can send back a partial response of other work that was completed rather
|
|
28
|
-
than just "hanging" or failing completely.
|
|
29
|
-
|
|
30
|
-
Logging
|
|
31
|
-
|
|
32
|
-
- Built-in support to log the duration of all minion tasks to make future analysis
|
|
33
|
-
of performance issues much easier
|
|
34
|
-
- Logs any exceptions thrown to assist with problem diagnosis
|
|
35
|
-
- Logging tags from the current thread are propagated to the minions thread
|
|
36
|
-
- The name of the thread in log entries is set to the description supplied for
|
|
37
|
-
the minion to make it easy to distinguish log entries by minion / thread
|
|
38
|
-
|
|
39
|
-
Rails Support
|
|
40
|
-
|
|
41
|
-
- When used in a Rails environment the current scope of specified models can be
|
|
42
|
-
propagated to the minions thread
|
|
8
|
+
Parallel Minion allows you to take existing blocks of code and wrap them in a minion
|
|
9
|
+
so that they can run asynchronously in a separate thread.
|
|
10
|
+
The minion then passes back the result to the caller when or if requested.
|
|
11
|
+
If any exceptions were thrown during the minion processing, it will be re-raised
|
|
12
|
+
in the callers thread so that no additional work needs to be done when converting
|
|
13
|
+
existing code to use minions.
|
|
43
14
|
|
|
44
15
|
## Example
|
|
45
16
|
|
|
46
|
-
Simple example
|
|
47
|
-
|
|
48
17
|
```ruby
|
|
49
|
-
minion = ParallelMinion::Minion.new(10.days.ago,
|
|
18
|
+
minion = ParallelMinion::Minion.new(10.days.ago,
|
|
19
|
+
description: 'Doing something else in parallel',
|
|
20
|
+
timeout: 1000) do |date|
|
|
50
21
|
MyTable.where('created_at <= ?', date).count
|
|
51
22
|
end
|
|
52
23
|
|
|
@@ -58,196 +29,22 @@ count = minion.result
|
|
|
58
29
|
puts "Found #{count} records"
|
|
59
30
|
```
|
|
60
31
|
|
|
61
|
-
##
|
|
62
|
-
|
|
63
|
-
For example, in the code below there are several steps that are performed
|
|
64
|
-
sequentially and does not yet use minions:
|
|
65
|
-
|
|
66
|
-
```ruby
|
|
67
|
-
# Contrived example to show how to do parallel code execution
|
|
68
|
-
# with (unreal) sample durations in the comments
|
|
69
|
-
|
|
70
|
-
def process_request(request)
|
|
71
|
-
# Count number of entries in a table.
|
|
72
|
-
# Average response time 150ms
|
|
73
|
-
person_count = Person.where(state: 'FL').count
|
|
74
|
-
|
|
75
|
-
# Count the number of requests for this user (usually more complex with were clauses etc.)
|
|
76
|
-
# Average response time 320ms
|
|
77
|
-
request_count = Requests.where(user_id: request.user.id).count
|
|
78
|
-
|
|
79
|
-
# Call an external provider
|
|
80
|
-
# Average response time 1800ms ( Sometimes "hangs" when supplier does not respond )
|
|
81
|
-
inventory = inventory_supplier.check_inventory(request.product.id)
|
|
82
|
-
|
|
83
|
-
# Call another provider for more info on the user
|
|
84
|
-
# Average response time 1500ms
|
|
85
|
-
user_info = user_supplier.more_info(request.user.name)
|
|
32
|
+
## Documentation
|
|
86
33
|
|
|
87
|
-
|
|
88
|
-
reply = MyReply.new(user_id: request.user.id)
|
|
34
|
+
For complete documentation see: http://reidmorrison.github.io/parallel_minion
|
|
89
35
|
|
|
90
|
-
|
|
91
|
-
reply.number_of_requests = request_count
|
|
92
|
-
reply.user_details = user_info.details
|
|
93
|
-
if inventory.product_available?
|
|
94
|
-
reply.available = true
|
|
95
|
-
reply.quantity = 100
|
|
96
|
-
else
|
|
97
|
-
reply.available = false
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
reply
|
|
101
|
-
end
|
|
102
|
-
```
|
|
103
|
-
The average response time when calling #process_request is around 3,780 milli-seconds.
|
|
104
|
-
|
|
105
|
-
The first step could be to run the supplier calls in parallel.
|
|
106
|
-
Through log analysis we have determined that the first supplier call takes on average
|
|
107
|
-
1,800 ms and we have decided that it should not wait longer than 2,200 ms for a response.
|
|
108
|
-
|
|
109
|
-
```ruby
|
|
110
|
-
# Now with a single parallel call
|
|
111
|
-
|
|
112
|
-
def process_request(request)
|
|
113
|
-
# Count number of entries in a table.
|
|
114
|
-
# Average response time 150ms
|
|
115
|
-
person_count = Person.where(state: 'FL').count
|
|
116
|
-
|
|
117
|
-
# Count the number of requests for this user (usually more complex with were clauses etc.)
|
|
118
|
-
# Average response time 320ms
|
|
119
|
-
request_count = Requests.where(user_id: request.user.id).count
|
|
120
|
-
|
|
121
|
-
# Call an external provider
|
|
122
|
-
# Average response time 1800ms ( Sometimes "hangs" when supplier does not respond )
|
|
123
|
-
inventory_minion = ParallelMinion::Minion.new(request.product.id, description: 'Inventory Lookup', timeout: 2200) do |product_id|
|
|
124
|
-
inventory_supplier.check_inventory(product_id)
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
# Call another provider for more info on the user
|
|
128
|
-
# Average response time 1500ms
|
|
129
|
-
user_info = user_supplier.more_info(request.user.name)
|
|
130
|
-
|
|
131
|
-
# Build up the reply
|
|
132
|
-
reply = MyReply.new(user_id: request.user.id)
|
|
133
|
-
|
|
134
|
-
reply.number_of_people = person_count
|
|
135
|
-
reply.number_of_requests = request_count
|
|
136
|
-
reply.user_details = user_info.details
|
|
137
|
-
|
|
138
|
-
# Get inventory result from Inventory Lookup minion
|
|
139
|
-
inventory = inventory_minion.result
|
|
140
|
-
|
|
141
|
-
if inventory.product_available?
|
|
142
|
-
reply.available = true
|
|
143
|
-
reply.quantity = 100
|
|
144
|
-
else
|
|
145
|
-
reply.available = false
|
|
146
|
-
end
|
|
147
|
-
|
|
148
|
-
reply
|
|
149
|
-
end
|
|
150
|
-
```
|
|
151
|
-
|
|
152
|
-
The above changes drop the average processing time from 3,780 milli-seconds to
|
|
153
|
-
2,280 milli-seconds.
|
|
154
|
-
|
|
155
|
-
By moving the supplier call to the top of the function call it can be optimized
|
|
156
|
-
to about 1,970 milli-seconds.
|
|
157
|
-
|
|
158
|
-
We can further parallelize the processing to gain even greater performance.
|
|
159
|
-
|
|
160
|
-
```ruby
|
|
161
|
-
# Now with two parallel calls
|
|
162
|
-
|
|
163
|
-
def process_request(request)
|
|
164
|
-
# Call an external provider
|
|
165
|
-
# Average response time 1800ms ( Sometimes "hangs" when supplier does not respond )
|
|
166
|
-
inventory_minion = ParallelMinion::Minion.new(request.product.id, description: 'Inventory Lookup', timeout: 2200) do |product_id|
|
|
167
|
-
inventory_supplier.check_inventory(product_id)
|
|
168
|
-
end
|
|
169
|
-
|
|
170
|
-
# Count the number of requests for this user (usually more complex with were clauses etc.)
|
|
171
|
-
# Average response time 320ms
|
|
172
|
-
request_count_minion = ParallelMinion::Minion.new(request.user.id, description: 'Request Count', timeout: 500) do |user_id|
|
|
173
|
-
Requests.where(user_id: user_id).count
|
|
174
|
-
end
|
|
175
|
-
|
|
176
|
-
# Leave the current thread some work to do too
|
|
177
|
-
|
|
178
|
-
# Count number of entries in a table.
|
|
179
|
-
# Average response time 150ms
|
|
180
|
-
person_count = Person.where(state: 'FL').count
|
|
181
|
-
|
|
182
|
-
# Call another provider for more info on the user
|
|
183
|
-
# Average response time 1500ms
|
|
184
|
-
user_info = user_supplier.more_info(request.user.name)
|
|
185
|
-
|
|
186
|
-
# Build up the reply
|
|
187
|
-
reply = MyReply.new(user_id: request.user.id)
|
|
188
|
-
|
|
189
|
-
reply.number_of_people = person_count
|
|
190
|
-
# The request_count is retrieved from the request_count_minion first since it
|
|
191
|
-
# should complete before the inventory_minion
|
|
192
|
-
reply.number_of_requests = request_count_minion.result
|
|
193
|
-
reply.user_details = user_info.details
|
|
194
|
-
|
|
195
|
-
# Get inventory result from Inventory Lookup minion
|
|
196
|
-
inventory = inventory_minion.result
|
|
197
|
-
|
|
198
|
-
if inventory.product_available?
|
|
199
|
-
reply.available = true
|
|
200
|
-
reply.quantity = 100
|
|
201
|
-
else
|
|
202
|
-
reply.available = false
|
|
203
|
-
end
|
|
204
|
-
|
|
205
|
-
reply
|
|
206
|
-
end
|
|
207
|
-
```
|
|
208
|
-
|
|
209
|
-
The above #process_request method should now take on average 1,810 milli-seconds
|
|
210
|
-
which is significantly faster than the 3,780 milli-seconds it took to perform
|
|
211
|
-
the exact same request prior to using minions.
|
|
212
|
-
|
|
213
|
-
The exact breakdown of which calls to do in the main thread versus a minion is determined
|
|
214
|
-
through experience and trial and error over time. The key is logging the duration
|
|
215
|
-
of each call which minion does by default so that the exact processing breakdown
|
|
216
|
-
can be fine-tuned over time.
|
|
217
|
-
|
|
218
|
-
## Disabling Minions
|
|
219
|
-
|
|
220
|
-
In the event that strange problems are occurring in production and no one is
|
|
221
|
-
sure if it is due to running the minion tasks in parallel, it is simple to make
|
|
222
|
-
all minion tasks run in the calling thread.
|
|
223
|
-
|
|
224
|
-
It may also be useful to disable minions on a single production server to compare
|
|
225
|
-
its performance to that of the servers running with minions active.
|
|
226
|
-
|
|
227
|
-
To disable minions / make them run in the calling thread, add the following
|
|
228
|
-
lines to config/environments/production.rb:
|
|
229
|
-
|
|
230
|
-
```ruby
|
|
231
|
-
# Make minions run immediately in the current thread
|
|
232
|
-
config.parallel_minion.enabled = false
|
|
233
|
-
```
|
|
234
|
-
|
|
235
|
-
## Notes:
|
|
236
|
-
|
|
237
|
-
- When using JRuby it is important to enable it's built-in thread-pooling by
|
|
238
|
-
adding the following line to .jrubyrc, or setting the appropriate command line option:
|
|
239
|
-
|
|
240
|
-
```ruby
|
|
241
|
-
thread.pool.enabled=true
|
|
242
|
-
```
|
|
36
|
+
## Production Use
|
|
243
37
|
|
|
38
|
+
Parallel Minion is being used in a high performance, highly concurrent
|
|
39
|
+
production environment running JRuby with Ruby on Rails on a Puma web server.
|
|
40
|
+
Significant reduction in the time it takes to complete rails request processing
|
|
41
|
+
has been achieved by moving existing blocks of code into Minions.
|
|
244
42
|
|
|
245
|
-
##
|
|
43
|
+
## Installation
|
|
246
44
|
|
|
247
|
-
|
|
45
|
+
gem install parallel_minion
|
|
248
46
|
|
|
249
|
-
Meta
|
|
250
|
-
----
|
|
47
|
+
## Meta
|
|
251
48
|
|
|
252
49
|
* Code: `git clone git://github.com/reidmorrison/parallel_minion.git`
|
|
253
50
|
* Home: <https://github.com/reidmorrison/parallel_minion>
|
|
@@ -256,15 +53,13 @@ Meta
|
|
|
256
53
|
|
|
257
54
|
This project uses [Semantic Versioning](http://semver.org/).
|
|
258
55
|
|
|
259
|
-
Author
|
|
260
|
-
-------
|
|
56
|
+
## Author
|
|
261
57
|
|
|
262
|
-
Reid Morrison
|
|
58
|
+
[Reid Morrison](https://github.com/reidmorrison) :: @reidmorrison
|
|
263
59
|
|
|
264
|
-
License
|
|
265
|
-
-------
|
|
60
|
+
## License
|
|
266
61
|
|
|
267
|
-
Copyright 2013 Reid Morrison
|
|
62
|
+
Copyright 2013, 2014 Reid Morrison
|
|
268
63
|
|
|
269
64
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
270
65
|
you may not use this file except in compliance with the License.
|
data/Rakefile
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
|
-
lib = File.expand_path('../lib/', __FILE__)
|
|
2
|
-
$:.unshift lib unless $:.include?(lib)
|
|
3
|
-
|
|
4
|
-
require 'rubygems'
|
|
5
|
-
require 'rubygems/package'
|
|
6
1
|
require 'rake/clean'
|
|
7
2
|
require 'rake/testtask'
|
|
3
|
+
|
|
4
|
+
$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
|
|
8
5
|
require 'parallel_minion/version'
|
|
9
6
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
7
|
+
task :gem do
|
|
8
|
+
system "gem build parallel_minion.gemspec"
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
task :publish => :gem do
|
|
12
|
+
system "git tag -a v#{ParallelMinion::VERSION} -m 'Tagging #{ParallelMinion::VERSION}'"
|
|
13
|
+
system "git push --tags"
|
|
14
|
+
system "gem push parallel_minion-#{ParallelMinion::VERSION}.gem"
|
|
15
|
+
system "rm parallel_minion-#{ParallelMinion::VERSION}.gem"
|
|
13
16
|
end
|
|
14
17
|
|
|
15
18
|
desc "Run Test Suite"
|
|
@@ -21,3 +24,5 @@ task :test do
|
|
|
21
24
|
|
|
22
25
|
Rake::Task['functional'].invoke
|
|
23
26
|
end
|
|
27
|
+
|
|
28
|
+
task :default => :test
|
|
@@ -12,6 +12,9 @@ module ParallelMinion
|
|
|
12
12
|
# Returns [Integer] the maximum duration in milli-seconds that the Minion may take to complete the task
|
|
13
13
|
attr_reader :timeout
|
|
14
14
|
|
|
15
|
+
# Returns [Array<Object>] list of arguments in the order they were passed into the initializer
|
|
16
|
+
attr_reader :arguments
|
|
17
|
+
|
|
15
18
|
# Give an infinite amount of time to wait for a Minion to complete a task
|
|
16
19
|
INFINITE = -1
|
|
17
20
|
|
|
@@ -79,6 +82,8 @@ module ParallelMinion
|
|
|
79
82
|
# in the order they are listed
|
|
80
83
|
# It is recommended to duplicate and/or freeze objects passed as arguments
|
|
81
84
|
# so that they are not modified at the same time by multiple threads
|
|
85
|
+
# These arguments are accessible while and after the minion is running
|
|
86
|
+
# by calling #arguments
|
|
82
87
|
#
|
|
83
88
|
# Proc / lambda
|
|
84
89
|
# A block of code must be supplied that the Minion will execute
|
|
@@ -106,11 +111,10 @@ module ParallelMinion
|
|
|
106
111
|
# end
|
|
107
112
|
def initialize(*args, &block)
|
|
108
113
|
raise "Missing mandatory block that Minion must perform" unless block
|
|
109
|
-
@start_time
|
|
110
|
-
@exception
|
|
111
|
-
|
|
112
|
-
options
|
|
113
|
-
|
|
114
|
+
@start_time = Time.now
|
|
115
|
+
@exception = nil
|
|
116
|
+
@arguments = args.dup
|
|
117
|
+
options = self.class.extract_options!(@arguments)
|
|
114
118
|
@timeout = (options.delete(:timeout) || Minion::INFINITE).to_f
|
|
115
119
|
@description = (options.delete(:description) || 'Minion').to_s
|
|
116
120
|
@metric = options.delete(:metric)
|
|
@@ -119,7 +123,7 @@ module ParallelMinion
|
|
|
119
123
|
@enabled = self.class.enabled? if @enabled.nil?
|
|
120
124
|
|
|
121
125
|
# Warn about any unknown options.
|
|
122
|
-
options.each_pair do |key,val|
|
|
126
|
+
options.each_pair do | key, val |
|
|
123
127
|
logger.warn "Ignoring unknown option: #{key.inspect} => #{val.inspect}"
|
|
124
128
|
warn "ParallelMinion::Minion Ignoring unknown option: #{key.inspect} => #{val.inspect}"
|
|
125
129
|
end
|
|
@@ -130,7 +134,7 @@ module ParallelMinion
|
|
|
130
134
|
begin
|
|
131
135
|
logger.info("Started in the current thread: #{@description}")
|
|
132
136
|
logger.benchmark_info("Completed in the current thread: #{@description}", log_exception: @log_exception, metric: @metric) do
|
|
133
|
-
@result = instance_exec(
|
|
137
|
+
@result = instance_exec(*@arguments, &block)
|
|
134
138
|
end
|
|
135
139
|
rescue Exception => exc
|
|
136
140
|
@exception = exc
|
|
@@ -143,7 +147,7 @@ module ParallelMinion
|
|
|
143
147
|
# Copy current scopes for new thread. Only applicable for AR models
|
|
144
148
|
scopes = self.class.current_scopes if defined?(ActiveRecord::Base)
|
|
145
149
|
|
|
146
|
-
@thread = Thread.new(
|
|
150
|
+
@thread = Thread.new(*@arguments) do
|
|
147
151
|
# Copy logging tags from parent thread
|
|
148
152
|
logger.tagged(*tags) do
|
|
149
153
|
# Set the current thread name to the description for this Minion
|
|
@@ -155,10 +159,10 @@ module ParallelMinion
|
|
|
155
159
|
logger.benchmark_info("Completed #{@description}", log_exception: @log_exception, metric: @metric) do
|
|
156
160
|
# Use the current scope for the duration of the task execution
|
|
157
161
|
if scopes.nil? || (scopes.size == 0)
|
|
158
|
-
@result = instance_exec(
|
|
162
|
+
@result = instance_exec(*@arguments, &block)
|
|
159
163
|
else
|
|
160
164
|
# Each Class to scope requires passing a block to .scoping
|
|
161
|
-
proc = Proc.new { instance_exec(
|
|
165
|
+
proc = Proc.new { instance_exec(*@arguments, &block) }
|
|
162
166
|
first = scopes.shift
|
|
163
167
|
scopes.each {|scope| proc = Proc.new { scope.scoping(&proc) } }
|
|
164
168
|
@result = first.scoping(&proc)
|
data/test/minion_test.rb
CHANGED
|
@@ -145,6 +145,13 @@ class MinionTest < Test::Unit::TestCase
|
|
|
145
145
|
assert_equal enabled, minion.result
|
|
146
146
|
end
|
|
147
147
|
|
|
148
|
+
should 'keep the original arguments' do
|
|
149
|
+
minion = ParallelMinion::Minion.new(1, 'data', 14.1, description: 'Test') do | num, str, float |
|
|
150
|
+
num + float
|
|
151
|
+
end
|
|
152
|
+
assert_equal 15.1, minion.result
|
|
153
|
+
assert_equal [ 1, 'data', 14.1 ], minion.arguments
|
|
154
|
+
end
|
|
148
155
|
end
|
|
149
156
|
|
|
150
157
|
end
|
data/test/test_db.sqlite3
CHANGED
|
Binary file
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: parallel_minion
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Reid Morrison
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2014-
|
|
11
|
+
date: 2014-11-03 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: semantic_logger
|
|
@@ -24,19 +24,18 @@ dependencies:
|
|
|
24
24
|
- - ">="
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '0'
|
|
27
|
-
description: Parallel Minion
|
|
28
|
-
|
|
29
|
-
in parallel
|
|
27
|
+
description: Parallel Minion allows you to take existing blocks of code and wrap them
|
|
28
|
+
in a minion so that they can run asynchronously in a separate thread
|
|
30
29
|
email:
|
|
31
30
|
- reidmo@gmail.com
|
|
32
31
|
executables: []
|
|
33
32
|
extensions: []
|
|
34
33
|
extra_rdoc_files: []
|
|
35
34
|
files:
|
|
35
|
+
- LICENSE.txt
|
|
36
36
|
- README.md
|
|
37
37
|
- Rakefile
|
|
38
38
|
- lib/parallel_minion.rb
|
|
39
|
-
- lib/parallel_minion/horde.rb
|
|
40
39
|
- lib/parallel_minion/minion.rb
|
|
41
40
|
- lib/parallel_minion/railtie.rb
|
|
42
41
|
- lib/parallel_minion/version.rb
|
|
@@ -67,7 +66,7 @@ rubyforge_project:
|
|
|
67
66
|
rubygems_version: 2.2.2
|
|
68
67
|
signing_key:
|
|
69
68
|
specification_version: 4
|
|
70
|
-
summary:
|
|
69
|
+
summary: Pragmatic approach to parallel and asynchronous processing in Ruby
|
|
71
70
|
test_files:
|
|
72
71
|
- test/config/database.yml
|
|
73
72
|
- test/minion_scope_test.rb
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
require 'minion'
|
|
2
|
-
|
|
3
|
-
# A Horde is way to manage a group of Minions
|
|
4
|
-
#
|
|
5
|
-
# Horde supports the following features for minions
|
|
6
|
-
# - Limit the number of Minions in the Horde to prevent overloading the system
|
|
7
|
-
# - Queue up requests for Minions when the lmit is reached
|
|
8
|
-
# - Optionally block submitting work for Minions when the queued up requests
|
|
9
|
-
# reach a specified number
|
|
10
|
-
class Horde
|
|
11
|
-
include SemanticLogger::Loggable
|
|
12
|
-
|
|
13
|
-
# Returns the description for this Horde
|
|
14
|
-
attr_reader :description
|
|
15
|
-
|
|
16
|
-
# Returns the maximum number of Minions active in this Horde at any time
|
|
17
|
-
attr_reader :capacity
|
|
18
|
-
|
|
19
|
-
# Returns the maximum number of queued up requests before blocking
|
|
20
|
-
# new requests for work
|
|
21
|
-
attr_reader :max_queue_size
|
|
22
|
-
|
|
23
|
-
# Create a new Horde of Minions
|
|
24
|
-
#
|
|
25
|
-
# Parameters
|
|
26
|
-
# :description
|
|
27
|
-
# Description for this task that the Minion is performing
|
|
28
|
-
# Put in the log file along with the time take to complete the task
|
|
29
|
-
#
|
|
30
|
-
# :capacity
|
|
31
|
-
# Maximum number of Minions active in this Horde at any time
|
|
32
|
-
# Default: 10
|
|
33
|
-
#
|
|
34
|
-
# :max_queue_size
|
|
35
|
-
# Maximum number of queued up requests before blocking
|
|
36
|
-
# new requests for work
|
|
37
|
-
# Default: -1 (unlimited)
|
|
38
|
-
#
|
|
39
|
-
def initialize(params={})
|
|
40
|
-
end
|
|
41
|
-
end
|