team_effort 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +70 -22
- data/lib/team_effort.rb +3 -3
- data/lib/team_effort/version.rb +1 -1
- data/test/test_team_effort.rb +66 -0
- metadata +12 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 75c7884021dc7ff32714dd57b1d642ba110817a4
|
4
|
+
data.tar.gz: 0129f103f4db6e91a423dab711c9de64c7094dfb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 42c57584ea793a16e6888535e29e94ac44aeb4a1a580c001cc4ec48e53b4f4637411bb8584b87c0e7d7e2c838673aa2eb5c733620fc26cd163575d08af4cd642
|
7
|
+
data.tar.gz: 5ff868836356131210df02424b805807cf04bba59a9e717c1340de88248faa724c640b830fef13b05ed91320d83465331e13ad94d127cbdb0de6bc17a1237529
|
data/README.md
CHANGED
@@ -1,8 +1,23 @@
|
|
1
1
|
# TeamEffort
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
3
|
+
TeamEffort makes it easy to process a collection with child processes
|
4
|
+
allowing you to take advantage of multiple cores. By replacing
|
5
|
+
|
6
|
+
```ruby
|
7
|
+
collection.each do |item|
|
8
|
+
# do some work on item
|
9
|
+
end
|
10
|
+
```
|
11
|
+
|
12
|
+
with
|
13
|
+
|
14
|
+
```ruby
|
15
|
+
TeamEffort.work(collection) do |item|
|
16
|
+
# do some work on item
|
17
|
+
end
|
18
|
+
```
|
19
|
+
|
20
|
+
you get each item processed in a new child process.
|
6
21
|
|
7
22
|
## Installation
|
8
23
|
|
@@ -20,39 +35,72 @@ Or install it yourself as:
|
|
20
35
|
|
21
36
|
## Usage
|
22
37
|
|
23
|
-
To do work in child processes just call `TeamEffort.work` with a
|
24
|
-
of items to process and a block:
|
38
|
+
To do work in child processes just call `TeamEffort.work` with a
|
39
|
+
collection of items to process and a block:
|
25
40
|
|
26
41
|
```ruby
|
27
|
-
class ProcessALotOfStuff
|
28
|
-
|
29
|
-
def some_method
|
30
|
-
# collection = a lot of stuff from somewhere
|
31
42
|
TeamEffort.work(collection) do |item|
|
32
43
|
# do some work on item
|
33
44
|
end
|
34
|
-
end
|
35
|
-
|
36
|
-
end
|
37
45
|
```
|
38
46
|
|
39
47
|
You may specify the number of child processes with the work method:
|
40
48
|
|
41
49
|
```ruby
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
# do some work on item
|
46
|
-
end
|
47
|
-
end
|
50
|
+
TeamEffort.work(collection, 3) do |item| # do the work using 3 child processes
|
51
|
+
# do some work on item
|
52
|
+
end
|
48
53
|
```
|
49
54
|
|
50
55
|
The number of child processes defaults to 4.
|
51
56
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
57
|
+
In rails you need to reestablish your ActiveRecord connection in the
|
58
|
+
child process:
|
59
|
+
|
60
|
+
```ruby
|
61
|
+
ActiveRecord::Base.clear_all_connections!
|
62
|
+
begin
|
63
|
+
TeamEffort.work(collection, max_process_count) do |item|
|
64
|
+
|
65
|
+
ActiveRecord::Base.establish_connection
|
66
|
+
|
67
|
+
# do some work with active record
|
68
|
+
|
69
|
+
end
|
70
|
+
ensure
|
71
|
+
ActiveRecord::Base.establish_connection
|
72
|
+
end
|
73
|
+
```
|
74
|
+
|
75
|
+
Logging can be messy when multiple processes are writing to the same
|
76
|
+
io channel. One approach is to wrap logging statements in a
|
77
|
+
synchronize block:
|
78
|
+
|
79
|
+
```ruby
|
80
|
+
def pid_logger(msg)
|
81
|
+
@mutex ||= Mutex.new
|
82
|
+
@mutex.synchronize do
|
83
|
+
puts "[#{Process.pid} at #{Time.now.strftime('%H:%M:%S')}] #{msg}"
|
84
|
+
end
|
85
|
+
end
|
86
|
+
```
|
87
|
+
|
88
|
+
## Discussion
|
89
|
+
|
90
|
+
TeamEffort uses child processes to do concurrent processing. To review
|
91
|
+
the unix process model I recommend Jesse Storimer's
|
92
|
+
[Working With Unix Processes][1].
|
93
|
+
|
94
|
+
[1]: http://www.jstorimer.com/products/working-with-unix-processes
|
95
|
+
|
96
|
+
A disadvantage to using child processes for concurrent processing is
|
97
|
+
the work required to create the child process and the duplication of
|
98
|
+
memory. For this reason you should only use TeamEffort on collections
|
99
|
+
where there is significant processing to be performed on each item.
|
100
|
+
|
101
|
+
An advantage of using child processes is that the memory is reclaimed
|
102
|
+
when the child process goes away. This can make long running jobs
|
103
|
+
resilient to memory leaks.
|
56
104
|
|
57
105
|
## Contributing
|
58
106
|
|
data/lib/team_effort.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
|
1
|
+
require_relative "team_effort/version"
|
2
2
|
|
3
3
|
module TeamEffort
|
4
4
|
def self.work(enumerable, max_process_count = 4)
|
5
5
|
pids = []
|
6
6
|
|
7
7
|
enumerable.each do |args|
|
8
|
-
|
8
|
+
while pids.size == max_process_count
|
9
9
|
finished_pid = Process.wait
|
10
10
|
pids.delete finished_pid
|
11
11
|
end
|
@@ -20,4 +20,4 @@ module TeamEffort
|
|
20
20
|
pids.delete finished_pid
|
21
21
|
end
|
22
22
|
end
|
23
|
-
end
|
23
|
+
end
|
data/lib/team_effort/version.rb
CHANGED
data/test/test_team_effort.rb
CHANGED
@@ -29,5 +29,71 @@ describe TeamEffort do
|
|
29
29
|
line.must_match /^\d+$/
|
30
30
|
end
|
31
31
|
end
|
32
|
+
|
33
|
+
|
34
|
+
it 'ignores other child process completions' do
|
35
|
+
output_io_class = Class.new do
|
36
|
+
def initialize
|
37
|
+
@mutex = Mutex.new
|
38
|
+
@io = Tempfile.new('mumble')
|
39
|
+
end
|
40
|
+
|
41
|
+
def log(text)
|
42
|
+
@mutex.synchronize do
|
43
|
+
@io.puts text
|
44
|
+
@io.flush
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def lines
|
49
|
+
@io.rewind
|
50
|
+
@io.read.split(/\n/)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
output_io = output_io_class.new
|
55
|
+
|
56
|
+
require 'socket'
|
57
|
+
|
58
|
+
unmanaged_child_reader, unmanaged_child_writer = Socket.pair(:UNIX, :DGRAM, 0)
|
59
|
+
maxlen = 1000
|
60
|
+
|
61
|
+
fork do
|
62
|
+
output_io.log "unmanaged starting"
|
63
|
+
unmanaged_child_writer.close
|
64
|
+
output_io.log "unmanaged waiting for IO"
|
65
|
+
message = unmanaged_child_reader.recv(maxlen)
|
66
|
+
output_io.log "unmanaged received >#{message}< and exiting"
|
67
|
+
output_io.log "unmanaged finishing"
|
68
|
+
end
|
69
|
+
|
70
|
+
sleep 1
|
71
|
+
|
72
|
+
TeamEffort.work([1, 2], 1) do |index|
|
73
|
+
output_io.log "task #{index} starting"
|
74
|
+
unmanaged_child_reader.close
|
75
|
+
if index == 1
|
76
|
+
output_io.log "task 1 waking unmanaged process"
|
77
|
+
unmanaged_child_writer.send("wake up", 0)
|
78
|
+
unmanaged_child_writer.close
|
79
|
+
sleep 1
|
80
|
+
end
|
81
|
+
output_io.log "task #{index} finishing"
|
82
|
+
end
|
83
|
+
|
84
|
+
lines = output_io.lines
|
85
|
+
|
86
|
+
lines.must_equal [
|
87
|
+
'unmanaged starting',
|
88
|
+
'unmanaged waiting for IO',
|
89
|
+
'task 1 starting',
|
90
|
+
'task 1 waking unmanaged process',
|
91
|
+
'unmanaged received >wake up< and exiting',
|
92
|
+
'unmanaged finishing',
|
93
|
+
'task 1 finishing',
|
94
|
+
'task 2 starting',
|
95
|
+
'task 2 finishing',
|
96
|
+
]
|
97
|
+
end
|
32
98
|
end
|
33
99
|
end
|
metadata
CHANGED
@@ -1,41 +1,41 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: team_effort
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kelly Felkins
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.3'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.3'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
description: " Team Effort provides a simple wrapper to ruby's process management
|
@@ -46,9 +46,9 @@ executables: []
|
|
46
46
|
extensions: []
|
47
47
|
extra_rdoc_files: []
|
48
48
|
files:
|
49
|
-
- .gitignore
|
50
|
-
- .ruby-gemset
|
51
|
-
- .ruby-version
|
49
|
+
- ".gitignore"
|
50
|
+
- ".ruby-gemset"
|
51
|
+
- ".ruby-version"
|
52
52
|
- LICENSE
|
53
53
|
- README.md
|
54
54
|
- Rakefile
|
@@ -66,17 +66,17 @@ require_paths:
|
|
66
66
|
- lib
|
67
67
|
required_ruby_version: !ruby/object:Gem::Requirement
|
68
68
|
requirements:
|
69
|
-
- -
|
69
|
+
- - ">="
|
70
70
|
- !ruby/object:Gem::Version
|
71
71
|
version: '0'
|
72
72
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
73
73
|
requirements:
|
74
|
-
- -
|
74
|
+
- - ">="
|
75
75
|
- !ruby/object:Gem::Version
|
76
76
|
version: '0'
|
77
77
|
requirements: []
|
78
78
|
rubyforge_project:
|
79
|
-
rubygems_version: 2.
|
79
|
+
rubygems_version: 2.4.3
|
80
80
|
signing_key:
|
81
81
|
specification_version: 4
|
82
82
|
summary: Use child processes to process a collection in parallel
|