multiprocessing 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +3 -1
- data/.yardopts +4 -0
- data/README.ja.md +121 -0
- data/README.md +85 -91
- data/Rakefile +24 -1
- data/lib/multiprocessing.rb +41 -1
- data/lib/multiprocessing/conditionvariable.rb +82 -60
- data/lib/multiprocessing/externalobject.rb +5 -3
- data/lib/multiprocessing/mutex.rb +126 -96
- data/lib/multiprocessing/processerror.rb +5 -0
- data/lib/multiprocessing/queue.rb +95 -27
- data/lib/multiprocessing/semaphore.rb +78 -27
- data/lib/multiprocessing/version.rb +12 -1
- data/spec/multiprocessing/conditionvariable_spec.rb +206 -0
- data/spec/multiprocessing/mutex_spec.rb +468 -74
- data/spec/multiprocessing/queue_spec.rb +341 -97
- data/spec/multiprocessing/semaphore_spec.rb +136 -33
- data/spec/spec_helper.rb +15 -1
- metadata +11 -12
- data/lib/multiprocessing/process.rb +0 -43
- data/spec/multiprocessing/process_spec.rb +0 -51
@@ -1,6 +1,5 @@
|
|
1
1
|
require File.expand_path(File.dirname(__FILE__) + '/mutex')
|
2
2
|
require File.expand_path(File.dirname(__FILE__) + '/queue')
|
3
|
-
require File.expand_path(File.dirname(__FILE__) + '/process')
|
4
3
|
|
5
4
|
module MultiProcessing
|
6
5
|
class ExternalObject < BasicObject
|
@@ -10,7 +9,7 @@ module MultiProcessing
|
|
10
9
|
@result_queue = Queue.new
|
11
10
|
@mutex = Mutex.new
|
12
11
|
@closed = false
|
13
|
-
@
|
12
|
+
@pid = fork{|obj| process_loop obj }
|
14
13
|
end
|
15
14
|
|
16
15
|
def process_loop obj
|
@@ -38,7 +37,10 @@ module MultiProcessing
|
|
38
37
|
def close
|
39
38
|
@mutex.synchronize do
|
40
39
|
@closed = true
|
41
|
-
|
40
|
+
begin
|
41
|
+
Process.kill :TERM, @pid
|
42
|
+
rescue
|
43
|
+
end
|
42
44
|
end
|
43
45
|
end
|
44
46
|
|
@@ -3,136 +3,166 @@ require File.expand_path(File.dirname(__FILE__) + '/processerror')
|
|
3
3
|
|
4
4
|
module MultiProcessing
|
5
5
|
|
6
|
+
##
|
7
|
+
#
|
8
|
+
# Process version of Mutex.
|
9
|
+
# This can be used like ::Mutex in Ruby standard library.
|
10
|
+
#
|
11
|
+
# Do not fork in #synchronize block or before #unlock.
|
12
|
+
# Forking and forked process run in parallel.
|
13
|
+
#
|
14
|
+
# Note that Mutex uses 1 pipe.
|
15
|
+
#
|
16
|
+
# @example
|
17
|
+
# require 'multiprocessing'
|
18
|
+
#
|
19
|
+
# mutex = MultiProcessing::Mutex.new
|
20
|
+
# 3.times do
|
21
|
+
# fork do
|
22
|
+
# mutex.synchronize do
|
23
|
+
# # critical section
|
24
|
+
# puts Process.pid
|
25
|
+
# sleep 1
|
26
|
+
# end
|
27
|
+
# end
|
28
|
+
# end
|
29
|
+
# Process.waitall
|
30
|
+
# # => prints 3 pids of forked process in 1 sec interval
|
31
|
+
#
|
6
32
|
class Mutex
|
7
33
|
|
8
34
|
def initialize
|
9
35
|
@pout,@pin = IO.pipe
|
10
36
|
@pin.syswrite 1
|
11
|
-
#@pin.write 1
|
12
|
-
#@pin.flush
|
13
37
|
end
|
14
38
|
|
39
|
+
##
|
40
|
+
#
|
41
|
+
# Attempts to grab the lock and waits if it isn't available.
|
42
|
+
# Raises ProcessError if mutex was locked by the current thread.
|
43
|
+
#
|
44
|
+
# @return [Mutex] self
|
45
|
+
# @raise [ProcessError]
|
46
|
+
#
|
15
47
|
def lock
|
16
|
-
|
48
|
+
MultiProcessing.try_handle_interrupt(RuntimeError => :on_blocking) do
|
49
|
+
raise ProcessError.new "mutex was tried locking twice" if owned?
|
17
50
|
@pout.readpartial 1
|
18
|
-
@locking_pid =
|
51
|
+
@locking_pid = Process.pid
|
19
52
|
@locking_thread = Thread.current
|
20
|
-
|
21
|
-
raise ProcessError.new "mutex was tried locking twice"
|
53
|
+
self
|
22
54
|
end
|
23
|
-
self
|
24
55
|
end
|
25
56
|
|
57
|
+
##
|
58
|
+
#
|
59
|
+
# Returns true if this lock is currently held by some thread.
|
60
|
+
#
|
61
|
+
# @return [Boolean]
|
62
|
+
#
|
26
63
|
def locked?
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
64
|
+
MultiProcessing.try_handle_interrupt(RuntimeError => :never) do
|
65
|
+
begin
|
66
|
+
@pout.read_nonblock 1
|
67
|
+
@pin.syswrite 1
|
68
|
+
false
|
69
|
+
rescue Errno::EAGAIN => e
|
70
|
+
true
|
71
|
+
end
|
35
72
|
end
|
36
73
|
end
|
37
74
|
|
75
|
+
##
|
76
|
+
#
|
77
|
+
# Attempts to obtain the lock and returns immediately.
|
78
|
+
# Returns true if the lock was granted.
|
79
|
+
#
|
80
|
+
# @return [Boolean]
|
81
|
+
#
|
38
82
|
def try_lock
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
83
|
+
MultiProcessing.try_handle_interrupt(RuntimeError => :never) do
|
84
|
+
begin
|
85
|
+
@pout.read_nonblock 1
|
86
|
+
@locking_thread = Thread.current
|
87
|
+
@locking_pid = Process.pid
|
88
|
+
return true
|
89
|
+
rescue Errno::EAGAIN
|
90
|
+
return false
|
91
|
+
end
|
46
92
|
end
|
47
93
|
end
|
48
94
|
|
95
|
+
##
|
96
|
+
#
|
97
|
+
# Returns true if the lock is locked by current thread on current process
|
98
|
+
#
|
99
|
+
# @return [Boolean]
|
100
|
+
#
|
101
|
+
def owned?
|
102
|
+
@locking_pid == Process.pid && @locking_thread == Thread.current
|
103
|
+
end
|
104
|
+
|
105
|
+
##
|
106
|
+
#
|
107
|
+
# Releases the lock.
|
108
|
+
# Raises ProcessError if mutex wasn't locked by the current thread.
|
109
|
+
#
|
110
|
+
# @note An order of restarting thread is indefinite.
|
111
|
+
#
|
112
|
+
# @return [Mutex] self
|
113
|
+
# @raise [ProcessError]
|
114
|
+
#
|
49
115
|
def unlock
|
50
|
-
|
51
|
-
|
52
|
-
@
|
53
|
-
#@pin.write 1
|
54
|
-
#@pin.flush
|
116
|
+
MultiProcessing.try_handle_interrupt(RuntimeError => :never) do
|
117
|
+
raise ProcessError.new("Attempt to unlock a mutex which is not locked") unless locked?
|
118
|
+
raise ProcessError.new("Mutex was tried being unlocked in process/thread which didn't lock this mutex: locking[pid:#{(@locking_pid||'nil')}, thread:#{@locking_thread.inspect}] current[pid:#{Process.pid}, thread:#{Thread.current.inspect}]") unless owned?
|
55
119
|
@locking_pid = nil
|
56
120
|
@locking_thread = nil
|
57
|
-
|
58
|
-
|
59
|
-
#return nil
|
60
|
-
raise ProcessError.new("mutex was tried unlocking in process/thread which didn't lock this mutex #{@locking_pid} #{::Process.pid}")
|
121
|
+
@pin.syswrite 1
|
122
|
+
self
|
61
123
|
end
|
62
124
|
end
|
63
125
|
|
126
|
+
##
|
127
|
+
#
|
128
|
+
# Obtains a lock, runs the block, and releases the lock when the block completes.
|
129
|
+
#
|
130
|
+
# @return [Object] returned value of block
|
131
|
+
#
|
64
132
|
def synchronize
|
65
|
-
|
66
|
-
|
67
|
-
ret =
|
68
|
-
|
69
|
-
|
133
|
+
MultiProcessing.try_handle_interrupt(RuntimeError => :on_blocking) do
|
134
|
+
lock
|
135
|
+
ret = nil
|
136
|
+
begin
|
137
|
+
MultiProcessing.try_handle_interrupt(RuntimeError => :immediate) do
|
138
|
+
ret = yield
|
139
|
+
end
|
140
|
+
ensure
|
141
|
+
unlock
|
142
|
+
end
|
143
|
+
ret
|
70
144
|
end
|
71
|
-
return ret
|
72
145
|
end
|
73
146
|
|
147
|
+
##
|
148
|
+
#
|
149
|
+
# Releases the lock and sleeps timeout seconds if it is given and non-nil or forever.
|
150
|
+
# Raises ProcessError if mutex wasn't locked by the current thread.
|
151
|
+
#
|
152
|
+
# @param [Numeric,nil] timeout
|
153
|
+
# @raise [ProcessError]
|
154
|
+
#
|
74
155
|
def sleep timeout=nil
|
75
|
-
|
76
|
-
|
77
|
-
|
156
|
+
MultiProcessing.try_handle_interrupt(RuntimeError => :on_blocking) do
|
157
|
+
unlock
|
158
|
+
begin
|
159
|
+
timeout ? Kernel.sleep(timeout) : Kernel.sleep
|
160
|
+
ensure
|
161
|
+
lock
|
162
|
+
end
|
163
|
+
end
|
78
164
|
end
|
79
|
-
end
|
80
|
-
end
|
81
165
|
|
82
|
-
if __FILE__ == $0
|
83
|
-
|
84
|
-
puts "use lock and unlock"
|
85
|
-
m = MultiProcessing::Mutex.new
|
86
|
-
puts "locking mutex in main process(pid:#{Process.pid})"
|
87
|
-
m.lock
|
88
|
-
puts "locked mutex in main process(pid:#{Process.pid})"
|
89
|
-
pid1 = fork do
|
90
|
-
puts "locking mutex in child process(pid:#{Process.pid})"
|
91
|
-
m.lock
|
92
|
-
puts "locked mutex in child process(pid:#{Process.pid})"
|
93
|
-
sleep 1
|
94
|
-
puts "unlocking mutex in child process(pid:#{Process.pid})"
|
95
|
-
m.unlock
|
96
|
-
puts "unlocked mutex in child process(pid:#{Process.pid})"
|
97
|
-
exit
|
98
|
-
end
|
99
|
-
pid2 = fork do
|
100
|
-
puts "locking mutex in child process(pid:#{Process.pid})"
|
101
|
-
m.lock
|
102
|
-
puts "locked mutex in child process(pid:#{Process.pid})"
|
103
|
-
sleep 1
|
104
|
-
puts "unlocking mutex in child process(pid:#{Process.pid})"
|
105
|
-
m.unlock
|
106
|
-
puts "unlocked mutex in child process(pid:#{Process.pid})"
|
107
|
-
exit
|
108
|
-
end
|
109
|
-
|
110
|
-
sleep 1
|
111
|
-
puts "unlocking mutex in main process(pid:#{Process.pid})"
|
112
|
-
m.unlock
|
113
|
-
puts "unlocked mutex in main process(pid:#{Process.pid})"
|
114
|
-
Process.waitall
|
115
|
-
|
116
|
-
|
117
|
-
puts ""
|
118
|
-
puts "use synchrnize"
|
119
|
-
m = MultiProcessing::Mutex.new
|
120
|
-
if pid = fork
|
121
|
-
puts "synchronizing in main process(pid:#{Process.pid})"
|
122
|
-
m.synchronize do
|
123
|
-
puts "something to do in main process(pid:#{Process.pid})"
|
124
|
-
sleep 2
|
125
|
-
puts "end something in main process(pid:#{Process.pid})"
|
126
|
-
end
|
127
|
-
Process.waitpid pid
|
128
|
-
else
|
129
|
-
sleep 1
|
130
|
-
puts "synchronizing in child process(pid:#{Process.pid})"
|
131
|
-
m.synchronize do
|
132
|
-
puts "something to do in child process(pid:#{Process.pid})"
|
133
|
-
sleep 1
|
134
|
-
puts "end something in child process(pid:#{Process.pid})"
|
135
|
-
end
|
136
166
|
end
|
137
167
|
end
|
138
168
|
|
@@ -4,8 +4,38 @@ require File.expand_path(File.dirname(__FILE__) + '/semaphore')
|
|
4
4
|
|
5
5
|
module MultiProcessing
|
6
6
|
|
7
|
+
##
|
8
|
+
#
|
9
|
+
# Raised when an invalid operation is attempted on a queue.
|
10
|
+
#
|
7
11
|
class QueueError < StandardError; end
|
8
12
|
|
13
|
+
##
|
14
|
+
#
|
15
|
+
# This class provides a way to synchronize communication between process.
|
16
|
+
#
|
17
|
+
# Queue uses pipes to communicate with other processes.
|
18
|
+
# {Queue#push} starts background thread to write data to the pipe.
|
19
|
+
# Avoiding to exit process before writing to the pipe, use {#close} and {#join_thread}.
|
20
|
+
#
|
21
|
+
# q.close.join_thread
|
22
|
+
#
|
23
|
+
# {#join_thread} waits until all data is written to the pipe.
|
24
|
+
#
|
25
|
+
# Note that Queue uses 8 pipes ( 2 pipes, 2 Mutex, 1 Semaphore).
|
26
|
+
#
|
27
|
+
# @example
|
28
|
+
# require 'multiprocessing'
|
29
|
+
#
|
30
|
+
# q = MultiProcessing::Queue.new
|
31
|
+
# fork do
|
32
|
+
# q.push :nyan
|
33
|
+
# q.push :wan
|
34
|
+
# q.close.join_thread
|
35
|
+
# end
|
36
|
+
# q.pop # => :nyan
|
37
|
+
# q.pop # => :wan
|
38
|
+
#
|
9
39
|
class Queue
|
10
40
|
|
11
41
|
def initialize
|
@@ -19,34 +49,62 @@ module MultiProcessing
|
|
19
49
|
@closed = false
|
20
50
|
end
|
21
51
|
|
52
|
+
##
|
53
|
+
#
|
54
|
+
# Removes all objects from the queue
|
55
|
+
#
|
56
|
+
# @return [Queue] self
|
57
|
+
#
|
22
58
|
def clear
|
23
59
|
begin
|
24
60
|
loop do
|
25
|
-
|
26
|
-
self.deq(true)
|
27
|
-
end
|
61
|
+
self.deq(true)
|
28
62
|
end
|
29
|
-
rescue
|
63
|
+
rescue QueueError
|
30
64
|
end
|
65
|
+
self
|
31
66
|
end
|
32
67
|
|
68
|
+
##
|
69
|
+
#
|
70
|
+
# Returns true if the queue is empty.
|
71
|
+
#
|
72
|
+
# @return [Boolean]
|
73
|
+
#
|
33
74
|
def empty?
|
34
75
|
length == 0
|
35
76
|
end
|
36
77
|
|
78
|
+
##
|
79
|
+
#
|
80
|
+
# Returns number of items in the queue.
|
81
|
+
#
|
82
|
+
# @return [Fixnum]
|
83
|
+
#
|
37
84
|
def length
|
38
|
-
|
85
|
+
@count.value
|
39
86
|
end
|
40
87
|
alias :size :length
|
41
88
|
alias :count :length
|
42
89
|
|
90
|
+
##
|
91
|
+
#
|
92
|
+
# Retrieves data from the queue.
|
93
|
+
# If the queue is empty, the calling thread is suspended until data is pushed onto the queue.
|
94
|
+
# If non_block is true, thread isn't suspended, and exception is raised.
|
95
|
+
#
|
96
|
+
# @param [Boolean] non_block
|
97
|
+
# @return [Object]
|
98
|
+
#
|
43
99
|
def deq non_block=false
|
44
100
|
data = ""
|
45
101
|
@read_mutex.synchronize do
|
46
102
|
unless non_block
|
47
103
|
@count.wait
|
48
104
|
else
|
49
|
-
@count.
|
105
|
+
unless @count.try_wait
|
106
|
+
raise QueueError.new("Queue is empty")
|
107
|
+
end
|
50
108
|
end
|
51
109
|
|
52
110
|
buf = ""
|
@@ -72,6 +130,17 @@ module MultiProcessing
|
|
72
130
|
alias :pop :deq
|
73
131
|
alias :shift :deq
|
74
132
|
|
133
|
+
##
|
134
|
+
#
|
135
|
+
# Pushes object to the queue.
|
136
|
+
# Raise QueueError if the queue is already closed.
|
137
|
+
# Raise TypeError if the object passed cannot be dumped with Marshal.
|
138
|
+
#
|
139
|
+
# @param [Object] obj
|
140
|
+
# @return [Queue] self
|
141
|
+
# @raise [QueueError] the queue is already closed.
|
142
|
+
# @raise [TypeError] object cannot be dumped with Marshal.
|
143
|
+
#
|
75
144
|
def enq obj
|
76
145
|
raise QueueError.new("already closed") if @closed
|
77
146
|
unless(@enq_thread && @enq_thread.alive?)
|
@@ -79,16 +148,16 @@ module MultiProcessing
|
|
79
148
|
@enq_thread = Thread.new &method(:enq_loop)
|
80
149
|
end
|
81
150
|
@enq_queue.enq(Marshal.dump(obj))
|
82
|
-
|
151
|
+
@count.post
|
152
|
+
self
|
83
153
|
end
|
84
154
|
alias :push :enq
|
85
|
-
alias
|
155
|
+
alias :<< :enq
|
86
156
|
|
87
157
|
def enq_loop
|
88
158
|
loop do
|
89
159
|
data = @enq_queue.deq
|
90
160
|
@write_mutex.synchronize do
|
91
|
-
@count.post
|
92
161
|
@len_pin.write data.length.to_s + "\n"
|
93
162
|
@len_pin.flush
|
94
163
|
@data_pin.write data
|
@@ -99,36 +168,35 @@ module MultiProcessing
|
|
99
168
|
end
|
100
169
|
private :enq_loop
|
101
170
|
|
171
|
+
##
|
172
|
+
#
|
173
|
+
# Close the queue.
|
174
|
+
# After closing, the queue cannot be pushed any object.
|
175
|
+
# {#join_thread} can call only after closing the queue.
|
176
|
+
#
|
177
|
+
# @return [Queue] self
|
178
|
+
#
|
102
179
|
def close
|
103
180
|
@closed = true
|
104
181
|
self
|
105
182
|
end
|
106
183
|
|
184
|
+
##
|
185
|
+
#
|
186
|
+
# Waits until all data is written to the communication pipe.
|
187
|
+
# This can call only after closing({#close}) queue.
|
188
|
+
#
|
189
|
+
# @return [Queue] self
|
190
|
+
# @raise [QueueError] the queue is not closed.
|
191
|
+
#
|
107
192
|
def join_thread
|
108
193
|
raise QueueError.new("must be closed before join_thread") unless @closed
|
109
194
|
if @enq_thread && @enq_thread.alive?
|
110
195
|
@enq_thread.join
|
111
196
|
end
|
197
|
+
self
|
112
198
|
end
|
113
199
|
|
114
200
|
end
|
115
201
|
end
|
116
202
|
|
117
|
-
if __FILE__ == $0
|
118
|
-
|
119
|
-
q = MultiProcessing::Queue.new
|
120
|
-
|
121
|
-
q.push(0)
|
122
|
-
sleep 1
|
123
|
-
pid = fork
|
124
|
-
if !pid
|
125
|
-
q.push("111")
|
126
|
-
q.push({:a=>"a",:b=>123})
|
127
|
-
p q.pop
|
128
|
-
exit(0)
|
129
|
-
end
|
130
|
-
p q.pop
|
131
|
-
p q.pop
|
132
|
-
Process.waitall
|
133
|
-
|
134
|
-
end
|