multiprocessing 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,5 @@
1
1
  require File.expand_path(File.dirname(__FILE__) + '/mutex')
2
2
  require File.expand_path(File.dirname(__FILE__) + '/queue')
3
- require File.expand_path(File.dirname(__FILE__) + '/process')
4
3
 
5
4
  module MultiProcessing
6
5
  class ExternalObject < BasicObject
@@ -10,7 +9,7 @@ module MultiProcessing
10
9
  @result_queue = Queue.new
11
10
  @mutex = Mutex.new
12
11
  @closed = false
13
- @process = Process.new(obj){|obj| process_loop obj }
12
+ @pid = fork{|obj| process_loop obj }
14
13
  end
15
14
 
16
15
  def process_loop obj
@@ -38,7 +37,10 @@ module MultiProcessing
38
37
  def close
39
38
  @mutex.synchronize do
40
39
  @closed = true
41
- @process.kill :TERM
40
+ begin
41
+ Process.kill :TERM, @pid
42
+ rescue
43
+ end
42
44
  end
43
45
  end
44
46
 
@@ -3,136 +3,166 @@ require File.expand_path(File.dirname(__FILE__) + '/processerror')
3
3
 
4
4
  module MultiProcessing
5
5
 
6
+ ##
7
+ #
8
+ # Process version of Mutex.
9
+ # This can be used like ::Mutex in Ruby standard library.
10
+ #
11
+ # Do not fork in #synchronize block or before #unlock.
12
+ # Forking and forked process run in parallel.
13
+ #
14
+ # Note that Mutex uses 1 pipe.
15
+ #
16
+ # @example
17
+ # require 'multiprocessing'
18
+ #
19
+ # mutex = MultiProcessing::Mutex.new
20
+ # 3.times do
21
+ # fork do
22
+ # mutex.synchronize do
23
+ # # critical section
24
+ # puts Process.pid
25
+ # sleep 1
26
+ # end
27
+ # end
28
+ # end
29
+ # Process.waitall
30
+ # # => prints 3 pids of forked process in 1 sec interval
31
+ #
6
32
  class Mutex
7
33
 
8
34
  def initialize
9
35
  @pout,@pin = IO.pipe
10
36
  @pin.syswrite 1
11
- #@pin.write 1
12
- #@pin.flush
13
37
  end
14
38
 
39
+ ##
40
+ #
41
+ # Attempts to grab the lock and waits if it isn't available.
42
+ # Raises ProcessError if mutex was locked by the current thread.
43
+ #
44
+ # @return [Mutex] self
45
+ # @raise [ProcessError]
46
+ #
15
47
  def lock
16
- unless @locking_pid == ::Process.pid && @locking_thread == Thread.current
48
+ MultiProcessing.try_handle_interrupt(RuntimeError => :on_blocking) do
49
+ raise ProcessError.new "mutex was tried locking twice" if owned?
17
50
  @pout.readpartial 1
18
- @locking_pid = ::Process.pid
51
+ @locking_pid = Process.pid
19
52
  @locking_thread = Thread.current
20
- else
21
- raise ProcessError.new "mutex was tried locking twice"
53
+ self
22
54
  end
23
- self
24
55
  end
25
56
 
57
+ ##
58
+ #
59
+ # Returns true if this lock is currently held by some thread.
60
+ #
61
+ # @return [Boolean]
62
+ #
26
63
  def locked?
27
- begin
28
- @pout.read_nonblock 1
29
- @pin.syswrite 1
30
- #@pin.write 1
31
- #@pin.flush
32
- return false
33
- rescue Errno::EAGAIN => e
34
- return true
64
+ MultiProcessing.try_handle_interrupt(RuntimeError => :never) do
65
+ begin
66
+ @pout.read_nonblock 1
67
+ @pin.syswrite 1
68
+ false
69
+ rescue Errno::EAGAIN => e
70
+ true
71
+ end
35
72
  end
36
73
  end
37
74
 
75
+ ##
76
+ #
77
+ # Attempts to obtain the lock and returns immediately.
78
+ # Returns true if the lock was granted.
79
+ #
80
+ # @return [Boolean]
81
+ #
38
82
  def try_lock
39
- begin
40
- @pout.read_nonblock 1
41
- @locking_thread = Thread.current
42
- @locking_pid = ::Process.pid
43
- return true
44
- rescue Errno::EAGAIN
45
- return false
83
+ MultiProcessing.try_handle_interrupt(RuntimeError => :never) do
84
+ begin
85
+ @pout.read_nonblock 1
86
+ @locking_thread = Thread.current
87
+ @locking_pid = Process.pid
88
+ return true
89
+ rescue Errno::EAGAIN
90
+ return false
91
+ end
46
92
  end
47
93
  end
48
94
 
95
+ ##
96
+ #
97
+ # Returns true if the lock is locked by current thread on current process
98
+ #
99
+ # @return [Boolean]
100
+ #
101
+ def owned?
102
+ @locking_pid == Process.pid && @locking_thread == Thread.current
103
+ end
104
+
105
+ ##
106
+ #
107
+ # Releases the lock.
108
+ # Raises ProcessError if mutex wasn't locked by the current thread.
109
+ #
110
+ # @note An order of restarting thread is indefinite.
111
+ #
112
+ # @return [Mutex] self
113
+ # @raise [ProcessError]
114
+ #
49
115
  def unlock
50
- raise ProcessError.new("Attempt to unlock a mutex which is not locked") if !locked?
51
- if @locking_pid == ::Process.pid && @locking_thread == Thread.current
52
- @pin.syswrite 1
53
- #@pin.write 1
54
- #@pin.flush
116
+ MultiProcessing.try_handle_interrupt(RuntimeError => :never) do
117
+ raise ProcessError.new("Attempt to unlock a mutex which is not locked") unless locked?
118
+ raise ProcessError.new("Mutex was tried being unlocked in process/thread which didn't lock this mutex: locking[pid:#{(@locking_pid||'nil')}, thread:#{@locking_thread.inspect}] current[pid:#{Process.pid}, thread:#{Thread.current.inspect}]") unless owned?
55
119
  @locking_pid = nil
56
120
  @locking_thread = nil
57
- return self
58
- else
59
- #return nil
60
- raise ProcessError.new("mutex was tried unlocking in process/thread which didn't lock this mutex #{@locking_pid} #{::Process.pid}")
121
+ @pin.syswrite 1
122
+ self
61
123
  end
62
124
  end
63
125
 
126
+ ##
127
+ #
128
+ # Obtains a lock, runs the block, and releases the lock when the block completes.
129
+ #
130
+ # @return [Object] returned value of block
131
+ #
64
132
  def synchronize
65
- lock
66
- begin
67
- ret = yield
68
- ensure
69
- unlock if locked?
133
+ MultiProcessing.try_handle_interrupt(RuntimeError => :on_blocking) do
134
+ lock
135
+ ret = nil
136
+ begin
137
+ MultiProcessing.try_handle_interrupt(RuntimeError => :immediate) do
138
+ ret = yield
139
+ end
140
+ ensure
141
+ unlock
142
+ end
143
+ ret
70
144
  end
71
- return ret
72
145
  end
73
146
 
147
+ ##
148
+ #
149
+ # Releases the lock and sleeps timeout seconds if it is given and non-nil or forever.
150
+ # Raises ProcessError if mutex wasn't locked by the current thread.
151
+ #
152
+ # @param [Numeric,nil] timeout
153
+ # @raise [ProcessError]
154
+ #
74
155
  def sleep timeout=nil
75
- unlock
76
- timeout ? Kernel.sleep(timeout) : Kernel.sleep
77
- lock
156
+ MultiProcessing.try_handle_interrupt(RuntimeError => :on_blocking) do
157
+ unlock
158
+ begin
159
+ timeout ? Kernel.sleep(timeout) : Kernel.sleep
160
+ ensure
161
+ lock
162
+ end
163
+ end
78
164
  end
79
- end
80
- end
81
165
 
82
- if __FILE__ == $0
83
-
84
- puts "use lock and unlock"
85
- m = MultiProcessing::Mutex.new
86
- puts "locking mutex in main process(pid:#{Process.pid})"
87
- m.lock
88
- puts "locked mutex in main process(pid:#{Process.pid})"
89
- pid1 = fork do
90
- puts "locking mutex in child process(pid:#{Process.pid})"
91
- m.lock
92
- puts "locked mutex in child process(pid:#{Process.pid})"
93
- sleep 1
94
- puts "unlocking mutex in child process(pid:#{Process.pid})"
95
- m.unlock
96
- puts "unlocked mutex in child process(pid:#{Process.pid})"
97
- exit
98
- end
99
- pid2 = fork do
100
- puts "locking mutex in child process(pid:#{Process.pid})"
101
- m.lock
102
- puts "locked mutex in child process(pid:#{Process.pid})"
103
- sleep 1
104
- puts "unlocking mutex in child process(pid:#{Process.pid})"
105
- m.unlock
106
- puts "unlocked mutex in child process(pid:#{Process.pid})"
107
- exit
108
- end
109
-
110
- sleep 1
111
- puts "unlocking mutex in main process(pid:#{Process.pid})"
112
- m.unlock
113
- puts "unlocked mutex in main process(pid:#{Process.pid})"
114
- Process.waitall
115
-
116
-
117
- puts ""
118
- puts "use synchrnize"
119
- m = MultiProcessing::Mutex.new
120
- if pid = fork
121
- puts "synchronizing in main process(pid:#{Process.pid})"
122
- m.synchronize do
123
- puts "something to do in main process(pid:#{Process.pid})"
124
- sleep 2
125
- puts "end something in main process(pid:#{Process.pid})"
126
- end
127
- Process.waitpid pid
128
- else
129
- sleep 1
130
- puts "synchronizing in child process(pid:#{Process.pid})"
131
- m.synchronize do
132
- puts "something to do in child process(pid:#{Process.pid})"
133
- sleep 1
134
- puts "end something in child process(pid:#{Process.pid})"
135
- end
136
166
  end
137
167
  end
138
168
 
@@ -1,4 +1,9 @@
1
1
  module MultiProcessing
2
+
3
+ ##
4
+ #
5
+ # Raised when an invalid operation is attempted on process.
6
+ #
2
7
  class ProcessError < StandardError; end
3
8
  end
4
9
 
@@ -4,8 +4,38 @@ require File.expand_path(File.dirname(__FILE__) + '/semaphore')
4
4
 
5
5
  module MultiProcessing
6
6
 
7
+ ##
8
+ #
9
+ # Raised when an invalid operation is attempted on a queue.
10
+ #
7
11
  class QueueError < StandardError; end
8
12
 
13
+ ##
14
+ #
15
+ # This class provides a way to synchronize communication between process.
16
+ #
17
+ # Queue uses pipes to communicate with other processes.
18
+ # {Queue#push} starts background thread to write data to the pipe.
19
+ # Avoiding to exit process before writing to the pipe, use {#close} and {#join_thread}.
20
+ #
21
+ # q.close.join_thread
22
+ #
23
+ # {#join_thread} waits until all data is written to the pipe.
24
+ #
25
+ # Note that Queue uses 8 pipes ( 2 pipes, 2 Mutex, 1 Semaphore).
26
+ #
27
+ # @example
28
+ # require 'multiprocessing'
29
+ #
30
+ # q = MultiProcessing::Queue.new
31
+ # fork do
32
+ # q.push :nyan
33
+ # q.push :wan
34
+ # q.close.join_thread
35
+ # end
36
+ # q.pop # => :nyan
37
+ # q.pop # => :wan
38
+ #
9
39
  class Queue
10
40
 
11
41
  def initialize
@@ -19,34 +49,62 @@ module MultiProcessing
19
49
  @closed = false
20
50
  end
21
51
 
52
+ ##
53
+ #
54
+ # Removes all objects from the queue
55
+ #
56
+ # @return [Queue] self
57
+ #
22
58
  def clear
23
59
  begin
24
60
  loop do
25
- @read_mutex.synchronize do
26
- self.deq(true)
27
- end
61
+ self.deq(true)
28
62
  end
29
- rescue Errno::EAGAIN
63
+ rescue QueueError
30
64
  end
65
+ self
31
66
  end
32
67
 
68
+ ##
69
+ #
70
+ # Returns true if the queue is empty.
71
+ #
72
+ # @return [Boolean]
73
+ #
33
74
  def empty?
34
75
  length == 0
35
76
  end
36
77
 
78
+ ##
79
+ #
80
+ # Returns number of items in the queue.
81
+ #
82
+ # @return [Fixnum]
83
+ #
37
84
  def length
38
- return @count.value
85
+ @count.value
39
86
  end
40
87
  alias :size :length
41
88
  alias :count :length
42
89
 
90
+ ##
91
+ #
92
+ # Retrieves data from the queue.
93
+ # If the queue is empty, the calling thread is suspended until data is pushed onto the queue.
94
+ # If non_block is true, thread isn't suspended, and exception is raised.
95
+ #
96
+ # @param [Boolean] non_block
97
+ # @return [Object]
98
+ #
43
99
  def deq non_block=false
44
100
  data = ""
45
101
  @read_mutex.synchronize do
46
102
  unless non_block
47
103
  @count.wait
48
104
  else
49
- @count.trywait
105
+ unless @count.try_wait
106
+ raise QueueError.new("Queue is empty")
107
+ end
50
108
  end
51
109
 
52
110
  buf = ""
@@ -72,6 +130,17 @@ module MultiProcessing
72
130
  alias :pop :deq
73
131
  alias :shift :deq
74
132
 
133
+ ##
134
+ #
135
+ # Pushes object to the queue.
136
+ # Raise QueueError if the queue is already closed.
137
+ # Raise TypeError if the object passed cannot be dumped with Marshal.
138
+ #
139
+ # @param [Object] obj
140
+ # @return [Queue] self
141
+ # @raise [QueueError] the queue is already closed.
142
+ # @raise [TypeError] object cannot be dumped with Marshal.
143
+ #
75
144
  def enq obj
76
145
  raise QueueError.new("already closed") if @closed
77
146
  unless(@enq_thread && @enq_thread.alive?)
@@ -79,16 +148,16 @@ module MultiProcessing
79
148
  @enq_thread = Thread.new &method(:enq_loop)
80
149
  end
81
150
  @enq_queue.enq(Marshal.dump(obj))
82
- Thread.pass
151
+ @count.post
152
+ self
83
153
  end
84
154
  alias :push :enq
85
- alias :unshift :enq
155
+ alias :<< :enq
86
156
 
87
157
  def enq_loop
88
158
  loop do
89
159
  data = @enq_queue.deq
90
160
  @write_mutex.synchronize do
91
- @count.post
92
161
  @len_pin.write data.length.to_s + "\n"
93
162
  @len_pin.flush
94
163
  @data_pin.write data
@@ -99,36 +168,35 @@ module MultiProcessing
99
168
  end
100
169
  private :enq_loop
101
170
 
171
+ ##
172
+ #
173
+ # Close the queue.
174
+ # After closing, the queue cannot be pushed any object.
175
+ # {#join_thread} can call only after closing the queue.
176
+ #
177
+ # @return [Queue] self
178
+ #
102
179
  def close
103
180
  @closed = true
104
181
  self
105
182
  end
106
183
 
184
+ ##
185
+ #
186
+ # Waits until all data is written to the communication pipe.
187
+ # This can call only after closing({#close}) queue.
188
+ #
189
+ # @return [Queue] self
190
+ # @raise [QueueError] the queue is not closed.
191
+ #
107
192
  def join_thread
108
193
  raise QueueError.new("must be closed before join_thread") unless @closed
109
194
  if @enq_thread && @enq_thread.alive?
110
195
  @enq_thread.join
111
196
  end
197
+ self
112
198
  end
113
199
 
114
200
  end
115
201
  end
116
202
 
117
- if __FILE__ == $0
118
-
119
- q = MultiProcessing::Queue.new
120
-
121
- q.push(0)
122
- sleep 1
123
- pid = fork
124
- if !pid
125
- q.push("111")
126
- q.push({:a=>"a",:b=>123})
127
- p q.pop
128
- exit(0)
129
- end
130
- p q.pop
131
- p q.pop
132
- Process.waitall
133
-
134
- end