procrastinate 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
data/README CHANGED
@@ -20,7 +20,7 @@ SYNOPSIS
20
20
  end
21
21
  end
22
22
 
23
- scheduler = Scheduler.start(DispatchStrategy::Throttled.new(5))
23
+ scheduler = Scheduler.start(SpawnStrategy::Throttled.new(5))
24
24
  worker = scheduler.create_proxy(Worker.new)
25
25
 
26
26
  10.times do
@@ -54,22 +54,23 @@ The above example will output something like
54
54
 
55
55
  COMPATIBILITY
56
56
 
57
- This library runs with at least Ruby 1.8.7 and Ruby 1.9. For running it with
58
- Ruby 1.9, you need to patch it with the patch found at
59
-
60
- http://redmine.ruby-lang.org/issues/show/1525
61
-
62
- or run it with at least r25844 of Ruby trunk.
57
+ This library runs with at least Ruby 1.8.7 and Ruby 1.9. Ruby 1.9 support
58
+ might be spotty, because the threading primitives in Ruby 1.9 are still
59
+ somewhat buggy.
63
60
 
64
61
  KNOWN BUGS
65
62
 
66
63
  Due to the way we handle signal traps, you cannot start more than one
67
64
  Scheduler. We might allow that in the future.
68
65
 
66
+ Also: signal traps interact with other libraries and might cause things to
67
+ break. This is the real world.
68
+
69
69
  STATUS
70
70
 
71
- This code is not released as a gem yet. Once we iron out the bugs we'll
72
- release it as 0.1.0. Alpha quality code.
71
+ We're still adding features that we believe must be in 1.0. What is there
72
+ mostly works; Multi-{Processing, Threading} is always a difficult topic and
73
+ we're glad to receive bug reports.
73
74
 
74
75
  Please see the LICENSE file for license information.
75
76
 
data/Rakefile CHANGED
@@ -3,8 +3,6 @@ require 'rspec/core/rake_task'
3
3
  Rspec::Core::RakeTask.new
4
4
  task :default => :spec
5
5
 
6
- task :default => :spec
7
-
8
6
  require "rubygems"
9
7
  require "rake/gempackagetask"
10
8
  require "rake/rdoctask"
@@ -18,7 +16,7 @@ spec = Gem::Specification.new do |s|
18
16
 
19
17
  # Change these as appropriate
20
18
  s.name = "procrastinate"
21
- s.version = "0.1.0"
19
+ s.version = "0.2.0"
22
20
  s.summary = "Framework to run tasks in separate processes."
23
21
  s.authors = ['Kaspar Schiess', 'Patrick Marchi']
24
22
  s.email = ['kaspar.schiess@absurd.li', 'mail@patrickmarchi.ch']
@@ -34,13 +32,20 @@ spec = Gem::Specification.new do |s|
34
32
 
35
33
  # If you want to depend on other gems, add them here, along with any
36
34
  # relevant versions
37
- # s.add_dependency("blankslate", "~> 2.0")
35
+ s.add_dependency("state_machine", "~> 0.9.4")
38
36
 
39
37
  # If your tests use any gems, include them here
40
38
  s.add_development_dependency("rspec")
41
39
  s.add_development_dependency("flexmock")
42
40
  end
43
41
 
42
+ desc "Regenerate the .gemspec file for github/bundler."
43
+ task :gemspec do
44
+ # Generate the gemspec file for github.
45
+ file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
46
+ File.open(file, "w") {|f| f << spec.to_ruby }
47
+ end
48
+
44
49
  # This task actually builds the gem. We also regenerate a static
45
50
  # .gemspec file, which is useful if something (i.e. GitHub) will
46
51
  # be automatically building a gem for this project. If you're not
@@ -50,10 +55,6 @@ end
50
55
  # about that here: http://gemcutter.org/pages/gem_docs
51
56
  Rake::GemPackageTask.new(spec) do |pkg|
52
57
  pkg.gem_spec = spec
53
-
54
- # Generate the gemspec file for github.
55
- file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
56
- File.open(file, "w") {|f| f << spec.to_ruby }
57
58
  end
58
59
 
59
60
  # Generate documentation
@@ -0,0 +1,106 @@
1
+
2
+ # A communication endpoint. This acts as a factory and hub for the whole
3
+ # IPC library.
4
+ #
5
+ module Procrastinate::IPC::Endpoint
6
+ def anonymous
7
+ Anonymous.new
8
+ end
9
+ module_function :anonymous
10
+
11
+ # Works the same as IO.select, only that it doesn't care about write and
12
+ # error readiness, only read. You can mix IPC::Endpoints and normal IO
13
+ # instances freely.
14
+ #
15
+ def select(read_array, timeout=nil)
16
+ # This maps real system IO instances to wrapper objects. Return the thing
17
+ # to the right if IO.select returns the thing to the left.
18
+ mapping = Hash.new
19
+ waiting = []
20
+
21
+ read_array.each { |io_or_endpoint|
22
+ if io_or_endpoint.respond_to?(:select_ios)
23
+ waiting << io_or_endpoint if io_or_endpoint.waiting?
24
+
25
+ io_or_endpoint.select_ios.each do |io|
26
+ mapping[io] = io_or_endpoint
27
+ end
28
+ else
29
+ mapping[io_or_endpoint] = io_or_endpoint
30
+ end
31
+ }
32
+
33
+ return waiting unless waiting.empty?
34
+
35
+ system_io = IO.select(mapping.keys, nil, nil, timeout)
36
+ if system_io
37
+ return system_io.first.
38
+ # Map returned selectors to their object counterparts and then only
39
+ # return once (if more than one was returned).
40
+ map { |e| mapping[e] }.uniq
41
+ end
42
+ end
43
+ module_function :select
44
+
45
+ class Anonymous
46
+ def initialize
47
+ @re, @we = IO.pipe
48
+ end
49
+
50
+ def server
51
+ Server.new(@re)
52
+ end
53
+ def client
54
+ Client.new(@we)
55
+ end
56
+ end
57
+
58
+ class Anonymous::Server
59
+ attr_reader :pipe
60
+ attr_reader :waiting
61
+ def initialize(pipe)
62
+ @pipe = pipe
63
+ @waiting = Array.new
64
+ end
65
+
66
+ def receive(timeout=nil)
67
+ return waiting.shift if waiting?
68
+
69
+ loop do
70
+ buffer = pipe.read_nonblock(1024*1024*1024)
71
+
72
+ while buffer.size > 0
73
+ size = buffer.slice!(0...4).unpack('l').first
74
+ waiting << buffer.slice!(0...size)
75
+ end
76
+
77
+ return waiting.shift if waiting?
78
+ end
79
+ end
80
+
81
+ # True if there are queued messages in the Endpoint stack. If this is
82
+ # false, a receive might block.
83
+ #
84
+ def waiting?
85
+ not waiting.empty?
86
+ end
87
+
88
+ # Return underlying IOs for select.
89
+ #
90
+ def select_ios
91
+ [@pipe]
92
+ end
93
+ end
94
+
95
+ class Anonymous::Client
96
+ attr_reader :pipe
97
+ def initialize(pipe)
98
+ @pipe = pipe
99
+ end
100
+
101
+ def send(msg)
102
+ buffer = [msg.size].pack('l') + msg
103
+ pipe.write(buffer)
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,6 @@
1
+
2
+ # Holds classes for interprocess communication.
3
+ #
4
+ module Procrastinate::IPC
5
+ autoload :Endpoint, 'procrastinate/ipc/endpoint'
6
+ end
@@ -0,0 +1,247 @@
1
+
2
+ require 'state_machine'
3
+
4
+ # Dispatches and handles tasks and task completion. Only low level unixy
5
+ # manipulation here, no strategy. The only methods you should call from the
6
+ # outside are #setup, #step, #wakeup and #shutdown.
7
+ #
8
+ class Procrastinate::ProcessManager
9
+ include Procrastinate::IPC
10
+
11
+ # This pipe is used to wait for events in the master process.
12
+ attr_reader :control_pipe
13
+
14
+ # A hash of <pid, callback> that contains callbacks for all the child
15
+ # processes we spawn. Once the process is complete, the callback is called
16
+ # in the procrastinate thread.
17
+ attr_reader :children
18
+
19
+ # A class that acts as a filter between ProcessManager and the endpoint it
20
+ # uses to communicate with its children. This converts Ruby objects into
21
+ # Strings and also sends process id.
22
+ #
23
+ class ObjectEndpoint < Struct.new(:endpoint, :pid)
24
+ def send(obj)
25
+ msg = Marshal.dump([pid, obj])
26
+ endpoint.send(msg)
27
+ end
28
+ end
29
+
30
+ # A <completion handler, result> tuple that stores the handler to call when
31
+ # a child exits and the object that will handle child-master communication
32
+ # if desired.
33
+ #
34
+ class Child < Struct.new(:handler, :result, :state)
35
+ state_machine :state, :initial => :new do
36
+ event(:start) { transition :new => :running }
37
+ event(:died) { transition :running => :dead }
38
+
39
+ after_transition :on => :died, :do => :call_completion_handlers
40
+ end
41
+
42
+ # Calls the completion handler for the child. This is triggered by the
43
+ # transition into the 'dead' state.
44
+ #
45
+ def call_completion_handlers
46
+ result.process_died if result
47
+ handler.call if handler
48
+ end
49
+
50
+ # Handles incoming messages from the tasks process.
51
+ #
52
+ def incoming_message(obj)
53
+ result.incoming_message(obj) if result
54
+ end
55
+ end
56
+
57
+ def initialize
58
+ # This controls process manager wakeup
59
+ @control_pipe = IO.pipe
60
+
61
+ # All presently running children
62
+ @children = {}
63
+
64
+ # Child Master Communication (cmc)
65
+ endpoint = Endpoint.anonymous
66
+ @cmc_server = endpoint.server
67
+ @cmc_client = endpoint.client
68
+ end
69
+
70
+ # Sets up resource usage for dispatcher. You must call this before dispatcher
71
+ # can start its work.
72
+ #
73
+ def setup
74
+ register_signals
75
+ end
76
+
77
+ # Performs one step in the dispatchers work. This will sleep and wait
78
+ # for work to be done, then wake up and reap processes that are still
79
+ # pending. This method will mostly sleep.
80
+ #
81
+ def step
82
+ # Sleep until either work arrives or we receive a SIGCHLD
83
+ wait_for_event
84
+ # Reap all processes that have terminated in the meantime.
85
+ reap_childs
86
+ end
87
+
88
+ # Tears down the dispatcher. This frees resources that have been allocated
89
+ # and waits for all children to terminate.
90
+ #
91
+ def teardown
92
+ wait_for_all_childs
93
+ unregister_signals
94
+ end
95
+
96
+ # Wake up the dispatcher thread.
97
+ #
98
+ def wakeup
99
+ control_pipe.last.write '.'
100
+ # rescue IOError
101
+ # Ignore:
102
+ end
103
+
104
+ # Internal methods below this point. ---------------------------------------
105
+
106
+ # Register signals that aid in child care. NB: Because we do this globally,
107
+ # holding more than one dispatcher in a process will not work yet.
108
+ #
109
+ def register_signals
110
+ trap('CHLD') { wakeup }
111
+ end
112
+
113
+ # Unregister signals. Process should be as before.
114
+ #
115
+ def unregister_signals
116
+ trap('CHLD', 'DEFAULT')
117
+ end
118
+
119
+ # Called from the child management thread, will put that thread to sleep
120
+ # until someone requests it to become active again. See #wakeup.
121
+ #
122
+ def wait_for_event
123
+ cp_read_end = control_pipe.first
124
+
125
+ loop do # until we have input in the cp_read_end (control_pipe)
126
+ ready = Endpoint.select([cp_read_end, @cmc_server])
127
+
128
+ read_child_messages if ready.include? @cmc_server
129
+
130
+ # Kill children here, since we've just depleted the communication
131
+ # endpoint. This avoids the situation where the child process
132
+ # communicates but we remove it from our records before it can be told
133
+ # about it.
134
+ kill_children
135
+
136
+ if ready.include? cp_read_end
137
+ # Consume the data (not important)
138
+ cp_read_end.read_nonblock(1024)
139
+ return
140
+ end
141
+ end
142
+
143
+ # rescue Errno::EAGAIN, Errno::EINTR
144
+ # TODO Is this needed?
145
+ # A signal has been received. Mostly, this is as if we had received
146
+ # something in the control pipe.
147
+ end
148
+
149
+ def kill_children
150
+ children.delete_if { |pid, child| child.dead? }
151
+ end
152
+
153
+ # Once the @cmc_server endpoint is ready, loops and reads all child communication.
154
+ #
155
+ def read_child_messages
156
+ loop do
157
+ msg = @cmc_server.receive
158
+ decode_and_handle_message(msg)
159
+
160
+ break unless @cmc_server.waiting?
161
+ end
162
+ end
163
+
164
+ # Called for every message sent from a child. The +msg+ param here is a string
165
+ # that still needs decoding.
166
+ #
167
+ def decode_and_handle_message(msg)
168
+ pid, obj = Marshal.load(msg)
169
+ if child=children[pid]
170
+ child.incoming_message(obj)
171
+ else
172
+ warn "Communication from child #{pid} received, but child is gone."
173
+ end
174
+ rescue => b
175
+ # Messages that cannot be unmarshalled will be ignored.
176
+ warn "Can't unmarshal child communication."
177
+ end
178
+
179
+ # Calls completion handlers for all the childs that have now exited.
180
+ #
181
+ def reap_childs
182
+ loop do
183
+ child_pid, status = Process.waitpid(-1, Process::WNOHANG)
184
+ break unless child_pid
185
+
186
+ # Trigger the completion callback
187
+ if child=children[child_pid]
188
+ child.died
189
+ end
190
+ end
191
+ rescue Errno::ECHILD
192
+ # Ignore: This means that no childs remain.
193
+ end
194
+
195
+ # Spawns a process to work on +task+. If a block is given, it is called
196
+ # when the task completes. This method should only be called from a strategy
197
+ # inside the dispatchers thread. Otherwise it will expose threading issues.
198
+ #
199
+ # Example:
200
+ #
201
+ # spawn(wi) { |pid| puts "Task is complete" }
202
+ #
203
+ def create_process(task, &completion_handler)
204
+ # Tasks that are interested in getting messages from their childs must
205
+ # provide a result object that handles incoming 'result' messages.
206
+ result = task.result
207
+
208
+ pid = fork do
209
+ cleanup
210
+
211
+ if result
212
+ endpoint = ObjectEndpoint.new(@cmc_client, Process.pid)
213
+ task.run(endpoint)
214
+ else
215
+ task.run(nil)
216
+ end
217
+
218
+ exit! # this seems to be needed to avoid rspecs cleanup tasks
219
+ end
220
+
221
+ # The spawning is done in the same thread as the reaping is done. This is
222
+ # why no race condition to the following line exists. (or in other code,
223
+ # for that matter.)
224
+ children[pid] = Child.new(completion_handler, result).tap { |s| s.start }
225
+ end
226
+
227
+ # Gets executed in child process to clean up file handles and pipes that the
228
+ # master holds.
229
+ #
230
+ def cleanup
231
+ # Children dont need the parents signal handler
232
+ unregister_signals
233
+
234
+ # The child doesn't need the control pipe for now.
235
+ control_pipe.each { |io| io.close }
236
+ end
237
+
238
+ # Waits for all childs to complete.
239
+ #
240
+ def wait_for_all_childs
241
+ # TODO Maybe signal KILL to children after some time.
242
+ until children.all? { |p, c| c.dead? }
243
+ wait_for_event
244
+ reap_childs
245
+ end
246
+ end
247
+ end
@@ -1,6 +1,12 @@
1
-
1
+ # A proxy class that will translate all method calls made on it to method
2
+ # calls inside their own process via the Scheduler.
3
+ #
2
4
  class Procrastinate::Proxy
3
- def initialize(worker, scheduler)
5
+ # Create a new proxy class. +worker+ is an instance of the class that we
6
+ # want to perform work in, +scheduler+ is where the work will be scheduled.
7
+ # Don't call this on your own, instead use Scheduler#create_proxy.
8
+ #
9
+ def initialize(worker, scheduler) # :nodoc:
4
10
  @worker = worker
5
11
  @scheduler = scheduler
6
12
  end
@@ -11,8 +17,10 @@ class Procrastinate::Proxy
11
17
 
12
18
  def method_missing(name, *args, &block)
13
19
  if respond_to? name
14
- @scheduler.schedule(
15
- Procrastinate::Task::MethodCall.new(@worker, name, args, block))
20
+ task = Procrastinate::Task::MethodCall.new(@worker, name, args, block)
21
+ @scheduler.schedule(task)
22
+
23
+ return task.result
16
24
  else
17
25
  super
18
26
  end
@@ -1,41 +1,141 @@
1
+
2
+ require 'thread'
3
+
4
+ # API Frontend for the procrastinate library. Allows scheduling of tasks and
5
+ # workers in seperate processes and provides minimal locking primitives.
6
+ #
7
+ # Each scheduler owns its own thread that does all the processing. The
8
+ # interface between your main thread and the procrastinate thread is defined
9
+ # in this class.
10
+ #
1
11
  class Procrastinate::Scheduler
2
- attr_reader :dispatcher
12
+ attr_reader :manager
3
13
  attr_reader :strategy
4
-
5
- def initialize
6
- @shutdown_requested = false
14
+ attr_reader :task_queue
15
+
16
+ def initialize(strategy)
17
+ @strategy = strategy || Procrastinate::SpawnStrategy::Simple.new
18
+ @manager = Procrastinate::ProcessManager.new
19
+
20
+ # State takes three values: :running, :soft_shutdown, :real_shutdown
21
+ # :soft_shutdown will not accept any new tasks and wait for completion
22
+ # :real_shutdown will stop as soon as possible (still closing down nicely)
23
+ @state = :running
24
+ @task_queue = Queue.new
7
25
  end
8
26
 
9
- # Start a new scheduler
27
+ # Starts a new scheduler
28
+ #
10
29
  def self.start(strategy=nil)
11
- new.start(strategy)
30
+ new(strategy).
31
+ tap { |obj| obj.start }
12
32
  end
13
- def start(strategy=nil)
14
- @strategy = strategy || Procrastinate::DispatchStrategy::Simple.new
15
- @dispatcher = Procrastinate::Dispatcher.start(@strategy)
16
-
17
- self
33
+ def start
34
+ start_thread
18
35
  end
19
36
 
37
+ # Returns a proxy for the +worker+ instance that will allow executing its
38
+ # methods in a new process.
39
+ #
40
+ # Example:
41
+ #
42
+ # proxy = scheduler.create_proxy(worker)
43
+ # status = proxy.do_some_work # will execute later and in its own process
44
+ #
20
45
  def create_proxy(worker)
21
46
  return Procrastinate::Proxy.new(worker, self)
22
47
  end
23
48
 
24
- # Returns a runtime linked to this scheduler.
49
+ # Returns a runtime linked to this scheduler. This method should only be
50
+ # used inside task execution processes; If you call it from your main
51
+ # process, the result is undefined.
25
52
  #
26
53
  def runtime
27
54
  Procrastinate::Runtime.new
28
55
  end
29
56
 
30
- # Called by the proxy to schedule work.
57
+ # Called by the proxy to schedule work. You can implement your own Task
58
+ # classes; the relevant interface consists of only a #run method.
31
59
  #
32
60
  def schedule(task)
33
- strategy.schedule(task)
34
- dispatcher.wakeup
61
+ fail "Shutting down..." if @state != :running
62
+ task_queue << task
63
+
64
+ # Create an occasion for spawning
65
+ manager.wakeup
66
+ end
67
+
68
+ # Immediately shuts down the procrastinate thread and frees resources.
69
+ # If there are any tasks left in the queue, they will NOT be executed.
70
+ #
71
+ def shutdown(hard=false)
72
+ unless hard
73
+ @state = :soft_shutdown
74
+ loop do
75
+ manager.wakeup
76
+ break if task_queue.empty?
77
+ end
78
+ end
79
+
80
+ # Set the flag that will provoke shutdown
81
+ @state = :real_shutdown
82
+ # Wake the manager up, making it check the flag
83
+ manager.wakeup
84
+ # Wait for the manager to finish its work. This waits for child processes
85
+ # and then reaps their result, avoiding zombies.
86
+ @thread.join
35
87
  end
36
88
 
37
- def shutdown
38
- strategy.shutdown
39
- dispatcher.join
89
+ private
90
+ # Spawns new tasks (if needed). This is only ever called from the control
91
+ # thread (see below).
92
+ #
93
+ def spawn
94
+ while strategy.should_spawn? && !task_queue.empty?
95
+ task = task_queue.pop
96
+ manager.create_process(task) do
97
+ strategy.notify_dead
98
+ end
99
+ strategy.notify_spawn
100
+ end
101
+ end
102
+
103
+ # This is the content of the control thread that is spawned with
104
+ # #start_thread
105
+ #
106
+ def run
107
+ # Start managers work
108
+ manager.setup
109
+
110
+ # Loop until someone requests a shutdown.
111
+ loop do
112
+ manager.step
113
+
114
+ break if @state == :real_shutdown
115
+ spawn
116
+ end
117
+
118
+ manager.teardown
119
+ rescue => ex
120
+ # Sometimes exceptions vanish silently. This will avoid that, even though
121
+ # they should abort the whole process.
122
+
123
+ warn "Exception #{ex.inspect} caught."
124
+ ex.backtrace.first(5).each do |line|
125
+ warn line
126
+ end
127
+
128
+ raise
129
+ end
130
+
131
+ # Hosts the control thread that runs in parallel with your code. This thread
132
+ # handles child spawning and reaping.
133
+ #
134
+ def start_thread # :nodoc:
135
+ Thread.abort_on_exception = true
136
+
137
+ @thread = Thread.new do
138
+ run
139
+ end
40
140
  end
41
141
  end
@@ -0,0 +1,12 @@
1
+
2
+ class Procrastinate::SpawnStrategy::Simple
3
+ def should_spawn?
4
+ true
5
+ end
6
+
7
+ def notify_spawn
8
+ end
9
+
10
+ def notify_dead
11
+ end
12
+ end
@@ -2,7 +2,7 @@
2
2
  # A dispatcher strategy that throttles tasks starting and ensures that no
3
3
  # more than limit processes run concurrently.
4
4
  #
5
- class Procrastinate::DispatchStrategy::Throttled < Procrastinate::DispatchStrategy::Simple
5
+ class Procrastinate::SpawnStrategy::Throttled < Procrastinate::SpawnStrategy::Simple
6
6
  attr_reader :limit, :current
7
7
 
8
8
  # Client thread
@@ -13,13 +13,17 @@ class Procrastinate::DispatchStrategy::Throttled < Procrastinate::DispatchStrate
13
13
  @current = 0
14
14
  end
15
15
 
16
- # Dispatcher thread
17
- def spawn(dispatcher)
18
- super(dispatcher) { @current -= 1 }
16
+ def should_spawn?
17
+ current < limit
18
+ end
19
+
20
+ def notify_spawn
19
21
  @current += 1
22
+ warn "Throttled reports too many births!" if current > limit
20
23
  end
21
- def should_spawn?
22
- super &&
23
- current < limit
24
+
25
+ def notify_dead
26
+ @current -= 1
27
+ warn "Throttled reports more deaths than births?!" if current < 0
24
28
  end
25
29
  end
@@ -0,0 +1,5 @@
1
+
2
+ module Procrastinate::SpawnStrategy
3
+ autoload :Simple, 'procrastinate/spawn_strategy/simple'
4
+ autoload :Throttled, 'procrastinate/spawn_strategy/throttled'
5
+ end
@@ -0,0 +1,35 @@
1
+
2
+ require 'procrastinate/task/result'
3
+
4
+ # Constructs an object of type +klass+ and calls a method on it.
5
+ #
6
+ class Procrastinate::Task::MethodCall
7
+ include Procrastinate::Task
8
+
9
+ attr_reader :i
10
+ attr_reader :m
11
+ attr_reader :a
12
+ attr_reader :b
13
+
14
+ def initialize(instance, method, arguments, block)
15
+ @i = instance
16
+ @m = method
17
+ @a = arguments
18
+ @b = block
19
+ end
20
+
21
+ # Runs this task. Gets passed an endpoint that can be used to communicate
22
+ # values back to the master. Every time you write a value to that endpoint
23
+ # (using #send), the server will call #incoming_message on the task object
24
+ # in the master process. This allows return values and other communication
25
+ # from children to the master (and to the caller in this case).
26
+ #
27
+ def run(endpoint)
28
+ r = @i.send(@m, *@a, &@b)
29
+ endpoint.send r if endpoint
30
+ end
31
+
32
+ def result
33
+ @result ||= Result.new
34
+ end
35
+ end
@@ -0,0 +1,47 @@
1
+
2
+ require 'procrastinate/utils'
3
+
4
+ # A single value result, like from a normal method call. Return an instance of
5
+ # this from your task#result method to enable result handling.
6
+ #
7
+ class Procrastinate::Task::Result
8
+ def initialize
9
+ @value_ready = Procrastinate::Utils::OneTimeFlag.new
10
+ @value = nil
11
+ @exception = false
12
+ end
13
+
14
+ # Gets passed all messages sent by the child process for this task.
15
+ #
16
+ def incoming_message(obj)
17
+ return if ready?
18
+
19
+ @value = obj
20
+ @value_ready.set
21
+ end
22
+
23
+ # Notifies this result that the process has died. If this happens before
24
+ # a process result is passed to #incoming_message, that message will never
25
+ # arrive.
26
+ #
27
+ def process_died
28
+ return if ready?
29
+
30
+ @exception = true
31
+ @value_ready.set
32
+ end
33
+
34
+ def value
35
+ @value_ready.wait
36
+
37
+ if @exception
38
+ raise Procrastinate::ChildDeath, "Child process died before producing a value."
39
+ else
40
+ @value
41
+ end
42
+ end
43
+
44
+ def ready?
45
+ @value_ready.set?
46
+ end
47
+ end
@@ -0,0 +1,6 @@
1
+
2
+ # A collection of tasks that can be performed with procrastinate.
3
+ #
4
+ module Procrastinate::Task
5
+ autoload :MethodCall, 'procrastinate/task/method_call'
6
+ end
@@ -0,0 +1,39 @@
1
+ class Procrastinate::Utils::OneTimeFlag
2
+ def initialize
3
+ @waiting = []
4
+ @waiting_m = Mutex.new
5
+ @set = false
6
+ end
7
+
8
+ # If the flag is set, does nothing. If it isn't, it blocks until the flag
9
+ # is set.
10
+ def wait
11
+ return if set?
12
+
13
+ @waiting_m.synchronize do
14
+ @waiting << Thread.current
15
+ @waiting_m.sleep(0.001) until set?
16
+ end
17
+ end
18
+
19
+ # Sets the flag and releases all waiting threads.
20
+ #
21
+ def set
22
+ @set = true
23
+ @waiting_m.synchronize do
24
+ @waiting.each { |t| t.run }
25
+ @waiting = [] # cleanup
26
+ end
27
+ end
28
+
29
+ # Non blocking: Is the flag set?
30
+ #
31
+ def set?
32
+ @set
33
+ end
34
+ end
35
+
36
+ if RUBY_VERSION =~ /^1.8/
37
+ require 'procrastinate/utils/one_time_flag_ruby18_shim'
38
+ end
39
+
@@ -0,0 +1,32 @@
1
+ class Procrastinate::Utils::OneTimeFlag
2
+ def initialize
3
+ @waiting_m = Mutex.new
4
+ @waiting_cv = ConditionVariable.new
5
+ @set = false
6
+ end
7
+
8
+ # If the flag is set, does nothing. If it isn't, it blocks until the flag
9
+ # is set.
10
+ def wait
11
+ return if set?
12
+
13
+ @waiting_m.synchronize do
14
+ @waiting_cv.wait(@waiting_m)
15
+ end
16
+ end
17
+
18
+ # Sets the flag and releases all waiting threads.
19
+ #
20
+ def set
21
+ @set = true
22
+ @waiting_m.synchronize do
23
+ @waiting_cv.broadcast
24
+ end
25
+ end
26
+
27
+ # Non blocking: Is the flag set?
28
+ #
29
+ def set?
30
+ @set
31
+ end
32
+ end
@@ -0,0 +1,5 @@
1
+
2
+ module Procrastinate::Utils
3
+ end
4
+
5
+ require 'procrastinate/utils/one_time_flag'
data/lib/procrastinate.rb CHANGED
@@ -1,10 +1,17 @@
1
1
 
2
- module Procrastinate; end
2
+ module Procrastinate
3
+ # Raised when you try to access a future value that belongs to a process
4
+ # that died before producing a value.
5
+ #
6
+ class ChildDeath < StandardError; end
7
+
8
+ autoload :Lock, 'procrastinate/lock'
9
+ autoload :Runtime, 'procrastinate/runtime'
10
+ autoload :IPC, 'procrastinate/ipc'
11
+ autoload :Task, 'procrastinate/task'
12
+ end
3
13
 
4
- require 'procrastinate/runtime'
5
- require 'procrastinate/lock'
6
- require 'procrastinate/dispatch_strategies'
7
- require 'procrastinate/tasks'
14
+ require 'procrastinate/spawn_strategy'
8
15
  require 'procrastinate/proxy'
9
- require 'procrastinate/dispatcher'
16
+ require 'procrastinate/process_manager'
10
17
  require 'procrastinate/scheduler'
metadata CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
4
4
  prerelease: false
5
5
  segments:
6
6
  - 0
7
- - 1
7
+ - 2
8
8
  - 0
9
- version: 0.1.0
9
+ version: 0.2.0
10
10
  platform: ruby
11
11
  authors:
12
12
  - Kaspar Schiess
@@ -15,13 +15,28 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2010-12-10 00:00:00 +01:00
18
+ date: 2010-12-22 00:00:00 +01:00
19
19
  default_executable:
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
22
- name: rspec
22
+ name: state_machine
23
23
  prerelease: false
24
24
  requirement: &id001 !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ segments:
30
+ - 0
31
+ - 9
32
+ - 4
33
+ version: 0.9.4
34
+ type: :runtime
35
+ version_requirements: *id001
36
+ - !ruby/object:Gem::Dependency
37
+ name: rspec
38
+ prerelease: false
39
+ requirement: &id002 !ruby/object:Gem::Requirement
25
40
  none: false
26
41
  requirements:
27
42
  - - ">="
@@ -30,11 +45,11 @@ dependencies:
30
45
  - 0
31
46
  version: "0"
32
47
  type: :development
33
- version_requirements: *id001
48
+ version_requirements: *id002
34
49
  - !ruby/object:Gem::Dependency
35
50
  name: flexmock
36
51
  prerelease: false
37
- requirement: &id002 !ruby/object:Gem::Requirement
52
+ requirement: &id003 !ruby/object:Gem::Requirement
38
53
  none: false
39
54
  requirements:
40
55
  - - ">="
@@ -43,7 +58,7 @@ dependencies:
43
58
  - 0
44
59
  version: "0"
45
60
  type: :development
46
- version_requirements: *id002
61
+ version_requirements: *id003
47
62
  description:
48
63
  email:
49
64
  - kaspar.schiess@absurd.li
@@ -58,15 +73,22 @@ files:
58
73
  - LICENSE
59
74
  - Rakefile
60
75
  - README
61
- - lib/procrastinate/dispatch_strategies.rb
62
- - lib/procrastinate/dispatch_strategy/simple.rb
63
- - lib/procrastinate/dispatch_strategy/throttled.rb
64
- - lib/procrastinate/dispatcher.rb
76
+ - lib/procrastinate/ipc/endpoint.rb
77
+ - lib/procrastinate/ipc.rb
65
78
  - lib/procrastinate/lock.rb
79
+ - lib/procrastinate/process_manager.rb
66
80
  - lib/procrastinate/proxy.rb
67
81
  - lib/procrastinate/runtime.rb
68
82
  - lib/procrastinate/scheduler.rb
69
- - lib/procrastinate/tasks.rb
83
+ - lib/procrastinate/spawn_strategy/simple.rb
84
+ - lib/procrastinate/spawn_strategy/throttled.rb
85
+ - lib/procrastinate/spawn_strategy.rb
86
+ - lib/procrastinate/task/method_call.rb
87
+ - lib/procrastinate/task/result.rb
88
+ - lib/procrastinate/task.rb
89
+ - lib/procrastinate/utils/one_time_flag.rb
90
+ - lib/procrastinate/utils/one_time_flag_ruby18_shim.rb
91
+ - lib/procrastinate/utils.rb
70
92
  - lib/procrastinate.rb
71
93
  has_rdoc: true
72
94
  homepage: http://github.com/kschiess/procrastinate
@@ -1,9 +0,0 @@
1
-
2
- module Procrastinate::DispatchStrategy
3
- # Raised when you request a shutdown and then schedule new work.
4
- #
5
- class ShutdownRequested < StandardError; end
6
- end
7
-
8
- require 'procrastinate/dispatch_strategy/simple'
9
- require 'procrastinate/dispatch_strategy/throttled'
@@ -1,47 +0,0 @@
1
-
2
- require 'thread'
3
-
4
- class Procrastinate::DispatchStrategy::Simple
5
- attr_reader :queue
6
-
7
- # Client thread
8
- def initialize
9
- @queue = Queue.new
10
- @shutdown_requested = false
11
- end
12
-
13
- def shutdown_requested?
14
- @shutdown_requested
15
- end
16
-
17
- # Client thread
18
- def schedule(task)
19
- raise ::ShutdownRequested if shutdown_requested?
20
-
21
- queue.push task
22
- end
23
-
24
- # Dispatcher thread
25
- def spawn_new_workers(dispatcher)
26
- # Spawn tasks
27
- spawn(dispatcher) while should_spawn?
28
-
29
- # If the queue is empty now, maybe shutdown the dispatcher
30
- dispatcher.request_stop if shutdown_requested? && queue.empty?
31
- end
32
-
33
-
34
- # Spawn a new task from the job queue.
35
- # Dispatcher thread
36
- #
37
- def spawn(dispatcher, &block)
38
- dispatcher.spawn(queue.pop, &block)
39
- end
40
- def should_spawn?
41
- not queue.empty?
42
- end
43
-
44
- def shutdown
45
- @shutdown_requested = true
46
- end
47
- end
@@ -1,164 +0,0 @@
1
-
2
- # Dispatches and handles tasks and task completion. Only low level unixy
3
- # manipulation here, no strategy. The only method you should call from the
4
- # outside is #wakeup.
5
- #
6
- class Procrastinate::Dispatcher
7
- # The dispatcher runs in its own thread, which sleeps most of the time.
8
- attr_reader :thread
9
-
10
- # This pipe is used to wait for events in the master process.
11
- attr_reader :control_pipe
12
-
13
- # A hash of <pid, callback> that contains callbacks for all the child
14
- # processes we spawn. Once the process is complete, the callback is called
15
- # in the dispatcher/strategy's thread.
16
- attr_reader :handlers
17
-
18
- # The strategy for dispatching new tasks. Makes all the decisions about
19
- # when to launch what process.
20
- #
21
- attr_reader :strategy
22
-
23
- def initialize(strategy)
24
- @strategy = strategy
25
-
26
- @control_pipe = IO.pipe
27
- @handlers = {}
28
- @stop_requested = false
29
- end
30
-
31
- def self.start(strategy)
32
- new(strategy).tap do |dispatcher|
33
- dispatcher.start
34
- end
35
- end
36
- def start
37
- register_signals
38
- start_thread
39
- end
40
-
41
- # Called from anywhere, will complete all running tasks and stop the
42
- # dispatcher.
43
- #
44
- def stop
45
- request_stop
46
- join
47
- unregister_signals
48
- end
49
-
50
- # Called from the dispatcher thread, will cause the dispatcher to wait on
51
- # all running tasks and then stop dispatching.
52
- #
53
- def request_stop
54
- @stop_requested = true
55
- wakeup
56
- end
57
-
58
- def stop_requested?
59
- @stop_requested
60
- end
61
-
62
- def register_signals
63
- trap('CHLD') { wakeup }
64
- end
65
- def unregister_signals
66
- trap('CHLD', 'DEFAULT')
67
- end
68
-
69
- def start_thread
70
- @thread = Thread.new do
71
- Thread.current.abort_on_exception = true
72
-
73
- # Loop until someone requests a shutdown.
74
- loop do
75
- wait_for_event
76
- reap_workers
77
-
78
- break if stop_requested?
79
-
80
- strategy.spawn_new_workers(self)
81
- end
82
-
83
- wait_for_all_childs
84
- end
85
- end
86
-
87
- def wait_for_event
88
- # Returns array<ready_for_read, ..., ...>
89
- IO.select([control_pipe.first], nil, nil)
90
-
91
- # Consume the data (not important)
92
- control_pipe.first.read_nonblock(1024)
93
- rescue Errno::EAGAIN, Errno::EINTR
94
- end
95
-
96
- # Wake up the dispatcher thread.
97
- #
98
- def wakeup
99
- control_pipe.last.write '.'
100
- # rescue IOError
101
- # Ignore:
102
- end
103
-
104
- # Waits until the dispatcher completes its work. If you don't initiate a
105
- # shutdown, this may be forever.
106
- #
107
- def join
108
- @thread.join
109
- end
110
-
111
- # Calls completion handlers for all the childs that have now exited.
112
- #
113
- def reap_workers
114
- loop do
115
- child_pid, status = Process.waitpid2(-1, Process::WNOHANG)
116
- break unless child_pid
117
-
118
- # Trigger the completion callback
119
- handler = handlers.delete(child_pid)
120
- handler.call if handler
121
- end
122
- rescue Errno::ECHILD
123
- # Ignored: Child status has been reaped by someone else
124
- end
125
-
126
- # Spawns a process to work on +task+. If a block is given, it is called
127
- # when the task completes.
128
- #
129
- # Example:
130
- #
131
- # spawn(wi) { puts "Task is complete" }
132
- #
133
- def spawn(task, &completion_handler)
134
- pid = fork do
135
- cleanup
136
-
137
- task.run
138
-
139
- exit! # this seems to be needed to avoid rspecs cleanup tasks
140
- end
141
-
142
- handlers[pid] = completion_handler
143
- end
144
-
145
- # Gets executed in child process to clean up file handles and pipes that the
146
- # master holds.
147
- #
148
- def cleanup
149
- # Children dont need the parents signal handler
150
- trap(:CHLD, 'DEFAULT')
151
-
152
- # The child doesn't need the control pipe for now.
153
- control_pipe.each { |io| io.close }
154
- end
155
-
156
- # Waits for all childs to complete.
157
- #
158
- def wait_for_all_childs
159
- until handlers.empty?
160
- sleep 0.01
161
- reap_workers
162
- end
163
- end
164
- end
@@ -1,16 +0,0 @@
1
- module Procrastinate::Task
2
- # Constructs an object of type +klass+ and calls a method on it.
3
- #
4
- class MethodCall
5
- def initialize(instance, method, arguments, block)
6
- @instance = instance
7
- @method = method
8
- @arguments = arguments
9
- @block = block
10
- end
11
-
12
- def run
13
- @instance.send(@method, *@arguments, &@block)
14
- end
15
- end
16
- end