procrastinate 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/HISTORY.txt +5 -1
- data/lib/procrastinate.rb +0 -1
- data/lib/procrastinate/process_manager.rb +18 -26
- data/lib/procrastinate/task/callable.rb +1 -1
- metadata +56 -12
- data/lib/procrastinate/ipc.rb +0 -6
- data/lib/procrastinate/ipc/endpoint.rb +0 -106
- data/lib/procrastinate/process_manager/object_endpoint.rb +0 -10
data/HISTORY.txt
CHANGED
data/lib/procrastinate.rb
CHANGED
@@ -1,14 +1,12 @@
|
|
1
1
|
|
2
2
|
require 'state_machine'
|
3
|
+
require 'cod'
|
3
4
|
|
4
5
|
# Dispatches and handles tasks and task completion. Only low level unixy
|
5
6
|
# manipulation here, no strategy. The only methods you should call from the
|
6
7
|
# outside are #setup, #step, #wakeup and #shutdown.
|
7
8
|
#
|
8
9
|
class Procrastinate::ProcessManager
|
9
|
-
include Procrastinate::IPC
|
10
|
-
|
11
|
-
autoload :ObjectEndpoint, 'procrastinate/process_manager/object_endpoint'
|
12
10
|
autoload :ChildProcess, 'procrastinate/process_manager/child_process'
|
13
11
|
|
14
12
|
# This pipe is used to wait for events in the master process.
|
@@ -25,11 +23,9 @@ class Procrastinate::ProcessManager
|
|
25
23
|
|
26
24
|
# All presently running children
|
27
25
|
@children = {}
|
28
|
-
|
29
|
-
#
|
30
|
-
|
31
|
-
@cmc_server = endpoint.server
|
32
|
-
@cmc_client = endpoint.client
|
26
|
+
|
27
|
+
# Master should read from @master, Children write to @child
|
28
|
+
@master, @child = Cod.pipe.split
|
33
29
|
end
|
34
30
|
|
35
31
|
# Sets up resource usage for dispatcher. You must call this before dispatcher
|
@@ -99,16 +95,19 @@ class Procrastinate::ProcessManager
|
|
99
95
|
cp_read_end = control_pipe.first
|
100
96
|
|
101
97
|
loop do # until we have input in the cp_read_end (control_pipe)
|
102
|
-
|
98
|
+
# TODO Why does procrastinate (cod) hang sometimes when there is no
|
99
|
+
# timeout here? What messages are we missing?
|
100
|
+
ready = Cod.select(1,
|
101
|
+
:child_msgs => @master, :control_pipe => cp_read_end)
|
103
102
|
|
104
|
-
read_child_messages if ready.
|
103
|
+
read_child_messages if ready.has_key?(:child_msgs)
|
105
104
|
|
106
105
|
# Send the tracking code for the child processes the final notifications
|
107
106
|
# and remove them from the children hash. At this point we know that
|
108
107
|
# no messages are waiting in the child queue.
|
109
108
|
finalize_children
|
110
109
|
|
111
|
-
if ready.
|
110
|
+
if ready.has_key?(:control_pipe)
|
112
111
|
# Consume the data (not important)
|
113
112
|
cp_read_end.read_nonblock(1024)
|
114
113
|
return
|
@@ -125,28 +124,20 @@ class Procrastinate::ProcessManager
|
|
125
124
|
child.removable? }
|
126
125
|
end
|
127
126
|
|
128
|
-
# Once the @cmc_server endpoint is ready, loops and reads all child
|
129
|
-
# communication.
|
130
|
-
#
|
131
127
|
def read_child_messages
|
132
128
|
loop do
|
133
|
-
|
134
|
-
|
129
|
+
ready = Cod.select(0.1, @master)
|
130
|
+
break unless ready
|
135
131
|
|
136
|
-
|
132
|
+
handle_message @master.get
|
137
133
|
end
|
138
134
|
end
|
139
135
|
|
140
136
|
# Called for every message sent from a child. The +msg+ param here is a string
|
141
137
|
# that still needs decoding.
|
142
138
|
#
|
143
|
-
def
|
144
|
-
pid, obj =
|
145
|
-
Marshal.load(msg)
|
146
|
-
rescue => b
|
147
|
-
# Messages that cannot be unmarshalled will be ignored.
|
148
|
-
warn "Can't unmarshal child communication: #{b}"
|
149
|
-
end
|
139
|
+
def handle_message(msg)
|
140
|
+
pid, obj = msg
|
150
141
|
|
151
142
|
if child=children[pid]
|
152
143
|
child.incoming_message(obj)
|
@@ -192,7 +183,7 @@ class Procrastinate::ProcessManager
|
|
192
183
|
cleanup
|
193
184
|
|
194
185
|
if result
|
195
|
-
endpoint =
|
186
|
+
endpoint = lambda { |obj| @child.put [Process.pid, obj] }
|
196
187
|
task.run(endpoint)
|
197
188
|
else
|
198
189
|
task.run(nil)
|
@@ -227,9 +218,10 @@ class Procrastinate::ProcessManager
|
|
227
218
|
#
|
228
219
|
def wait_for_all_childs
|
229
220
|
# TODO Maybe signal KILL to children after some time.
|
230
|
-
until children.
|
221
|
+
until children.empty?
|
231
222
|
wait_for_event
|
232
223
|
reap_childs
|
224
|
+
finalize_children
|
233
225
|
end
|
234
226
|
end
|
235
227
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: procrastinate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,11 +10,11 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2012-01-23 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: state_machine
|
17
|
-
requirement: &
|
17
|
+
requirement: &70180063938160 !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
20
20
|
- - ~>
|
@@ -22,10 +22,32 @@ dependencies:
|
|
22
22
|
version: 0.9.4
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
|
-
version_requirements: *
|
25
|
+
version_requirements: *70180063938160
|
26
|
+
- !ruby/object:Gem::Dependency
|
27
|
+
name: cod
|
28
|
+
requirement: &70180063966580 !ruby/object:Gem::Requirement
|
29
|
+
none: false
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.4'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: *70180063966580
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: rake
|
39
|
+
requirement: &70180063963140 !ruby/object:Gem::Requirement
|
40
|
+
none: false
|
41
|
+
requirements:
|
42
|
+
- - ! '>='
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: '0'
|
45
|
+
type: :development
|
46
|
+
prerelease: false
|
47
|
+
version_requirements: *70180063963140
|
26
48
|
- !ruby/object:Gem::Dependency
|
27
49
|
name: rspec
|
28
|
-
requirement: &
|
50
|
+
requirement: &70180063959700 !ruby/object:Gem::Requirement
|
29
51
|
none: false
|
30
52
|
requirements:
|
31
53
|
- - ! '>='
|
@@ -33,10 +55,10 @@ dependencies:
|
|
33
55
|
version: '0'
|
34
56
|
type: :development
|
35
57
|
prerelease: false
|
36
|
-
version_requirements: *
|
58
|
+
version_requirements: *70180063959700
|
37
59
|
- !ruby/object:Gem::Dependency
|
38
60
|
name: flexmock
|
39
|
-
requirement: &
|
61
|
+
requirement: &70180063973380 !ruby/object:Gem::Requirement
|
40
62
|
none: false
|
41
63
|
requirements:
|
42
64
|
- - ! '>='
|
@@ -44,7 +66,29 @@ dependencies:
|
|
44
66
|
version: '0'
|
45
67
|
type: :development
|
46
68
|
prerelease: false
|
47
|
-
version_requirements: *
|
69
|
+
version_requirements: *70180063973380
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: guard
|
72
|
+
requirement: &70180063972060 !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
type: :development
|
79
|
+
prerelease: false
|
80
|
+
version_requirements: *70180063972060
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: growl
|
83
|
+
requirement: &70180063970680 !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
85
|
+
requirements:
|
86
|
+
- - ! '>='
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
type: :development
|
90
|
+
prerelease: false
|
91
|
+
version_requirements: *70180063970680
|
48
92
|
description:
|
49
93
|
email:
|
50
94
|
- kaspar.schiess@absurd.li
|
@@ -59,11 +103,8 @@ files:
|
|
59
103
|
- Rakefile
|
60
104
|
- README
|
61
105
|
- lib/procrastinate/implicit.rb
|
62
|
-
- lib/procrastinate/ipc/endpoint.rb
|
63
|
-
- lib/procrastinate/ipc.rb
|
64
106
|
- lib/procrastinate/lock.rb
|
65
107
|
- lib/procrastinate/process_manager/child_process.rb
|
66
|
-
- lib/procrastinate/process_manager/object_endpoint.rb
|
67
108
|
- lib/procrastinate/process_manager.rb
|
68
109
|
- lib/procrastinate/proxy.rb
|
69
110
|
- lib/procrastinate/runtime.rb
|
@@ -99,13 +140,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
99
140
|
version: '0'
|
100
141
|
segments:
|
101
142
|
- 0
|
102
|
-
hash:
|
143
|
+
hash: 3407763989240963082
|
103
144
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
104
145
|
none: false
|
105
146
|
requirements:
|
106
147
|
- - ! '>='
|
107
148
|
- !ruby/object:Gem::Version
|
108
149
|
version: '0'
|
150
|
+
segments:
|
151
|
+
- 0
|
152
|
+
hash: 3407763989240963082
|
109
153
|
requirements: []
|
110
154
|
rubyforge_project:
|
111
155
|
rubygems_version: 1.8.10
|
data/lib/procrastinate/ipc.rb
DELETED
@@ -1,106 +0,0 @@
|
|
1
|
-
|
2
|
-
# A communication endpoint. This acts as a factory and hub for the whole
|
3
|
-
# IPC library.
|
4
|
-
#
|
5
|
-
module Procrastinate::IPC::Endpoint
|
6
|
-
def anonymous
|
7
|
-
Anonymous.new
|
8
|
-
end
|
9
|
-
module_function :anonymous
|
10
|
-
|
11
|
-
# Works the same as IO.select, only that it doesn't care about write and
|
12
|
-
# error readiness, only read. You can mix IPC::Endpoints and normal IO
|
13
|
-
# instances freely.
|
14
|
-
#
|
15
|
-
def select(read_array, timeout=nil)
|
16
|
-
# This maps real system IO instances to wrapper objects. Return the thing
|
17
|
-
# to the right if IO.select returns the thing to the left.
|
18
|
-
mapping = Hash.new
|
19
|
-
waiting = []
|
20
|
-
|
21
|
-
read_array.each { |io_or_endpoint|
|
22
|
-
if io_or_endpoint.respond_to?(:select_ios)
|
23
|
-
waiting << io_or_endpoint if io_or_endpoint.waiting?
|
24
|
-
|
25
|
-
io_or_endpoint.select_ios.each do |io|
|
26
|
-
mapping[io] = io_or_endpoint
|
27
|
-
end
|
28
|
-
else
|
29
|
-
mapping[io_or_endpoint] = io_or_endpoint
|
30
|
-
end
|
31
|
-
}
|
32
|
-
|
33
|
-
return waiting unless waiting.empty?
|
34
|
-
|
35
|
-
system_io = IO.select(mapping.keys, nil, nil, timeout)
|
36
|
-
if system_io
|
37
|
-
return system_io.first.
|
38
|
-
# Map returned selectors to their object counterparts and then only
|
39
|
-
# return once (if more than one was returned).
|
40
|
-
map { |e| mapping[e] }.uniq
|
41
|
-
end
|
42
|
-
end
|
43
|
-
module_function :select
|
44
|
-
|
45
|
-
class Anonymous
|
46
|
-
def initialize
|
47
|
-
@re, @we = IO.pipe
|
48
|
-
end
|
49
|
-
|
50
|
-
def server
|
51
|
-
Server.new(@re)
|
52
|
-
end
|
53
|
-
def client
|
54
|
-
Client.new(@we)
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
class Anonymous::Server
|
59
|
-
attr_reader :pipe
|
60
|
-
attr_reader :waiting
|
61
|
-
def initialize(pipe)
|
62
|
-
@pipe = pipe
|
63
|
-
@waiting = Array.new
|
64
|
-
end
|
65
|
-
|
66
|
-
def receive(timeout=nil)
|
67
|
-
return waiting.shift if waiting?
|
68
|
-
|
69
|
-
loop do
|
70
|
-
buffer = pipe.read_nonblock(1024*1024*1024)
|
71
|
-
|
72
|
-
while buffer.size > 0
|
73
|
-
size = buffer.slice!(0...4).unpack('l').first
|
74
|
-
waiting << buffer.slice!(0...size)
|
75
|
-
end
|
76
|
-
|
77
|
-
return waiting.shift if waiting?
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
# True if there are queued messages in the Endpoint stack. If this is
|
82
|
-
# false, a receive might block.
|
83
|
-
#
|
84
|
-
def waiting?
|
85
|
-
not waiting.empty?
|
86
|
-
end
|
87
|
-
|
88
|
-
# Return underlying IOs for select.
|
89
|
-
#
|
90
|
-
def select_ios
|
91
|
-
[@pipe]
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
class Anonymous::Client
|
96
|
-
attr_reader :pipe
|
97
|
-
def initialize(pipe)
|
98
|
-
@pipe = pipe
|
99
|
-
end
|
100
|
-
|
101
|
-
def send(msg)
|
102
|
-
buffer = [msg.size].pack('l') + msg
|
103
|
-
pipe.write(buffer)
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
@@ -1,10 +0,0 @@
|
|
1
|
-
# A class that acts as a filter between ProcessManager and the endpoint it
|
2
|
-
# uses to communicate with its children. This converts Ruby objects into
|
3
|
-
# Strings and also sends process id.
|
4
|
-
#
|
5
|
-
class Procrastinate::ProcessManager::ObjectEndpoint < Struct.new(:endpoint, :pid)
|
6
|
-
def send(obj)
|
7
|
-
msg = Marshal.dump([pid, obj])
|
8
|
-
endpoint.send(msg)
|
9
|
-
end
|
10
|
-
end
|