tubes 0.1.12 → 0.1.13
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/tubes.rb +37 -36
- metadata +2 -2
data/lib/tubes.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'json'
|
2
|
+
require 'monitor'
|
2
3
|
|
3
4
|
class Tube
|
4
5
|
attr :dir
|
@@ -24,7 +25,7 @@ class Tube
|
|
24
25
|
@serial_count = 0
|
25
26
|
@parallel_count = 0
|
26
27
|
|
27
|
-
@thread_lock = @parent ? @parent.thread_lock :
|
28
|
+
@thread_lock = @parent ? @parent.thread_lock : Monitor.new
|
28
29
|
@stats = @parent ? @parent.stats : {}
|
29
30
|
|
30
31
|
@name = underscore self.class.name.split('::')[-1]
|
@@ -34,6 +35,7 @@ class Tube
|
|
34
35
|
@output = serial? ? nil : []
|
35
36
|
|
36
37
|
@threads = []
|
38
|
+
Thread.abort_on_exception = true
|
37
39
|
|
38
40
|
@options = options
|
39
41
|
@step = nil
|
@@ -72,13 +74,10 @@ class Tube
|
|
72
74
|
|
73
75
|
def invoke(klass, *args)
|
74
76
|
@invocations += 1
|
77
|
+
dir = File.join(@dir, "#{@order}-#{underscore(klass.name.split('::')[-1])}")
|
78
|
+
segment = klass.new dir, :order => @order, :parent => self
|
75
79
|
|
76
|
-
|
77
|
-
segment = klass.new @dir, options
|
78
|
-
|
79
|
-
step = segment.name
|
80
|
-
|
81
|
-
output_file = segment_cache self, step
|
80
|
+
output_file = segment_cache self, segment.name
|
82
81
|
if output_file && File.exists?(output_file)
|
83
82
|
self.puts "Skipping: #{step}"
|
84
83
|
output = JSON.load(File.read(output_file))["data"]
|
@@ -92,23 +91,18 @@ class Tube
|
|
92
91
|
end
|
93
92
|
end
|
94
93
|
else
|
95
|
-
self.puts "Running: #{
|
94
|
+
self.puts "Running: #{segment.name}"
|
96
95
|
|
97
96
|
if serial?
|
98
97
|
dispatch(segment, output_file, *args)
|
99
98
|
@input = @output
|
100
99
|
elsif parallel?
|
101
|
-
thread = Thread.new
|
102
|
-
Thread.current[:lock] = lock
|
103
|
-
Thread.current.abort_on_exception = true
|
104
|
-
|
100
|
+
thread = Thread.new do
|
105
101
|
dispatch(segment, output_file, *args)
|
106
102
|
end
|
107
103
|
@threads << thread
|
108
104
|
end
|
109
105
|
end
|
110
|
-
|
111
|
-
Thread.current[:step] = step
|
112
106
|
end
|
113
107
|
|
114
108
|
|
@@ -144,10 +138,7 @@ class Tube
|
|
144
138
|
def tube(mode, args=nil, &block)
|
145
139
|
begin
|
146
140
|
if parallel? # When inside parallel.
|
147
|
-
thread = Thread.new
|
148
|
-
Thread.current[:lock] = lock
|
149
|
-
Thread.current.abort_on_exception = true
|
150
|
-
|
141
|
+
thread = Thread.new do
|
151
142
|
tube = child(mode, args)
|
152
143
|
tube.instance_eval &block
|
153
144
|
tube.threads.each { |thread| thread.join } # Could be a parallel block inside a parallel block.
|
@@ -178,29 +169,39 @@ class Tube
|
|
178
169
|
# This should be implemented in the subclasses.
|
179
170
|
end
|
180
171
|
|
172
|
+
def run_with_args(segment, args, options)
|
173
|
+
if args.empty?
|
174
|
+
if @invocations > 1
|
175
|
+
if options.present?
|
176
|
+
segment.send :run, @input, options
|
177
|
+
else
|
178
|
+
segment.send :run, @input
|
179
|
+
end
|
180
|
+
else
|
181
|
+
if options.present?
|
182
|
+
segment.send :run, options
|
183
|
+
else
|
184
|
+
segment.send :run
|
185
|
+
end
|
186
|
+
end
|
187
|
+
else
|
188
|
+
segment.send :run, *args
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
181
192
|
def dispatch(segment, output_file, *args)
|
193
|
+
options = args.last.is_a?(Hash) ? args.pop : {}
|
194
|
+
|
182
195
|
output = if segment.method(:run).arity > 0 # Optional arguments result in negative arity.
|
183
|
-
|
184
|
-
if @invocations > 1
|
185
|
-
segment.send :run, @input
|
186
|
-
else
|
187
|
-
segment.send :run # This should raise an argument mismatch error.
|
188
|
-
end
|
189
|
-
else
|
190
|
-
segment.send :run, *args
|
191
|
-
end
|
196
|
+
run_with_args(segment, args, options)
|
192
197
|
elsif segment.method(:run).arity < 0
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
segment.send :run
|
198
|
-
end
|
198
|
+
run_with_args(segment, args, options)
|
199
|
+
else
|
200
|
+
if options.present?
|
201
|
+
segment.send :run, options
|
199
202
|
else
|
200
|
-
segment.send :run
|
203
|
+
segment.send :run
|
201
204
|
end
|
202
|
-
else
|
203
|
-
segment.send :run
|
204
205
|
end
|
205
206
|
|
206
207
|
if output_file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tubes
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.13
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-11-09 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: A simple way to build a pipeline of tasks. These tasks can be configured
|
15
15
|
to run in serial, parallel or any combination thereof.
|