tubes 0.1.12 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/tubes.rb +37 -36
- metadata +2 -2
data/lib/tubes.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'json'
|
2
|
+
require 'monitor'
|
2
3
|
|
3
4
|
class Tube
|
4
5
|
attr :dir
|
@@ -24,7 +25,7 @@ class Tube
|
|
24
25
|
@serial_count = 0
|
25
26
|
@parallel_count = 0
|
26
27
|
|
27
|
-
@thread_lock = @parent ? @parent.thread_lock :
|
28
|
+
@thread_lock = @parent ? @parent.thread_lock : Monitor.new
|
28
29
|
@stats = @parent ? @parent.stats : {}
|
29
30
|
|
30
31
|
@name = underscore self.class.name.split('::')[-1]
|
@@ -34,6 +35,7 @@ class Tube
|
|
34
35
|
@output = serial? ? nil : []
|
35
36
|
|
36
37
|
@threads = []
|
38
|
+
Thread.abort_on_exception = true
|
37
39
|
|
38
40
|
@options = options
|
39
41
|
@step = nil
|
@@ -72,13 +74,10 @@ class Tube
|
|
72
74
|
|
73
75
|
def invoke(klass, *args)
|
74
76
|
@invocations += 1
|
77
|
+
dir = File.join(@dir, "#{@order}-#{underscore(klass.name.split('::')[-1])}")
|
78
|
+
segment = klass.new dir, :order => @order, :parent => self
|
75
79
|
|
76
|
-
|
77
|
-
segment = klass.new @dir, options
|
78
|
-
|
79
|
-
step = segment.name
|
80
|
-
|
81
|
-
output_file = segment_cache self, step
|
80
|
+
output_file = segment_cache self, segment.name
|
82
81
|
if output_file && File.exists?(output_file)
|
83
82
|
self.puts "Skipping: #{step}"
|
84
83
|
output = JSON.load(File.read(output_file))["data"]
|
@@ -92,23 +91,18 @@ class Tube
|
|
92
91
|
end
|
93
92
|
end
|
94
93
|
else
|
95
|
-
self.puts "Running: #{
|
94
|
+
self.puts "Running: #{segment.name}"
|
96
95
|
|
97
96
|
if serial?
|
98
97
|
dispatch(segment, output_file, *args)
|
99
98
|
@input = @output
|
100
99
|
elsif parallel?
|
101
|
-
thread = Thread.new
|
102
|
-
Thread.current[:lock] = lock
|
103
|
-
Thread.current.abort_on_exception = true
|
104
|
-
|
100
|
+
thread = Thread.new do
|
105
101
|
dispatch(segment, output_file, *args)
|
106
102
|
end
|
107
103
|
@threads << thread
|
108
104
|
end
|
109
105
|
end
|
110
|
-
|
111
|
-
Thread.current[:step] = step
|
112
106
|
end
|
113
107
|
|
114
108
|
|
@@ -144,10 +138,7 @@ class Tube
|
|
144
138
|
def tube(mode, args=nil, &block)
|
145
139
|
begin
|
146
140
|
if parallel? # When inside parallel.
|
147
|
-
thread = Thread.new
|
148
|
-
Thread.current[:lock] = lock
|
149
|
-
Thread.current.abort_on_exception = true
|
150
|
-
|
141
|
+
thread = Thread.new do
|
151
142
|
tube = child(mode, args)
|
152
143
|
tube.instance_eval &block
|
153
144
|
tube.threads.each { |thread| thread.join } # Could be a parallel block inside a parallel block.
|
@@ -178,29 +169,39 @@ class Tube
|
|
178
169
|
# This should be implemented in the subclasses.
|
179
170
|
end
|
180
171
|
|
172
|
+
def run_with_args(segment, args, options)
|
173
|
+
if args.empty?
|
174
|
+
if @invocations > 1
|
175
|
+
if options.present?
|
176
|
+
segment.send :run, @input, options
|
177
|
+
else
|
178
|
+
segment.send :run, @input
|
179
|
+
end
|
180
|
+
else
|
181
|
+
if options.present?
|
182
|
+
segment.send :run, options
|
183
|
+
else
|
184
|
+
segment.send :run
|
185
|
+
end
|
186
|
+
end
|
187
|
+
else
|
188
|
+
segment.send :run, *args
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
181
192
|
def dispatch(segment, output_file, *args)
|
193
|
+
options = args.last.is_a?(Hash) ? args.pop : {}
|
194
|
+
|
182
195
|
output = if segment.method(:run).arity > 0 # Optional arguments result in negative arity.
|
183
|
-
|
184
|
-
if @invocations > 1
|
185
|
-
segment.send :run, @input
|
186
|
-
else
|
187
|
-
segment.send :run # This should raise an argument mismatch error.
|
188
|
-
end
|
189
|
-
else
|
190
|
-
segment.send :run, *args
|
191
|
-
end
|
196
|
+
run_with_args(segment, args, options)
|
192
197
|
elsif segment.method(:run).arity < 0
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
segment.send :run
|
198
|
-
end
|
198
|
+
run_with_args(segment, args, options)
|
199
|
+
else
|
200
|
+
if options.present?
|
201
|
+
segment.send :run, options
|
199
202
|
else
|
200
|
-
segment.send :run
|
203
|
+
segment.send :run
|
201
204
|
end
|
202
|
-
else
|
203
|
-
segment.send :run
|
204
205
|
end
|
205
206
|
|
206
207
|
if output_file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tubes
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.13
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-11-09 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: A simple way to build a pipeline of tasks. These tasks can be configured
|
15
15
|
to run in serial, parallel or any combination thereof.
|