tubes 0.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/tubes.rb +199 -0
- metadata +45 -0
data/lib/tubes.rb
ADDED
@@ -0,0 +1,199 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
class Tube
|
4
|
+
attr :dir
|
5
|
+
attr :ended_at
|
6
|
+
attr :exception
|
7
|
+
attr :lock
|
8
|
+
attr :name
|
9
|
+
attr :order
|
10
|
+
attr :output
|
11
|
+
attr :started_at
|
12
|
+
attr :stats
|
13
|
+
attr :threads
|
14
|
+
|
15
|
+
def initialize(dir=nil, options={})
|
16
|
+
@dir = dir || File.join('/tmp', Time.now.strftime('%Y-%m-%d_%H:%M:%S'))
|
17
|
+
@type = options[:type] || :serial
|
18
|
+
@parent = options[:parent] # This is nil only for the top level tube.
|
19
|
+
@serial_count = 0
|
20
|
+
@parallel_count = 0
|
21
|
+
|
22
|
+
@lock = @parent ? @parent.lock : Mutex.new
|
23
|
+
@stats = @parent ? @parent.stats : {}
|
24
|
+
|
25
|
+
@name = underscore self.class.name.split('::')[-1]
|
26
|
+
@order = options[:order] || ""
|
27
|
+
|
28
|
+
@output = options[:output] || (@type == :serial ? nil : [])
|
29
|
+
@threads = []
|
30
|
+
|
31
|
+
@options = options
|
32
|
+
@step = nil
|
33
|
+
@ended_at = nil
|
34
|
+
@started_at = options[:started_at] || Time.now
|
35
|
+
@invocations = 0
|
36
|
+
|
37
|
+
Dir.mkdir(@dir) unless Dir.exists?(@dir)
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
def serial(args=nil, &block)
|
42
|
+
tube(:serial, args, &block)
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
def parallel(args=nil, &block)
|
47
|
+
tube(:parallel, args, &block)
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
def start
|
52
|
+
lock = File.join @dir, "lock"
|
53
|
+
if !@options[:force] && File.exists?(lock)
|
54
|
+
raise "Another instance of the tubes seems to be running.\nPlease remove #{lock} if that is not the case."
|
55
|
+
end
|
56
|
+
|
57
|
+
File.open(lock, "w") do |f|
|
58
|
+
f.write $$
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
def finish
|
64
|
+
lock = File.join @dir, "lock"
|
65
|
+
File.delete lock
|
66
|
+
|
67
|
+
@ended_at = Time.now
|
68
|
+
end
|
69
|
+
|
70
|
+
|
71
|
+
def invoke(klass, *args)
|
72
|
+
@invocations += 1
|
73
|
+
|
74
|
+
options = args.last.kind_of?(Hash) ? args.pop : {}
|
75
|
+
options.merge! :order => @order, :parent => @parent
|
76
|
+
segment = klass.new @dir, options
|
77
|
+
|
78
|
+
step = segment.name
|
79
|
+
|
80
|
+
output_file = segment_cache self, step
|
81
|
+
if File.exists?(output_file)
|
82
|
+
self.puts "Skipping: #{step}"
|
83
|
+
@output = JSON.load(File.read(output_file))["data"]
|
84
|
+
else
|
85
|
+
self.puts "Running: #{step}"
|
86
|
+
|
87
|
+
if @type == :serial
|
88
|
+
run(segment, output_file, *args)
|
89
|
+
elsif @type == :parallel
|
90
|
+
thread = Thread.new(@lock) do |lock|
|
91
|
+
Thread.current[:lock] = lock
|
92
|
+
Thread.current.abort_on_exception = true
|
93
|
+
|
94
|
+
# This clobbers the @output. Perhaps make @output an array instead of a value and append to it under a lock.
|
95
|
+
run(segment, output_file, *args)
|
96
|
+
end
|
97
|
+
@threads << thread
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
Thread.current[:step] = step
|
102
|
+
end
|
103
|
+
|
104
|
+
|
105
|
+
def puts(string="")
|
106
|
+
@lock.synchronize do
|
107
|
+
if self.class == Tube
|
108
|
+
Kernel.puts "\033[32m[#{@order}]\033[0m #{string}"
|
109
|
+
else
|
110
|
+
Kernel.puts "\033[32m[#{@order}]\033[0m\033[36m[#{@name}]\033[0m #{string}"
|
111
|
+
end
|
112
|
+
|
113
|
+
STDOUT.flush
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
|
118
|
+
private
|
119
|
+
|
120
|
+
def tube(mode, args=nil, &block)
|
121
|
+
begin
|
122
|
+
case @type
|
123
|
+
when :parallel # When inside parallel.
|
124
|
+
thread = Thread.new(@lock) do |lock|
|
125
|
+
Thread.current[:lock] = lock
|
126
|
+
Thread.current.abort_on_exception = true
|
127
|
+
child(mode, args).instance_eval &block
|
128
|
+
end
|
129
|
+
@threads << thread
|
130
|
+
when :serial # When inside serial.
|
131
|
+
tube = child(mode, args)
|
132
|
+
tube.instance_eval &block
|
133
|
+
tube.threads.each { |thread| thread.join }
|
134
|
+
end
|
135
|
+
rescue => e
|
136
|
+
@exception = e
|
137
|
+
notify
|
138
|
+
raise
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def notify
|
143
|
+
# This should be implemented in the subclasses.
|
144
|
+
end
|
145
|
+
|
146
|
+
def run(segment, output_file, *args)
|
147
|
+
output = if segment.method(:run).arity.abs > 0 # Optional arguments result in negative arity.
|
148
|
+
if args.empty?
|
149
|
+
segment.send :run, @output
|
150
|
+
else
|
151
|
+
segment.send :run, *args
|
152
|
+
end
|
153
|
+
else
|
154
|
+
segment.send :run
|
155
|
+
end
|
156
|
+
|
157
|
+
unless output_file.nil?
|
158
|
+
File.open(output_file, "w") do |f|
|
159
|
+
f.write({:data => output}.to_json)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
if @type == :serial
|
164
|
+
@output = output
|
165
|
+
elsif @type == :parallel
|
166
|
+
@lock.synchronize do
|
167
|
+
@output << output
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
|
173
|
+
def segment_cache(tube, segment)
|
174
|
+
File.join tube.dir, "#{tube.order}-#{@invocations}-#{segment}.json"
|
175
|
+
end
|
176
|
+
|
177
|
+
|
178
|
+
def child(type, args=nil)
|
179
|
+
output = args || @output
|
180
|
+
|
181
|
+
order = case type
|
182
|
+
when :serial
|
183
|
+
@serial_count += 1
|
184
|
+
"#{@order}S#{@serial_count}"
|
185
|
+
when :parallel
|
186
|
+
@parallel_count += 1
|
187
|
+
"#{@order}P#{@parallel_count}"
|
188
|
+
end
|
189
|
+
|
190
|
+
Tube.new(@dir, :type => type, :output => output, :parent => self, :order => order, :started_at => started_at)
|
191
|
+
end
|
192
|
+
|
193
|
+
def underscore(string)
|
194
|
+
string.gsub(/::/, '/').
|
195
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
196
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
197
|
+
tr("-", "_").downcase
|
198
|
+
end
|
199
|
+
end
|
metadata
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: tubes
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Sujoy Gupta
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-09-21 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description: A simple way to build a pipeline of tasks.
|
15
|
+
email: sujoyg@gmail.com
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- lib/tubes.rb
|
21
|
+
homepage: http://rubygems.org/gems/tubes
|
22
|
+
licenses: []
|
23
|
+
post_install_message:
|
24
|
+
rdoc_options: []
|
25
|
+
require_paths:
|
26
|
+
- lib
|
27
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
34
|
+
none: false
|
35
|
+
requirements:
|
36
|
+
- - ! '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
requirements: []
|
40
|
+
rubyforge_project:
|
41
|
+
rubygems_version: 1.8.24
|
42
|
+
signing_key:
|
43
|
+
specification_version: 3
|
44
|
+
summary: Tubes
|
45
|
+
test_files: []
|