tubes 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/tubes.rb +199 -0
- metadata +45 -0
data/lib/tubes.rb
ADDED
@@ -0,0 +1,199 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
class Tube
|
4
|
+
attr :dir
|
5
|
+
attr :ended_at
|
6
|
+
attr :exception
|
7
|
+
attr :lock
|
8
|
+
attr :name
|
9
|
+
attr :order
|
10
|
+
attr :output
|
11
|
+
attr :started_at
|
12
|
+
attr :stats
|
13
|
+
attr :threads
|
14
|
+
|
15
|
+
def initialize(dir=nil, options={})
|
16
|
+
@dir = dir || File.join('/tmp', Time.now.strftime('%Y-%m-%d_%H:%M:%S'))
|
17
|
+
@type = options[:type] || :serial
|
18
|
+
@parent = options[:parent] # This is nil only for the top level tube.
|
19
|
+
@serial_count = 0
|
20
|
+
@parallel_count = 0
|
21
|
+
|
22
|
+
@lock = @parent ? @parent.lock : Mutex.new
|
23
|
+
@stats = @parent ? @parent.stats : {}
|
24
|
+
|
25
|
+
@name = underscore self.class.name.split('::')[-1]
|
26
|
+
@order = options[:order] || ""
|
27
|
+
|
28
|
+
@output = options[:output] || (@type == :serial ? nil : [])
|
29
|
+
@threads = []
|
30
|
+
|
31
|
+
@options = options
|
32
|
+
@step = nil
|
33
|
+
@ended_at = nil
|
34
|
+
@started_at = options[:started_at] || Time.now
|
35
|
+
@invocations = 0
|
36
|
+
|
37
|
+
Dir.mkdir(@dir) unless Dir.exists?(@dir)
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
def serial(args=nil, &block)
|
42
|
+
tube(:serial, args, &block)
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
def parallel(args=nil, &block)
|
47
|
+
tube(:parallel, args, &block)
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
def start
|
52
|
+
lock = File.join @dir, "lock"
|
53
|
+
if !@options[:force] && File.exists?(lock)
|
54
|
+
raise "Another instance of the tubes seems to be running.\nPlease remove #{lock} if that is not the case."
|
55
|
+
end
|
56
|
+
|
57
|
+
File.open(lock, "w") do |f|
|
58
|
+
f.write $$
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
def finish
|
64
|
+
lock = File.join @dir, "lock"
|
65
|
+
File.delete lock
|
66
|
+
|
67
|
+
@ended_at = Time.now
|
68
|
+
end
|
69
|
+
|
70
|
+
|
71
|
+
def invoke(klass, *args)
|
72
|
+
@invocations += 1
|
73
|
+
|
74
|
+
options = args.last.kind_of?(Hash) ? args.pop : {}
|
75
|
+
options.merge! :order => @order, :parent => @parent
|
76
|
+
segment = klass.new @dir, options
|
77
|
+
|
78
|
+
step = segment.name
|
79
|
+
|
80
|
+
output_file = segment_cache self, step
|
81
|
+
if File.exists?(output_file)
|
82
|
+
self.puts "Skipping: #{step}"
|
83
|
+
@output = JSON.load(File.read(output_file))["data"]
|
84
|
+
else
|
85
|
+
self.puts "Running: #{step}"
|
86
|
+
|
87
|
+
if @type == :serial
|
88
|
+
run(segment, output_file, *args)
|
89
|
+
elsif @type == :parallel
|
90
|
+
thread = Thread.new(@lock) do |lock|
|
91
|
+
Thread.current[:lock] = lock
|
92
|
+
Thread.current.abort_on_exception = true
|
93
|
+
|
94
|
+
# This clobbers the @output. Perhaps make @output an array instead of a value and append to it under a lock.
|
95
|
+
run(segment, output_file, *args)
|
96
|
+
end
|
97
|
+
@threads << thread
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
Thread.current[:step] = step
|
102
|
+
end
|
103
|
+
|
104
|
+
|
105
|
+
def puts(string="")
|
106
|
+
@lock.synchronize do
|
107
|
+
if self.class == Tube
|
108
|
+
Kernel.puts "\033[32m[#{@order}]\033[0m #{string}"
|
109
|
+
else
|
110
|
+
Kernel.puts "\033[32m[#{@order}]\033[0m\033[36m[#{@name}]\033[0m #{string}"
|
111
|
+
end
|
112
|
+
|
113
|
+
STDOUT.flush
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
|
118
|
+
private
|
119
|
+
|
120
|
+
def tube(mode, args=nil, &block)
|
121
|
+
begin
|
122
|
+
case @type
|
123
|
+
when :parallel # When inside parallel.
|
124
|
+
thread = Thread.new(@lock) do |lock|
|
125
|
+
Thread.current[:lock] = lock
|
126
|
+
Thread.current.abort_on_exception = true
|
127
|
+
child(mode, args).instance_eval &block
|
128
|
+
end
|
129
|
+
@threads << thread
|
130
|
+
when :serial # When inside serial.
|
131
|
+
tube = child(mode, args)
|
132
|
+
tube.instance_eval &block
|
133
|
+
tube.threads.each { |thread| thread.join }
|
134
|
+
end
|
135
|
+
rescue => e
|
136
|
+
@exception = e
|
137
|
+
notify
|
138
|
+
raise
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def notify
|
143
|
+
# This should be implemented in the subclasses.
|
144
|
+
end
|
145
|
+
|
146
|
+
def run(segment, output_file, *args)
|
147
|
+
output = if segment.method(:run).arity.abs > 0 # Optional arguments result in negative arity.
|
148
|
+
if args.empty?
|
149
|
+
segment.send :run, @output
|
150
|
+
else
|
151
|
+
segment.send :run, *args
|
152
|
+
end
|
153
|
+
else
|
154
|
+
segment.send :run
|
155
|
+
end
|
156
|
+
|
157
|
+
unless output_file.nil?
|
158
|
+
File.open(output_file, "w") do |f|
|
159
|
+
f.write({:data => output}.to_json)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
if @type == :serial
|
164
|
+
@output = output
|
165
|
+
elsif @type == :parallel
|
166
|
+
@lock.synchronize do
|
167
|
+
@output << output
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
|
173
|
+
def segment_cache(tube, segment)
|
174
|
+
File.join tube.dir, "#{tube.order}-#{@invocations}-#{segment}.json"
|
175
|
+
end
|
176
|
+
|
177
|
+
|
178
|
+
def child(type, args=nil)
|
179
|
+
output = args || @output
|
180
|
+
|
181
|
+
order = case type
|
182
|
+
when :serial
|
183
|
+
@serial_count += 1
|
184
|
+
"#{@order}S#{@serial_count}"
|
185
|
+
when :parallel
|
186
|
+
@parallel_count += 1
|
187
|
+
"#{@order}P#{@parallel_count}"
|
188
|
+
end
|
189
|
+
|
190
|
+
Tube.new(@dir, :type => type, :output => output, :parent => self, :order => order, :started_at => started_at)
|
191
|
+
end
|
192
|
+
|
193
|
+
def underscore(string)
|
194
|
+
string.gsub(/::/, '/').
|
195
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
196
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
197
|
+
tr("-", "_").downcase
|
198
|
+
end
|
199
|
+
end
|
metadata
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: tubes
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Sujoy Gupta
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-09-21 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description: A simple way to build a pipeline of tasks.
|
15
|
+
email: sujoyg@gmail.com
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- lib/tubes.rb
|
21
|
+
homepage: http://rubygems.org/gems/tubes
|
22
|
+
licenses: []
|
23
|
+
post_install_message:
|
24
|
+
rdoc_options: []
|
25
|
+
require_paths:
|
26
|
+
- lib
|
27
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
34
|
+
none: false
|
35
|
+
requirements:
|
36
|
+
- - ! '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
requirements: []
|
40
|
+
rubyforge_project:
|
41
|
+
rubygems_version: 1.8.24
|
42
|
+
signing_key:
|
43
|
+
specification_version: 3
|
44
|
+
summary: Tubes
|
45
|
+
test_files: []
|