tubes 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. data/lib/tubes.rb +199 -0
  2. metadata +45 -0
@@ -0,0 +1,199 @@
1
+ require 'json'
2
+
3
+ class Tube
4
+ attr :dir
5
+ attr :ended_at
6
+ attr :exception
7
+ attr :lock
8
+ attr :name
9
+ attr :order
10
+ attr :output
11
+ attr :started_at
12
+ attr :stats
13
+ attr :threads
14
+
15
+ def initialize(dir=nil, options={})
16
+ @dir = dir || File.join('/tmp', Time.now.strftime('%Y-%m-%d_%H:%M:%S'))
17
+ @type = options[:type] || :serial
18
+ @parent = options[:parent] # This is nil only for the top level tube.
19
+ @serial_count = 0
20
+ @parallel_count = 0
21
+
22
+ @lock = @parent ? @parent.lock : Mutex.new
23
+ @stats = @parent ? @parent.stats : {}
24
+
25
+ @name = underscore self.class.name.split('::')[-1]
26
+ @order = options[:order] || ""
27
+
28
+ @output = options[:output] || (@type == :serial ? nil : [])
29
+ @threads = []
30
+
31
+ @options = options
32
+ @step = nil
33
+ @ended_at = nil
34
+ @started_at = options[:started_at] || Time.now
35
+ @invocations = 0
36
+
37
+ Dir.mkdir(@dir) unless Dir.exists?(@dir)
38
+ end
39
+
40
+
41
+ def serial(args=nil, &block)
42
+ tube(:serial, args, &block)
43
+ end
44
+
45
+
46
+ def parallel(args=nil, &block)
47
+ tube(:parallel, args, &block)
48
+ end
49
+
50
+
51
+ def start
52
+ lock = File.join @dir, "lock"
53
+ if !@options[:force] && File.exists?(lock)
54
+ raise "Another instance of the tubes seems to be running.\nPlease remove #{lock} if that is not the case."
55
+ end
56
+
57
+ File.open(lock, "w") do |f|
58
+ f.write $$
59
+ end
60
+ end
61
+
62
+
63
+ def finish
64
+ lock = File.join @dir, "lock"
65
+ File.delete lock
66
+
67
+ @ended_at = Time.now
68
+ end
69
+
70
+
71
+ def invoke(klass, *args)
72
+ @invocations += 1
73
+
74
+ options = args.last.kind_of?(Hash) ? args.pop : {}
75
+ options.merge! :order => @order, :parent => @parent
76
+ segment = klass.new @dir, options
77
+
78
+ step = segment.name
79
+
80
+ output_file = segment_cache self, step
81
+ if File.exists?(output_file)
82
+ self.puts "Skipping: #{step}"
83
+ @output = JSON.load(File.read(output_file))["data"]
84
+ else
85
+ self.puts "Running: #{step}"
86
+
87
+ if @type == :serial
88
+ run(segment, output_file, *args)
89
+ elsif @type == :parallel
90
+ thread = Thread.new(@lock) do |lock|
91
+ Thread.current[:lock] = lock
92
+ Thread.current.abort_on_exception = true
93
+
94
+ # This clobbers the @output. Perhaps make @output an array instead of a value and append to it under a lock.
95
+ run(segment, output_file, *args)
96
+ end
97
+ @threads << thread
98
+ end
99
+ end
100
+
101
+ Thread.current[:step] = step
102
+ end
103
+
104
+
105
+ def puts(string="")
106
+ @lock.synchronize do
107
+ if self.class == Tube
108
+ Kernel.puts "\033[32m[#{@order}]\033[0m #{string}"
109
+ else
110
+ Kernel.puts "\033[32m[#{@order}]\033[0m\033[36m[#{@name}]\033[0m #{string}"
111
+ end
112
+
113
+ STDOUT.flush
114
+ end
115
+ end
116
+
117
+
118
+ private
119
+
120
+ def tube(mode, args=nil, &block)
121
+ begin
122
+ case @type
123
+ when :parallel # When inside parallel.
124
+ thread = Thread.new(@lock) do |lock|
125
+ Thread.current[:lock] = lock
126
+ Thread.current.abort_on_exception = true
127
+ child(mode, args).instance_eval &block
128
+ end
129
+ @threads << thread
130
+ when :serial # When inside serial.
131
+ tube = child(mode, args)
132
+ tube.instance_eval &block
133
+ tube.threads.each { |thread| thread.join }
134
+ end
135
+ rescue => e
136
+ @exception = e
137
+ notify
138
+ raise
139
+ end
140
+ end
141
+
142
+ def notify
143
+ # This should be implemented in the subclasses.
144
+ end
145
+
146
+ def run(segment, output_file, *args)
147
+ output = if segment.method(:run).arity.abs > 0 # Optional arguments result in negative arity.
148
+ if args.empty?
149
+ segment.send :run, @output
150
+ else
151
+ segment.send :run, *args
152
+ end
153
+ else
154
+ segment.send :run
155
+ end
156
+
157
+ unless output_file.nil?
158
+ File.open(output_file, "w") do |f|
159
+ f.write({:data => output}.to_json)
160
+ end
161
+ end
162
+
163
+ if @type == :serial
164
+ @output = output
165
+ elsif @type == :parallel
166
+ @lock.synchronize do
167
+ @output << output
168
+ end
169
+ end
170
+ end
171
+
172
+
173
+ def segment_cache(tube, segment)
174
+ File.join tube.dir, "#{tube.order}-#{@invocations}-#{segment}.json"
175
+ end
176
+
177
+
178
+ def child(type, args=nil)
179
+ output = args || @output
180
+
181
+ order = case type
182
+ when :serial
183
+ @serial_count += 1
184
+ "#{@order}S#{@serial_count}"
185
+ when :parallel
186
+ @parallel_count += 1
187
+ "#{@order}P#{@parallel_count}"
188
+ end
189
+
190
+ Tube.new(@dir, :type => type, :output => output, :parent => self, :order => order, :started_at => started_at)
191
+ end
192
+
193
+ def underscore(string)
194
+ string.gsub(/::/, '/').
195
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
196
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
197
+ tr("-", "_").downcase
198
+ end
199
+ end
metadata ADDED
@@ -0,0 +1,45 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: tubes
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Sujoy Gupta
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-09-21 00:00:00.000000000 Z
13
+ dependencies: []
14
+ description: A simple way to build a pipeline of tasks.
15
+ email: sujoyg@gmail.com
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - lib/tubes.rb
21
+ homepage: http://rubygems.org/gems/tubes
22
+ licenses: []
23
+ post_install_message:
24
+ rdoc_options: []
25
+ require_paths:
26
+ - lib
27
+ required_ruby_version: !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '0'
33
+ required_rubygems_version: !ruby/object:Gem::Requirement
34
+ none: false
35
+ requirements:
36
+ - - ! '>='
37
+ - !ruby/object:Gem::Version
38
+ version: '0'
39
+ requirements: []
40
+ rubyforge_project:
41
+ rubygems_version: 1.8.24
42
+ signing_key:
43
+ specification_version: 3
44
+ summary: Tubes
45
+ test_files: []