rbbt-util 5.11.2 → 5.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/persist.rb +46 -38
- data/lib/rbbt/tsv/parallel/traverse.rb +1 -4
- data/lib/rbbt/util/concurrency/processes/socket.rb +1 -1
- data/lib/rbbt/util/misc.rb +2 -1
- data/lib/rbbt/workflow.rb +2 -1
- data/lib/rbbt/workflow/accessor.rb +24 -2
- data/lib/rbbt/workflow/step.rb +51 -19
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: add772f9d207913045a9042098c44b2c5549215c
|
4
|
+
data.tar.gz: 5932efc5c957bc4cf93de0228d4e654a47a788ce
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 864893d125d7ca6e03736829a5c77d11e936229dfd3569139c22b1acd10d8430156603b8a23f3ecf5c9b0602040278b6fff1482ccddde3cc9ca3686f40c16791
|
7
|
+
data.tar.gz: 71da4b77c4fad1c83fcaa068118150bd6cfbc0e6e029a2bd3ffc2867293fa585cba4a4638eff98df9d4120fe2a95847c290ae7f6b07bfad2699ba15566f1a99c
|
data/lib/rbbt/persist.rb
CHANGED
@@ -83,48 +83,53 @@ module Persist
|
|
83
83
|
|
84
84
|
TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
|
85
85
|
def self.load_file(path, type)
|
86
|
-
|
87
|
-
|
88
|
-
nil
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
86
|
+
begin
|
87
|
+
case (type || :marshal).to_sym
|
88
|
+
when :nil
|
89
|
+
nil
|
90
|
+
when :boolean
|
91
|
+
TRUE_STRINGS.include? Open.read(path).chomp.strip
|
92
|
+
when :annotations
|
93
|
+
Annotated.load_tsv TSV.open(path)
|
94
|
+
when :tsv
|
95
|
+
TSV.open(path)
|
96
|
+
when :marshal_tsv
|
97
|
+
TSV.setup(Marshal.load(Open.open(path)))
|
98
|
+
when :fwt
|
99
|
+
FixWidthTable.get(path)
|
100
|
+
when :string, :text
|
101
|
+
Open.read(path)
|
102
|
+
when :binary
|
103
|
+
f = File.open(path, 'rb')
|
104
|
+
res = f.read
|
105
|
+
f.close
|
106
|
+
res.force_encoding("ASCII-8BIT") if res.respond_to? :force_encoding
|
107
|
+
res
|
108
|
+
when :array
|
109
|
+
res = Open.read(path).split("\n", -1)
|
110
|
+
res.pop
|
111
|
+
res
|
112
|
+
when :marshal
|
113
|
+
Marshal.load(Open.open(path))
|
114
|
+
when :yaml
|
115
|
+
YAML.load(Open.open(path))
|
116
|
+
when :float
|
117
|
+
Open.read(path).to_f
|
118
|
+
when :integer
|
119
|
+
Open.read(path).to_i
|
120
|
+
else
|
121
|
+
raise "Unknown persistence: #{ type }"
|
122
|
+
end
|
123
|
+
rescue
|
124
|
+
Log.error "Exception loading #{ type } #{ path }: #{$!.message}"
|
125
|
+
raise $!
|
121
126
|
end
|
122
127
|
end
|
123
128
|
|
124
129
|
def self.save_file(path, type, content)
|
125
130
|
|
126
131
|
return if content.nil?
|
127
|
-
|
132
|
+
|
128
133
|
case (type || :marshal).to_sym
|
129
134
|
when :nil
|
130
135
|
nil
|
@@ -248,9 +253,11 @@ module Persist
|
|
248
253
|
end
|
249
254
|
rescue Aborted
|
250
255
|
Log.error "Tee stream thread aborted"
|
251
|
-
|
256
|
+
sout.abort if sout.respond_to? :abort
|
257
|
+
sin.abort if sin.respond_to? :abort
|
252
258
|
rescue Exception
|
253
|
-
|
259
|
+
sin.abort if sin.respond_to? :abort
|
260
|
+
sout.abort if sout.respond_to? :abort
|
254
261
|
Log.exception $!
|
255
262
|
parent.raise $!
|
256
263
|
ensure
|
@@ -413,6 +420,7 @@ module Persist
|
|
413
420
|
res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
|
414
421
|
ConcurrentStream.setup res do
|
415
422
|
begin
|
423
|
+
iii [:unlock, lockfile.path]
|
416
424
|
lockfile.unlock
|
417
425
|
rescue
|
418
426
|
Log.warn "Lockfile exception: " << $!.message
|
@@ -199,12 +199,9 @@ module TSV
|
|
199
199
|
end
|
200
200
|
end
|
201
201
|
|
202
|
-
#stream = obj_stream(obj)
|
203
|
-
#stream.close if stream
|
204
|
-
|
205
202
|
Process.waitpid pid
|
206
203
|
rescue Exception
|
207
|
-
Log.
|
204
|
+
Log.error "Exception traversin in cpus: #{$!.message}"
|
208
205
|
q.abort
|
209
206
|
raise $!
|
210
207
|
ensure
|
data/lib/rbbt/util/misc.rb
CHANGED
@@ -35,7 +35,7 @@ module LaterString
|
|
35
35
|
end
|
36
36
|
|
37
37
|
module ConcurrentStream
|
38
|
-
attr_accessor :threads, :pids, :callback, :filename, :joined
|
38
|
+
attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined
|
39
39
|
|
40
40
|
def joined?
|
41
41
|
@joined
|
@@ -72,6 +72,7 @@ module ConcurrentStream
|
|
72
72
|
def abort
|
73
73
|
@threads.each{|t| t.raise Aborted.new } if @threads
|
74
74
|
@pids.each{|pid| Process.kill :INT, pid } if @pids
|
75
|
+
@abort_callback.call if @abort_callback
|
75
76
|
end
|
76
77
|
|
77
78
|
def self.setup(stream, options = {}, &block)
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -234,7 +234,8 @@ module Workflow
|
|
234
234
|
def get_job_step(step_path, task = nil, input_values = nil, dependencies = nil)
|
235
235
|
step_path = step_path.call if Proc === step_path
|
236
236
|
persist = input_values.nil? ? false : true
|
237
|
-
|
237
|
+
key = Path === step_path ? step_path.find : step_path
|
238
|
+
step = Persist.memory("Step", :key => key, :repo => step_cache, :persist => persist ) do
|
238
239
|
step = Step.new step_path, task, input_values, dependencies
|
239
240
|
|
240
241
|
helpers.each do |name, block|
|
@@ -105,10 +105,21 @@ class Step
|
|
105
105
|
def self.log(status, message, path, &block)
|
106
106
|
if block_given?
|
107
107
|
start = Time.now
|
108
|
+
status = status.to_s
|
109
|
+
status_color = case status
|
110
|
+
when "starting"
|
111
|
+
:yellow
|
112
|
+
when "error"
|
113
|
+
:red
|
114
|
+
when "done"
|
115
|
+
:green
|
116
|
+
else
|
117
|
+
:cyan
|
118
|
+
end
|
108
119
|
Log.info do
|
109
120
|
now = Time.now
|
110
121
|
str = Log.color :reset
|
111
|
-
str << "#{ Log.color
|
122
|
+
str << "#{ Log.color status_color, status}"
|
112
123
|
str << ": #{ message }" if message
|
113
124
|
str << " -- #{Log.color :blue, path.to_s}" if path
|
114
125
|
str
|
@@ -123,10 +134,21 @@ class Step
|
|
123
134
|
end
|
124
135
|
res
|
125
136
|
else
|
137
|
+
status = status.to_s
|
138
|
+
status_color = case status
|
139
|
+
when "starting"
|
140
|
+
:yellow
|
141
|
+
when "error"
|
142
|
+
:red
|
143
|
+
when "done"
|
144
|
+
:green
|
145
|
+
else
|
146
|
+
:cyan
|
147
|
+
end
|
126
148
|
Log.info do
|
127
149
|
now = Time.now
|
128
150
|
str = Log.color :reset
|
129
|
-
str << "#{ Log.color
|
151
|
+
str << "#{ Log.color status_color, status}"
|
130
152
|
str << ": #{ message }" if message
|
131
153
|
str << " -- #{Log.color :blue, path.to_s}" if path
|
132
154
|
str
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -52,7 +52,7 @@ class Step
|
|
52
52
|
def log(status, message = nil)
|
53
53
|
self.status = status
|
54
54
|
message message
|
55
|
-
relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message )
|
55
|
+
relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless relay_step.done? or relay_step.error? or relay_step.aborted?
|
56
56
|
end
|
57
57
|
end
|
58
58
|
end
|
@@ -80,8 +80,8 @@ class Step
|
|
80
80
|
rescue Exception
|
81
81
|
value.abort if value.respond_to? :abort
|
82
82
|
ensure
|
83
|
-
value.close unless value.closed?
|
84
83
|
value.join if value.respond_to? :join
|
84
|
+
value.close unless value.closed?
|
85
85
|
end
|
86
86
|
when (not defined? Entity or description.nil? or not Entity.formats.include? description)
|
87
87
|
value
|
@@ -152,10 +152,26 @@ class Step
|
|
152
152
|
rec_dependencies.collect{|dependency| dependency.path }.uniq
|
153
153
|
end
|
154
154
|
|
155
|
+
def kill_children
|
156
|
+
children_pids = info[:children_pids]
|
157
|
+
if children_pids and children_pids.any?
|
158
|
+
Log.medium("Killing children: #{ children_pids * ", " }")
|
159
|
+
children_pids.each do |pid|
|
160
|
+
Log.medium("Killing child #{ pid }")
|
161
|
+
begin
|
162
|
+
Process.kill "INT", pid
|
163
|
+
rescue Exception
|
164
|
+
Log.medium("Exception killing child #{ pid }: #{$!.message}")
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
155
170
|
def run(no_load = false)
|
156
171
|
|
157
172
|
@mutex.synchronize do
|
158
|
-
|
173
|
+
no_load = no_load ? :stream : false
|
174
|
+
result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do |lockfile|
|
159
175
|
if Step === Step.log_relay_step and not self == Step.log_relay_step
|
160
176
|
relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
|
161
177
|
end
|
@@ -176,7 +192,8 @@ class Step
|
|
176
192
|
dependency.relay_log self
|
177
193
|
dependency.clean if not dependency.done? and dependency.error?
|
178
194
|
dependency.clean if dependency.streaming? and not dependency.running?
|
179
|
-
dependency.run true unless dependency.result
|
195
|
+
dependency.run true unless dependency.result or dependency.done?
|
196
|
+
seen_deps << dependency.path
|
180
197
|
seen_deps.concat dependency.rec_dependencies.collect{|d| d.path}
|
181
198
|
rescue Exception
|
182
199
|
backtrace = $!.backtrace
|
@@ -196,29 +213,30 @@ class Step
|
|
196
213
|
rescue Aborted
|
197
214
|
log(:error, "Aborted")
|
198
215
|
|
199
|
-
children_pids = info[:children_pids]
|
200
|
-
if children_pids and children_pids.any?
|
201
|
-
Log.medium("Killing children: #{ children_pids * ", " }")
|
202
|
-
children_pids.each do |pid|
|
203
|
-
Log.medium("Killing child #{ pid }")
|
204
|
-
begin
|
205
|
-
Process.kill "INT", pid
|
206
|
-
rescue Exception
|
207
|
-
Log.medium("Exception killing child #{ pid }: #{$!.message}")
|
208
|
-
end
|
209
|
-
end
|
210
|
-
end
|
211
|
-
|
216
|
+
# children_pids = info[:children_pids]
|
217
|
+
# if children_pids and children_pids.any?
|
218
|
+
# Log.medium("Killing children: #{ children_pids * ", " }")
|
219
|
+
# children_pids.each do |pid|
|
220
|
+
# Log.medium("Killing child #{ pid }")
|
221
|
+
# begin
|
222
|
+
# Process.kill "INT", pid
|
223
|
+
# rescue Exception
|
224
|
+
# Log.medium("Exception killing child #{ pid }: #{$!.message}")
|
225
|
+
# end
|
226
|
+
# end
|
227
|
+
# end
|
228
|
+
|
229
|
+
kill_children
|
212
230
|
raise $!
|
213
231
|
rescue Exception
|
214
232
|
backtrace = $!.backtrace
|
215
233
|
|
216
234
|
# HACK: This fixes an strange behaviour in 1.9.3 where some
|
217
235
|
# backtrace strings are coded in ASCII-8BIT
|
218
|
-
|
219
|
-
|
236
|
+
kill_children
|
220
237
|
set_info :backtrace, backtrace
|
221
238
|
log(:error, "#{$!.class}: #{$!.message}")
|
239
|
+
backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
|
222
240
|
raise $!
|
223
241
|
end
|
224
242
|
|
@@ -234,6 +252,13 @@ class Step
|
|
234
252
|
Log.exception $!
|
235
253
|
end
|
236
254
|
end
|
255
|
+
result.abort_callback = Proc.new do
|
256
|
+
begin
|
257
|
+
log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"
|
258
|
+
rescue
|
259
|
+
Log.exception $!
|
260
|
+
end
|
261
|
+
end
|
237
262
|
when TSV::Dumper
|
238
263
|
log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
|
239
264
|
ConcurrentStream.setup result.stream do
|
@@ -246,6 +271,13 @@ class Step
|
|
246
271
|
Log.exception $!
|
247
272
|
end
|
248
273
|
end
|
274
|
+
result.stream.abort_callback = Proc.new do
|
275
|
+
begin
|
276
|
+
log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"
|
277
|
+
rescue
|
278
|
+
Log.exception $!
|
279
|
+
end
|
280
|
+
end
|
249
281
|
else
|
250
282
|
set_info :done, (done_time = Time.now)
|
251
283
|
set_info :time_elapsed, (time_elapsed = done_time - start_time)
|