rbbt-util 5.28.10 → 5.28.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/persist.rb +8 -3
- data/lib/rbbt/resource.rb +12 -6
- data/lib/rbbt/util/R.rb +2 -2
- data/lib/rbbt/util/cmd.rb +9 -0
- data/lib/rbbt/util/misc/inspect.rb +11 -7
- data/lib/rbbt/workflow/accessor.rb +1 -1
- data/lib/rbbt/workflow/step.rb +1 -1
- data/lib/rbbt/workflow/step/accessor.rb +4 -4
- data/lib/rbbt/workflow/util/provenance.rb +7 -3
- data/share/rbbt_commands/system/status +22 -22
- data/share/rbbt_commands/workflow/prov +2 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6ae5e7a2e944e3aabababe528af35f6d0b9002db33c661cf75dd3f6f49bd1b94
|
4
|
+
data.tar.gz: 94a6ec3c4a32e3632d59872c17b449a5c6be624cf999b2bb3469647933a0ba28
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 433ee5aa750efe1f609d6ee37a30b4438fc512719de2a7062d8696e2e1f15dec5caa33e2e63dce2ea7082d9f741fadf3314991a6a5e4a30d770f9fe6c7a04207
|
7
|
+
data.tar.gz: d16905de756df32abacd314b8793d4792c9852d76267be7ef71b6c5e0ae4e08f8b9ce04f34f2ceba3cd36ed8cda18966e737a893439b2451a9cde43a334afcb6
|
data/lib/rbbt/persist.rb
CHANGED
@@ -26,12 +26,17 @@ module Persist
|
|
26
26
|
MAX_FILE_LENGTH = 150
|
27
27
|
|
28
28
|
# Is 'file' newer than 'path'? return non-true if path is newer than file
|
29
|
-
def self.newer?(path, file)
|
29
|
+
def self.newer?(path, file, by_link = false)
|
30
30
|
return true if not Open.exists?(file)
|
31
31
|
path = path.find if Path === path
|
32
32
|
file = file.find if Path === file
|
33
|
-
|
34
|
-
|
33
|
+
if by_link
|
34
|
+
patht = File.lstat(path).mtime
|
35
|
+
filet = File.lstat(file).mtime
|
36
|
+
else
|
37
|
+
patht = Open.mtime(path)
|
38
|
+
filet = Open.mtime(file)
|
39
|
+
end
|
35
40
|
return true if patht.nil? || filet.nil?
|
36
41
|
diff = patht - filet
|
37
42
|
return diff if diff < 0
|
data/lib/rbbt/resource.rb
CHANGED
@@ -112,13 +112,19 @@ module Resource
|
|
112
112
|
end
|
113
113
|
when Net::HTTPRedirection, Net::HTTPFound
|
114
114
|
location = response['location']
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
115
|
+
if location.include? 'get_directory'
|
116
|
+
Log.debug("Feching directory from: #{location}. Into: #{final_path}")
|
117
|
+
FileUtils.mkdir_p final_path unless File.exist? final_path
|
118
|
+
TmpFile.with_file do |tmp_dir|
|
119
|
+
Misc.in_dir tmp_dir do
|
120
|
+
CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
|
121
|
+
end
|
122
|
+
FileUtils.mv tmp_dir, final_path
|
123
|
+
end
|
124
|
+
else
|
125
|
+
Open.open(location, :nocache => true) do |s|
|
126
|
+
Misc.sensiblewrite(final_path, s)
|
120
127
|
end
|
121
|
-
FileUtils.mv tmp_dir, final_path
|
122
128
|
end
|
123
129
|
when Net::HTTPInternalServerError
|
124
130
|
@server_missing_resource_cache << url
|
data/lib/rbbt/util/R.rb
CHANGED
@@ -41,7 +41,7 @@ source('#{UTIL}');
|
|
41
41
|
|
42
42
|
if monitor
|
43
43
|
#io = CMD.cmd('R --no-save --quiet', options.merge(:in => cmd, :pipe => true, :log => true))
|
44
|
-
io = CMD.cmd('R --no-save --quiet', options.merge(:in => cmd, :pipe => true, :log => true))
|
44
|
+
io = CMD.cmd('R --no-save --quiet', options.merge(:in => cmd, :pipe => true, :log => true, :xvfb => true))
|
45
45
|
while line = io.gets
|
46
46
|
case monitor
|
47
47
|
when Proc
|
@@ -52,7 +52,7 @@ source('#{UTIL}');
|
|
52
52
|
end
|
53
53
|
nil
|
54
54
|
else
|
55
|
-
CMD.cmd('R --no-save --slave --quiet', options.merge(:in => cmd))
|
55
|
+
CMD.cmd('R --no-save --slave --quiet', options.merge(:in => cmd, :xvfb => true))
|
56
56
|
end
|
57
57
|
end
|
58
58
|
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -100,6 +100,7 @@ module CMD
|
|
100
100
|
no_fail = options.delete(:no_fail)
|
101
101
|
no_fail = options.delete(:nofail) if no_fail.nil?
|
102
102
|
no_wait = options.delete(:no_wait)
|
103
|
+
xvfb = options.delete(:xvfb)
|
103
104
|
|
104
105
|
dont_close_in = options.delete(:dont_close_in)
|
105
106
|
|
@@ -117,6 +118,14 @@ module CMD
|
|
117
118
|
|
118
119
|
end
|
119
120
|
|
121
|
+
case xvfb
|
122
|
+
when TrueClass
|
123
|
+
cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
|
124
|
+
when String
|
125
|
+
cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
|
126
|
+
when String
|
127
|
+
end
|
128
|
+
|
120
129
|
if stderr == true
|
121
130
|
stderr = Log::HIGH
|
122
131
|
end
|
@@ -287,15 +287,19 @@ module Misc
|
|
287
287
|
when Symbol
|
288
288
|
obj.to_s
|
289
289
|
when (defined?(Path) and Path)
|
290
|
-
if obj.
|
291
|
-
|
292
|
-
|
293
|
-
|
290
|
+
if Step === obj.resource
|
291
|
+
"Step file: " + obj
|
292
|
+
else
|
293
|
+
if obj.exists?
|
294
|
+
if obj.directory?
|
295
|
+
files = obj.glob("**/*")
|
296
|
+
"directory: #{Misc.fingerprint(files)}"
|
297
|
+
else
|
298
|
+
"file: " << Open.realpath(obj) << "--" << mtime_str(obj)
|
299
|
+
end
|
294
300
|
else
|
295
|
-
|
301
|
+
obj + " (file missing)"
|
296
302
|
end
|
297
|
-
else
|
298
|
-
obj + " (file missing)"
|
299
303
|
end
|
300
304
|
when String
|
301
305
|
if Misc.is_filename?(obj) and ! %w(. ..).include?(obj)
|
@@ -360,7 +360,7 @@ module Workflow
|
|
360
360
|
compute = options[:compute]
|
361
361
|
|
362
362
|
options = IndiferentHash.setup(options.dup)
|
363
|
-
dep = dependency.call jobname,
|
363
|
+
dep = dependency.call jobname, _inputs.merge(options), real_dependencies
|
364
364
|
|
365
365
|
dep = [dep] unless Array === dep
|
366
366
|
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -156,7 +156,7 @@ class Step
|
|
156
156
|
|
157
157
|
def dependencies=(dependencies)
|
158
158
|
@dependencies = dependencies
|
159
|
-
set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
|
159
|
+
set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
|
160
160
|
end
|
161
161
|
|
162
162
|
def recursive_inputs
|
@@ -188,7 +188,7 @@ class Step
|
|
188
188
|
info_lock.lock if check_lock and false
|
189
189
|
begin
|
190
190
|
Open.open(info_file, :mode => 'rb') do |file|
|
191
|
-
INFO_SERIALIZER.load(file) #|| {}
|
191
|
+
IndiferentHash.setup(INFO_SERIALIZER.load(file)) #|| {}
|
192
192
|
end
|
193
193
|
ensure
|
194
194
|
info_lock.unlock if check_lock and false
|
@@ -227,7 +227,7 @@ class Step
|
|
227
227
|
Open.lock(info_file, :lock => info_lock) do
|
228
228
|
i = info(false).dup
|
229
229
|
i[key] = value
|
230
|
-
@info_cache = i
|
230
|
+
@info_cache = IndiferentHash.setup(i)
|
231
231
|
dump = INFO_SERIALIZER.dump(i)
|
232
232
|
Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
|
233
233
|
@info_cache_time = Time.now
|
@@ -242,7 +242,7 @@ class Step
|
|
242
242
|
Open.lock(info_file, :lock => info_lock) do
|
243
243
|
i = info(false)
|
244
244
|
i.merge! hash
|
245
|
-
@info_cache = i
|
245
|
+
@info_cache = IndiferentHash.setup(i)
|
246
246
|
dump = INFO_SERIALIZER.dump(i)
|
247
247
|
Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
|
248
248
|
@info_cache_time = Time.now
|
@@ -537,7 +537,7 @@ class Step
|
|
537
537
|
end
|
538
538
|
|
539
539
|
def file(name)
|
540
|
-
Path.setup(File.join(files_dir, name.to_s))
|
540
|
+
Path.setup(File.join(files_dir, name.to_s), workflow, self)
|
541
541
|
end
|
542
542
|
|
543
543
|
def save_file(name, content)
|
@@ -66,7 +66,7 @@ class Step
|
|
66
66
|
str << "\n"
|
67
67
|
end
|
68
68
|
|
69
|
-
def self.prov_report(step, offset = 0, task = nil, seen = [])
|
69
|
+
def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false)
|
70
70
|
info = step.info || {}
|
71
71
|
info[:task_name] = task
|
72
72
|
path = step.path
|
@@ -82,9 +82,13 @@ class Step
|
|
82
82
|
new = ! seen.include?(path)
|
83
83
|
if new
|
84
84
|
seen << path
|
85
|
-
str << prov_report(dep, offset + 1, task, seen)
|
85
|
+
str << prov_report(dep, offset + 1, task, seen, expand_repeats)
|
86
86
|
else
|
87
|
-
|
87
|
+
if expand_repeats
|
88
|
+
str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
|
89
|
+
else
|
90
|
+
str << Log.color(:green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
|
91
|
+
end
|
88
92
|
end
|
89
93
|
end if step.dependencies
|
90
94
|
str
|
@@ -49,27 +49,27 @@ def pid_msg(pid)
|
|
49
49
|
end
|
50
50
|
|
51
51
|
|
52
|
-
def status_msg(status)
|
53
|
-
color = case status.to_sym
|
54
|
-
when :error, :aborted, :missing, :dead, :broken
|
55
|
-
:red
|
56
|
-
when :streaming, :started
|
57
|
-
:cyan
|
58
|
-
when :done
|
59
|
-
:green
|
60
|
-
when :noinfo
|
61
|
-
:blue
|
62
|
-
when :dependencies, :waiting, :setup
|
63
|
-
:yellow
|
64
|
-
else
|
65
|
-
if status.to_s.index ">"
|
66
|
-
:cyan
|
67
|
-
else
|
68
|
-
:cyan
|
69
|
-
end
|
70
|
-
end
|
71
|
-
Log.color(color, status.to_s)
|
72
|
-
end
|
52
|
+
#def status_msg(status)
|
53
|
+
# color = case status.to_sym
|
54
|
+
# when :error, :aborted, :missing, :dead, :broken
|
55
|
+
# :red
|
56
|
+
# when :streaming, :started
|
57
|
+
# :cyan
|
58
|
+
# when :done
|
59
|
+
# :green
|
60
|
+
# when :noinfo, :notfound
|
61
|
+
# :blue
|
62
|
+
# when :dependencies, :waiting, :setup
|
63
|
+
# :yellow
|
64
|
+
# else
|
65
|
+
# if status.to_s.index ">"
|
66
|
+
# :cyan
|
67
|
+
# else
|
68
|
+
# :cyan
|
69
|
+
# end
|
70
|
+
# end
|
71
|
+
# Log.color(color, status.to_s)
|
72
|
+
#end
|
73
73
|
|
74
74
|
def input_msg(file, inputs)
|
75
75
|
|
@@ -218,7 +218,7 @@ workflows.sort.each do |workflow,tasks|
|
|
218
218
|
status << Log.color(:red, " (dead)")
|
219
219
|
end
|
220
220
|
end
|
221
|
-
str << " #{
|
221
|
+
str << " #{ Step.prov_status_msg status }"
|
222
222
|
str << " (dirty)" if status == 'done' && Workflow.load_step(file).dirty?
|
223
223
|
|
224
224
|
if inputs and inputs.any?
|
@@ -24,6 +24,7 @@ $ rbbt workflow prov <job-result>
|
|
24
24
|
-i--inputs* List of inputs to print
|
25
25
|
-if--info_fields* List of info fields to print
|
26
26
|
-t--touch Update modification times to be consistent
|
27
|
+
-e--expand_repeats Show all the dependency tree even if reapeated dependencies have already been seen before
|
27
28
|
EOF
|
28
29
|
|
29
30
|
SOPT.usage if options[:help]
|
@@ -130,6 +131,6 @@ if options[:plot]
|
|
130
131
|
end
|
131
132
|
|
132
133
|
else
|
133
|
-
puts Step.prov_report(step).strip
|
134
|
+
puts Step.prov_report(step, 0, nil, [], options[:expand_repeats]).strip
|
134
135
|
end
|
135
136
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.28.
|
4
|
+
version: 5.28.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-11-
|
11
|
+
date: 2020-11-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|