rbbt-util 5.20.9 → 5.20.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/rbbt +1 -1
- data/bin/rbbt_query.rb +1 -1
- data/lib/rbbt/association/index.rb +3 -3
- data/lib/rbbt/fix_width_table.rb +4 -4
- data/lib/rbbt/persist/tsv.rb +4 -4
- data/lib/rbbt/persist/tsv/cdb.rb +3 -3
- data/lib/rbbt/persist/tsv/fix_width_table.rb +2 -2
- data/lib/rbbt/persist/tsv/kyotocabinet.rb +2 -2
- data/lib/rbbt/persist/tsv/leveldb.rb +2 -2
- data/lib/rbbt/persist/tsv/lmdb.rb +2 -2
- data/lib/rbbt/persist/tsv/packed_index.rb +2 -2
- data/lib/rbbt/persist/tsv/sharder.rb +4 -4
- data/lib/rbbt/persist/tsv/tokyocabinet.rb +2 -2
- data/lib/rbbt/resource.rb +6 -6
- data/lib/rbbt/resource/path.rb +7 -7
- data/lib/rbbt/resource/util.rb +9 -9
- data/lib/rbbt/rest/client/step.rb +8 -4
- data/lib/rbbt/tsv/attach.rb +21 -19
- data/lib/rbbt/tsv/parallel/traverse.rb +2 -2
- data/lib/rbbt/tsv/util.rb +1 -1
- data/lib/rbbt/util/R.rb +1 -1
- data/lib/rbbt/util/R/eval.rb +10 -10
- data/lib/rbbt/util/R/model.rb +2 -2
- data/lib/rbbt/util/docker.rb +3 -3
- data/lib/rbbt/util/filecache.rb +2 -2
- data/lib/rbbt/util/misc/development.rb +6 -4
- data/lib/rbbt/util/misc/lock.rb +1 -1
- data/lib/rbbt/util/misc/multipart_payload.rb +1 -1
- data/lib/rbbt/util/misc/pipes.rb +8 -8
- data/lib/rbbt/util/misc/system.rb +3 -3
- data/lib/rbbt/util/open.rb +9 -9
- data/lib/rbbt/util/simpleDSL.rb +2 -2
- data/lib/rbbt/util/tar.rb +1 -1
- data/lib/rbbt/util/tmpfile.rb +2 -2
- data/lib/rbbt/workflow.rb +26 -12
- data/lib/rbbt/workflow/accessor.rb +4 -4
- data/lib/rbbt/workflow/step/dependencies.rb +1 -1
- data/lib/rbbt/workflow/step/run.rb +6 -6
- data/share/rbbt_commands/app/install +2 -2
- data/share/rbbt_commands/app/start +1 -1
- data/share/rbbt_commands/log +5 -1
- data/share/rbbt_commands/resource/exists +1 -1
- data/share/rbbt_commands/study/task +3 -3
- data/share/rbbt_commands/system/clean +2 -2
- data/share/rbbt_commands/system/deleted_files +1 -1
- data/share/rbbt_commands/system/status +3 -3
- data/share/rbbt_commands/tsv/info +1 -1
- data/share/rbbt_commands/tsv/subset +1 -1
- data/share/rbbt_commands/workflow/example +1 -1
- data/share/rbbt_commands/workflow/install +3 -3
- data/share/rbbt_commands/workflow/monitor +6 -6
- data/share/rbbt_commands/workflow/prov +2 -2
- data/share/rbbt_commands/workflow/server +2 -2
- data/share/rbbt_commands/workflow/task +3 -3
- data/test/rbbt/test_resource.rb +2 -2
- data/test/rbbt/workflow/test_step.rb +3 -6
- metadata +2 -3
- data/lib/rbbt/util/misc/progress.rb +0 -0
@@ -324,7 +324,7 @@ class Step
|
|
324
324
|
end
|
325
325
|
|
326
326
|
def done?
|
327
|
-
path and File.
|
327
|
+
path and File.exist? path
|
328
328
|
end
|
329
329
|
|
330
330
|
def streaming?
|
@@ -426,8 +426,8 @@ class Step
|
|
426
426
|
provenance = {}
|
427
427
|
dependencies.each do |dep|
|
428
428
|
next unless dep.path.exists?
|
429
|
-
if File.
|
430
|
-
provenance[dep.path] = dep.provenance if File.
|
429
|
+
if File.exist? dep.info_file
|
430
|
+
provenance[dep.path] = dep.provenance if File.exist? dep.path
|
431
431
|
else
|
432
432
|
provenance[dep.path] = nil
|
433
433
|
end
|
@@ -438,7 +438,7 @@ class Step
|
|
438
438
|
def provenance_paths
|
439
439
|
provenance = {}
|
440
440
|
dependencies.each do |dep|
|
441
|
-
provenance[dep.path] = dep.provenance_paths if File.
|
441
|
+
provenance[dep.path] = dep.provenance_paths if File.exist? dep.path
|
442
442
|
end
|
443
443
|
provenance
|
444
444
|
end
|
@@ -116,7 +116,7 @@ class Step
|
|
116
116
|
begin
|
117
117
|
run_dependencies
|
118
118
|
rescue Exception
|
119
|
-
FileUtils.rm pid_file if File.
|
119
|
+
FileUtils.rm pid_file if File.exist?(pid_file)
|
120
120
|
stop_dependencies
|
121
121
|
raise $!
|
122
122
|
end
|
@@ -174,7 +174,7 @@ class Step
|
|
174
174
|
ensure
|
175
175
|
join
|
176
176
|
Step.purge_stream_cache
|
177
|
-
FileUtils.rm pid_file if File.
|
177
|
+
FileUtils.rm pid_file if File.exist?(pid_file)
|
178
178
|
end
|
179
179
|
end
|
180
180
|
stream.abort_callback = Proc.new do
|
@@ -183,7 +183,7 @@ class Step
|
|
183
183
|
rescue
|
184
184
|
Log.exception $!
|
185
185
|
stop_dependencies
|
186
|
-
FileUtils.rm pid_file if File.
|
186
|
+
FileUtils.rm pid_file if File.exist?(pid_file)
|
187
187
|
end
|
188
188
|
end
|
189
189
|
else
|
@@ -192,7 +192,7 @@ class Step
|
|
192
192
|
set_info :time_elapsed, (time_elapsed = done_time - start_time)
|
193
193
|
log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
|
194
194
|
Step.purge_stream_cache
|
195
|
-
FileUtils.rm pid_file if File.
|
195
|
+
FileUtils.rm pid_file if File.exist?(pid_file)
|
196
196
|
end
|
197
197
|
|
198
198
|
result
|
@@ -243,7 +243,7 @@ class Step
|
|
243
243
|
Misc.pre_fork
|
244
244
|
begin
|
245
245
|
RbbtSemaphore.wait_semaphore(semaphore) if semaphore
|
246
|
-
FileUtils.mkdir_p File.dirname(path) unless File.
|
246
|
+
FileUtils.mkdir_p File.dirname(path) unless File.exist? File.dirname(path)
|
247
247
|
begin
|
248
248
|
@forked = true
|
249
249
|
res = run true
|
@@ -377,7 +377,7 @@ class Step
|
|
377
377
|
end
|
378
378
|
|
379
379
|
def soft_grace
|
380
|
-
until done? or File.
|
380
|
+
until done? or File.exist?(info_file)
|
381
381
|
sleep 1
|
382
382
|
end
|
383
383
|
self
|
@@ -11,10 +11,10 @@ raise "No app specified" if app.nil?
|
|
11
11
|
|
12
12
|
app_dir = Rbbt.etc.app_dir.exists? ? Rbbt.etc.app_dir.read.strip : Rbbt.apps.find(:user)
|
13
13
|
|
14
|
-
FileUtils.mkdir_p app_dir unless File.
|
14
|
+
FileUtils.mkdir_p app_dir unless File.exist? app_dir
|
15
15
|
|
16
16
|
Misc.in_dir(app_dir) do
|
17
|
-
if File.
|
17
|
+
if File.exist? app
|
18
18
|
Misc.in_dir(app) do
|
19
19
|
`git pull`
|
20
20
|
end
|
@@ -62,7 +62,7 @@ Misc.in_dir(app_dir) do
|
|
62
62
|
ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
|
63
63
|
end
|
64
64
|
|
65
|
-
config_ru_file = File.
|
65
|
+
config_ru_file = File.exist?('./config.ru') ? './config.ru' : Rbbt.share['config.ru'].find
|
66
66
|
|
67
67
|
if options[:options]
|
68
68
|
options[:options].split(";").each do |pair|
|
data/share/rbbt_commands/log
CHANGED
@@ -24,7 +24,11 @@ ERROR
|
|
24
24
|
EOF
|
25
25
|
rbbt_usage and exit 0 if options[:help]
|
26
26
|
if ARGV.empty?
|
27
|
-
|
27
|
+
if Rbbt.etc.log_severity.exists?
|
28
|
+
puts Rbbt.etc.log_severity.read
|
29
|
+
else
|
30
|
+
puts Rbbt.etc.log_severity.find + ' does not exist'
|
31
|
+
end
|
28
32
|
else
|
29
33
|
Open.write(Rbbt.etc.log_severity, ARGV[0].upcase)
|
30
34
|
end
|
@@ -59,7 +59,7 @@ def fix_options(workflow, task, job_options)
|
|
59
59
|
case
|
60
60
|
when value == '-'
|
61
61
|
STDIN.read
|
62
|
-
when (String === value and File.
|
62
|
+
when (String === value and File.exist?(value) and not File.directory?(value))
|
63
63
|
Open.read(value)
|
64
64
|
else
|
65
65
|
value
|
@@ -71,7 +71,7 @@ def fix_options(workflow, task, job_options)
|
|
71
71
|
str = case
|
72
72
|
when value == '-'
|
73
73
|
STDIN.read
|
74
|
-
when (String === value and File.
|
74
|
+
when (String === value and File.exist?(value))
|
75
75
|
Open.read(value)
|
76
76
|
else
|
77
77
|
value
|
@@ -214,7 +214,7 @@ else
|
|
214
214
|
end
|
215
215
|
|
216
216
|
if Step === res
|
217
|
-
puts Open.read(res.path) if File.
|
217
|
+
puts Open.read(res.path) if File.exist? res.path
|
218
218
|
else
|
219
219
|
if res.respond_to? :gets
|
220
220
|
while line = res.gets
|
@@ -87,9 +87,9 @@ jobs.each do |file,i|
|
|
87
87
|
pid = info[:pid]
|
88
88
|
|
89
89
|
status = info[:status].to_s
|
90
|
-
status = :missing if status == "done" and not File.
|
90
|
+
status = :missing if status == "done" and not File.exist? file
|
91
91
|
status = :dead if status != "done" and pid and not Misc.pid_exists?(pid)
|
92
|
-
status = :sync if status != "done" and File.
|
92
|
+
status = :sync if status != "done" and File.exist? file
|
93
93
|
|
94
94
|
status = status.to_s
|
95
95
|
next unless status =~ /\berror$/ or status =~ /\bmissing$/ or status =~ /\baborted$/ or status =~ /\bdead$/ or status =~ /\bsync$/ or status == "" or (force and status == 'noinfo')
|
@@ -116,7 +116,7 @@ jobs = Rbbt.job_info workflow, task
|
|
116
116
|
workflows = {}
|
117
117
|
|
118
118
|
jobs.each do |file,info|
|
119
|
-
next unless all or not info[:done] or not File.
|
119
|
+
next unless all or not info[:done] or not File.exist? file
|
120
120
|
workflow = info[:workflow]
|
121
121
|
task = info[:task]
|
122
122
|
workflows[workflow] ||= {}
|
@@ -139,10 +139,10 @@ workflows.sort.each do |workflow,tasks|
|
|
139
139
|
|
140
140
|
pid = info[:pid]
|
141
141
|
status = info[:status]
|
142
|
-
status = :missing if status == :done and not File.
|
142
|
+
status = :missing if status == :done and not File.exist? file
|
143
143
|
status = status.to_s
|
144
144
|
if status != "done" and pid and not Misc.pid_exists?(pid)
|
145
|
-
if File.
|
145
|
+
if File.exist? file
|
146
146
|
status << Log.color(:red, " (out of sync)")
|
147
147
|
else
|
148
148
|
status << Log.color(:red, " (dead)")
|
@@ -56,7 +56,7 @@ else
|
|
56
56
|
end
|
57
57
|
end
|
58
58
|
|
59
|
-
if String === file and not Open.remote? file and File.
|
59
|
+
if String === file and not Open.remote? file and File.exist? file
|
60
60
|
rows = `wc -l '#{ file }' 2>/dev/null|cut -f 1 -d' '`
|
61
61
|
else
|
62
62
|
rows = "Could not get rows of #{Misc.fingerprint file}"
|
@@ -30,7 +30,7 @@ raise ParameterException, "Please specify the tsv file as argument" if file.nil?
|
|
30
30
|
subset = options[:subset]
|
31
31
|
raise ParameterException, "Please specify a subset of keys" if subset.nil?
|
32
32
|
|
33
|
-
if File.
|
33
|
+
if File.exist?(subset)
|
34
34
|
subset = Open.read(subset).split("\n")
|
35
35
|
else
|
36
36
|
subset = subset.split(',')
|
@@ -37,7 +37,7 @@ raise "No workflow specified" if workflow.nil?
|
|
37
37
|
|
38
38
|
workflow_dir = Rbbt.etc.workflow_dir.exists? ? Path.setup(Rbbt.etc.workflow_dir.read.strip) : Rbbt.workflows.find(:user)
|
39
39
|
|
40
|
-
FileUtils.mkdir_p workflow_dir unless File.
|
40
|
+
FileUtils.mkdir_p workflow_dir unless File.exist? workflow_dir
|
41
41
|
|
42
42
|
all_workflows = workflow_dir.glob("*/.git").collect{|d| File.basename(File.dirname(d))}
|
43
43
|
workflows = workflow == 'all' ? all_workflows : workflow.split(",")
|
@@ -45,12 +45,12 @@ workflows = workflow == 'all' ? all_workflows : workflow.split(",")
|
|
45
45
|
Misc.in_dir(workflow_dir) do
|
46
46
|
workflows.each do |workflow|
|
47
47
|
case
|
48
|
-
when File.
|
48
|
+
when File.exist?(Misc.snake_case(workflow))
|
49
49
|
Log.info "Updating: " + workflow
|
50
50
|
Misc.in_dir(Misc.snake_case(workflow)) do
|
51
51
|
`git pull`
|
52
52
|
end
|
53
|
-
when File.
|
53
|
+
when File.exist?(workflow)
|
54
54
|
Misc.in_dir(workflow) do
|
55
55
|
Log.info "Updating: " + workflow
|
56
56
|
`git pull`
|
@@ -40,9 +40,9 @@ def list_jobs(options)
|
|
40
40
|
info_files.each do |file|
|
41
41
|
clean_file = file.sub('.info','')
|
42
42
|
begin
|
43
|
-
next if File.
|
43
|
+
next if File.exist? clean_file and $quick
|
44
44
|
info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
|
45
|
-
next if (File.
|
45
|
+
next if (File.exist?(clean_file) or info[:status] == :done) and (info[:children_pids].nil? or info[:children_done] or info[:children_pids].select{|pid| Misc.pid_exists? pid}.empty?)
|
46
46
|
rescue Exception
|
47
47
|
puts "Error parsing info file: #{ file }"
|
48
48
|
info = nil
|
@@ -75,16 +75,16 @@ end
|
|
75
75
|
|
76
76
|
def remove_job(file)
|
77
77
|
clean_file = file.sub('.info','')
|
78
|
-
FileUtils.rm file if File.
|
79
|
-
FileUtils.rm clean_file if File.
|
80
|
-
FileUtils.rm_rf clean_file + '.files' if File.
|
78
|
+
FileUtils.rm file if File.exist? file
|
79
|
+
FileUtils.rm clean_file if File.exist? clean_file
|
80
|
+
FileUtils.rm_rf clean_file + '.files' if File.exist? clean_file + '.files'
|
81
81
|
end
|
82
82
|
|
83
83
|
def clean_jobs(options)
|
84
84
|
info_files.each do |file|
|
85
85
|
clean_file = file.sub('.info','')
|
86
86
|
info = nil
|
87
|
-
next if File.
|
87
|
+
next if File.exist? clean_file
|
88
88
|
begin
|
89
89
|
info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
|
90
90
|
rescue Exception
|
@@ -64,7 +64,7 @@ def report(step, offset = 0)
|
|
64
64
|
status = info[:status] || :missing
|
65
65
|
status = :remote if Open.remote?(path)
|
66
66
|
name = info[:name] || File.basename(path)
|
67
|
-
status = :unsync if status == :done and not File.
|
67
|
+
status = :unsync if status == :done and not File.exist? path
|
68
68
|
str = " " * offset
|
69
69
|
str << report_msg(status, name, path)
|
70
70
|
info[:dependencies].each do |task,name,path|
|
@@ -80,6 +80,6 @@ def report(step, offset = 0)
|
|
80
80
|
end
|
81
81
|
|
82
82
|
step = get_step file
|
83
|
-
$main_mtime = File.
|
83
|
+
$main_mtime = File.exist?(step.path) ? File.mtime(step.path) : nil
|
84
84
|
|
85
85
|
puts report(step).strip
|
@@ -46,7 +46,7 @@ options[:Bind] ||= "0.0.0.0"
|
|
46
46
|
workflow = ARGV.shift
|
47
47
|
workflows = options[:workflows] || ""
|
48
48
|
|
49
|
-
workflow = File.expand_path(workflow) if File.
|
49
|
+
workflow = File.expand_path(workflow) if File.exist?(workflow)
|
50
50
|
|
51
51
|
ENV["RServe-session"] = options[:RServe_session] || workflow
|
52
52
|
|
@@ -68,7 +68,7 @@ TmpFile.with_file do |app_dir|
|
|
68
68
|
ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
|
69
69
|
end
|
70
70
|
|
71
|
-
config_ru_file = File.
|
71
|
+
config_ru_file = File.exist?('./workflow_config.ru') ? './config.ru' : Rbbt.share['workflow_config.ru'].find
|
72
72
|
|
73
73
|
|
74
74
|
if options[:options]
|
@@ -95,7 +95,7 @@ def fix_options(workflow, task, job_options)
|
|
95
95
|
case
|
96
96
|
when value == '-'
|
97
97
|
STDIN.read
|
98
|
-
when (String === value and File.
|
98
|
+
when (String === value and File.exist?(value) and not File.directory?(value))
|
99
99
|
Open.read(value)
|
100
100
|
else
|
101
101
|
value
|
@@ -113,7 +113,7 @@ def fix_options(workflow, task, job_options)
|
|
113
113
|
when value == '-'
|
114
114
|
array_separator ||= "\n"
|
115
115
|
STDIN.read
|
116
|
-
when (String === value and File.
|
116
|
+
when (String === value and File.exist?(value))
|
117
117
|
array_separator ||= "\n"
|
118
118
|
Open.read(value)
|
119
119
|
else
|
@@ -487,7 +487,7 @@ when Step
|
|
487
487
|
end
|
488
488
|
else
|
489
489
|
res.join
|
490
|
-
out.puts Open.read(res.path) if File.
|
490
|
+
out.puts Open.read(res.path) if File.exist? res.path
|
491
491
|
end
|
492
492
|
else
|
493
493
|
out.puts res.to_s
|
data/test/rbbt/test_resource.rb
CHANGED
@@ -69,8 +69,8 @@ class TestTSV < Test::Unit::TestCase
|
|
69
69
|
end
|
70
70
|
|
71
71
|
def test_libdir
|
72
|
-
assert File.
|
73
|
-
assert File.
|
72
|
+
assert File.exist? TestResource[].share.Rlib["util.R"].find :lib
|
73
|
+
assert File.exist? TestResource[].share.Rlib["util.R"].find
|
74
74
|
end
|
75
75
|
|
76
76
|
def __test_server
|
@@ -12,7 +12,7 @@ class TestStep < Test::Unit::TestCase
|
|
12
12
|
TmpFile.with_file do |tmp|
|
13
13
|
step = Step.new tmp, task
|
14
14
|
assert_equal "TEST", step.run
|
15
|
-
assert File.
|
15
|
+
assert File.exist? tmp
|
16
16
|
step = Step.new tmp, task2
|
17
17
|
assert_equal "TEST", step.run
|
18
18
|
end
|
@@ -102,7 +102,7 @@ class TestStep < Test::Unit::TestCase
|
|
102
102
|
|
103
103
|
def __test_fork
|
104
104
|
TmpFile.with_file do |lock|
|
105
|
-
task = Task.setup do while not File.
|
105
|
+
task = Task.setup do while not File.exist?(lock) do sleep 1; end; "TEST" end
|
106
106
|
TmpFile.with_file do |tmp|
|
107
107
|
step = Step.new tmp, task
|
108
108
|
job = step.fork
|
@@ -118,7 +118,7 @@ class TestStep < Test::Unit::TestCase
|
|
118
118
|
|
119
119
|
def __test_abort
|
120
120
|
TmpFile.with_file do |lock|
|
121
|
-
task = Task.setup do while not File.
|
121
|
+
task = Task.setup do while not File.exist?(lock) do sleep 1; end; "TEST" end
|
122
122
|
TmpFile.with_file do |tmp|
|
123
123
|
step = Step.new tmp, task
|
124
124
|
job = step.fork
|
@@ -148,9 +148,6 @@ class TestStep < Test::Unit::TestCase
|
|
148
148
|
end
|
149
149
|
end
|
150
150
|
end
|
151
|
-
|
152
|
-
def test_messages
|
153
|
-
TmpFile.with_file do |lock|
|
154
151
|
task = Task.setup do
|
155
152
|
message "WRITE"
|
156
153
|
Open.write(file("test"),"TEST")
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.20.
|
4
|
+
version: 5.20.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-06-
|
11
|
+
date: 2016-06-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -280,7 +280,6 @@ files:
|
|
280
280
|
- lib/rbbt/util/misc/omics.rb
|
281
281
|
- lib/rbbt/util/misc/options.rb
|
282
282
|
- lib/rbbt/util/misc/pipes.rb
|
283
|
-
- lib/rbbt/util/misc/progress.rb
|
284
283
|
- lib/rbbt/util/misc/system.rb
|
285
284
|
- lib/rbbt/util/named_array.rb
|
286
285
|
- lib/rbbt/util/open.rb
|
File without changes
|