rbbt-util 5.20.9 → 5.20.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/bin/rbbt +1 -1
  3. data/bin/rbbt_query.rb +1 -1
  4. data/lib/rbbt/association/index.rb +3 -3
  5. data/lib/rbbt/fix_width_table.rb +4 -4
  6. data/lib/rbbt/persist/tsv.rb +4 -4
  7. data/lib/rbbt/persist/tsv/cdb.rb +3 -3
  8. data/lib/rbbt/persist/tsv/fix_width_table.rb +2 -2
  9. data/lib/rbbt/persist/tsv/kyotocabinet.rb +2 -2
  10. data/lib/rbbt/persist/tsv/leveldb.rb +2 -2
  11. data/lib/rbbt/persist/tsv/lmdb.rb +2 -2
  12. data/lib/rbbt/persist/tsv/packed_index.rb +2 -2
  13. data/lib/rbbt/persist/tsv/sharder.rb +4 -4
  14. data/lib/rbbt/persist/tsv/tokyocabinet.rb +2 -2
  15. data/lib/rbbt/resource.rb +6 -6
  16. data/lib/rbbt/resource/path.rb +7 -7
  17. data/lib/rbbt/resource/util.rb +9 -9
  18. data/lib/rbbt/rest/client/step.rb +8 -4
  19. data/lib/rbbt/tsv/attach.rb +21 -19
  20. data/lib/rbbt/tsv/parallel/traverse.rb +2 -2
  21. data/lib/rbbt/tsv/util.rb +1 -1
  22. data/lib/rbbt/util/R.rb +1 -1
  23. data/lib/rbbt/util/R/eval.rb +10 -10
  24. data/lib/rbbt/util/R/model.rb +2 -2
  25. data/lib/rbbt/util/docker.rb +3 -3
  26. data/lib/rbbt/util/filecache.rb +2 -2
  27. data/lib/rbbt/util/misc/development.rb +6 -4
  28. data/lib/rbbt/util/misc/lock.rb +1 -1
  29. data/lib/rbbt/util/misc/multipart_payload.rb +1 -1
  30. data/lib/rbbt/util/misc/pipes.rb +8 -8
  31. data/lib/rbbt/util/misc/system.rb +3 -3
  32. data/lib/rbbt/util/open.rb +9 -9
  33. data/lib/rbbt/util/simpleDSL.rb +2 -2
  34. data/lib/rbbt/util/tar.rb +1 -1
  35. data/lib/rbbt/util/tmpfile.rb +2 -2
  36. data/lib/rbbt/workflow.rb +26 -12
  37. data/lib/rbbt/workflow/accessor.rb +4 -4
  38. data/lib/rbbt/workflow/step/dependencies.rb +1 -1
  39. data/lib/rbbt/workflow/step/run.rb +6 -6
  40. data/share/rbbt_commands/app/install +2 -2
  41. data/share/rbbt_commands/app/start +1 -1
  42. data/share/rbbt_commands/log +5 -1
  43. data/share/rbbt_commands/resource/exists +1 -1
  44. data/share/rbbt_commands/study/task +3 -3
  45. data/share/rbbt_commands/system/clean +2 -2
  46. data/share/rbbt_commands/system/deleted_files +1 -1
  47. data/share/rbbt_commands/system/status +3 -3
  48. data/share/rbbt_commands/tsv/info +1 -1
  49. data/share/rbbt_commands/tsv/subset +1 -1
  50. data/share/rbbt_commands/workflow/example +1 -1
  51. data/share/rbbt_commands/workflow/install +3 -3
  52. data/share/rbbt_commands/workflow/monitor +6 -6
  53. data/share/rbbt_commands/workflow/prov +2 -2
  54. data/share/rbbt_commands/workflow/server +2 -2
  55. data/share/rbbt_commands/workflow/task +3 -3
  56. data/test/rbbt/test_resource.rb +2 -2
  57. data/test/rbbt/workflow/test_step.rb +3 -6
  58. metadata +2 -3
  59. data/lib/rbbt/util/misc/progress.rb +0 -0
@@ -324,7 +324,7 @@ class Step
324
324
  end
325
325
 
326
326
  def done?
327
- path and File.exists? path
327
+ path and File.exist? path
328
328
  end
329
329
 
330
330
  def streaming?
@@ -426,8 +426,8 @@ class Step
426
426
  provenance = {}
427
427
  dependencies.each do |dep|
428
428
  next unless dep.path.exists?
429
- if File.exists? dep.info_file
430
- provenance[dep.path] = dep.provenance if File.exists? dep.path
429
+ if File.exist? dep.info_file
430
+ provenance[dep.path] = dep.provenance if File.exist? dep.path
431
431
  else
432
432
  provenance[dep.path] = nil
433
433
  end
@@ -438,7 +438,7 @@ class Step
438
438
  def provenance_paths
439
439
  provenance = {}
440
440
  dependencies.each do |dep|
441
- provenance[dep.path] = dep.provenance_paths if File.exists? dep.path
441
+ provenance[dep.path] = dep.provenance_paths if File.exist? dep.path
442
442
  end
443
443
  provenance
444
444
  end
@@ -4,7 +4,7 @@ class Step
4
4
  STREAM_CACHE = {}
5
5
  STREAM_CACHE_MUTEX = Mutex.new
6
6
  def self.purge_stream_cache
7
- Log.medium "Purging dup. stream cache"
7
+ Log.debug "Purging dup. stream cache"
8
8
  STREAM_CACHE_MUTEX.synchronize do
9
9
  #STREAM_CACHE.collect{|k,s|
10
10
  # Thread.new do
@@ -116,7 +116,7 @@ class Step
116
116
  begin
117
117
  run_dependencies
118
118
  rescue Exception
119
- FileUtils.rm pid_file if File.exists?(pid_file)
119
+ FileUtils.rm pid_file if File.exist?(pid_file)
120
120
  stop_dependencies
121
121
  raise $!
122
122
  end
@@ -174,7 +174,7 @@ class Step
174
174
  ensure
175
175
  join
176
176
  Step.purge_stream_cache
177
- FileUtils.rm pid_file if File.exists?(pid_file)
177
+ FileUtils.rm pid_file if File.exist?(pid_file)
178
178
  end
179
179
  end
180
180
  stream.abort_callback = Proc.new do
@@ -183,7 +183,7 @@ class Step
183
183
  rescue
184
184
  Log.exception $!
185
185
  stop_dependencies
186
- FileUtils.rm pid_file if File.exists?(pid_file)
186
+ FileUtils.rm pid_file if File.exist?(pid_file)
187
187
  end
188
188
  end
189
189
  else
@@ -192,7 +192,7 @@ class Step
192
192
  set_info :time_elapsed, (time_elapsed = done_time - start_time)
193
193
  log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
194
194
  Step.purge_stream_cache
195
- FileUtils.rm pid_file if File.exists?(pid_file)
195
+ FileUtils.rm pid_file if File.exist?(pid_file)
196
196
  end
197
197
 
198
198
  result
@@ -243,7 +243,7 @@ class Step
243
243
  Misc.pre_fork
244
244
  begin
245
245
  RbbtSemaphore.wait_semaphore(semaphore) if semaphore
246
- FileUtils.mkdir_p File.dirname(path) unless File.exists? File.dirname(path)
246
+ FileUtils.mkdir_p File.dirname(path) unless File.exist? File.dirname(path)
247
247
  begin
248
248
  @forked = true
249
249
  res = run true
@@ -377,7 +377,7 @@ class Step
377
377
  end
378
378
 
379
379
  def soft_grace
380
- until done? or File.exists?(info_file)
380
+ until done? or File.exist?(info_file)
381
381
  sleep 1
382
382
  end
383
383
  self
@@ -11,10 +11,10 @@ raise "No app specified" if app.nil?
11
11
 
12
12
  app_dir = Rbbt.etc.app_dir.exists? ? Rbbt.etc.app_dir.read.strip : Rbbt.apps.find(:user)
13
13
 
14
- FileUtils.mkdir_p app_dir unless File.exists? app_dir
14
+ FileUtils.mkdir_p app_dir unless File.exist? app_dir
15
15
 
16
16
  Misc.in_dir(app_dir) do
17
- if File.exists? app
17
+ if File.exist? app
18
18
  Misc.in_dir(app) do
19
19
  `git pull`
20
20
  end
@@ -62,7 +62,7 @@ Misc.in_dir(app_dir) do
62
62
  ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
63
63
  end
64
64
 
65
- config_ru_file = File.exists?('./config.ru') ? './config.ru' : Rbbt.share['config.ru'].find
65
+ config_ru_file = File.exist?('./config.ru') ? './config.ru' : Rbbt.share['config.ru'].find
66
66
 
67
67
  if options[:options]
68
68
  options[:options].split(";").each do |pair|
@@ -24,7 +24,11 @@ ERROR
24
24
  EOF
25
25
  rbbt_usage and exit 0 if options[:help]
26
26
  if ARGV.empty?
27
- puts Rbbt.etc.log_severity.read
27
+ if Rbbt.etc.log_severity.exists?
28
+ puts Rbbt.etc.log_severity.read
29
+ else
30
+ puts Rbbt.etc.log_severity.find + ' does not exist'
31
+ end
28
32
  else
29
33
  Open.write(Rbbt.etc.log_severity, ARGV[0].upcase)
30
34
  end
@@ -47,7 +47,7 @@ resource = Kernel.const_get(resource)
47
47
  if options[:create]
48
48
  exists = resource[path].exists?
49
49
  else
50
- exists = File.exists? resource[path].find
50
+ exists = File.exist? resource[path].find
51
51
  end
52
52
 
53
53
  if exists
@@ -59,7 +59,7 @@ def fix_options(workflow, task, job_options)
59
59
  case
60
60
  when value == '-'
61
61
  STDIN.read
62
- when (String === value and File.exists?(value) and not File.directory?(value))
62
+ when (String === value and File.exist?(value) and not File.directory?(value))
63
63
  Open.read(value)
64
64
  else
65
65
  value
@@ -71,7 +71,7 @@ def fix_options(workflow, task, job_options)
71
71
  str = case
72
72
  when value == '-'
73
73
  STDIN.read
74
- when (String === value and File.exists?(value))
74
+ when (String === value and File.exist?(value))
75
75
  Open.read(value)
76
76
  else
77
77
  value
@@ -214,7 +214,7 @@ else
214
214
  end
215
215
 
216
216
  if Step === res
217
- puts Open.read(res.path) if File.exists? res.path
217
+ puts Open.read(res.path) if File.exist? res.path
218
218
  else
219
219
  if res.respond_to? :gets
220
220
  while line = res.gets
@@ -87,9 +87,9 @@ jobs.each do |file,i|
87
87
  pid = info[:pid]
88
88
 
89
89
  status = info[:status].to_s
90
- status = :missing if status == "done" and not File.exists? file
90
+ status = :missing if status == "done" and not File.exist? file
91
91
  status = :dead if status != "done" and pid and not Misc.pid_exists?(pid)
92
- status = :sync if status != "done" and File.exists? file
92
+ status = :sync if status != "done" and File.exist? file
93
93
 
94
94
  status = status.to_s
95
95
  next unless status =~ /\berror$/ or status =~ /\bmissing$/ or status =~ /\baborted$/ or status =~ /\bdead$/ or status =~ /\bsync$/ or status == "" or (force and status == 'noinfo')
@@ -25,7 +25,7 @@ Dir.glob('/proc/*/fd/*').each do |file|
25
25
  rescue
26
26
  next
27
27
  end
28
- next if File.exists? dest
28
+ next if File.exist? dest
29
29
  pids[pid] ||= []
30
30
  pids[pid] << dest
31
31
  end
@@ -116,7 +116,7 @@ jobs = Rbbt.job_info workflow, task
116
116
  workflows = {}
117
117
 
118
118
  jobs.each do |file,info|
119
- next unless all or not info[:done] or not File.exists? file
119
+ next unless all or not info[:done] or not File.exist? file
120
120
  workflow = info[:workflow]
121
121
  task = info[:task]
122
122
  workflows[workflow] ||= {}
@@ -139,10 +139,10 @@ workflows.sort.each do |workflow,tasks|
139
139
 
140
140
  pid = info[:pid]
141
141
  status = info[:status]
142
- status = :missing if status == :done and not File.exists? file
142
+ status = :missing if status == :done and not File.exist? file
143
143
  status = status.to_s
144
144
  if status != "done" and pid and not Misc.pid_exists?(pid)
145
- if File.exists? file
145
+ if File.exist? file
146
146
  status << Log.color(:red, " (out of sync)")
147
147
  else
148
148
  status << Log.color(:red, " (dead)")
@@ -56,7 +56,7 @@ else
56
56
  end
57
57
  end
58
58
 
59
- if String === file and not Open.remote? file and File.exists? file
59
+ if String === file and not Open.remote? file and File.exist? file
60
60
  rows = `wc -l '#{ file }' 2>/dev/null|cut -f 1 -d' '`
61
61
  else
62
62
  rows = "Could not get rows of #{Misc.fingerprint file}"
@@ -30,7 +30,7 @@ raise ParameterException, "Please specify the tsv file as argument" if file.nil?
30
30
  subset = options[:subset]
31
31
  raise ParameterException, "Please specify a subset of keys" if subset.nil?
32
32
 
33
- if File.exists?(subset)
33
+ if File.exist?(subset)
34
34
  subset = Open.read(subset).split("\n")
35
35
  else
36
36
  subset = subset.split(',')
@@ -36,7 +36,7 @@ def run_task(workflow, task, name)
36
36
  end
37
37
  end
38
38
  sleep 0.5
39
- path = Open.read(res).strip.split("\n").last if File.exists? res
39
+ path = Open.read(res).strip.split("\n").last if File.exist? res
40
40
  end
41
41
  path = "NO RESULT" if path.nil? or path.empty?
42
42
 
@@ -37,7 +37,7 @@ raise "No workflow specified" if workflow.nil?
37
37
 
38
38
  workflow_dir = Rbbt.etc.workflow_dir.exists? ? Path.setup(Rbbt.etc.workflow_dir.read.strip) : Rbbt.workflows.find(:user)
39
39
 
40
- FileUtils.mkdir_p workflow_dir unless File.exists? workflow_dir
40
+ FileUtils.mkdir_p workflow_dir unless File.exist? workflow_dir
41
41
 
42
42
  all_workflows = workflow_dir.glob("*/.git").collect{|d| File.basename(File.dirname(d))}
43
43
  workflows = workflow == 'all' ? all_workflows : workflow.split(",")
@@ -45,12 +45,12 @@ workflows = workflow == 'all' ? all_workflows : workflow.split(",")
45
45
  Misc.in_dir(workflow_dir) do
46
46
  workflows.each do |workflow|
47
47
  case
48
- when File.exists?(Misc.snake_case(workflow))
48
+ when File.exist?(Misc.snake_case(workflow))
49
49
  Log.info "Updating: " + workflow
50
50
  Misc.in_dir(Misc.snake_case(workflow)) do
51
51
  `git pull`
52
52
  end
53
- when File.exists?(workflow)
53
+ when File.exist?(workflow)
54
54
  Misc.in_dir(workflow) do
55
55
  Log.info "Updating: " + workflow
56
56
  `git pull`
@@ -40,9 +40,9 @@ def list_jobs(options)
40
40
  info_files.each do |file|
41
41
  clean_file = file.sub('.info','')
42
42
  begin
43
- next if File.exists? clean_file and $quick
43
+ next if File.exist? clean_file and $quick
44
44
  info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
45
- next if (File.exists?(clean_file) or info[:status] == :done) and (info[:children_pids].nil? or info[:children_done] or info[:children_pids].select{|pid| Misc.pid_exists? pid}.empty?)
45
+ next if (File.exist?(clean_file) or info[:status] == :done) and (info[:children_pids].nil? or info[:children_done] or info[:children_pids].select{|pid| Misc.pid_exists? pid}.empty?)
46
46
  rescue Exception
47
47
  puts "Error parsing info file: #{ file }"
48
48
  info = nil
@@ -75,16 +75,16 @@ end
75
75
 
76
76
  def remove_job(file)
77
77
  clean_file = file.sub('.info','')
78
- FileUtils.rm file if File.exists? file
79
- FileUtils.rm clean_file if File.exists? clean_file
80
- FileUtils.rm_rf clean_file + '.files' if File.exists? clean_file + '.files'
78
+ FileUtils.rm file if File.exist? file
79
+ FileUtils.rm clean_file if File.exist? clean_file
80
+ FileUtils.rm_rf clean_file + '.files' if File.exist? clean_file + '.files'
81
81
  end
82
82
 
83
83
  def clean_jobs(options)
84
84
  info_files.each do |file|
85
85
  clean_file = file.sub('.info','')
86
86
  info = nil
87
- next if File.exists? clean_file
87
+ next if File.exist? clean_file
88
88
  begin
89
89
  info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
90
90
  rescue Exception
@@ -64,7 +64,7 @@ def report(step, offset = 0)
64
64
  status = info[:status] || :missing
65
65
  status = :remote if Open.remote?(path)
66
66
  name = info[:name] || File.basename(path)
67
- status = :unsync if status == :done and not File.exists? path
67
+ status = :unsync if status == :done and not File.exist? path
68
68
  str = " " * offset
69
69
  str << report_msg(status, name, path)
70
70
  info[:dependencies].each do |task,name,path|
@@ -80,6 +80,6 @@ def report(step, offset = 0)
80
80
  end
81
81
 
82
82
  step = get_step file
83
- $main_mtime = File.exists?(step.path) ? File.mtime(step.path) : nil
83
+ $main_mtime = File.exist?(step.path) ? File.mtime(step.path) : nil
84
84
 
85
85
  puts report(step).strip
@@ -46,7 +46,7 @@ options[:Bind] ||= "0.0.0.0"
46
46
  workflow = ARGV.shift
47
47
  workflows = options[:workflows] || ""
48
48
 
49
- workflow = File.expand_path(workflow) if File.exists?(workflow)
49
+ workflow = File.expand_path(workflow) if File.exist?(workflow)
50
50
 
51
51
  ENV["RServe-session"] = options[:RServe_session] || workflow
52
52
 
@@ -68,7 +68,7 @@ TmpFile.with_file do |app_dir|
68
68
  ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
69
69
  end
70
70
 
71
- config_ru_file = File.exists?('./workflow_config.ru') ? './config.ru' : Rbbt.share['workflow_config.ru'].find
71
+ config_ru_file = File.exist?('./workflow_config.ru') ? './config.ru' : Rbbt.share['workflow_config.ru'].find
72
72
 
73
73
 
74
74
  if options[:options]
@@ -95,7 +95,7 @@ def fix_options(workflow, task, job_options)
95
95
  case
96
96
  when value == '-'
97
97
  STDIN.read
98
- when (String === value and File.exists?(value) and not File.directory?(value))
98
+ when (String === value and File.exist?(value) and not File.directory?(value))
99
99
  Open.read(value)
100
100
  else
101
101
  value
@@ -113,7 +113,7 @@ def fix_options(workflow, task, job_options)
113
113
  when value == '-'
114
114
  array_separator ||= "\n"
115
115
  STDIN.read
116
- when (String === value and File.exists?(value))
116
+ when (String === value and File.exist?(value))
117
117
  array_separator ||= "\n"
118
118
  Open.read(value)
119
119
  else
@@ -487,7 +487,7 @@ when Step
487
487
  end
488
488
  else
489
489
  res.join
490
- out.puts Open.read(res.path) if File.exists? res.path
490
+ out.puts Open.read(res.path) if File.exist? res.path
491
491
  end
492
492
  else
493
493
  out.puts res.to_s
@@ -69,8 +69,8 @@ class TestTSV < Test::Unit::TestCase
69
69
  end
70
70
 
71
71
  def test_libdir
72
- assert File.exists? TestResource[].share.Rlib["util.R"].find :lib
73
- assert File.exists? TestResource[].share.Rlib["util.R"].find
72
+ assert File.exist? TestResource[].share.Rlib["util.R"].find :lib
73
+ assert File.exist? TestResource[].share.Rlib["util.R"].find
74
74
  end
75
75
 
76
76
  def __test_server
@@ -12,7 +12,7 @@ class TestStep < Test::Unit::TestCase
12
12
  TmpFile.with_file do |tmp|
13
13
  step = Step.new tmp, task
14
14
  assert_equal "TEST", step.run
15
- assert File.exists? tmp
15
+ assert File.exist? tmp
16
16
  step = Step.new tmp, task2
17
17
  assert_equal "TEST", step.run
18
18
  end
@@ -102,7 +102,7 @@ class TestStep < Test::Unit::TestCase
102
102
 
103
103
  def __test_fork
104
104
  TmpFile.with_file do |lock|
105
- task = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
105
+ task = Task.setup do while not File.exist?(lock) do sleep 1; end; "TEST" end
106
106
  TmpFile.with_file do |tmp|
107
107
  step = Step.new tmp, task
108
108
  job = step.fork
@@ -118,7 +118,7 @@ class TestStep < Test::Unit::TestCase
118
118
 
119
119
  def __test_abort
120
120
  TmpFile.with_file do |lock|
121
- task = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
121
+ task = Task.setup do while not File.exist?(lock) do sleep 1; end; "TEST" end
122
122
  TmpFile.with_file do |tmp|
123
123
  step = Step.new tmp, task
124
124
  job = step.fork
@@ -148,9 +148,6 @@ class TestStep < Test::Unit::TestCase
148
148
  end
149
149
  end
150
150
  end
151
-
152
- def test_messages
153
- TmpFile.with_file do |lock|
154
151
  task = Task.setup do
155
152
  message "WRITE"
156
153
  Open.write(file("test"),"TEST")
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.20.9
4
+ version: 5.20.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-08 00:00:00.000000000 Z
11
+ date: 2016-06-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -280,7 +280,6 @@ files:
280
280
  - lib/rbbt/util/misc/omics.rb
281
281
  - lib/rbbt/util/misc/options.rb
282
282
  - lib/rbbt/util/misc/pipes.rb
283
- - lib/rbbt/util/misc/progress.rb
284
283
  - lib/rbbt/util/misc/system.rb
285
284
  - lib/rbbt/util/named_array.rb
286
285
  - lib/rbbt/util/open.rb
File without changes