rbbt-util 5.44.1 → 6.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/rbbt +67 -90
- data/etc/app.d/base.rb +2 -2
- data/etc/app.d/semaphores.rb +3 -3
- data/lib/rbbt/annotations/annotated_array.rb +207 -207
- data/lib/rbbt/annotations/refactor.rb +27 -0
- data/lib/rbbt/annotations/util.rb +282 -282
- data/lib/rbbt/annotations.rb +343 -320
- data/lib/rbbt/association/database.rb +200 -225
- data/lib/rbbt/association/index.rb +294 -291
- data/lib/rbbt/association/item.rb +227 -227
- data/lib/rbbt/association/open.rb +35 -34
- data/lib/rbbt/association/util.rb +0 -169
- data/lib/rbbt/association.rb +2 -4
- data/lib/rbbt/entity/identifiers.rb +119 -118
- data/lib/rbbt/entity/refactor.rb +12 -0
- data/lib/rbbt/entity.rb +319 -315
- data/lib/rbbt/hpc/batch.rb +72 -53
- data/lib/rbbt/hpc/lsf.rb +2 -2
- data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
- data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
- data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
- data/lib/rbbt/hpc/orchestrate.rb +19 -13
- data/lib/rbbt/hpc/slurm.rb +18 -18
- data/lib/rbbt/knowledge_base/entity.rb +13 -5
- data/lib/rbbt/knowledge_base/query.rb +2 -2
- data/lib/rbbt/knowledge_base/registry.rb +32 -31
- data/lib/rbbt/knowledge_base/traverse.rb +1 -1
- data/lib/rbbt/knowledge_base.rb +1 -1
- data/lib/rbbt/monitor.rb +36 -25
- data/lib/rbbt/persist/refactor.rb +166 -0
- data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
- data/lib/rbbt/persist/tsv.rb +187 -185
- data/lib/rbbt/persist.rb +556 -551
- data/lib/rbbt/refactor.rb +20 -0
- data/lib/rbbt/resource/path/refactor.rb +178 -0
- data/lib/rbbt/resource/path.rb +317 -497
- data/lib/rbbt/resource/util.rb +0 -48
- data/lib/rbbt/resource.rb +3 -390
- data/lib/rbbt/tsv/accessor.rb +2 -838
- data/lib/rbbt/tsv/attach.rb +303 -299
- data/lib/rbbt/tsv/change_id.rb +244 -245
- data/lib/rbbt/tsv/csv.rb +87 -85
- data/lib/rbbt/tsv/dumper.rb +2 -100
- data/lib/rbbt/tsv/excel.rb +26 -24
- data/lib/rbbt/tsv/field_index.rb +4 -1
- data/lib/rbbt/tsv/filter.rb +3 -2
- data/lib/rbbt/tsv/index.rb +2 -284
- data/lib/rbbt/tsv/manipulate.rb +750 -747
- data/lib/rbbt/tsv/marshal.rb +3 -3
- data/lib/rbbt/tsv/matrix.rb +2 -2
- data/lib/rbbt/tsv/parallel/through.rb +2 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
- data/lib/rbbt/tsv/parser.rb +678 -678
- data/lib/rbbt/tsv/refactor.rb +195 -0
- data/lib/rbbt/tsv/stream.rb +253 -251
- data/lib/rbbt/tsv/util.rb +420 -420
- data/lib/rbbt/tsv.rb +210 -208
- data/lib/rbbt/util/R/eval.rb +4 -4
- data/lib/rbbt/util/R/plot.rb +62 -166
- data/lib/rbbt/util/R.rb +21 -18
- data/lib/rbbt/util/cmd.rb +2 -318
- data/lib/rbbt/util/color.rb +269 -269
- data/lib/rbbt/util/colorize.rb +89 -89
- data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
- data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
- data/lib/rbbt/util/concurrency/processes.rb +389 -386
- data/lib/rbbt/util/config.rb +169 -167
- data/lib/rbbt/util/iruby.rb +20 -0
- data/lib/rbbt/util/log/progress/report.rb +241 -241
- data/lib/rbbt/util/log/progress/util.rb +99 -99
- data/lib/rbbt/util/log/progress.rb +102 -102
- data/lib/rbbt/util/log/refactor.rb +49 -0
- data/lib/rbbt/util/log.rb +486 -532
- data/lib/rbbt/util/migrate.rb +1 -1
- data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
- data/lib/rbbt/util/misc/development.rb +12 -11
- data/lib/rbbt/util/misc/exceptions.rb +117 -112
- data/lib/rbbt/util/misc/format.rb +2 -230
- data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
- data/lib/rbbt/util/misc/inspect.rb +2 -476
- data/lib/rbbt/util/misc/lock.rb +109 -106
- data/lib/rbbt/util/misc/omics.rb +9 -1
- data/lib/rbbt/util/misc/pipes.rb +765 -793
- data/lib/rbbt/util/misc/refactor.rb +20 -0
- data/lib/rbbt/util/misc/ssw.rb +27 -17
- data/lib/rbbt/util/misc/system.rb +0 -15
- data/lib/rbbt/util/misc.rb +39 -20
- data/lib/rbbt/util/named_array/refactor.rb +4 -0
- data/lib/rbbt/util/named_array.rb +3 -220
- data/lib/rbbt/util/open/refactor.rb +7 -0
- data/lib/rbbt/util/open.rb +3 -857
- data/lib/rbbt/util/procpath.rb +6 -6
- data/lib/rbbt/util/python/paths.rb +27 -0
- data/lib/rbbt/util/python/run.rb +115 -0
- data/lib/rbbt/util/python/script.rb +110 -0
- data/lib/rbbt/util/python/util.rb +3 -3
- data/lib/rbbt/util/python.rb +22 -81
- data/lib/rbbt/util/semaphore.rb +152 -148
- data/lib/rbbt/util/simpleopt.rb +9 -8
- data/lib/rbbt/util/ssh/refactor.rb +19 -0
- data/lib/rbbt/util/ssh.rb +122 -118
- data/lib/rbbt/util/tar.rb +117 -115
- data/lib/rbbt/util/tmpfile.rb +69 -67
- data/lib/rbbt/util/version.rb +2 -0
- data/lib/rbbt/workflow/refactor/entity.rb +11 -0
- data/lib/rbbt/workflow/refactor/export.rb +66 -0
- data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
- data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
- data/lib/rbbt/workflow/refactor/task_info.rb +65 -0
- data/lib/rbbt/workflow/refactor.rb +153 -0
- data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
- data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
- data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
- data/lib/rbbt/workflow/remote_workflow.rb +6 -1
- data/lib/rbbt/workflow/step/run.rb +766 -766
- data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
- data/lib/rbbt/workflow/step.rb +2 -362
- data/lib/rbbt/workflow/task.rb +118 -118
- data/lib/rbbt/workflow/usage.rb +289 -287
- data/lib/rbbt/workflow/util/archive.rb +6 -5
- data/lib/rbbt/workflow/util/data.rb +1 -1
- data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
- data/lib/rbbt/workflow/util/trace.rb +79 -44
- data/lib/rbbt/workflow.rb +4 -882
- data/lib/rbbt-util.rb +21 -13
- data/lib/rbbt.rb +16 -3
- data/python/rbbt/__init__.py +19 -1
- data/share/Rlib/plot.R +37 -37
- data/share/Rlib/svg.R +22 -5
- data/share/install/software/lib/install_helpers +1 -1
- data/share/rbbt_commands/hpc/list +2 -3
- data/share/rbbt_commands/hpc/orchestrate +4 -4
- data/share/rbbt_commands/hpc/tail +2 -0
- data/share/rbbt_commands/hpc/task +10 -7
- data/share/rbbt_commands/lsf/list +2 -3
- data/share/rbbt_commands/lsf/orchestrate +4 -4
- data/share/rbbt_commands/lsf/tail +2 -0
- data/share/rbbt_commands/lsf/task +10 -7
- data/share/rbbt_commands/migrate +1 -1
- data/share/rbbt_commands/pbs/list +2 -3
- data/share/rbbt_commands/pbs/orchestrate +4 -4
- data/share/rbbt_commands/pbs/tail +2 -0
- data/share/rbbt_commands/pbs/task +10 -7
- data/share/rbbt_commands/resource/produce +8 -1
- data/share/rbbt_commands/slurm/list +2 -3
- data/share/rbbt_commands/slurm/orchestrate +4 -4
- data/share/rbbt_commands/slurm/tail +2 -0
- data/share/rbbt_commands/slurm/task +10 -7
- data/share/rbbt_commands/system/clean +5 -5
- data/share/rbbt_commands/system/status +5 -5
- data/share/rbbt_commands/tsv/get +2 -3
- data/share/rbbt_commands/tsv/info +10 -13
- data/share/rbbt_commands/tsv/keys +18 -14
- data/share/rbbt_commands/tsv/slice +2 -2
- data/share/rbbt_commands/tsv/transpose +6 -2
- data/share/rbbt_commands/workflow/info +20 -24
- data/share/rbbt_commands/workflow/list +1 -1
- data/share/rbbt_commands/workflow/prov +20 -13
- data/share/rbbt_commands/workflow/server +11 -1
- data/share/rbbt_commands/workflow/task +76 -71
- data/share/rbbt_commands/workflow/write_info +26 -9
- data/share/software/opt/ssw/ssw.c +861 -0
- data/share/software/opt/ssw/ssw.h +130 -0
- data/share/workflow_config.ru +3 -3
- metadata +40 -2
data/lib/rbbt/util/config.rb
CHANGED
@@ -1,167 +1,169 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
when "
|
52
|
-
|
53
|
-
when "
|
54
|
-
|
55
|
-
when "
|
56
|
-
|
57
|
-
when "
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
tokens = tokens.
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
l =~ /
|
101
|
-
l =~ /
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
Rbbt::Config.load_file(
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
1
|
+
require_relative '../refactor'
|
2
|
+
Rbbt.require_instead 'scout/config'
|
3
|
+
#require 'rbbt-util'
|
4
|
+
#require 'rbbt/resource/path'
|
5
|
+
#
|
6
|
+
#module Rbbt::Config
|
7
|
+
#
|
8
|
+
# CACHE = IndiferentHash.setup({})
|
9
|
+
#
|
10
|
+
# GOT_KEYS=[]
|
11
|
+
#
|
12
|
+
# def self.add_entry(key, value, tokens)
|
13
|
+
# tokens = [tokens] unless Array === tokens
|
14
|
+
# tokens << "key:#{key}" unless tokens.include?("key:#{key}")
|
15
|
+
# CACHE[key.to_s] ||= []
|
16
|
+
# CACHE[key.to_s] << [tokens, value]
|
17
|
+
# end
|
18
|
+
#
|
19
|
+
# def self.load_file(file)
|
20
|
+
# Log.debug "Loading config file: #{ file }"
|
21
|
+
# TSV.traverse file, :type => :array do |line|
|
22
|
+
# next if line =~ /^#/
|
23
|
+
# key, value, *tokens = line.strip.split(/\s/)
|
24
|
+
#
|
25
|
+
# self.add_entry(key, value, tokens) if key
|
26
|
+
# end
|
27
|
+
# end
|
28
|
+
#
|
29
|
+
# def self.load_config
|
30
|
+
# Rbbt.etc.config.find_all.reverse.each do |file|
|
31
|
+
# self.load_file(file)
|
32
|
+
# end
|
33
|
+
# end
|
34
|
+
#
|
35
|
+
# def self.set(values, *tokens)
|
36
|
+
# if not Hash === values
|
37
|
+
# values = {values => tokens.shift}
|
38
|
+
# end
|
39
|
+
#
|
40
|
+
# values.each do |key,value|
|
41
|
+
# add_entry key, value, tokens
|
42
|
+
# end
|
43
|
+
# end
|
44
|
+
#
|
45
|
+
# def self.token_priority(token)
|
46
|
+
# token, _sep, priority = token.to_s.partition("::")
|
47
|
+
#
|
48
|
+
# if priority.nil? || priority.empty?
|
49
|
+
# type, _sep, rest = token.partition(":")
|
50
|
+
# priority = case type
|
51
|
+
# when "workflow"
|
52
|
+
# 4
|
53
|
+
# when "task"
|
54
|
+
# 3
|
55
|
+
# when "file"
|
56
|
+
# 2
|
57
|
+
# when "line"
|
58
|
+
# 1
|
59
|
+
# when "key"
|
60
|
+
# 20
|
61
|
+
# else
|
62
|
+
# 10
|
63
|
+
# end
|
64
|
+
# else
|
65
|
+
# priority = priority.to_i
|
66
|
+
# end
|
67
|
+
#
|
68
|
+
# [token, priority]
|
69
|
+
# end
|
70
|
+
#
|
71
|
+
# def self.match(entries, give_token)
|
72
|
+
# priorities = {}
|
73
|
+
# entries.each do |tokens, value|
|
74
|
+
# best_prio = nil
|
75
|
+
# tokens = [tokens] unless Array === tokens
|
76
|
+
# tokens.each do |tok|
|
77
|
+
# tok, prio = token_priority tok
|
78
|
+
# next unless tok == give_token
|
79
|
+
#
|
80
|
+
# best_prio = prio if best_prio.nil? or best_prio > prio
|
81
|
+
# next if prio > best_prio
|
82
|
+
#
|
83
|
+
# priorities[prio] ||= []
|
84
|
+
# priorities[prio].unshift value
|
85
|
+
# end
|
86
|
+
# end if entries
|
87
|
+
# priorities
|
88
|
+
# end
|
89
|
+
#
|
90
|
+
# # For equal priorities the matching prioritizes tokens ealier in the list
|
91
|
+
# def self.get(key, *tokens)
|
92
|
+
# options = tokens.pop if Hash === tokens.last
|
93
|
+
# default = options.nil? ? nil : options[:default]
|
94
|
+
#
|
95
|
+
# tokens = ["key:" + key] if tokens.empty?
|
96
|
+
#
|
97
|
+
# tokens = tokens.flatten
|
98
|
+
# file, _sep, line = caller.reject{|l|
|
99
|
+
# l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
|
100
|
+
# l =~ /rbbt\/resource\/path\.rb/ or
|
101
|
+
# l =~ /rbbt\/util\/misc\.rb/ or
|
102
|
+
# l =~ /accessor\.rb/ or
|
103
|
+
# l =~ /progress-monitor\.rb/
|
104
|
+
# }.first.partition(":")
|
105
|
+
#
|
106
|
+
# File.expand_path(file)
|
107
|
+
#
|
108
|
+
# tokens << ("file:" << file)
|
109
|
+
# tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
|
110
|
+
#
|
111
|
+
# entries = CACHE[key.to_s]
|
112
|
+
# priorities = {}
|
113
|
+
# tokens.each do |token|
|
114
|
+
# token_prio = match entries, token.to_s
|
115
|
+
# token_prio.each do |prio, values|
|
116
|
+
# priorities[prio] ||= []
|
117
|
+
# priorities[prio].concat(values)
|
118
|
+
# end
|
119
|
+
# end
|
120
|
+
#
|
121
|
+
# value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
|
122
|
+
# value = false if value == 'false'
|
123
|
+
#
|
124
|
+
# Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
|
125
|
+
# GOT_KEYS << [key, value, tokens]
|
126
|
+
#
|
127
|
+
# if String === value && m = value.match(/^env:(.*)/)
|
128
|
+
# variable = m.captures.first
|
129
|
+
# ENV[variable]
|
130
|
+
# elsif value == 'nil'
|
131
|
+
# nil
|
132
|
+
# else
|
133
|
+
# value
|
134
|
+
# end
|
135
|
+
# end
|
136
|
+
#
|
137
|
+
# def self.with_config
|
138
|
+
# saved_config = {}
|
139
|
+
# CACHE.each do |k,v|
|
140
|
+
# saved_config[k] = v.dup
|
141
|
+
# end
|
142
|
+
# saved_got_keys = GOT_KEYS.dup
|
143
|
+
# begin
|
144
|
+
# yield
|
145
|
+
# ensure
|
146
|
+
# CACHE.replace(saved_config)
|
147
|
+
# GOT_KEYS.replace(saved_got_keys)
|
148
|
+
# end
|
149
|
+
# end
|
150
|
+
#
|
151
|
+
# def self.process_config(config)
|
152
|
+
# if Misc.is_filename?(config) && File.exist?(config)
|
153
|
+
# Rbbt::Config.load_file(config)
|
154
|
+
# elsif Rbbt.etc.config_profile[config].exists?
|
155
|
+
# Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)
|
156
|
+
# else
|
157
|
+
# key, value, *tokens = config.split(/\s/)
|
158
|
+
# tokens = tokens.collect do |tok|
|
159
|
+
# tok, _sep, prio = tok.partition("::")
|
160
|
+
# prio = "0" if prio.nil? or prio.empty?
|
161
|
+
# [tok, prio] * "::"
|
162
|
+
# end
|
163
|
+
# Rbbt::Config.set({key => value}, *tokens)
|
164
|
+
# end
|
165
|
+
# end
|
166
|
+
#
|
167
|
+
#
|
168
|
+
# self.load_config
|
169
|
+
#end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'iruby'
|
2
|
+
require 'rbbt/util/R'
|
3
|
+
|
4
|
+
module IRuby
|
5
|
+
def self.img(file)
|
6
|
+
IRuby.html("<img src='#{file}'/>")
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.plot(...)
|
10
|
+
filename = Rbbt.iruby[rand(10000).to_s + ".png"]
|
11
|
+
Open.mkdir File.dirname(filename)
|
12
|
+
R::PNG.plot(filename, ...)
|
13
|
+
img(filename)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.ggplot(...)
|
17
|
+
svg = R::SVG.ggplot(...)
|
18
|
+
IRuby.html(svg)
|
19
|
+
end
|
20
|
+
end
|