rbbt-util 5.44.1 → 6.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (167) hide show
  1. checksums.yaml +4 -4
  2. data/bin/rbbt +67 -90
  3. data/etc/app.d/base.rb +2 -2
  4. data/etc/app.d/semaphores.rb +3 -3
  5. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  6. data/lib/rbbt/annotations/refactor.rb +27 -0
  7. data/lib/rbbt/annotations/util.rb +282 -282
  8. data/lib/rbbt/annotations.rb +343 -320
  9. data/lib/rbbt/association/database.rb +200 -225
  10. data/lib/rbbt/association/index.rb +294 -291
  11. data/lib/rbbt/association/item.rb +227 -227
  12. data/lib/rbbt/association/open.rb +35 -34
  13. data/lib/rbbt/association/util.rb +0 -169
  14. data/lib/rbbt/association.rb +2 -4
  15. data/lib/rbbt/entity/identifiers.rb +119 -118
  16. data/lib/rbbt/entity/refactor.rb +12 -0
  17. data/lib/rbbt/entity.rb +319 -315
  18. data/lib/rbbt/hpc/batch.rb +72 -53
  19. data/lib/rbbt/hpc/lsf.rb +2 -2
  20. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  21. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  22. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  24. data/lib/rbbt/hpc/slurm.rb +18 -18
  25. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  26. data/lib/rbbt/knowledge_base/query.rb +2 -2
  27. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  28. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  29. data/lib/rbbt/knowledge_base.rb +1 -1
  30. data/lib/rbbt/monitor.rb +36 -25
  31. data/lib/rbbt/persist/refactor.rb +166 -0
  32. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  33. data/lib/rbbt/persist/tsv.rb +187 -185
  34. data/lib/rbbt/persist.rb +556 -551
  35. data/lib/rbbt/refactor.rb +20 -0
  36. data/lib/rbbt/resource/path/refactor.rb +178 -0
  37. data/lib/rbbt/resource/path.rb +317 -497
  38. data/lib/rbbt/resource/util.rb +0 -48
  39. data/lib/rbbt/resource.rb +3 -390
  40. data/lib/rbbt/tsv/accessor.rb +2 -838
  41. data/lib/rbbt/tsv/attach.rb +303 -299
  42. data/lib/rbbt/tsv/change_id.rb +244 -245
  43. data/lib/rbbt/tsv/csv.rb +87 -85
  44. data/lib/rbbt/tsv/dumper.rb +2 -100
  45. data/lib/rbbt/tsv/excel.rb +26 -24
  46. data/lib/rbbt/tsv/field_index.rb +4 -1
  47. data/lib/rbbt/tsv/filter.rb +3 -2
  48. data/lib/rbbt/tsv/index.rb +2 -284
  49. data/lib/rbbt/tsv/manipulate.rb +750 -747
  50. data/lib/rbbt/tsv/marshal.rb +3 -3
  51. data/lib/rbbt/tsv/matrix.rb +2 -2
  52. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  53. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  54. data/lib/rbbt/tsv/parser.rb +678 -678
  55. data/lib/rbbt/tsv/refactor.rb +195 -0
  56. data/lib/rbbt/tsv/stream.rb +253 -251
  57. data/lib/rbbt/tsv/util.rb +420 -420
  58. data/lib/rbbt/tsv.rb +210 -208
  59. data/lib/rbbt/util/R/eval.rb +4 -4
  60. data/lib/rbbt/util/R/plot.rb +62 -166
  61. data/lib/rbbt/util/R.rb +21 -18
  62. data/lib/rbbt/util/cmd.rb +2 -318
  63. data/lib/rbbt/util/color.rb +269 -269
  64. data/lib/rbbt/util/colorize.rb +89 -89
  65. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  66. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  67. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  68. data/lib/rbbt/util/config.rb +169 -167
  69. data/lib/rbbt/util/iruby.rb +20 -0
  70. data/lib/rbbt/util/log/progress/report.rb +241 -241
  71. data/lib/rbbt/util/log/progress/util.rb +99 -99
  72. data/lib/rbbt/util/log/progress.rb +102 -102
  73. data/lib/rbbt/util/log/refactor.rb +49 -0
  74. data/lib/rbbt/util/log.rb +486 -532
  75. data/lib/rbbt/util/migrate.rb +1 -1
  76. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  77. data/lib/rbbt/util/misc/development.rb +12 -11
  78. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  79. data/lib/rbbt/util/misc/format.rb +2 -230
  80. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  81. data/lib/rbbt/util/misc/inspect.rb +2 -476
  82. data/lib/rbbt/util/misc/lock.rb +109 -106
  83. data/lib/rbbt/util/misc/omics.rb +9 -1
  84. data/lib/rbbt/util/misc/pipes.rb +765 -793
  85. data/lib/rbbt/util/misc/refactor.rb +20 -0
  86. data/lib/rbbt/util/misc/ssw.rb +27 -17
  87. data/lib/rbbt/util/misc/system.rb +0 -15
  88. data/lib/rbbt/util/misc.rb +39 -20
  89. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  90. data/lib/rbbt/util/named_array.rb +3 -220
  91. data/lib/rbbt/util/open/refactor.rb +7 -0
  92. data/lib/rbbt/util/open.rb +3 -857
  93. data/lib/rbbt/util/procpath.rb +6 -6
  94. data/lib/rbbt/util/python/paths.rb +27 -0
  95. data/lib/rbbt/util/python/run.rb +115 -0
  96. data/lib/rbbt/util/python/script.rb +110 -0
  97. data/lib/rbbt/util/python/util.rb +3 -3
  98. data/lib/rbbt/util/python.rb +22 -81
  99. data/lib/rbbt/util/semaphore.rb +152 -148
  100. data/lib/rbbt/util/simpleopt.rb +9 -8
  101. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  102. data/lib/rbbt/util/ssh.rb +122 -118
  103. data/lib/rbbt/util/tar.rb +117 -115
  104. data/lib/rbbt/util/tmpfile.rb +69 -67
  105. data/lib/rbbt/util/version.rb +2 -0
  106. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  107. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  108. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  109. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  110. data/lib/rbbt/workflow/refactor/task_info.rb +65 -0
  111. data/lib/rbbt/workflow/refactor.rb +153 -0
  112. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  113. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  114. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  115. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  116. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  117. data/lib/rbbt/workflow/step/run.rb +766 -766
  118. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  119. data/lib/rbbt/workflow/step.rb +2 -362
  120. data/lib/rbbt/workflow/task.rb +118 -118
  121. data/lib/rbbt/workflow/usage.rb +289 -287
  122. data/lib/rbbt/workflow/util/archive.rb +6 -5
  123. data/lib/rbbt/workflow/util/data.rb +1 -1
  124. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  125. data/lib/rbbt/workflow/util/trace.rb +79 -44
  126. data/lib/rbbt/workflow.rb +4 -882
  127. data/lib/rbbt-util.rb +21 -13
  128. data/lib/rbbt.rb +16 -3
  129. data/python/rbbt/__init__.py +19 -1
  130. data/share/Rlib/plot.R +37 -37
  131. data/share/Rlib/svg.R +22 -5
  132. data/share/install/software/lib/install_helpers +1 -1
  133. data/share/rbbt_commands/hpc/list +2 -3
  134. data/share/rbbt_commands/hpc/orchestrate +4 -4
  135. data/share/rbbt_commands/hpc/tail +2 -0
  136. data/share/rbbt_commands/hpc/task +10 -7
  137. data/share/rbbt_commands/lsf/list +2 -3
  138. data/share/rbbt_commands/lsf/orchestrate +4 -4
  139. data/share/rbbt_commands/lsf/tail +2 -0
  140. data/share/rbbt_commands/lsf/task +10 -7
  141. data/share/rbbt_commands/migrate +1 -1
  142. data/share/rbbt_commands/pbs/list +2 -3
  143. data/share/rbbt_commands/pbs/orchestrate +4 -4
  144. data/share/rbbt_commands/pbs/tail +2 -0
  145. data/share/rbbt_commands/pbs/task +10 -7
  146. data/share/rbbt_commands/resource/produce +8 -1
  147. data/share/rbbt_commands/slurm/list +2 -3
  148. data/share/rbbt_commands/slurm/orchestrate +4 -4
  149. data/share/rbbt_commands/slurm/tail +2 -0
  150. data/share/rbbt_commands/slurm/task +10 -7
  151. data/share/rbbt_commands/system/clean +5 -5
  152. data/share/rbbt_commands/system/status +5 -5
  153. data/share/rbbt_commands/tsv/get +2 -3
  154. data/share/rbbt_commands/tsv/info +10 -13
  155. data/share/rbbt_commands/tsv/keys +18 -14
  156. data/share/rbbt_commands/tsv/slice +2 -2
  157. data/share/rbbt_commands/tsv/transpose +6 -2
  158. data/share/rbbt_commands/workflow/info +20 -24
  159. data/share/rbbt_commands/workflow/list +1 -1
  160. data/share/rbbt_commands/workflow/prov +20 -13
  161. data/share/rbbt_commands/workflow/server +11 -1
  162. data/share/rbbt_commands/workflow/task +76 -71
  163. data/share/rbbt_commands/workflow/write_info +26 -9
  164. data/share/software/opt/ssw/ssw.c +861 -0
  165. data/share/software/opt/ssw/ssw.h +130 -0
  166. data/share/workflow_config.ru +3 -3
  167. metadata +40 -2
@@ -1,167 +1,169 @@
1
- require 'rbbt-util'
2
- require 'rbbt/resource/path'
3
-
4
- module Rbbt::Config
5
-
6
- CACHE = IndiferentHash.setup({})
7
-
8
- GOT_KEYS=[]
9
-
10
- def self.add_entry(key, value, tokens)
11
- tokens = [tokens] unless Array === tokens
12
- tokens << "key:#{key}" unless tokens.include?("key:#{key}")
13
- CACHE[key.to_s] ||= []
14
- CACHE[key.to_s] << [tokens, value]
15
- end
16
-
17
- def self.load_file(file)
18
- Log.debug "Loading config file: #{ file }"
19
- TSV.traverse file, :type => :array do |line|
20
- next if line =~ /^#/
21
- key, value, *tokens = line.strip.split(/\s+/)
22
-
23
- self.add_entry(key, value, tokens) if key
24
- end
25
- end
26
-
27
- def self.load_config
28
- Rbbt.etc.config.find_all.reverse.each do |file|
29
- self.load_file(file)
30
- end
31
- end
32
-
33
- def self.set(values, *tokens)
34
- if not Hash === values
35
- values = {values => tokens.shift}
36
- end
37
-
38
- values.each do |key,value|
39
- add_entry key, value, tokens
40
- end
41
- end
42
-
43
- def self.token_priority(token)
44
- token, _sep, priority = token.to_s.partition("::")
45
-
46
- if priority.nil? || priority.empty?
47
- type, _sep, rest = token.partition(":")
48
- priority = case type
49
- when "workflow"
50
- 4
51
- when "task"
52
- 3
53
- when "file"
54
- 2
55
- when "line"
56
- 1
57
- when "key"
58
- 20
59
- else
60
- 10
61
- end
62
- else
63
- priority = priority.to_i
64
- end
65
-
66
- [token, priority]
67
- end
68
-
69
- def self.match(entries, give_token)
70
- priorities = {}
71
- entries.each do |tokens, value|
72
- best_prio = nil
73
- tokens = [tokens] unless Array === tokens
74
- tokens.each do |tok|
75
- tok, prio = token_priority tok
76
- next unless tok == give_token
77
-
78
- best_prio = prio if best_prio.nil? or best_prio > prio
79
- next if prio > best_prio
80
-
81
- priorities[prio] ||= []
82
- priorities[prio].unshift value
83
- end
84
- end if entries
85
- priorities
86
- end
87
-
88
- # For equal priorities the matching prioritizes tokens ealier in the list
89
- def self.get(key, *tokens)
90
- options = tokens.pop if Hash === tokens.last
91
- default = options.nil? ? nil : options[:default]
92
-
93
- tokens = ["key:" + key] if tokens.empty?
94
-
95
- tokens = tokens.flatten
96
- file, _sep, line = caller.reject{|l|
97
- l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
98
- l =~ /rbbt\/resource\/path\.rb/ or
99
- l =~ /rbbt\/util\/misc\.rb/ or
100
- l =~ /accessor\.rb/ or
101
- l =~ /progress-monitor\.rb/
102
- }.first.partition(":")
103
-
104
- File.expand_path(file)
105
-
106
- tokens << ("file:" << file)
107
- tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
108
-
109
- entries = CACHE[key.to_s]
110
- priorities = {}
111
- tokens.each do |token|
112
- token_prio = match entries, token.to_s
113
- token_prio.each do |prio, values|
114
- priorities[prio] ||= []
115
- priorities[prio].concat(values)
116
- end
117
- end
118
-
119
- value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
120
- value = false if value == 'false'
121
-
122
- Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
123
- GOT_KEYS << [key, value, tokens]
124
-
125
- if String === value && m = value.match(/^env:(.*)/)
126
- variable = m.captures.first
127
- ENV[variable]
128
- elsif value == 'nil'
129
- nil
130
- else
131
- value
132
- end
133
- end
134
-
135
- def self.with_config
136
- saved_config = {}
137
- CACHE.each do |k,v|
138
- saved_config[k] = v.dup
139
- end
140
- saved_got_keys = GOT_KEYS.dup
141
- begin
142
- yield
143
- ensure
144
- CACHE.replace(saved_config)
145
- GOT_KEYS.replace(saved_got_keys)
146
- end
147
- end
148
-
149
- def self.process_config(config)
150
- if Misc.is_filename?(config) && File.exist?(config)
151
- Rbbt::Config.load_file(config)
152
- elsif Rbbt.etc.config_profile[config].exists?
153
- Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)
154
- else
155
- key, value, *tokens = config.split(/\s/)
156
- tokens = tokens.collect do |tok|
157
- tok, _sep, prio = tok.partition("::")
158
- prio = "0" if prio.nil? or prio.empty?
159
- [tok, prio] * "::"
160
- end
161
- Rbbt::Config.set({key => value}, *tokens)
162
- end
163
- end
164
-
165
-
166
- self.load_config
167
- end
1
+ require_relative '../refactor'
2
+ Rbbt.require_instead 'scout/config'
3
+ #require 'rbbt-util'
4
+ #require 'rbbt/resource/path'
5
+ #
6
+ #module Rbbt::Config
7
+ #
8
+ # CACHE = IndiferentHash.setup({})
9
+ #
10
+ # GOT_KEYS=[]
11
+ #
12
+ # def self.add_entry(key, value, tokens)
13
+ # tokens = [tokens] unless Array === tokens
14
+ # tokens << "key:#{key}" unless tokens.include?("key:#{key}")
15
+ # CACHE[key.to_s] ||= []
16
+ # CACHE[key.to_s] << [tokens, value]
17
+ # end
18
+ #
19
+ # def self.load_file(file)
20
+ # Log.debug "Loading config file: #{ file }"
21
+ # TSV.traverse file, :type => :array do |line|
22
+ # next if line =~ /^#/
23
+ # key, value, *tokens = line.strip.split(/\s/)
24
+ #
25
+ # self.add_entry(key, value, tokens) if key
26
+ # end
27
+ # end
28
+ #
29
+ # def self.load_config
30
+ # Rbbt.etc.config.find_all.reverse.each do |file|
31
+ # self.load_file(file)
32
+ # end
33
+ # end
34
+ #
35
+ # def self.set(values, *tokens)
36
+ # if not Hash === values
37
+ # values = {values => tokens.shift}
38
+ # end
39
+ #
40
+ # values.each do |key,value|
41
+ # add_entry key, value, tokens
42
+ # end
43
+ # end
44
+ #
45
+ # def self.token_priority(token)
46
+ # token, _sep, priority = token.to_s.partition("::")
47
+ #
48
+ # if priority.nil? || priority.empty?
49
+ # type, _sep, rest = token.partition(":")
50
+ # priority = case type
51
+ # when "workflow"
52
+ # 4
53
+ # when "task"
54
+ # 3
55
+ # when "file"
56
+ # 2
57
+ # when "line"
58
+ # 1
59
+ # when "key"
60
+ # 20
61
+ # else
62
+ # 10
63
+ # end
64
+ # else
65
+ # priority = priority.to_i
66
+ # end
67
+ #
68
+ # [token, priority]
69
+ # end
70
+ #
71
+ # def self.match(entries, give_token)
72
+ # priorities = {}
73
+ # entries.each do |tokens, value|
74
+ # best_prio = nil
75
+ # tokens = [tokens] unless Array === tokens
76
+ # tokens.each do |tok|
77
+ # tok, prio = token_priority tok
78
+ # next unless tok == give_token
79
+ #
80
+ # best_prio = prio if best_prio.nil? or best_prio > prio
81
+ # next if prio > best_prio
82
+ #
83
+ # priorities[prio] ||= []
84
+ # priorities[prio].unshift value
85
+ # end
86
+ # end if entries
87
+ # priorities
88
+ # end
89
+ #
90
+ # # For equal priorities the matching prioritizes tokens ealier in the list
91
+ # def self.get(key, *tokens)
92
+ # options = tokens.pop if Hash === tokens.last
93
+ # default = options.nil? ? nil : options[:default]
94
+ #
95
+ # tokens = ["key:" + key] if tokens.empty?
96
+ #
97
+ # tokens = tokens.flatten
98
+ # file, _sep, line = caller.reject{|l|
99
+ # l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
100
+ # l =~ /rbbt\/resource\/path\.rb/ or
101
+ # l =~ /rbbt\/util\/misc\.rb/ or
102
+ # l =~ /accessor\.rb/ or
103
+ # l =~ /progress-monitor\.rb/
104
+ # }.first.partition(":")
105
+ #
106
+ # File.expand_path(file)
107
+ #
108
+ # tokens << ("file:" << file)
109
+ # tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
110
+ #
111
+ # entries = CACHE[key.to_s]
112
+ # priorities = {}
113
+ # tokens.each do |token|
114
+ # token_prio = match entries, token.to_s
115
+ # token_prio.each do |prio, values|
116
+ # priorities[prio] ||= []
117
+ # priorities[prio].concat(values)
118
+ # end
119
+ # end
120
+ #
121
+ # value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
122
+ # value = false if value == 'false'
123
+ #
124
+ # Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
125
+ # GOT_KEYS << [key, value, tokens]
126
+ #
127
+ # if String === value && m = value.match(/^env:(.*)/)
128
+ # variable = m.captures.first
129
+ # ENV[variable]
130
+ # elsif value == 'nil'
131
+ # nil
132
+ # else
133
+ # value
134
+ # end
135
+ # end
136
+ #
137
+ # def self.with_config
138
+ # saved_config = {}
139
+ # CACHE.each do |k,v|
140
+ # saved_config[k] = v.dup
141
+ # end
142
+ # saved_got_keys = GOT_KEYS.dup
143
+ # begin
144
+ # yield
145
+ # ensure
146
+ # CACHE.replace(saved_config)
147
+ # GOT_KEYS.replace(saved_got_keys)
148
+ # end
149
+ # end
150
+ #
151
+ # def self.process_config(config)
152
+ # if Misc.is_filename?(config) && File.exist?(config)
153
+ # Rbbt::Config.load_file(config)
154
+ # elsif Rbbt.etc.config_profile[config].exists?
155
+ # Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)
156
+ # else
157
+ # key, value, *tokens = config.split(/\s/)
158
+ # tokens = tokens.collect do |tok|
159
+ # tok, _sep, prio = tok.partition("::")
160
+ # prio = "0" if prio.nil? or prio.empty?
161
+ # [tok, prio] * "::"
162
+ # end
163
+ # Rbbt::Config.set({key => value}, *tokens)
164
+ # end
165
+ # end
166
+ #
167
+ #
168
+ # self.load_config
169
+ #end
@@ -0,0 +1,20 @@
1
+ require 'iruby'
2
+ require 'rbbt/util/R'
3
+
4
+ module IRuby
5
+ def self.img(file)
6
+ IRuby.html("<img src='#{file}'/>")
7
+ end
8
+
9
+ def self.plot(...)
10
+ filename = Rbbt.iruby[rand(10000).to_s + ".png"]
11
+ Open.mkdir File.dirname(filename)
12
+ R::PNG.plot(filename, ...)
13
+ img(filename)
14
+ end
15
+
16
+ def self.ggplot(...)
17
+ svg = R::SVG.ggplot(...)
18
+ IRuby.html(svg)
19
+ end
20
+ end