rbbt-util 5.44.1 → 6.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/bin/rbbt +67 -90
  4. data/bin/rbbt_exec.rb +2 -2
  5. data/etc/app.d/base.rb +2 -2
  6. data/etc/app.d/semaphores.rb +3 -3
  7. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  8. data/lib/rbbt/annotations/refactor.rb +27 -0
  9. data/lib/rbbt/annotations/util.rb +282 -282
  10. data/lib/rbbt/annotations.rb +343 -320
  11. data/lib/rbbt/association/database.rb +200 -225
  12. data/lib/rbbt/association/index.rb +294 -291
  13. data/lib/rbbt/association/item.rb +227 -227
  14. data/lib/rbbt/association/open.rb +35 -34
  15. data/lib/rbbt/association/util.rb +0 -169
  16. data/lib/rbbt/association.rb +2 -4
  17. data/lib/rbbt/entity/identifiers.rb +119 -118
  18. data/lib/rbbt/entity/refactor.rb +12 -0
  19. data/lib/rbbt/entity.rb +319 -315
  20. data/lib/rbbt/hpc/batch.rb +72 -53
  21. data/lib/rbbt/hpc/lsf.rb +2 -2
  22. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  24. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  25. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  26. data/lib/rbbt/hpc/slurm.rb +18 -18
  27. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  28. data/lib/rbbt/knowledge_base/query.rb +2 -2
  29. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  30. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  31. data/lib/rbbt/knowledge_base.rb +1 -1
  32. data/lib/rbbt/monitor.rb +36 -25
  33. data/lib/rbbt/persist/refactor.rb +166 -0
  34. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  35. data/lib/rbbt/persist/tsv.rb +187 -185
  36. data/lib/rbbt/persist.rb +556 -551
  37. data/lib/rbbt/refactor.rb +20 -0
  38. data/lib/rbbt/resource/path/refactor.rb +178 -0
  39. data/lib/rbbt/resource/path.rb +317 -497
  40. data/lib/rbbt/resource/util.rb +0 -48
  41. data/lib/rbbt/resource.rb +3 -390
  42. data/lib/rbbt/tsv/accessor.rb +2 -838
  43. data/lib/rbbt/tsv/attach.rb +303 -299
  44. data/lib/rbbt/tsv/change_id.rb +244 -245
  45. data/lib/rbbt/tsv/csv.rb +87 -85
  46. data/lib/rbbt/tsv/dumper.rb +2 -100
  47. data/lib/rbbt/tsv/excel.rb +26 -24
  48. data/lib/rbbt/tsv/field_index.rb +4 -1
  49. data/lib/rbbt/tsv/filter.rb +3 -2
  50. data/lib/rbbt/tsv/index.rb +2 -284
  51. data/lib/rbbt/tsv/manipulate.rb +750 -747
  52. data/lib/rbbt/tsv/marshal.rb +3 -3
  53. data/lib/rbbt/tsv/matrix.rb +2 -2
  54. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  55. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  56. data/lib/rbbt/tsv/parser.rb +678 -678
  57. data/lib/rbbt/tsv/refactor.rb +195 -0
  58. data/lib/rbbt/tsv/stream.rb +253 -251
  59. data/lib/rbbt/tsv/util.rb +420 -420
  60. data/lib/rbbt/tsv.rb +210 -208
  61. data/lib/rbbt/util/R/eval.rb +4 -4
  62. data/lib/rbbt/util/R/plot.rb +62 -166
  63. data/lib/rbbt/util/R.rb +21 -18
  64. data/lib/rbbt/util/cmd.rb +2 -318
  65. data/lib/rbbt/util/color.rb +269 -269
  66. data/lib/rbbt/util/colorize.rb +89 -89
  67. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  68. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  69. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  70. data/lib/rbbt/util/config.rb +169 -167
  71. data/lib/rbbt/util/filecache.rb +1 -1
  72. data/lib/rbbt/util/iruby.rb +20 -0
  73. data/lib/rbbt/util/log/progress/report.rb +241 -241
  74. data/lib/rbbt/util/log/progress/util.rb +99 -99
  75. data/lib/rbbt/util/log/progress.rb +102 -102
  76. data/lib/rbbt/util/log/refactor.rb +49 -0
  77. data/lib/rbbt/util/log.rb +486 -532
  78. data/lib/rbbt/util/migrate.rb +2 -2
  79. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  80. data/lib/rbbt/util/misc/development.rb +12 -11
  81. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  82. data/lib/rbbt/util/misc/format.rb +2 -230
  83. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  84. data/lib/rbbt/util/misc/inspect.rb +2 -476
  85. data/lib/rbbt/util/misc/lock.rb +109 -106
  86. data/lib/rbbt/util/misc/omics.rb +9 -1
  87. data/lib/rbbt/util/misc/pipes.rb +765 -793
  88. data/lib/rbbt/util/misc/refactor.rb +20 -0
  89. data/lib/rbbt/util/misc/ssw.rb +27 -17
  90. data/lib/rbbt/util/misc/system.rb +92 -105
  91. data/lib/rbbt/util/misc.rb +39 -20
  92. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  93. data/lib/rbbt/util/named_array.rb +3 -220
  94. data/lib/rbbt/util/open/refactor.rb +7 -0
  95. data/lib/rbbt/util/open.rb +3 -857
  96. data/lib/rbbt/util/procpath.rb +6 -6
  97. data/lib/rbbt/util/python/paths.rb +27 -0
  98. data/lib/rbbt/util/python/run.rb +115 -0
  99. data/lib/rbbt/util/python/script.rb +110 -0
  100. data/lib/rbbt/util/python/util.rb +3 -3
  101. data/lib/rbbt/util/python.rb +22 -81
  102. data/lib/rbbt/util/semaphore.rb +152 -148
  103. data/lib/rbbt/util/simpleopt.rb +9 -8
  104. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  105. data/lib/rbbt/util/ssh.rb +122 -118
  106. data/lib/rbbt/util/tar.rb +117 -115
  107. data/lib/rbbt/util/tmpfile.rb +69 -67
  108. data/lib/rbbt/util/version.rb +2 -0
  109. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  110. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  111. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  112. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  113. data/lib/rbbt/workflow/refactor/task_info.rb +66 -0
  114. data/lib/rbbt/workflow/refactor.rb +150 -0
  115. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +1 -2
  116. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  117. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  118. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  119. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  120. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  121. data/lib/rbbt/workflow/step/run.rb +766 -766
  122. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  123. data/lib/rbbt/workflow/step.rb +2 -362
  124. data/lib/rbbt/workflow/task.rb +118 -118
  125. data/lib/rbbt/workflow/usage.rb +289 -287
  126. data/lib/rbbt/workflow/util/archive.rb +6 -5
  127. data/lib/rbbt/workflow/util/data.rb +1 -1
  128. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  129. data/lib/rbbt/workflow/util/trace.rb +79 -44
  130. data/lib/rbbt/workflow.rb +4 -882
  131. data/lib/rbbt-util.rb +21 -13
  132. data/lib/rbbt.rb +16 -3
  133. data/python/rbbt/__init__.py +96 -4
  134. data/python/rbbt/workflow/remote.py +104 -0
  135. data/python/rbbt/workflow.py +64 -0
  136. data/python/test.py +10 -0
  137. data/share/Rlib/plot.R +37 -37
  138. data/share/Rlib/svg.R +22 -5
  139. data/share/install/software/lib/install_helpers +1 -1
  140. data/share/rbbt_commands/hpc/list +2 -3
  141. data/share/rbbt_commands/hpc/orchestrate +4 -4
  142. data/share/rbbt_commands/hpc/tail +2 -0
  143. data/share/rbbt_commands/hpc/task +10 -7
  144. data/share/rbbt_commands/lsf/list +2 -3
  145. data/share/rbbt_commands/lsf/orchestrate +4 -4
  146. data/share/rbbt_commands/lsf/tail +2 -0
  147. data/share/rbbt_commands/lsf/task +10 -7
  148. data/share/rbbt_commands/migrate +1 -1
  149. data/share/rbbt_commands/pbs/list +2 -3
  150. data/share/rbbt_commands/pbs/orchestrate +4 -4
  151. data/share/rbbt_commands/pbs/tail +2 -0
  152. data/share/rbbt_commands/pbs/task +10 -7
  153. data/share/rbbt_commands/resource/produce +8 -1
  154. data/share/rbbt_commands/slurm/list +2 -3
  155. data/share/rbbt_commands/slurm/orchestrate +4 -4
  156. data/share/rbbt_commands/slurm/tail +2 -0
  157. data/share/rbbt_commands/slurm/task +10 -7
  158. data/share/rbbt_commands/system/clean +5 -5
  159. data/share/rbbt_commands/system/status +5 -5
  160. data/share/rbbt_commands/tsv/get +2 -3
  161. data/share/rbbt_commands/tsv/info +10 -13
  162. data/share/rbbt_commands/tsv/keys +18 -14
  163. data/share/rbbt_commands/tsv/slice +2 -2
  164. data/share/rbbt_commands/tsv/transpose +6 -2
  165. data/share/rbbt_commands/workflow/info +20 -24
  166. data/share/rbbt_commands/workflow/list +1 -1
  167. data/share/rbbt_commands/workflow/prov +20 -13
  168. data/share/rbbt_commands/workflow/retry +43 -0
  169. data/share/rbbt_commands/workflow/server +12 -2
  170. data/share/rbbt_commands/workflow/task +80 -73
  171. data/share/rbbt_commands/workflow/write_info +26 -9
  172. data/share/software/opt/ssw/ssw.c +861 -0
  173. data/share/software/opt/ssw/ssw.h +130 -0
  174. data/share/workflow_config.ru +3 -3
  175. metadata +45 -6
@@ -1,167 +1,169 @@
1
- require 'rbbt-util'
2
- require 'rbbt/resource/path'
3
-
4
- module Rbbt::Config
5
-
6
- CACHE = IndiferentHash.setup({})
7
-
8
- GOT_KEYS=[]
9
-
10
- def self.add_entry(key, value, tokens)
11
- tokens = [tokens] unless Array === tokens
12
- tokens << "key:#{key}" unless tokens.include?("key:#{key}")
13
- CACHE[key.to_s] ||= []
14
- CACHE[key.to_s] << [tokens, value]
15
- end
16
-
17
- def self.load_file(file)
18
- Log.debug "Loading config file: #{ file }"
19
- TSV.traverse file, :type => :array do |line|
20
- next if line =~ /^#/
21
- key, value, *tokens = line.strip.split(/\s+/)
22
-
23
- self.add_entry(key, value, tokens) if key
24
- end
25
- end
26
-
27
- def self.load_config
28
- Rbbt.etc.config.find_all.reverse.each do |file|
29
- self.load_file(file)
30
- end
31
- end
32
-
33
- def self.set(values, *tokens)
34
- if not Hash === values
35
- values = {values => tokens.shift}
36
- end
37
-
38
- values.each do |key,value|
39
- add_entry key, value, tokens
40
- end
41
- end
42
-
43
- def self.token_priority(token)
44
- token, _sep, priority = token.to_s.partition("::")
45
-
46
- if priority.nil? || priority.empty?
47
- type, _sep, rest = token.partition(":")
48
- priority = case type
49
- when "workflow"
50
- 4
51
- when "task"
52
- 3
53
- when "file"
54
- 2
55
- when "line"
56
- 1
57
- when "key"
58
- 20
59
- else
60
- 10
61
- end
62
- else
63
- priority = priority.to_i
64
- end
65
-
66
- [token, priority]
67
- end
68
-
69
- def self.match(entries, give_token)
70
- priorities = {}
71
- entries.each do |tokens, value|
72
- best_prio = nil
73
- tokens = [tokens] unless Array === tokens
74
- tokens.each do |tok|
75
- tok, prio = token_priority tok
76
- next unless tok == give_token
77
-
78
- best_prio = prio if best_prio.nil? or best_prio > prio
79
- next if prio > best_prio
80
-
81
- priorities[prio] ||= []
82
- priorities[prio].unshift value
83
- end
84
- end if entries
85
- priorities
86
- end
87
-
88
- # For equal priorities the matching prioritizes tokens ealier in the list
89
- def self.get(key, *tokens)
90
- options = tokens.pop if Hash === tokens.last
91
- default = options.nil? ? nil : options[:default]
92
-
93
- tokens = ["key:" + key] if tokens.empty?
94
-
95
- tokens = tokens.flatten
96
- file, _sep, line = caller.reject{|l|
97
- l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
98
- l =~ /rbbt\/resource\/path\.rb/ or
99
- l =~ /rbbt\/util\/misc\.rb/ or
100
- l =~ /accessor\.rb/ or
101
- l =~ /progress-monitor\.rb/
102
- }.first.partition(":")
103
-
104
- File.expand_path(file)
105
-
106
- tokens << ("file:" << file)
107
- tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
108
-
109
- entries = CACHE[key.to_s]
110
- priorities = {}
111
- tokens.each do |token|
112
- token_prio = match entries, token.to_s
113
- token_prio.each do |prio, values|
114
- priorities[prio] ||= []
115
- priorities[prio].concat(values)
116
- end
117
- end
118
-
119
- value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
120
- value = false if value == 'false'
121
-
122
- Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
123
- GOT_KEYS << [key, value, tokens]
124
-
125
- if String === value && m = value.match(/^env:(.*)/)
126
- variable = m.captures.first
127
- ENV[variable]
128
- elsif value == 'nil'
129
- nil
130
- else
131
- value
132
- end
133
- end
134
-
135
- def self.with_config
136
- saved_config = {}
137
- CACHE.each do |k,v|
138
- saved_config[k] = v.dup
139
- end
140
- saved_got_keys = GOT_KEYS.dup
141
- begin
142
- yield
143
- ensure
144
- CACHE.replace(saved_config)
145
- GOT_KEYS.replace(saved_got_keys)
146
- end
147
- end
148
-
149
- def self.process_config(config)
150
- if Misc.is_filename?(config) && File.exist?(config)
151
- Rbbt::Config.load_file(config)
152
- elsif Rbbt.etc.config_profile[config].exists?
153
- Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)
154
- else
155
- key, value, *tokens = config.split(/\s/)
156
- tokens = tokens.collect do |tok|
157
- tok, _sep, prio = tok.partition("::")
158
- prio = "0" if prio.nil? or prio.empty?
159
- [tok, prio] * "::"
160
- end
161
- Rbbt::Config.set({key => value}, *tokens)
162
- end
163
- end
164
-
165
-
166
- self.load_config
167
- end
1
+ require_relative '../refactor'
2
+ Rbbt.require_instead 'scout/config'
3
+ #require 'rbbt-util'
4
+ #require 'rbbt/resource/path'
5
+ #
6
+ #module Rbbt::Config
7
+ #
8
+ # CACHE = IndiferentHash.setup({})
9
+ #
10
+ # GOT_KEYS=[]
11
+ #
12
+ # def self.add_entry(key, value, tokens)
13
+ # tokens = [tokens] unless Array === tokens
14
+ # tokens << "key:#{key}" unless tokens.include?("key:#{key}")
15
+ # CACHE[key.to_s] ||= []
16
+ # CACHE[key.to_s] << [tokens, value]
17
+ # end
18
+ #
19
+ # def self.load_file(file)
20
+ # Log.debug "Loading config file: #{ file }"
21
+ # TSV.traverse file, :type => :array do |line|
22
+ # next if line =~ /^#/
23
+ # key, value, *tokens = line.strip.split(/\s/)
24
+ #
25
+ # self.add_entry(key, value, tokens) if key
26
+ # end
27
+ # end
28
+ #
29
+ # def self.load_config
30
+ # Rbbt.etc.config.find_all.reverse.each do |file|
31
+ # self.load_file(file)
32
+ # end
33
+ # end
34
+ #
35
+ # def self.set(values, *tokens)
36
+ # if not Hash === values
37
+ # values = {values => tokens.shift}
38
+ # end
39
+ #
40
+ # values.each do |key,value|
41
+ # add_entry key, value, tokens
42
+ # end
43
+ # end
44
+ #
45
+ # def self.token_priority(token)
46
+ # token, _sep, priority = token.to_s.partition("::")
47
+ #
48
+ # if priority.nil? || priority.empty?
49
+ # type, _sep, rest = token.partition(":")
50
+ # priority = case type
51
+ # when "workflow"
52
+ # 4
53
+ # when "task"
54
+ # 3
55
+ # when "file"
56
+ # 2
57
+ # when "line"
58
+ # 1
59
+ # when "key"
60
+ # 20
61
+ # else
62
+ # 10
63
+ # end
64
+ # else
65
+ # priority = priority.to_i
66
+ # end
67
+ #
68
+ # [token, priority]
69
+ # end
70
+ #
71
+ # def self.match(entries, give_token)
72
+ # priorities = {}
73
+ # entries.each do |tokens, value|
74
+ # best_prio = nil
75
+ # tokens = [tokens] unless Array === tokens
76
+ # tokens.each do |tok|
77
+ # tok, prio = token_priority tok
78
+ # next unless tok == give_token
79
+ #
80
+ # best_prio = prio if best_prio.nil? or best_prio > prio
81
+ # next if prio > best_prio
82
+ #
83
+ # priorities[prio] ||= []
84
+ # priorities[prio].unshift value
85
+ # end
86
+ # end if entries
87
+ # priorities
88
+ # end
89
+ #
90
+ # # For equal priorities the matching prioritizes tokens ealier in the list
91
+ # def self.get(key, *tokens)
92
+ # options = tokens.pop if Hash === tokens.last
93
+ # default = options.nil? ? nil : options[:default]
94
+ #
95
+ # tokens = ["key:" + key] if tokens.empty?
96
+ #
97
+ # tokens = tokens.flatten
98
+ # file, _sep, line = caller.reject{|l|
99
+ # l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
100
+ # l =~ /rbbt\/resource\/path\.rb/ or
101
+ # l =~ /rbbt\/util\/misc\.rb/ or
102
+ # l =~ /accessor\.rb/ or
103
+ # l =~ /progress-monitor\.rb/
104
+ # }.first.partition(":")
105
+ #
106
+ # File.expand_path(file)
107
+ #
108
+ # tokens << ("file:" << file)
109
+ # tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
110
+ #
111
+ # entries = CACHE[key.to_s]
112
+ # priorities = {}
113
+ # tokens.each do |token|
114
+ # token_prio = match entries, token.to_s
115
+ # token_prio.each do |prio, values|
116
+ # priorities[prio] ||= []
117
+ # priorities[prio].concat(values)
118
+ # end
119
+ # end
120
+ #
121
+ # value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
122
+ # value = false if value == 'false'
123
+ #
124
+ # Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
125
+ # GOT_KEYS << [key, value, tokens]
126
+ #
127
+ # if String === value && m = value.match(/^env:(.*)/)
128
+ # variable = m.captures.first
129
+ # ENV[variable]
130
+ # elsif value == 'nil'
131
+ # nil
132
+ # else
133
+ # value
134
+ # end
135
+ # end
136
+ #
137
+ # def self.with_config
138
+ # saved_config = {}
139
+ # CACHE.each do |k,v|
140
+ # saved_config[k] = v.dup
141
+ # end
142
+ # saved_got_keys = GOT_KEYS.dup
143
+ # begin
144
+ # yield
145
+ # ensure
146
+ # CACHE.replace(saved_config)
147
+ # GOT_KEYS.replace(saved_got_keys)
148
+ # end
149
+ # end
150
+ #
151
+ # def self.process_config(config)
152
+ # if Misc.is_filename?(config) && File.exist?(config)
153
+ # Rbbt::Config.load_file(config)
154
+ # elsif Rbbt.etc.config_profile[config].exists?
155
+ # Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)
156
+ # else
157
+ # key, value, *tokens = config.split(/\s/)
158
+ # tokens = tokens.collect do |tok|
159
+ # tok, _sep, prio = tok.partition("::")
160
+ # prio = "0" if prio.nil? or prio.empty?
161
+ # [tok, prio] * "::"
162
+ # end
163
+ # Rbbt::Config.set({key => value}, *tokens)
164
+ # end
165
+ # end
166
+ #
167
+ #
168
+ # self.load_config
169
+ #end
@@ -12,7 +12,7 @@ module FileCache
12
12
 
13
13
  def self.cachedir=(cachedir)
14
14
  CACHEDIR.replace cachedir
15
- FileUtils.mkdir_p CACHEDIR unless File.exist? CACHEDIR
15
+ Open.mkdir CACHEDIR unless Open.exist? CACHEDIR
16
16
  end
17
17
 
18
18
  def self.cachedir
@@ -0,0 +1,20 @@
1
+ require 'iruby'
2
+ require 'rbbt/util/R'
3
+
4
+ module IRuby
5
+ def self.img(file)
6
+ IRuby.html("<img src='#{file}'/>")
7
+ end
8
+
9
+ def self.plot(...)
10
+ filename = Rbbt.iruby[rand(10000).to_s + ".png"]
11
+ Open.mkdir File.dirname(filename)
12
+ R::PNG.plot(filename, ...)
13
+ img(filename)
14
+ end
15
+
16
+ def self.ggplot(...)
17
+ svg = R::SVG.ggplot(...)
18
+ IRuby.html(svg)
19
+ end
20
+ end