rbbt-util 5.44.1 → 6.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/bin/rbbt +67 -90
  4. data/bin/rbbt_exec.rb +2 -2
  5. data/etc/app.d/base.rb +2 -2
  6. data/etc/app.d/semaphores.rb +3 -3
  7. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  8. data/lib/rbbt/annotations/refactor.rb +27 -0
  9. data/lib/rbbt/annotations/util.rb +282 -282
  10. data/lib/rbbt/annotations.rb +343 -320
  11. data/lib/rbbt/association/database.rb +200 -225
  12. data/lib/rbbt/association/index.rb +294 -291
  13. data/lib/rbbt/association/item.rb +227 -227
  14. data/lib/rbbt/association/open.rb +35 -34
  15. data/lib/rbbt/association/util.rb +0 -169
  16. data/lib/rbbt/association.rb +2 -4
  17. data/lib/rbbt/entity/identifiers.rb +119 -118
  18. data/lib/rbbt/entity/refactor.rb +12 -0
  19. data/lib/rbbt/entity.rb +319 -315
  20. data/lib/rbbt/hpc/batch.rb +72 -53
  21. data/lib/rbbt/hpc/lsf.rb +2 -2
  22. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  24. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  25. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  26. data/lib/rbbt/hpc/slurm.rb +18 -18
  27. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  28. data/lib/rbbt/knowledge_base/query.rb +2 -2
  29. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  30. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  31. data/lib/rbbt/knowledge_base.rb +1 -1
  32. data/lib/rbbt/monitor.rb +36 -25
  33. data/lib/rbbt/persist/refactor.rb +166 -0
  34. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  35. data/lib/rbbt/persist/tsv.rb +187 -185
  36. data/lib/rbbt/persist.rb +556 -551
  37. data/lib/rbbt/refactor.rb +20 -0
  38. data/lib/rbbt/resource/path/refactor.rb +178 -0
  39. data/lib/rbbt/resource/path.rb +317 -497
  40. data/lib/rbbt/resource/util.rb +0 -48
  41. data/lib/rbbt/resource.rb +3 -390
  42. data/lib/rbbt/tsv/accessor.rb +2 -838
  43. data/lib/rbbt/tsv/attach.rb +303 -299
  44. data/lib/rbbt/tsv/change_id.rb +244 -245
  45. data/lib/rbbt/tsv/csv.rb +87 -85
  46. data/lib/rbbt/tsv/dumper.rb +2 -100
  47. data/lib/rbbt/tsv/excel.rb +26 -24
  48. data/lib/rbbt/tsv/field_index.rb +4 -1
  49. data/lib/rbbt/tsv/filter.rb +3 -2
  50. data/lib/rbbt/tsv/index.rb +2 -284
  51. data/lib/rbbt/tsv/manipulate.rb +750 -747
  52. data/lib/rbbt/tsv/marshal.rb +3 -3
  53. data/lib/rbbt/tsv/matrix.rb +2 -2
  54. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  55. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  56. data/lib/rbbt/tsv/parser.rb +678 -678
  57. data/lib/rbbt/tsv/refactor.rb +195 -0
  58. data/lib/rbbt/tsv/stream.rb +253 -251
  59. data/lib/rbbt/tsv/util.rb +420 -420
  60. data/lib/rbbt/tsv.rb +210 -208
  61. data/lib/rbbt/util/R/eval.rb +4 -4
  62. data/lib/rbbt/util/R/plot.rb +62 -166
  63. data/lib/rbbt/util/R.rb +21 -18
  64. data/lib/rbbt/util/cmd.rb +2 -318
  65. data/lib/rbbt/util/color.rb +269 -269
  66. data/lib/rbbt/util/colorize.rb +89 -89
  67. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  68. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  69. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  70. data/lib/rbbt/util/config.rb +169 -167
  71. data/lib/rbbt/util/filecache.rb +1 -1
  72. data/lib/rbbt/util/iruby.rb +20 -0
  73. data/lib/rbbt/util/log/progress/report.rb +241 -241
  74. data/lib/rbbt/util/log/progress/util.rb +99 -99
  75. data/lib/rbbt/util/log/progress.rb +102 -102
  76. data/lib/rbbt/util/log/refactor.rb +49 -0
  77. data/lib/rbbt/util/log.rb +486 -532
  78. data/lib/rbbt/util/migrate.rb +2 -2
  79. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  80. data/lib/rbbt/util/misc/development.rb +12 -11
  81. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  82. data/lib/rbbt/util/misc/format.rb +2 -230
  83. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  84. data/lib/rbbt/util/misc/inspect.rb +2 -476
  85. data/lib/rbbt/util/misc/lock.rb +109 -106
  86. data/lib/rbbt/util/misc/omics.rb +9 -1
  87. data/lib/rbbt/util/misc/pipes.rb +765 -793
  88. data/lib/rbbt/util/misc/refactor.rb +20 -0
  89. data/lib/rbbt/util/misc/ssw.rb +27 -17
  90. data/lib/rbbt/util/misc/system.rb +92 -105
  91. data/lib/rbbt/util/misc.rb +39 -20
  92. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  93. data/lib/rbbt/util/named_array.rb +3 -220
  94. data/lib/rbbt/util/open/refactor.rb +7 -0
  95. data/lib/rbbt/util/open.rb +3 -857
  96. data/lib/rbbt/util/procpath.rb +6 -6
  97. data/lib/rbbt/util/python/paths.rb +27 -0
  98. data/lib/rbbt/util/python/run.rb +115 -0
  99. data/lib/rbbt/util/python/script.rb +110 -0
  100. data/lib/rbbt/util/python/util.rb +3 -3
  101. data/lib/rbbt/util/python.rb +22 -81
  102. data/lib/rbbt/util/semaphore.rb +152 -148
  103. data/lib/rbbt/util/simpleopt.rb +9 -8
  104. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  105. data/lib/rbbt/util/ssh.rb +122 -118
  106. data/lib/rbbt/util/tar.rb +117 -115
  107. data/lib/rbbt/util/tmpfile.rb +69 -67
  108. data/lib/rbbt/util/version.rb +2 -0
  109. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  110. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  111. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  112. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  113. data/lib/rbbt/workflow/refactor/task_info.rb +66 -0
  114. data/lib/rbbt/workflow/refactor.rb +150 -0
  115. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +1 -2
  116. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  117. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  118. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  119. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  120. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  121. data/lib/rbbt/workflow/step/run.rb +766 -766
  122. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  123. data/lib/rbbt/workflow/step.rb +2 -362
  124. data/lib/rbbt/workflow/task.rb +118 -118
  125. data/lib/rbbt/workflow/usage.rb +289 -287
  126. data/lib/rbbt/workflow/util/archive.rb +6 -5
  127. data/lib/rbbt/workflow/util/data.rb +1 -1
  128. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  129. data/lib/rbbt/workflow/util/trace.rb +79 -44
  130. data/lib/rbbt/workflow.rb +4 -882
  131. data/lib/rbbt-util.rb +21 -13
  132. data/lib/rbbt.rb +16 -3
  133. data/python/rbbt/__init__.py +96 -4
  134. data/python/rbbt/workflow/remote.py +104 -0
  135. data/python/rbbt/workflow.py +64 -0
  136. data/python/test.py +10 -0
  137. data/share/Rlib/plot.R +37 -37
  138. data/share/Rlib/svg.R +22 -5
  139. data/share/install/software/lib/install_helpers +1 -1
  140. data/share/rbbt_commands/hpc/list +2 -3
  141. data/share/rbbt_commands/hpc/orchestrate +4 -4
  142. data/share/rbbt_commands/hpc/tail +2 -0
  143. data/share/rbbt_commands/hpc/task +10 -7
  144. data/share/rbbt_commands/lsf/list +2 -3
  145. data/share/rbbt_commands/lsf/orchestrate +4 -4
  146. data/share/rbbt_commands/lsf/tail +2 -0
  147. data/share/rbbt_commands/lsf/task +10 -7
  148. data/share/rbbt_commands/migrate +1 -1
  149. data/share/rbbt_commands/pbs/list +2 -3
  150. data/share/rbbt_commands/pbs/orchestrate +4 -4
  151. data/share/rbbt_commands/pbs/tail +2 -0
  152. data/share/rbbt_commands/pbs/task +10 -7
  153. data/share/rbbt_commands/resource/produce +8 -1
  154. data/share/rbbt_commands/slurm/list +2 -3
  155. data/share/rbbt_commands/slurm/orchestrate +4 -4
  156. data/share/rbbt_commands/slurm/tail +2 -0
  157. data/share/rbbt_commands/slurm/task +10 -7
  158. data/share/rbbt_commands/system/clean +5 -5
  159. data/share/rbbt_commands/system/status +5 -5
  160. data/share/rbbt_commands/tsv/get +2 -3
  161. data/share/rbbt_commands/tsv/info +10 -13
  162. data/share/rbbt_commands/tsv/keys +18 -14
  163. data/share/rbbt_commands/tsv/slice +2 -2
  164. data/share/rbbt_commands/tsv/transpose +6 -2
  165. data/share/rbbt_commands/workflow/info +20 -24
  166. data/share/rbbt_commands/workflow/list +1 -1
  167. data/share/rbbt_commands/workflow/prov +20 -13
  168. data/share/rbbt_commands/workflow/retry +43 -0
  169. data/share/rbbt_commands/workflow/server +12 -2
  170. data/share/rbbt_commands/workflow/task +80 -73
  171. data/share/rbbt_commands/workflow/write_info +26 -9
  172. data/share/software/opt/ssw/ssw.c +861 -0
  173. data/share/software/opt/ssw/ssw.h +130 -0
  174. data/share/workflow_config.ru +3 -3
  175. metadata +45 -6
@@ -1,207 +1,207 @@
1
- module AnnotatedArray
2
- attr_accessor :list_id
3
-
4
- def to_a
5
- self.collect{|i| i }
6
- end
7
-
8
- def double_array
9
- AnnotatedArray === self.send(:[], 0, true)
10
- end
11
-
12
- def first
13
- self[0]
14
- end
15
-
16
- def last
17
- self[-1]
18
- end
19
-
20
- def [](pos, clean = false)
21
-
22
- value = super(pos)
23
- return value if value.nil? or clean
24
-
25
- value = value.dup if value.frozen? and (String === value or Array === value)
26
-
27
- value = annotate(value)
28
-
29
- value.extend AnnotatedArray if Array === value and Annotated === value
30
-
31
- if value.respond_to? :container
32
- value.container = self
33
- value.container_index = pos
34
- end
35
-
36
- value
37
- end
38
-
39
- def each(&block)
40
-
41
- pos = 0
42
- super do |value|
43
-
44
- case value
45
- when Array
46
- value = value.dup if value.frozen?
47
-
48
- value = annotate(value)
49
-
50
- value.extend AnnotatedArray if Array === value and Annotated === value
51
-
52
- value.container = self
53
- value.container_index = pos
54
-
55
- pos += 1
56
-
57
- block.call value
58
- when String
59
-
60
- value = value.dup if value.frozen?
61
-
62
- value = annotate(value)
63
-
64
- value.container = self if value.respond_to?(:container)
65
- value.container_index = pos if value.respond_to?(:container_index)
66
-
67
- pos += 1
68
-
69
- block.call value
70
- else
71
- block.call value
72
- end
73
- end
74
- end
75
-
76
- def collect(&block)
77
-
78
- if block_given?
79
-
80
- res = []
81
- each do |value|
82
- res << yield(value)
83
- end
84
-
85
- res
86
- else
87
-
88
- res = []
89
- each do |value|
90
- res << value
91
- end
92
-
93
- res
94
- end
95
- end
96
-
97
- def +(other)
98
- self.annotate super(other)
99
- end
100
-
101
-
102
- def reject
103
- res = []
104
-
105
- each do |value|
106
- res << value unless yield(value)
107
- end
108
-
109
- annotate(res)
110
- res.extend AnnotatedArray
111
-
112
- res
113
- end
114
-
115
- def select(func_name = nil)
116
- res = []
117
-
118
- if func_name
119
- each do |elem|
120
- value = elem.send(func_name)
121
- if block_given?
122
- res << elem if yield(value)
123
- else
124
- res << elem if value
125
- end
126
- end
127
- else
128
- each do |elem|
129
- res << elem if yield(elem)
130
- end
131
- end
132
-
133
- annotate(res)
134
- res.extend AnnotatedArray
135
-
136
- res
137
- end
138
-
139
- def subset(list)
140
-
141
- res = (self & list)
142
-
143
- annotate(res)
144
- res.extend AnnotatedArray
145
-
146
- res
147
- end
148
-
149
- def remove(list)
150
-
151
- res = (self - list)
152
-
153
- annotate(res)
154
- res.extend AnnotatedArray
155
-
156
- res
157
- end
158
-
159
- def sort(&block)
160
- res = self.collect.sort(&block).collect{|value| value.respond_to?(:clean_annotations) ? value.clean_annotations.dup : value.dup }
161
-
162
- annotate(res)
163
- res.extend AnnotatedArray
164
-
165
- res
166
- end
167
-
168
- def select_by(method, *args, &block)
169
- return [] if self.empty? and not self.respond_to? :annotate
170
-
171
- values = self.send(method, *args)
172
- values = values.clean_annotations if values.respond_to? :clean_annotations
173
-
174
- new = []
175
- if block_given?
176
- self.clean_annotations.each_with_index do |e,i|
177
- new << e if yield(values[i])
178
- end
179
- else
180
- self.clean_annotations.each_with_index do |e,i|
181
- new << e if values[i]
182
- end
183
- end
184
- self.annotate(new)
185
- new.extend AnnotatedArray
186
-
187
- new
188
- end
189
-
190
- %w(compact uniq flatten reverse sort_by).each do |method|
191
-
192
- self.module_eval <<-EOC
193
-
194
- def #{method}
195
- res = super
196
-
197
- annotate(res)
198
- res.extend AnnotatedArray
199
-
200
- res
201
- end
202
-
203
- EOC
204
- end
205
- end
206
-
207
-
1
+ #module AnnotatedArray
2
+ # attr_accessor :list_id
3
+ #
4
+ # def to_a
5
+ # self.collect{|i| i }
6
+ # end
7
+ #
8
+ # def double_array
9
+ # AnnotatedArray === self.send(:[], 0, true)
10
+ # end
11
+ #
12
+ # def first
13
+ # self[0]
14
+ # end
15
+ #
16
+ # def last
17
+ # self[-1]
18
+ # end
19
+ #
20
+ # def [](pos, clean = false)
21
+ #
22
+ # value = super(pos)
23
+ # return value if value.nil? or clean
24
+ #
25
+ # value = value.dup if value.frozen? and (String === value or Array === value)
26
+ #
27
+ # value = annotate(value)
28
+ #
29
+ # value.extend AnnotatedArray if Array === value and Annotated === value
30
+ #
31
+ # if value.respond_to? :container
32
+ # value.container = self
33
+ # value.container_index = pos
34
+ # end
35
+ #
36
+ # value
37
+ # end
38
+ #
39
+ # def each(&block)
40
+ #
41
+ # pos = 0
42
+ # super do |value|
43
+ #
44
+ # case value
45
+ # when Array
46
+ # value = value.dup if value.frozen?
47
+ #
48
+ # value = annotate(value)
49
+ #
50
+ # value.extend AnnotatedArray if Array === value and Annotated === value
51
+ #
52
+ # value.container = self
53
+ # value.container_index = pos
54
+ #
55
+ # pos += 1
56
+ #
57
+ # block.call value
58
+ # when String
59
+ #
60
+ # value = value.dup if value.frozen?
61
+ #
62
+ # value = annotate(value)
63
+ #
64
+ # value.container = self if value.respond_to?(:container)
65
+ # value.container_index = pos if value.respond_to?(:container_index)
66
+ #
67
+ # pos += 1
68
+ #
69
+ # block.call value
70
+ # else
71
+ # block.call value
72
+ # end
73
+ # end
74
+ # end
75
+ #
76
+ # def collect(&block)
77
+ #
78
+ # if block_given?
79
+ #
80
+ # res = []
81
+ # each do |value|
82
+ # res << yield(value)
83
+ # end
84
+ #
85
+ # res
86
+ # else
87
+ #
88
+ # res = []
89
+ # each do |value|
90
+ # res << value
91
+ # end
92
+ #
93
+ # res
94
+ # end
95
+ # end
96
+ #
97
+ # def +(other)
98
+ # self.annotate super(other)
99
+ # end
100
+ #
101
+ #
102
+ # def reject
103
+ # res = []
104
+ #
105
+ # each do |value|
106
+ # res << value unless yield(value)
107
+ # end
108
+ #
109
+ # annotate(res)
110
+ # res.extend AnnotatedArray
111
+ #
112
+ # res
113
+ # end
114
+ #
115
+ # def select(func_name = nil)
116
+ # res = []
117
+ #
118
+ # if func_name
119
+ # each do |elem|
120
+ # value = elem.send(func_name)
121
+ # if block_given?
122
+ # res << elem if yield(value)
123
+ # else
124
+ # res << elem if value
125
+ # end
126
+ # end
127
+ # else
128
+ # each do |elem|
129
+ # res << elem if yield(elem)
130
+ # end
131
+ # end
132
+ #
133
+ # annotate(res)
134
+ # res.extend AnnotatedArray
135
+ #
136
+ # res
137
+ # end
138
+ #
139
+ # def subset(list)
140
+ #
141
+ # res = (self & list)
142
+ #
143
+ # annotate(res)
144
+ # res.extend AnnotatedArray
145
+ #
146
+ # res
147
+ # end
148
+ #
149
+ # def remove(list)
150
+ #
151
+ # res = (self - list)
152
+ #
153
+ # annotate(res)
154
+ # res.extend AnnotatedArray
155
+ #
156
+ # res
157
+ # end
158
+ #
159
+ # def sort(&block)
160
+ # res = self.collect.sort(&block).collect{|value| value.respond_to?(:clean_annotations) ? value.clean_annotations.dup : value.dup }
161
+ #
162
+ # annotate(res)
163
+ # res.extend AnnotatedArray
164
+ #
165
+ # res
166
+ # end
167
+ #
168
+ # def select_by(method, *args, &block)
169
+ # return [] if self.empty? and not self.respond_to? :annotate
170
+ #
171
+ # values = self.send(method, *args)
172
+ # values = values.clean_annotations if values.respond_to? :clean_annotations
173
+ #
174
+ # new = []
175
+ # if block_given?
176
+ # self.clean_annotations.each_with_index do |e,i|
177
+ # new << e if yield(values[i])
178
+ # end
179
+ # else
180
+ # self.clean_annotations.each_with_index do |e,i|
181
+ # new << e if values[i]
182
+ # end
183
+ # end
184
+ # self.annotate(new)
185
+ # new.extend AnnotatedArray
186
+ #
187
+ # new
188
+ # end
189
+ #
190
+ # %w(compact uniq flatten reverse sort_by).each do |method|
191
+ #
192
+ # self.module_eval <<-EOC
193
+ #
194
+ # def #{method}
195
+ # res = super
196
+ #
197
+ # annotate(res)
198
+ # res.extend AnnotatedArray
199
+ #
200
+ # res
201
+ # end
202
+ #
203
+ # EOC
204
+ # end
205
+ #end
206
+ #
207
+ #
@@ -0,0 +1,27 @@
1
+ require_relative '../refactor'
2
+ module Annotation
3
+ module AnnotatedObject
4
+ alias id annotation_id
5
+ alias clean_annotations purge
6
+ alias annotation_values annotation_hash
7
+ def info
8
+ annotation_hash.merge(:annotation_types => annotation_types, :annotated_array => (Array === self))
9
+ end
10
+ end
11
+ end
12
+
13
+ Annotated = Annotation::AnnotatedObject
14
+
15
+ module Annotation::AnnotatedObject
16
+ alias make_list make_array
17
+ end
18
+
19
+ module Annotation
20
+ def self.load_entity(obj, info)
21
+ self.setup(obj, info[:annotation_types], info)
22
+ end
23
+ end
24
+
25
+ Rbbt.relay_module_method Annotated, :tsv, Annotation, :tsv
26
+ Rbbt.relay_module_method Annotated, :load_tsv, Annotation, :load_tsv
27
+ Rbbt.relay_module_method Annotated, :load_entity, Annotation, :load_entity