rbbt-util 5.44.1 → 6.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (167) hide show
  1. checksums.yaml +4 -4
  2. data/bin/rbbt +67 -90
  3. data/etc/app.d/base.rb +2 -2
  4. data/etc/app.d/semaphores.rb +3 -3
  5. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  6. data/lib/rbbt/annotations/refactor.rb +27 -0
  7. data/lib/rbbt/annotations/util.rb +282 -282
  8. data/lib/rbbt/annotations.rb +343 -320
  9. data/lib/rbbt/association/database.rb +200 -225
  10. data/lib/rbbt/association/index.rb +294 -291
  11. data/lib/rbbt/association/item.rb +227 -227
  12. data/lib/rbbt/association/open.rb +35 -34
  13. data/lib/rbbt/association/util.rb +0 -169
  14. data/lib/rbbt/association.rb +2 -4
  15. data/lib/rbbt/entity/identifiers.rb +119 -118
  16. data/lib/rbbt/entity/refactor.rb +12 -0
  17. data/lib/rbbt/entity.rb +319 -315
  18. data/lib/rbbt/hpc/batch.rb +72 -53
  19. data/lib/rbbt/hpc/lsf.rb +2 -2
  20. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  21. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  22. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  24. data/lib/rbbt/hpc/slurm.rb +18 -18
  25. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  26. data/lib/rbbt/knowledge_base/query.rb +2 -2
  27. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  28. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  29. data/lib/rbbt/knowledge_base.rb +1 -1
  30. data/lib/rbbt/monitor.rb +36 -25
  31. data/lib/rbbt/persist/refactor.rb +166 -0
  32. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  33. data/lib/rbbt/persist/tsv.rb +187 -185
  34. data/lib/rbbt/persist.rb +556 -551
  35. data/lib/rbbt/refactor.rb +20 -0
  36. data/lib/rbbt/resource/path/refactor.rb +178 -0
  37. data/lib/rbbt/resource/path.rb +317 -497
  38. data/lib/rbbt/resource/util.rb +0 -48
  39. data/lib/rbbt/resource.rb +3 -390
  40. data/lib/rbbt/tsv/accessor.rb +2 -838
  41. data/lib/rbbt/tsv/attach.rb +303 -299
  42. data/lib/rbbt/tsv/change_id.rb +244 -245
  43. data/lib/rbbt/tsv/csv.rb +87 -85
  44. data/lib/rbbt/tsv/dumper.rb +2 -100
  45. data/lib/rbbt/tsv/excel.rb +26 -24
  46. data/lib/rbbt/tsv/field_index.rb +4 -1
  47. data/lib/rbbt/tsv/filter.rb +3 -2
  48. data/lib/rbbt/tsv/index.rb +2 -284
  49. data/lib/rbbt/tsv/manipulate.rb +750 -747
  50. data/lib/rbbt/tsv/marshal.rb +3 -3
  51. data/lib/rbbt/tsv/matrix.rb +2 -2
  52. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  53. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  54. data/lib/rbbt/tsv/parser.rb +678 -678
  55. data/lib/rbbt/tsv/refactor.rb +195 -0
  56. data/lib/rbbt/tsv/stream.rb +253 -251
  57. data/lib/rbbt/tsv/util.rb +420 -420
  58. data/lib/rbbt/tsv.rb +210 -208
  59. data/lib/rbbt/util/R/eval.rb +4 -4
  60. data/lib/rbbt/util/R/plot.rb +62 -166
  61. data/lib/rbbt/util/R.rb +21 -18
  62. data/lib/rbbt/util/cmd.rb +2 -318
  63. data/lib/rbbt/util/color.rb +269 -269
  64. data/lib/rbbt/util/colorize.rb +89 -89
  65. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  66. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  67. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  68. data/lib/rbbt/util/config.rb +169 -167
  69. data/lib/rbbt/util/iruby.rb +20 -0
  70. data/lib/rbbt/util/log/progress/report.rb +241 -241
  71. data/lib/rbbt/util/log/progress/util.rb +99 -99
  72. data/lib/rbbt/util/log/progress.rb +102 -102
  73. data/lib/rbbt/util/log/refactor.rb +49 -0
  74. data/lib/rbbt/util/log.rb +486 -532
  75. data/lib/rbbt/util/migrate.rb +1 -1
  76. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  77. data/lib/rbbt/util/misc/development.rb +12 -11
  78. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  79. data/lib/rbbt/util/misc/format.rb +2 -230
  80. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  81. data/lib/rbbt/util/misc/inspect.rb +2 -476
  82. data/lib/rbbt/util/misc/lock.rb +109 -106
  83. data/lib/rbbt/util/misc/omics.rb +9 -1
  84. data/lib/rbbt/util/misc/pipes.rb +765 -793
  85. data/lib/rbbt/util/misc/refactor.rb +20 -0
  86. data/lib/rbbt/util/misc/ssw.rb +27 -17
  87. data/lib/rbbt/util/misc/system.rb +0 -15
  88. data/lib/rbbt/util/misc.rb +39 -20
  89. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  90. data/lib/rbbt/util/named_array.rb +3 -220
  91. data/lib/rbbt/util/open/refactor.rb +7 -0
  92. data/lib/rbbt/util/open.rb +3 -857
  93. data/lib/rbbt/util/procpath.rb +6 -6
  94. data/lib/rbbt/util/python/paths.rb +27 -0
  95. data/lib/rbbt/util/python/run.rb +115 -0
  96. data/lib/rbbt/util/python/script.rb +110 -0
  97. data/lib/rbbt/util/python/util.rb +3 -3
  98. data/lib/rbbt/util/python.rb +22 -81
  99. data/lib/rbbt/util/semaphore.rb +152 -148
  100. data/lib/rbbt/util/simpleopt.rb +9 -8
  101. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  102. data/lib/rbbt/util/ssh.rb +122 -118
  103. data/lib/rbbt/util/tar.rb +117 -115
  104. data/lib/rbbt/util/tmpfile.rb +69 -67
  105. data/lib/rbbt/util/version.rb +2 -0
  106. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  107. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  108. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  109. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  110. data/lib/rbbt/workflow/refactor/task_info.rb +65 -0
  111. data/lib/rbbt/workflow/refactor.rb +153 -0
  112. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  113. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  114. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  115. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  116. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  117. data/lib/rbbt/workflow/step/run.rb +766 -766
  118. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  119. data/lib/rbbt/workflow/step.rb +2 -362
  120. data/lib/rbbt/workflow/task.rb +118 -118
  121. data/lib/rbbt/workflow/usage.rb +289 -287
  122. data/lib/rbbt/workflow/util/archive.rb +6 -5
  123. data/lib/rbbt/workflow/util/data.rb +1 -1
  124. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  125. data/lib/rbbt/workflow/util/trace.rb +79 -44
  126. data/lib/rbbt/workflow.rb +4 -882
  127. data/lib/rbbt-util.rb +21 -13
  128. data/lib/rbbt.rb +16 -3
  129. data/python/rbbt/__init__.py +19 -1
  130. data/share/Rlib/plot.R +37 -37
  131. data/share/Rlib/svg.R +22 -5
  132. data/share/install/software/lib/install_helpers +1 -1
  133. data/share/rbbt_commands/hpc/list +2 -3
  134. data/share/rbbt_commands/hpc/orchestrate +4 -4
  135. data/share/rbbt_commands/hpc/tail +2 -0
  136. data/share/rbbt_commands/hpc/task +10 -7
  137. data/share/rbbt_commands/lsf/list +2 -3
  138. data/share/rbbt_commands/lsf/orchestrate +4 -4
  139. data/share/rbbt_commands/lsf/tail +2 -0
  140. data/share/rbbt_commands/lsf/task +10 -7
  141. data/share/rbbt_commands/migrate +1 -1
  142. data/share/rbbt_commands/pbs/list +2 -3
  143. data/share/rbbt_commands/pbs/orchestrate +4 -4
  144. data/share/rbbt_commands/pbs/tail +2 -0
  145. data/share/rbbt_commands/pbs/task +10 -7
  146. data/share/rbbt_commands/resource/produce +8 -1
  147. data/share/rbbt_commands/slurm/list +2 -3
  148. data/share/rbbt_commands/slurm/orchestrate +4 -4
  149. data/share/rbbt_commands/slurm/tail +2 -0
  150. data/share/rbbt_commands/slurm/task +10 -7
  151. data/share/rbbt_commands/system/clean +5 -5
  152. data/share/rbbt_commands/system/status +5 -5
  153. data/share/rbbt_commands/tsv/get +2 -3
  154. data/share/rbbt_commands/tsv/info +10 -13
  155. data/share/rbbt_commands/tsv/keys +18 -14
  156. data/share/rbbt_commands/tsv/slice +2 -2
  157. data/share/rbbt_commands/tsv/transpose +6 -2
  158. data/share/rbbt_commands/workflow/info +20 -24
  159. data/share/rbbt_commands/workflow/list +1 -1
  160. data/share/rbbt_commands/workflow/prov +20 -13
  161. data/share/rbbt_commands/workflow/server +11 -1
  162. data/share/rbbt_commands/workflow/task +76 -71
  163. data/share/rbbt_commands/workflow/write_info +26 -9
  164. data/share/software/opt/ssw/ssw.c +861 -0
  165. data/share/software/opt/ssw/ssw.h +130 -0
  166. data/share/workflow_config.ru +3 -3
  167. metadata +40 -2
@@ -1,207 +1,207 @@
1
- module AnnotatedArray
2
- attr_accessor :list_id
3
-
4
- def to_a
5
- self.collect{|i| i }
6
- end
7
-
8
- def double_array
9
- AnnotatedArray === self.send(:[], 0, true)
10
- end
11
-
12
- def first
13
- self[0]
14
- end
15
-
16
- def last
17
- self[-1]
18
- end
19
-
20
- def [](pos, clean = false)
21
-
22
- value = super(pos)
23
- return value if value.nil? or clean
24
-
25
- value = value.dup if value.frozen? and (String === value or Array === value)
26
-
27
- value = annotate(value)
28
-
29
- value.extend AnnotatedArray if Array === value and Annotated === value
30
-
31
- if value.respond_to? :container
32
- value.container = self
33
- value.container_index = pos
34
- end
35
-
36
- value
37
- end
38
-
39
- def each(&block)
40
-
41
- pos = 0
42
- super do |value|
43
-
44
- case value
45
- when Array
46
- value = value.dup if value.frozen?
47
-
48
- value = annotate(value)
49
-
50
- value.extend AnnotatedArray if Array === value and Annotated === value
51
-
52
- value.container = self
53
- value.container_index = pos
54
-
55
- pos += 1
56
-
57
- block.call value
58
- when String
59
-
60
- value = value.dup if value.frozen?
61
-
62
- value = annotate(value)
63
-
64
- value.container = self if value.respond_to?(:container)
65
- value.container_index = pos if value.respond_to?(:container_index)
66
-
67
- pos += 1
68
-
69
- block.call value
70
- else
71
- block.call value
72
- end
73
- end
74
- end
75
-
76
- def collect(&block)
77
-
78
- if block_given?
79
-
80
- res = []
81
- each do |value|
82
- res << yield(value)
83
- end
84
-
85
- res
86
- else
87
-
88
- res = []
89
- each do |value|
90
- res << value
91
- end
92
-
93
- res
94
- end
95
- end
96
-
97
- def +(other)
98
- self.annotate super(other)
99
- end
100
-
101
-
102
- def reject
103
- res = []
104
-
105
- each do |value|
106
- res << value unless yield(value)
107
- end
108
-
109
- annotate(res)
110
- res.extend AnnotatedArray
111
-
112
- res
113
- end
114
-
115
- def select(func_name = nil)
116
- res = []
117
-
118
- if func_name
119
- each do |elem|
120
- value = elem.send(func_name)
121
- if block_given?
122
- res << elem if yield(value)
123
- else
124
- res << elem if value
125
- end
126
- end
127
- else
128
- each do |elem|
129
- res << elem if yield(elem)
130
- end
131
- end
132
-
133
- annotate(res)
134
- res.extend AnnotatedArray
135
-
136
- res
137
- end
138
-
139
- def subset(list)
140
-
141
- res = (self & list)
142
-
143
- annotate(res)
144
- res.extend AnnotatedArray
145
-
146
- res
147
- end
148
-
149
- def remove(list)
150
-
151
- res = (self - list)
152
-
153
- annotate(res)
154
- res.extend AnnotatedArray
155
-
156
- res
157
- end
158
-
159
- def sort(&block)
160
- res = self.collect.sort(&block).collect{|value| value.respond_to?(:clean_annotations) ? value.clean_annotations.dup : value.dup }
161
-
162
- annotate(res)
163
- res.extend AnnotatedArray
164
-
165
- res
166
- end
167
-
168
- def select_by(method, *args, &block)
169
- return [] if self.empty? and not self.respond_to? :annotate
170
-
171
- values = self.send(method, *args)
172
- values = values.clean_annotations if values.respond_to? :clean_annotations
173
-
174
- new = []
175
- if block_given?
176
- self.clean_annotations.each_with_index do |e,i|
177
- new << e if yield(values[i])
178
- end
179
- else
180
- self.clean_annotations.each_with_index do |e,i|
181
- new << e if values[i]
182
- end
183
- end
184
- self.annotate(new)
185
- new.extend AnnotatedArray
186
-
187
- new
188
- end
189
-
190
- %w(compact uniq flatten reverse sort_by).each do |method|
191
-
192
- self.module_eval <<-EOC
193
-
194
- def #{method}
195
- res = super
196
-
197
- annotate(res)
198
- res.extend AnnotatedArray
199
-
200
- res
201
- end
202
-
203
- EOC
204
- end
205
- end
206
-
207
-
1
+ #module AnnotatedArray
2
+ # attr_accessor :list_id
3
+ #
4
+ # def to_a
5
+ # self.collect{|i| i }
6
+ # end
7
+ #
8
+ # def double_array
9
+ # AnnotatedArray === self.send(:[], 0, true)
10
+ # end
11
+ #
12
+ # def first
13
+ # self[0]
14
+ # end
15
+ #
16
+ # def last
17
+ # self[-1]
18
+ # end
19
+ #
20
+ # def [](pos, clean = false)
21
+ #
22
+ # value = super(pos)
23
+ # return value if value.nil? or clean
24
+ #
25
+ # value = value.dup if value.frozen? and (String === value or Array === value)
26
+ #
27
+ # value = annotate(value)
28
+ #
29
+ # value.extend AnnotatedArray if Array === value and Annotated === value
30
+ #
31
+ # if value.respond_to? :container
32
+ # value.container = self
33
+ # value.container_index = pos
34
+ # end
35
+ #
36
+ # value
37
+ # end
38
+ #
39
+ # def each(&block)
40
+ #
41
+ # pos = 0
42
+ # super do |value|
43
+ #
44
+ # case value
45
+ # when Array
46
+ # value = value.dup if value.frozen?
47
+ #
48
+ # value = annotate(value)
49
+ #
50
+ # value.extend AnnotatedArray if Array === value and Annotated === value
51
+ #
52
+ # value.container = self
53
+ # value.container_index = pos
54
+ #
55
+ # pos += 1
56
+ #
57
+ # block.call value
58
+ # when String
59
+ #
60
+ # value = value.dup if value.frozen?
61
+ #
62
+ # value = annotate(value)
63
+ #
64
+ # value.container = self if value.respond_to?(:container)
65
+ # value.container_index = pos if value.respond_to?(:container_index)
66
+ #
67
+ # pos += 1
68
+ #
69
+ # block.call value
70
+ # else
71
+ # block.call value
72
+ # end
73
+ # end
74
+ # end
75
+ #
76
+ # def collect(&block)
77
+ #
78
+ # if block_given?
79
+ #
80
+ # res = []
81
+ # each do |value|
82
+ # res << yield(value)
83
+ # end
84
+ #
85
+ # res
86
+ # else
87
+ #
88
+ # res = []
89
+ # each do |value|
90
+ # res << value
91
+ # end
92
+ #
93
+ # res
94
+ # end
95
+ # end
96
+ #
97
+ # def +(other)
98
+ # self.annotate super(other)
99
+ # end
100
+ #
101
+ #
102
+ # def reject
103
+ # res = []
104
+ #
105
+ # each do |value|
106
+ # res << value unless yield(value)
107
+ # end
108
+ #
109
+ # annotate(res)
110
+ # res.extend AnnotatedArray
111
+ #
112
+ # res
113
+ # end
114
+ #
115
+ # def select(func_name = nil)
116
+ # res = []
117
+ #
118
+ # if func_name
119
+ # each do |elem|
120
+ # value = elem.send(func_name)
121
+ # if block_given?
122
+ # res << elem if yield(value)
123
+ # else
124
+ # res << elem if value
125
+ # end
126
+ # end
127
+ # else
128
+ # each do |elem|
129
+ # res << elem if yield(elem)
130
+ # end
131
+ # end
132
+ #
133
+ # annotate(res)
134
+ # res.extend AnnotatedArray
135
+ #
136
+ # res
137
+ # end
138
+ #
139
+ # def subset(list)
140
+ #
141
+ # res = (self & list)
142
+ #
143
+ # annotate(res)
144
+ # res.extend AnnotatedArray
145
+ #
146
+ # res
147
+ # end
148
+ #
149
+ # def remove(list)
150
+ #
151
+ # res = (self - list)
152
+ #
153
+ # annotate(res)
154
+ # res.extend AnnotatedArray
155
+ #
156
+ # res
157
+ # end
158
+ #
159
+ # def sort(&block)
160
+ # res = self.collect.sort(&block).collect{|value| value.respond_to?(:clean_annotations) ? value.clean_annotations.dup : value.dup }
161
+ #
162
+ # annotate(res)
163
+ # res.extend AnnotatedArray
164
+ #
165
+ # res
166
+ # end
167
+ #
168
+ # def select_by(method, *args, &block)
169
+ # return [] if self.empty? and not self.respond_to? :annotate
170
+ #
171
+ # values = self.send(method, *args)
172
+ # values = values.clean_annotations if values.respond_to? :clean_annotations
173
+ #
174
+ # new = []
175
+ # if block_given?
176
+ # self.clean_annotations.each_with_index do |e,i|
177
+ # new << e if yield(values[i])
178
+ # end
179
+ # else
180
+ # self.clean_annotations.each_with_index do |e,i|
181
+ # new << e if values[i]
182
+ # end
183
+ # end
184
+ # self.annotate(new)
185
+ # new.extend AnnotatedArray
186
+ #
187
+ # new
188
+ # end
189
+ #
190
+ # %w(compact uniq flatten reverse sort_by).each do |method|
191
+ #
192
+ # self.module_eval <<-EOC
193
+ #
194
+ # def #{method}
195
+ # res = super
196
+ #
197
+ # annotate(res)
198
+ # res.extend AnnotatedArray
199
+ #
200
+ # res
201
+ # end
202
+ #
203
+ # EOC
204
+ # end
205
+ #end
206
+ #
207
+ #
@@ -0,0 +1,27 @@
1
+ require_relative '../refactor'
2
+ module Annotation
3
+ module AnnotatedObject
4
+ alias id annotation_id
5
+ alias clean_annotations purge
6
+ alias annotation_values annotation_hash
7
+ def info
8
+ annotation_hash.merge(:annotation_types => annotation_types, :annotated_array => (Array === self))
9
+ end
10
+ end
11
+ end
12
+
13
+ Annotated = Annotation::AnnotatedObject
14
+
15
+ module Annotation::AnnotatedObject
16
+ alias make_list make_array
17
+ end
18
+
19
+ module Annotation
20
+ def self.load_entity(obj, info)
21
+ self.setup(obj, info[:annotation_types], info)
22
+ end
23
+ end
24
+
25
+ Rbbt.relay_module_method Annotated, :tsv, Annotation, :tsv
26
+ Rbbt.relay_module_method Annotated, :load_tsv, Annotation, :load_tsv
27
+ Rbbt.relay_module_method Annotated, :load_entity, Annotation, :load_entity