rbbt-util 5.44.1 → 6.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/bin/rbbt +67 -90
  4. data/bin/rbbt_exec.rb +2 -2
  5. data/etc/app.d/base.rb +2 -2
  6. data/etc/app.d/semaphores.rb +3 -3
  7. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  8. data/lib/rbbt/annotations/refactor.rb +27 -0
  9. data/lib/rbbt/annotations/util.rb +282 -282
  10. data/lib/rbbt/annotations.rb +343 -320
  11. data/lib/rbbt/association/database.rb +200 -225
  12. data/lib/rbbt/association/index.rb +294 -291
  13. data/lib/rbbt/association/item.rb +227 -227
  14. data/lib/rbbt/association/open.rb +35 -34
  15. data/lib/rbbt/association/util.rb +0 -169
  16. data/lib/rbbt/association.rb +2 -4
  17. data/lib/rbbt/entity/identifiers.rb +119 -118
  18. data/lib/rbbt/entity/refactor.rb +12 -0
  19. data/lib/rbbt/entity.rb +319 -315
  20. data/lib/rbbt/hpc/batch.rb +72 -53
  21. data/lib/rbbt/hpc/lsf.rb +2 -2
  22. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  24. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  25. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  26. data/lib/rbbt/hpc/slurm.rb +18 -18
  27. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  28. data/lib/rbbt/knowledge_base/query.rb +2 -2
  29. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  30. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  31. data/lib/rbbt/knowledge_base.rb +1 -1
  32. data/lib/rbbt/monitor.rb +36 -25
  33. data/lib/rbbt/persist/refactor.rb +166 -0
  34. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  35. data/lib/rbbt/persist/tsv.rb +187 -185
  36. data/lib/rbbt/persist.rb +556 -551
  37. data/lib/rbbt/refactor.rb +20 -0
  38. data/lib/rbbt/resource/path/refactor.rb +178 -0
  39. data/lib/rbbt/resource/path.rb +317 -497
  40. data/lib/rbbt/resource/util.rb +0 -48
  41. data/lib/rbbt/resource.rb +3 -390
  42. data/lib/rbbt/tsv/accessor.rb +2 -838
  43. data/lib/rbbt/tsv/attach.rb +303 -299
  44. data/lib/rbbt/tsv/change_id.rb +244 -245
  45. data/lib/rbbt/tsv/csv.rb +87 -85
  46. data/lib/rbbt/tsv/dumper.rb +2 -100
  47. data/lib/rbbt/tsv/excel.rb +26 -24
  48. data/lib/rbbt/tsv/field_index.rb +4 -1
  49. data/lib/rbbt/tsv/filter.rb +3 -2
  50. data/lib/rbbt/tsv/index.rb +2 -284
  51. data/lib/rbbt/tsv/manipulate.rb +750 -747
  52. data/lib/rbbt/tsv/marshal.rb +3 -3
  53. data/lib/rbbt/tsv/matrix.rb +2 -2
  54. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  55. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  56. data/lib/rbbt/tsv/parser.rb +678 -678
  57. data/lib/rbbt/tsv/refactor.rb +195 -0
  58. data/lib/rbbt/tsv/stream.rb +253 -251
  59. data/lib/rbbt/tsv/util.rb +420 -420
  60. data/lib/rbbt/tsv.rb +210 -208
  61. data/lib/rbbt/util/R/eval.rb +4 -4
  62. data/lib/rbbt/util/R/plot.rb +62 -166
  63. data/lib/rbbt/util/R.rb +21 -18
  64. data/lib/rbbt/util/cmd.rb +2 -318
  65. data/lib/rbbt/util/color.rb +269 -269
  66. data/lib/rbbt/util/colorize.rb +89 -89
  67. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  68. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  69. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  70. data/lib/rbbt/util/config.rb +169 -167
  71. data/lib/rbbt/util/filecache.rb +1 -1
  72. data/lib/rbbt/util/iruby.rb +20 -0
  73. data/lib/rbbt/util/log/progress/report.rb +241 -241
  74. data/lib/rbbt/util/log/progress/util.rb +99 -99
  75. data/lib/rbbt/util/log/progress.rb +102 -102
  76. data/lib/rbbt/util/log/refactor.rb +49 -0
  77. data/lib/rbbt/util/log.rb +486 -532
  78. data/lib/rbbt/util/migrate.rb +2 -2
  79. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  80. data/lib/rbbt/util/misc/development.rb +12 -11
  81. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  82. data/lib/rbbt/util/misc/format.rb +2 -230
  83. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  84. data/lib/rbbt/util/misc/inspect.rb +2 -476
  85. data/lib/rbbt/util/misc/lock.rb +109 -106
  86. data/lib/rbbt/util/misc/omics.rb +9 -1
  87. data/lib/rbbt/util/misc/pipes.rb +765 -793
  88. data/lib/rbbt/util/misc/refactor.rb +20 -0
  89. data/lib/rbbt/util/misc/ssw.rb +27 -17
  90. data/lib/rbbt/util/misc/system.rb +92 -105
  91. data/lib/rbbt/util/misc.rb +39 -20
  92. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  93. data/lib/rbbt/util/named_array.rb +3 -220
  94. data/lib/rbbt/util/open/refactor.rb +7 -0
  95. data/lib/rbbt/util/open.rb +3 -857
  96. data/lib/rbbt/util/procpath.rb +6 -6
  97. data/lib/rbbt/util/python/paths.rb +27 -0
  98. data/lib/rbbt/util/python/run.rb +115 -0
  99. data/lib/rbbt/util/python/script.rb +110 -0
  100. data/lib/rbbt/util/python/util.rb +3 -3
  101. data/lib/rbbt/util/python.rb +22 -81
  102. data/lib/rbbt/util/semaphore.rb +152 -148
  103. data/lib/rbbt/util/simpleopt.rb +9 -8
  104. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  105. data/lib/rbbt/util/ssh.rb +122 -118
  106. data/lib/rbbt/util/tar.rb +117 -115
  107. data/lib/rbbt/util/tmpfile.rb +69 -67
  108. data/lib/rbbt/util/version.rb +2 -0
  109. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  110. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  111. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  112. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  113. data/lib/rbbt/workflow/refactor/task_info.rb +66 -0
  114. data/lib/rbbt/workflow/refactor.rb +150 -0
  115. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +1 -2
  116. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  117. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  118. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  119. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  120. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  121. data/lib/rbbt/workflow/step/run.rb +766 -766
  122. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  123. data/lib/rbbt/workflow/step.rb +2 -362
  124. data/lib/rbbt/workflow/task.rb +118 -118
  125. data/lib/rbbt/workflow/usage.rb +289 -287
  126. data/lib/rbbt/workflow/util/archive.rb +6 -5
  127. data/lib/rbbt/workflow/util/data.rb +1 -1
  128. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  129. data/lib/rbbt/workflow/util/trace.rb +79 -44
  130. data/lib/rbbt/workflow.rb +4 -882
  131. data/lib/rbbt-util.rb +21 -13
  132. data/lib/rbbt.rb +16 -3
  133. data/python/rbbt/__init__.py +96 -4
  134. data/python/rbbt/workflow/remote.py +104 -0
  135. data/python/rbbt/workflow.py +64 -0
  136. data/python/test.py +10 -0
  137. data/share/Rlib/plot.R +37 -37
  138. data/share/Rlib/svg.R +22 -5
  139. data/share/install/software/lib/install_helpers +1 -1
  140. data/share/rbbt_commands/hpc/list +2 -3
  141. data/share/rbbt_commands/hpc/orchestrate +4 -4
  142. data/share/rbbt_commands/hpc/tail +2 -0
  143. data/share/rbbt_commands/hpc/task +10 -7
  144. data/share/rbbt_commands/lsf/list +2 -3
  145. data/share/rbbt_commands/lsf/orchestrate +4 -4
  146. data/share/rbbt_commands/lsf/tail +2 -0
  147. data/share/rbbt_commands/lsf/task +10 -7
  148. data/share/rbbt_commands/migrate +1 -1
  149. data/share/rbbt_commands/pbs/list +2 -3
  150. data/share/rbbt_commands/pbs/orchestrate +4 -4
  151. data/share/rbbt_commands/pbs/tail +2 -0
  152. data/share/rbbt_commands/pbs/task +10 -7
  153. data/share/rbbt_commands/resource/produce +8 -1
  154. data/share/rbbt_commands/slurm/list +2 -3
  155. data/share/rbbt_commands/slurm/orchestrate +4 -4
  156. data/share/rbbt_commands/slurm/tail +2 -0
  157. data/share/rbbt_commands/slurm/task +10 -7
  158. data/share/rbbt_commands/system/clean +5 -5
  159. data/share/rbbt_commands/system/status +5 -5
  160. data/share/rbbt_commands/tsv/get +2 -3
  161. data/share/rbbt_commands/tsv/info +10 -13
  162. data/share/rbbt_commands/tsv/keys +18 -14
  163. data/share/rbbt_commands/tsv/slice +2 -2
  164. data/share/rbbt_commands/tsv/transpose +6 -2
  165. data/share/rbbt_commands/workflow/info +20 -24
  166. data/share/rbbt_commands/workflow/list +1 -1
  167. data/share/rbbt_commands/workflow/prov +20 -13
  168. data/share/rbbt_commands/workflow/retry +43 -0
  169. data/share/rbbt_commands/workflow/server +12 -2
  170. data/share/rbbt_commands/workflow/task +80 -73
  171. data/share/rbbt_commands/workflow/write_info +26 -9
  172. data/share/software/opt/ssw/ssw.c +861 -0
  173. data/share/software/opt/ssw/ssw.h +130 -0
  174. data/share/workflow_config.ru +3 -3
  175. metadata +45 -6
@@ -0,0 +1,20 @@
1
+ require 'scout/log'
2
+ require 'scout/exceptions'
3
+
4
+ module Rbbt
5
+ def self.require_instead(new_file)
6
+ Log.low "Requiring #{new_file} instead of #{caller.first}"
7
+ require new_file
8
+ end
9
+
10
+ def self.relay_module_method(new_mod, new_method, orig_mod, orig_method = nil)
11
+ orig_method = new_method if orig_method.nil?
12
+ method = orig_mod.method(orig_method)
13
+ class << new_mod
14
+ self
15
+ end.define_method(new_method, &method)
16
+ end
17
+ end
18
+
19
+ require 'rbbt/util/concurrency/processes/refactor'
20
+ require 'rbbt/util/named_array/refactor'
@@ -0,0 +1,178 @@
1
+ module Path
2
+
3
+ #def _exists?
4
+ # Open.exists? self.find
5
+ #end
6
+
7
+ #def exists?
8
+ # begin
9
+ # self.produce
10
+ # _exists?
11
+ # rescue Exception
12
+ # false
13
+ # end
14
+ #end
15
+
16
+ #def open(options = {}, &block)
17
+ # file = Open.remote?(self) || Open.ssh?(self) ? self : self.produce.find
18
+ # Open.open(file, options, &block)
19
+ #end
20
+
21
+ #def read(&block)
22
+ # Open.read(self.produce.find, &block)
23
+ #end
24
+
25
+ #def write(*args, &block)
26
+ # Open.write(self.find(:default), *args, &block)
27
+ #end
28
+
29
+ #def index(options = {})
30
+ # TSV.index(self.produce, **options)
31
+ #end
32
+
33
+ def basename
34
+ Path.setup(File.basename(self), self.resource, self.pkgdir)
35
+ end
36
+
37
+ #def tsv(*args)
38
+ # begin
39
+ # path = self.produce
40
+ # rescue ResourceNotFound => e
41
+ # begin
42
+ # path = self.set_extension('tsv').produce
43
+ # rescue ResourceNotFound
44
+ # raise e
45
+ # end
46
+ # end
47
+ # TSV.open(path, *args)
48
+ #end
49
+
50
+ #def tsv_options(options = {})
51
+ # self.open do |stream|
52
+ # TSV::Parser.new(stream, options).options
53
+ # end
54
+ #end
55
+
56
+ def traverse(options = {}, &block)
57
+ TSV::Parser.traverse(self.open, options, &block)
58
+ end
59
+
60
+ def keys(field = 0, sep = "\t")
61
+ Open.read(self.produce.find).split("\n").collect{|l| next if l =~ /^#/; l.split(sep, -1)[field]}.compact
62
+ end
63
+
64
+ def yaml
65
+ self.open do |f|
66
+ YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(f) : YAML.load(f)
67
+ end
68
+ end
69
+
70
+ def pipe_to(cmd, options = {})
71
+ CMD.cmd(cmd, {:in => self.open, :pipe => true}.merge(options))
72
+ end
73
+
74
+ def range_index(start, eend, options = {})
75
+ TSV.range_index(self.produce, start, eend, options)
76
+ end
77
+
78
+ def pos_index(pos, options = {})
79
+ TSV.pos_index(self.produce, pos, options)
80
+ end
81
+
82
+ def to_yaml(*args)
83
+ self.to_s.to_yaml(*args)
84
+ end
85
+
86
+ def fields
87
+ TSV.parse_header(self.open).fields
88
+ end
89
+
90
+ def all_fields
91
+ self.open do |stream|
92
+ TSV.parse_header(stream)["all_fields"]
93
+ end
94
+ end
95
+
96
+ def identifier_file_path
97
+ if self.dirname.identifiers.exists?
98
+ self.dirname.identifiers
99
+ else
100
+ nil
101
+ end
102
+ end
103
+
104
+ def identifier_files
105
+ if identifier_file_path.nil?
106
+ []
107
+ else
108
+ [identifier_file_path]
109
+ end
110
+ end
111
+
112
+ def set_extension(new_extension = nil)
113
+ new_path = self + "." + new_extension.to_s
114
+ self.annotate(new_path)
115
+ end
116
+
117
+ def remove_extension(new_extension = nil)
118
+ self.sub(/\.[^\.\/]{1,5}$/,'')
119
+ end
120
+
121
+
122
+ def self.get_extension(path)
123
+ path.match(/\.([^\.\/]{1,5})$/)[1]
124
+ end
125
+
126
+ def replace_extension(new_extension = nil, multiple = false)
127
+ if String === multiple
128
+ new_path = self.sub(/(\.[^\.\/]{1,5})(.#{multiple})?$/,'')
129
+ elsif multiple
130
+ new_path = self.sub(/(\.[^\.\/]{1,5})+$/,'')
131
+ else
132
+ new_path = self.sub(/\.[^\.\/]{1,5}$/,'')
133
+ end
134
+ new_path = new_path + "." + new_extension.to_s
135
+ self.annotate(new_path)
136
+ end
137
+
138
+ def doc_file(relative_to = 'lib')
139
+ if located?
140
+ lib_dir = Path.caller_lib_dir(self, relative_to)
141
+ relative_file = File.join( 'doc', self.sub(lib_dir,''))
142
+ Path.setup File.join(lib_dir, relative_file) , @pkgdir, @resource
143
+ else
144
+ Path.setup File.join('doc', self) , @pkgdir, @resource
145
+ end
146
+ end
147
+
148
+ def source_for_doc_file(relative_to = 'lib')
149
+ if located?
150
+ lib_dir = Path.caller_lib_dir(Path.caller_lib_dir(self, 'doc'), relative_to)
151
+ relative_file = self.sub(/(.*\/)doc\//, '\1').sub(lib_dir + "/",'')
152
+ file = File.join(lib_dir, relative_file)
153
+
154
+ if not File.exist?(file)
155
+ file= Dir.glob(file.sub(/\.[^\.\/]+$/, '.*')).first
156
+ end
157
+
158
+ Path.setup file, @pkgdir, @resource
159
+ else
160
+ relative_file = self.sub(/^doc\//, '\1')
161
+
162
+ if not File.exist?(relative_file)
163
+ relative_file = Dir.glob(relative_file.sub(/\.[^\.\/]+$/, '.*')).first
164
+ end
165
+
166
+ Path.setup relative_file , @pkgdir, @resource
167
+ end
168
+ end
169
+
170
+ #def prepend_search_path(name, dir)
171
+ # dir = File.join(dir, '{PATH}')
172
+ # new = {name => dir}
173
+ # path_maps.each{|n,d| new[n] = d}
174
+ # self.path_maps = new
175
+ #end
176
+
177
+
178
+ end