rbbt-util 5.44.1 → 6.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/bin/rbbt +67 -90
  4. data/bin/rbbt_exec.rb +2 -2
  5. data/etc/app.d/base.rb +2 -2
  6. data/etc/app.d/semaphores.rb +3 -3
  7. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  8. data/lib/rbbt/annotations/refactor.rb +27 -0
  9. data/lib/rbbt/annotations/util.rb +282 -282
  10. data/lib/rbbt/annotations.rb +343 -320
  11. data/lib/rbbt/association/database.rb +200 -225
  12. data/lib/rbbt/association/index.rb +294 -291
  13. data/lib/rbbt/association/item.rb +227 -227
  14. data/lib/rbbt/association/open.rb +35 -34
  15. data/lib/rbbt/association/util.rb +0 -169
  16. data/lib/rbbt/association.rb +2 -4
  17. data/lib/rbbt/entity/identifiers.rb +119 -118
  18. data/lib/rbbt/entity/refactor.rb +12 -0
  19. data/lib/rbbt/entity.rb +319 -315
  20. data/lib/rbbt/hpc/batch.rb +72 -53
  21. data/lib/rbbt/hpc/lsf.rb +2 -2
  22. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  24. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  25. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  26. data/lib/rbbt/hpc/slurm.rb +18 -18
  27. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  28. data/lib/rbbt/knowledge_base/query.rb +2 -2
  29. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  30. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  31. data/lib/rbbt/knowledge_base.rb +1 -1
  32. data/lib/rbbt/monitor.rb +36 -25
  33. data/lib/rbbt/persist/refactor.rb +166 -0
  34. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  35. data/lib/rbbt/persist/tsv.rb +187 -185
  36. data/lib/rbbt/persist.rb +556 -551
  37. data/lib/rbbt/refactor.rb +20 -0
  38. data/lib/rbbt/resource/path/refactor.rb +178 -0
  39. data/lib/rbbt/resource/path.rb +317 -497
  40. data/lib/rbbt/resource/util.rb +0 -48
  41. data/lib/rbbt/resource.rb +3 -390
  42. data/lib/rbbt/tsv/accessor.rb +2 -838
  43. data/lib/rbbt/tsv/attach.rb +303 -299
  44. data/lib/rbbt/tsv/change_id.rb +244 -245
  45. data/lib/rbbt/tsv/csv.rb +87 -85
  46. data/lib/rbbt/tsv/dumper.rb +2 -100
  47. data/lib/rbbt/tsv/excel.rb +26 -24
  48. data/lib/rbbt/tsv/field_index.rb +4 -1
  49. data/lib/rbbt/tsv/filter.rb +3 -2
  50. data/lib/rbbt/tsv/index.rb +2 -284
  51. data/lib/rbbt/tsv/manipulate.rb +750 -747
  52. data/lib/rbbt/tsv/marshal.rb +3 -3
  53. data/lib/rbbt/tsv/matrix.rb +2 -2
  54. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  55. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  56. data/lib/rbbt/tsv/parser.rb +678 -678
  57. data/lib/rbbt/tsv/refactor.rb +195 -0
  58. data/lib/rbbt/tsv/stream.rb +253 -251
  59. data/lib/rbbt/tsv/util.rb +420 -420
  60. data/lib/rbbt/tsv.rb +210 -208
  61. data/lib/rbbt/util/R/eval.rb +4 -4
  62. data/lib/rbbt/util/R/plot.rb +62 -166
  63. data/lib/rbbt/util/R.rb +21 -18
  64. data/lib/rbbt/util/cmd.rb +2 -318
  65. data/lib/rbbt/util/color.rb +269 -269
  66. data/lib/rbbt/util/colorize.rb +89 -89
  67. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  68. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  69. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  70. data/lib/rbbt/util/config.rb +169 -167
  71. data/lib/rbbt/util/filecache.rb +1 -1
  72. data/lib/rbbt/util/iruby.rb +20 -0
  73. data/lib/rbbt/util/log/progress/report.rb +241 -241
  74. data/lib/rbbt/util/log/progress/util.rb +99 -99
  75. data/lib/rbbt/util/log/progress.rb +102 -102
  76. data/lib/rbbt/util/log/refactor.rb +49 -0
  77. data/lib/rbbt/util/log.rb +486 -532
  78. data/lib/rbbt/util/migrate.rb +2 -2
  79. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  80. data/lib/rbbt/util/misc/development.rb +12 -11
  81. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  82. data/lib/rbbt/util/misc/format.rb +2 -230
  83. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  84. data/lib/rbbt/util/misc/inspect.rb +2 -476
  85. data/lib/rbbt/util/misc/lock.rb +109 -106
  86. data/lib/rbbt/util/misc/omics.rb +9 -1
  87. data/lib/rbbt/util/misc/pipes.rb +765 -793
  88. data/lib/rbbt/util/misc/refactor.rb +20 -0
  89. data/lib/rbbt/util/misc/ssw.rb +27 -17
  90. data/lib/rbbt/util/misc/system.rb +92 -105
  91. data/lib/rbbt/util/misc.rb +39 -20
  92. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  93. data/lib/rbbt/util/named_array.rb +3 -220
  94. data/lib/rbbt/util/open/refactor.rb +7 -0
  95. data/lib/rbbt/util/open.rb +3 -857
  96. data/lib/rbbt/util/procpath.rb +6 -6
  97. data/lib/rbbt/util/python/paths.rb +27 -0
  98. data/lib/rbbt/util/python/run.rb +115 -0
  99. data/lib/rbbt/util/python/script.rb +110 -0
  100. data/lib/rbbt/util/python/util.rb +3 -3
  101. data/lib/rbbt/util/python.rb +22 -81
  102. data/lib/rbbt/util/semaphore.rb +152 -148
  103. data/lib/rbbt/util/simpleopt.rb +9 -8
  104. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  105. data/lib/rbbt/util/ssh.rb +122 -118
  106. data/lib/rbbt/util/tar.rb +117 -115
  107. data/lib/rbbt/util/tmpfile.rb +69 -67
  108. data/lib/rbbt/util/version.rb +2 -0
  109. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  110. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  111. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  112. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  113. data/lib/rbbt/workflow/refactor/task_info.rb +66 -0
  114. data/lib/rbbt/workflow/refactor.rb +150 -0
  115. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +1 -2
  116. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  117. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  118. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  119. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  120. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  121. data/lib/rbbt/workflow/step/run.rb +766 -766
  122. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  123. data/lib/rbbt/workflow/step.rb +2 -362
  124. data/lib/rbbt/workflow/task.rb +118 -118
  125. data/lib/rbbt/workflow/usage.rb +289 -287
  126. data/lib/rbbt/workflow/util/archive.rb +6 -5
  127. data/lib/rbbt/workflow/util/data.rb +1 -1
  128. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  129. data/lib/rbbt/workflow/util/trace.rb +79 -44
  130. data/lib/rbbt/workflow.rb +4 -882
  131. data/lib/rbbt-util.rb +21 -13
  132. data/lib/rbbt.rb +16 -3
  133. data/python/rbbt/__init__.py +96 -4
  134. data/python/rbbt/workflow/remote.py +104 -0
  135. data/python/rbbt/workflow.py +64 -0
  136. data/python/test.py +10 -0
  137. data/share/Rlib/plot.R +37 -37
  138. data/share/Rlib/svg.R +22 -5
  139. data/share/install/software/lib/install_helpers +1 -1
  140. data/share/rbbt_commands/hpc/list +2 -3
  141. data/share/rbbt_commands/hpc/orchestrate +4 -4
  142. data/share/rbbt_commands/hpc/tail +2 -0
  143. data/share/rbbt_commands/hpc/task +10 -7
  144. data/share/rbbt_commands/lsf/list +2 -3
  145. data/share/rbbt_commands/lsf/orchestrate +4 -4
  146. data/share/rbbt_commands/lsf/tail +2 -0
  147. data/share/rbbt_commands/lsf/task +10 -7
  148. data/share/rbbt_commands/migrate +1 -1
  149. data/share/rbbt_commands/pbs/list +2 -3
  150. data/share/rbbt_commands/pbs/orchestrate +4 -4
  151. data/share/rbbt_commands/pbs/tail +2 -0
  152. data/share/rbbt_commands/pbs/task +10 -7
  153. data/share/rbbt_commands/resource/produce +8 -1
  154. data/share/rbbt_commands/slurm/list +2 -3
  155. data/share/rbbt_commands/slurm/orchestrate +4 -4
  156. data/share/rbbt_commands/slurm/tail +2 -0
  157. data/share/rbbt_commands/slurm/task +10 -7
  158. data/share/rbbt_commands/system/clean +5 -5
  159. data/share/rbbt_commands/system/status +5 -5
  160. data/share/rbbt_commands/tsv/get +2 -3
  161. data/share/rbbt_commands/tsv/info +10 -13
  162. data/share/rbbt_commands/tsv/keys +18 -14
  163. data/share/rbbt_commands/tsv/slice +2 -2
  164. data/share/rbbt_commands/tsv/transpose +6 -2
  165. data/share/rbbt_commands/workflow/info +20 -24
  166. data/share/rbbt_commands/workflow/list +1 -1
  167. data/share/rbbt_commands/workflow/prov +20 -13
  168. data/share/rbbt_commands/workflow/retry +43 -0
  169. data/share/rbbt_commands/workflow/server +12 -2
  170. data/share/rbbt_commands/workflow/task +80 -73
  171. data/share/rbbt_commands/workflow/write_info +26 -9
  172. data/share/software/opt/ssw/ssw.c +861 -0
  173. data/share/software/opt/ssw/ssw.h +130 -0
  174. data/share/workflow_config.ru +3 -3
  175. metadata +45 -6
@@ -1,89 +1,89 @@
1
- require 'rbbt/util/color'
2
-
3
- module Colorize
4
- def self.from_name(color)
5
- return color if color =~ /^#?[0-9A-F]+$/i
6
- colors = Rbbt.share.color.color_names.tsv :type => :single
7
- case color.to_s
8
- when "white"
9
- '#000'
10
- when "black"
11
- '#fff'
12
- when 'green'
13
- colors["green3"]
14
- when 'red'
15
- colors["red3"]
16
- when 'yellow'
17
- colors["gold1"]
18
- when 'blue'
19
- colors["RoyalBlue"]
20
- else
21
- colors[color.to_s] || color
22
- end
23
- end
24
-
25
- def self.continuous(array, start = "#40324F", eend = "#EABD5D", percent = false)
26
- start_color = Color.new from_name(start)
27
- end_color = Color.new from_name(eend)
28
-
29
- if percent
30
- array = array.collect{|v| n = v.to_f; n = n > 100 ? 100 : n; n < 0.001 ? 0.001 : n}
31
- else
32
- array = array.collect{|v| n = v.to_f; }
33
- end
34
- max = array.max
35
- min = array.min
36
- range = max - min
37
- array.collect do |v|
38
- ratio = (v.to_f-min) / range
39
- start_color.blend end_color, ratio
40
- end
41
- end
42
-
43
- def self.gradient(array, value, start = :green, eend = :red, percent = false)
44
- index = array.index value
45
- colors = continuous(array, start, eend, percent)
46
- colors[index]
47
- end
48
-
49
- def self.rank_gradient(array, value, start = :green, eend = :red, percent = false)
50
- index = array.index value
51
- sorted = array.sort
52
- array = array.collect{|e| sorted.index e}
53
- colors = continuous(array, start, eend, percent)
54
- colors[index]
55
- end
56
-
57
-
58
- def self.distinct(array)
59
- colors = Rbbt.share.color["diverging_colors.hex"].list.collect{|c| Color.new c}
60
-
61
- num = array.uniq.length
62
- times = num / 12
63
-
64
- all_colors = colors.dup
65
- factor = 0.3 / times
66
- times.times do
67
- all_colors.concat colors.collect{|n| n.darken(factor) }
68
- end
69
-
70
- value_color = Hash[*array.uniq.zip(all_colors).flatten]
71
-
72
- value_color.values_at *array
73
- end
74
-
75
- def self.tsv(tsv, options = {})
76
- values = tsv.values.flatten
77
- if Numeric === values.first or (values.first.to_f != 0 and values[0] != "0" and values[0] != "0.0")
78
- value_colors = Misc.process_to_hash(values){continuous(values)}
79
- else
80
- value_colors = Misc.process_to_hash(values){distinct(values)}
81
- end
82
-
83
- if tsv.type == :single
84
- Hash[*tsv.keys.zip(value_colors.values_at(*values)).flatten]
85
- else
86
- Hash[*tsv.keys.zip(values.collect{|vs| value_colors.values_at(*vs)}).flatten]
87
- end
88
- end
89
- end
1
+ #require 'rbbt/util/color'
2
+ #
3
+ #module Colorize
4
+ # def self.from_name(color)
5
+ # return color if color =~ /^#?[0-9A-F]+$/i
6
+ # colors = Rbbt.share.color.color_names.tsv :type => :single
7
+ # case color.to_s
8
+ # when "white"
9
+ # '#000'
10
+ # when "black"
11
+ # '#fff'
12
+ # when 'green'
13
+ # colors["green3"]
14
+ # when 'red'
15
+ # colors["red3"]
16
+ # when 'yellow'
17
+ # colors["gold1"]
18
+ # when 'blue'
19
+ # colors["RoyalBlue"]
20
+ # else
21
+ # colors[color.to_s] || color
22
+ # end
23
+ # end
24
+ #
25
+ # def self.continuous(array, start = "#40324F", eend = "#EABD5D", percent = false)
26
+ # start_color = Color.new from_name(start)
27
+ # end_color = Color.new from_name(eend)
28
+ #
29
+ # if percent
30
+ # array = array.collect{|v| n = v.to_f; n = n > 100 ? 100 : n; n < 0.001 ? 0.001 : n}
31
+ # else
32
+ # array = array.collect{|v| n = v.to_f; }
33
+ # end
34
+ # max = array.max
35
+ # min = array.min
36
+ # range = max - min
37
+ # array.collect do |v|
38
+ # ratio = (v.to_f-min) / range
39
+ # start_color.blend end_color, ratio
40
+ # end
41
+ # end
42
+ #
43
+ # def self.gradient(array, value, start = :green, eend = :red, percent = false)
44
+ # index = array.index value
45
+ # colors = continuous(array, start, eend, percent)
46
+ # colors[index]
47
+ # end
48
+ #
49
+ # def self.rank_gradient(array, value, start = :green, eend = :red, percent = false)
50
+ # index = array.index value
51
+ # sorted = array.sort
52
+ # array = array.collect{|e| sorted.index e}
53
+ # colors = continuous(array, start, eend, percent)
54
+ # colors[index]
55
+ # end
56
+ #
57
+ #
58
+ # def self.distinct(array)
59
+ # colors = Rbbt.share.color["diverging_colors.hex"].list.collect{|c| Color.new c}
60
+ #
61
+ # num = array.uniq.length
62
+ # times = num / 12
63
+ #
64
+ # all_colors = colors.dup
65
+ # factor = 0.3 / times
66
+ # times.times do
67
+ # all_colors.concat colors.collect{|n| n.darken(factor) }
68
+ # end
69
+ #
70
+ # value_color = Hash[*array.uniq.zip(all_colors).flatten]
71
+ #
72
+ # value_color.values_at *array
73
+ # end
74
+ #
75
+ # def self.tsv(tsv, options = {})
76
+ # values = tsv.values.flatten
77
+ # if Numeric === values.first or (values.first.to_f != 0 and values[0] != "0" and values[0] != "0.0")
78
+ # value_colors = Misc.process_to_hash(values){continuous(values)}
79
+ # else
80
+ # value_colors = Misc.process_to_hash(values){distinct(values)}
81
+ # end
82
+ #
83
+ # if tsv.type == :single
84
+ # Hash[*tsv.keys.zip(value_colors.values_at(*values)).flatten]
85
+ # else
86
+ # Hash[*tsv.keys.zip(values.collect{|vs| value_colors.values_at(*vs)}).flatten]
87
+ # end
88
+ # end
89
+ #end
@@ -0,0 +1,22 @@
1
+ require 'scout/work_queue'
2
+ class RbbtProcessQueue < WorkQueue
3
+
4
+ self.define_method(:start_process, WorkQueue.instance_method(:process))
5
+
6
+ def init(&block)
7
+ @worker_proc = block
8
+ start_process(&@callback)
9
+ end
10
+
11
+ def callback(&block)
12
+ @callback = block
13
+ end
14
+
15
+ def process(obj)
16
+ self.write(obj)
17
+ end
18
+
19
+ alias add_process add_worker
20
+ alias remove_process remove_one_worker
21
+ alias remove_process remove_one_worker
22
+ end
@@ -199,10 +199,10 @@ class RbbtProcessQueue
199
199
  @queue.close_write
200
200
 
201
201
  if @callback_queue
202
- Misc.purge_pipes(@callback_queue.swrite)
202
+ Open.purge_pipes(@callback_queue.swrite)
203
203
  @callback_queue.close_read
204
204
  else
205
- Misc.purge_pipes
205
+ Open.purge_pipes
206
206
  end
207
207
 
208
208
  if respawn