rbbt-util 5.44.0 → 6.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (169) hide show
  1. checksums.yaml +4 -4
  2. data/bin/rbbt +67 -90
  3. data/etc/app.d/base.rb +3 -3
  4. data/etc/app.d/semaphores.rb +3 -3
  5. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  6. data/lib/rbbt/annotations/refactor.rb +27 -0
  7. data/lib/rbbt/annotations/util.rb +282 -282
  8. data/lib/rbbt/annotations.rb +343 -320
  9. data/lib/rbbt/association/database.rb +200 -225
  10. data/lib/rbbt/association/index.rb +294 -291
  11. data/lib/rbbt/association/item.rb +227 -227
  12. data/lib/rbbt/association/open.rb +35 -34
  13. data/lib/rbbt/association/util.rb +0 -169
  14. data/lib/rbbt/association.rb +2 -4
  15. data/lib/rbbt/entity/identifiers.rb +119 -118
  16. data/lib/rbbt/entity/refactor.rb +12 -0
  17. data/lib/rbbt/entity.rb +319 -315
  18. data/lib/rbbt/hpc/batch.rb +72 -53
  19. data/lib/rbbt/hpc/lsf.rb +2 -2
  20. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  21. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  22. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  24. data/lib/rbbt/hpc/slurm.rb +18 -18
  25. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  26. data/lib/rbbt/knowledge_base/query.rb +2 -2
  27. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  28. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  29. data/lib/rbbt/knowledge_base.rb +1 -1
  30. data/lib/rbbt/monitor.rb +36 -25
  31. data/lib/rbbt/persist/refactor.rb +166 -0
  32. data/lib/rbbt/persist/tsv/sharder.rb +1 -1
  33. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  34. data/lib/rbbt/persist/tsv.rb +187 -185
  35. data/lib/rbbt/persist.rb +556 -551
  36. data/lib/rbbt/refactor.rb +20 -0
  37. data/lib/rbbt/resource/path/refactor.rb +178 -0
  38. data/lib/rbbt/resource/path.rb +317 -497
  39. data/lib/rbbt/resource/util.rb +0 -48
  40. data/lib/rbbt/resource.rb +3 -390
  41. data/lib/rbbt/tsv/accessor.rb +2 -838
  42. data/lib/rbbt/tsv/attach.rb +303 -299
  43. data/lib/rbbt/tsv/change_id.rb +244 -245
  44. data/lib/rbbt/tsv/csv.rb +87 -85
  45. data/lib/rbbt/tsv/dumper.rb +2 -100
  46. data/lib/rbbt/tsv/excel.rb +26 -24
  47. data/lib/rbbt/tsv/field_index.rb +4 -1
  48. data/lib/rbbt/tsv/filter.rb +3 -2
  49. data/lib/rbbt/tsv/index.rb +2 -284
  50. data/lib/rbbt/tsv/manipulate.rb +750 -747
  51. data/lib/rbbt/tsv/marshal.rb +3 -3
  52. data/lib/rbbt/tsv/matrix.rb +2 -2
  53. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  54. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  55. data/lib/rbbt/tsv/parser.rb +678 -678
  56. data/lib/rbbt/tsv/refactor.rb +195 -0
  57. data/lib/rbbt/tsv/stream.rb +253 -251
  58. data/lib/rbbt/tsv/util.rb +420 -420
  59. data/lib/rbbt/tsv.rb +210 -208
  60. data/lib/rbbt/util/R/eval.rb +4 -4
  61. data/lib/rbbt/util/R/plot.rb +62 -166
  62. data/lib/rbbt/util/R.rb +21 -18
  63. data/lib/rbbt/util/cmd.rb +2 -318
  64. data/lib/rbbt/util/color.rb +269 -269
  65. data/lib/rbbt/util/colorize.rb +89 -89
  66. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  67. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  68. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  69. data/lib/rbbt/util/config.rb +169 -167
  70. data/lib/rbbt/util/iruby.rb +20 -0
  71. data/lib/rbbt/util/log/progress/report.rb +241 -241
  72. data/lib/rbbt/util/log/progress/util.rb +99 -99
  73. data/lib/rbbt/util/log/progress.rb +102 -102
  74. data/lib/rbbt/util/log/refactor.rb +49 -0
  75. data/lib/rbbt/util/log.rb +486 -532
  76. data/lib/rbbt/util/migrate.rb +1 -1
  77. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  78. data/lib/rbbt/util/misc/development.rb +12 -11
  79. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  80. data/lib/rbbt/util/misc/format.rb +2 -230
  81. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  82. data/lib/rbbt/util/misc/inspect.rb +2 -476
  83. data/lib/rbbt/util/misc/lock.rb +109 -106
  84. data/lib/rbbt/util/misc/omics.rb +10 -1
  85. data/lib/rbbt/util/misc/pipes.rb +765 -793
  86. data/lib/rbbt/util/misc/refactor.rb +20 -0
  87. data/lib/rbbt/util/misc/ssw.rb +308 -0
  88. data/lib/rbbt/util/misc/system.rb +0 -15
  89. data/lib/rbbt/util/misc.rb +39 -20
  90. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  91. data/lib/rbbt/util/named_array.rb +3 -220
  92. data/lib/rbbt/util/open/refactor.rb +7 -0
  93. data/lib/rbbt/util/open.rb +3 -857
  94. data/lib/rbbt/util/procpath.rb +6 -6
  95. data/lib/rbbt/util/python/paths.rb +27 -0
  96. data/lib/rbbt/util/python/run.rb +115 -0
  97. data/lib/rbbt/util/python/script.rb +110 -0
  98. data/lib/rbbt/util/python/util.rb +3 -3
  99. data/lib/rbbt/util/python.rb +22 -81
  100. data/lib/rbbt/util/semaphore.rb +152 -148
  101. data/lib/rbbt/util/simpleopt.rb +9 -8
  102. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  103. data/lib/rbbt/util/ssh.rb +122 -118
  104. data/lib/rbbt/util/tar.rb +117 -115
  105. data/lib/rbbt/util/tmpfile.rb +69 -67
  106. data/lib/rbbt/util/version.rb +2 -0
  107. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  108. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  109. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  110. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  111. data/lib/rbbt/workflow/refactor/task_info.rb +65 -0
  112. data/lib/rbbt/workflow/refactor.rb +153 -0
  113. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  114. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  115. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  116. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  117. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  118. data/lib/rbbt/workflow/step/info.rb +2 -2
  119. data/lib/rbbt/workflow/step/run.rb +766 -766
  120. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  121. data/lib/rbbt/workflow/step.rb +2 -362
  122. data/lib/rbbt/workflow/task.rb +118 -118
  123. data/lib/rbbt/workflow/usage.rb +289 -287
  124. data/lib/rbbt/workflow/util/archive.rb +6 -5
  125. data/lib/rbbt/workflow/util/data.rb +1 -1
  126. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  127. data/lib/rbbt/workflow/util/trace.rb +79 -44
  128. data/lib/rbbt/workflow.rb +4 -882
  129. data/lib/rbbt-util.rb +21 -13
  130. data/lib/rbbt.rb +16 -3
  131. data/python/rbbt/__init__.py +19 -1
  132. data/share/Rlib/plot.R +37 -37
  133. data/share/Rlib/svg.R +22 -5
  134. data/share/install/software/lib/install_helpers +1 -1
  135. data/share/rbbt_commands/hpc/list +2 -3
  136. data/share/rbbt_commands/hpc/orchestrate +4 -4
  137. data/share/rbbt_commands/hpc/tail +2 -0
  138. data/share/rbbt_commands/hpc/task +10 -7
  139. data/share/rbbt_commands/lsf/list +2 -3
  140. data/share/rbbt_commands/lsf/orchestrate +4 -4
  141. data/share/rbbt_commands/lsf/tail +2 -0
  142. data/share/rbbt_commands/lsf/task +10 -7
  143. data/share/rbbt_commands/migrate +1 -1
  144. data/share/rbbt_commands/pbs/list +2 -3
  145. data/share/rbbt_commands/pbs/orchestrate +4 -4
  146. data/share/rbbt_commands/pbs/tail +2 -0
  147. data/share/rbbt_commands/pbs/task +10 -7
  148. data/share/rbbt_commands/resource/produce +8 -1
  149. data/share/rbbt_commands/slurm/list +2 -3
  150. data/share/rbbt_commands/slurm/orchestrate +4 -4
  151. data/share/rbbt_commands/slurm/tail +2 -0
  152. data/share/rbbt_commands/slurm/task +10 -7
  153. data/share/rbbt_commands/system/clean +5 -5
  154. data/share/rbbt_commands/system/status +5 -5
  155. data/share/rbbt_commands/tsv/get +2 -3
  156. data/share/rbbt_commands/tsv/info +10 -13
  157. data/share/rbbt_commands/tsv/keys +18 -14
  158. data/share/rbbt_commands/tsv/slice +2 -2
  159. data/share/rbbt_commands/tsv/transpose +6 -2
  160. data/share/rbbt_commands/workflow/info +20 -24
  161. data/share/rbbt_commands/workflow/list +1 -1
  162. data/share/rbbt_commands/workflow/prov +20 -13
  163. data/share/rbbt_commands/workflow/server +16 -1
  164. data/share/rbbt_commands/workflow/task +76 -71
  165. data/share/rbbt_commands/workflow/write_info +26 -9
  166. data/share/software/opt/ssw/ssw.c +861 -0
  167. data/share/software/opt/ssw/ssw.h +130 -0
  168. data/share/workflow_config.ru +3 -3
  169. metadata +42 -3
@@ -1,89 +1,89 @@
1
- require 'rbbt/util/color'
2
-
3
- module Colorize
4
- def self.from_name(color)
5
- return color if color =~ /^#?[0-9A-F]+$/i
6
- colors = Rbbt.share.color.color_names.tsv :type => :single
7
- case color.to_s
8
- when "white"
9
- '#000'
10
- when "black"
11
- '#fff'
12
- when 'green'
13
- colors["green3"]
14
- when 'red'
15
- colors["red3"]
16
- when 'yellow'
17
- colors["gold1"]
18
- when 'blue'
19
- colors["RoyalBlue"]
20
- else
21
- colors[color.to_s] || color
22
- end
23
- end
24
-
25
- def self.continuous(array, start = "#40324F", eend = "#EABD5D", percent = false)
26
- start_color = Color.new from_name(start)
27
- end_color = Color.new from_name(eend)
28
-
29
- if percent
30
- array = array.collect{|v| n = v.to_f; n = n > 100 ? 100 : n; n < 0.001 ? 0.001 : n}
31
- else
32
- array = array.collect{|v| n = v.to_f; }
33
- end
34
- max = array.max
35
- min = array.min
36
- range = max - min
37
- array.collect do |v|
38
- ratio = (v.to_f-min) / range
39
- start_color.blend end_color, ratio
40
- end
41
- end
42
-
43
- def self.gradient(array, value, start = :green, eend = :red, percent = false)
44
- index = array.index value
45
- colors = continuous(array, start, eend, percent)
46
- colors[index]
47
- end
48
-
49
- def self.rank_gradient(array, value, start = :green, eend = :red, percent = false)
50
- index = array.index value
51
- sorted = array.sort
52
- array = array.collect{|e| sorted.index e}
53
- colors = continuous(array, start, eend, percent)
54
- colors[index]
55
- end
56
-
57
-
58
- def self.distinct(array)
59
- colors = Rbbt.share.color["diverging_colors.hex"].list.collect{|c| Color.new c}
60
-
61
- num = array.uniq.length
62
- times = num / 12
63
-
64
- all_colors = colors.dup
65
- factor = 0.3 / times
66
- times.times do
67
- all_colors.concat colors.collect{|n| n.darken(factor) }
68
- end
69
-
70
- value_color = Hash[*array.uniq.zip(all_colors).flatten]
71
-
72
- value_color.values_at *array
73
- end
74
-
75
- def self.tsv(tsv, options = {})
76
- values = tsv.values.flatten
77
- if Numeric === values.first or (values.first.to_f != 0 and values[0] != "0" and values[0] != "0.0")
78
- value_colors = Misc.process_to_hash(values){continuous(values)}
79
- else
80
- value_colors = Misc.process_to_hash(values){distinct(values)}
81
- end
82
-
83
- if tsv.type == :single
84
- Hash[*tsv.keys.zip(value_colors.values_at(*values)).flatten]
85
- else
86
- Hash[*tsv.keys.zip(values.collect{|vs| value_colors.values_at(*vs)}).flatten]
87
- end
88
- end
89
- end
1
+ #require 'rbbt/util/color'
2
+ #
3
+ #module Colorize
4
+ # def self.from_name(color)
5
+ # return color if color =~ /^#?[0-9A-F]+$/i
6
+ # colors = Rbbt.share.color.color_names.tsv :type => :single
7
+ # case color.to_s
8
+ # when "white"
9
+ # '#000'
10
+ # when "black"
11
+ # '#fff'
12
+ # when 'green'
13
+ # colors["green3"]
14
+ # when 'red'
15
+ # colors["red3"]
16
+ # when 'yellow'
17
+ # colors["gold1"]
18
+ # when 'blue'
19
+ # colors["RoyalBlue"]
20
+ # else
21
+ # colors[color.to_s] || color
22
+ # end
23
+ # end
24
+ #
25
+ # def self.continuous(array, start = "#40324F", eend = "#EABD5D", percent = false)
26
+ # start_color = Color.new from_name(start)
27
+ # end_color = Color.new from_name(eend)
28
+ #
29
+ # if percent
30
+ # array = array.collect{|v| n = v.to_f; n = n > 100 ? 100 : n; n < 0.001 ? 0.001 : n}
31
+ # else
32
+ # array = array.collect{|v| n = v.to_f; }
33
+ # end
34
+ # max = array.max
35
+ # min = array.min
36
+ # range = max - min
37
+ # array.collect do |v|
38
+ # ratio = (v.to_f-min) / range
39
+ # start_color.blend end_color, ratio
40
+ # end
41
+ # end
42
+ #
43
+ # def self.gradient(array, value, start = :green, eend = :red, percent = false)
44
+ # index = array.index value
45
+ # colors = continuous(array, start, eend, percent)
46
+ # colors[index]
47
+ # end
48
+ #
49
+ # def self.rank_gradient(array, value, start = :green, eend = :red, percent = false)
50
+ # index = array.index value
51
+ # sorted = array.sort
52
+ # array = array.collect{|e| sorted.index e}
53
+ # colors = continuous(array, start, eend, percent)
54
+ # colors[index]
55
+ # end
56
+ #
57
+ #
58
+ # def self.distinct(array)
59
+ # colors = Rbbt.share.color["diverging_colors.hex"].list.collect{|c| Color.new c}
60
+ #
61
+ # num = array.uniq.length
62
+ # times = num / 12
63
+ #
64
+ # all_colors = colors.dup
65
+ # factor = 0.3 / times
66
+ # times.times do
67
+ # all_colors.concat colors.collect{|n| n.darken(factor) }
68
+ # end
69
+ #
70
+ # value_color = Hash[*array.uniq.zip(all_colors).flatten]
71
+ #
72
+ # value_color.values_at *array
73
+ # end
74
+ #
75
+ # def self.tsv(tsv, options = {})
76
+ # values = tsv.values.flatten
77
+ # if Numeric === values.first or (values.first.to_f != 0 and values[0] != "0" and values[0] != "0.0")
78
+ # value_colors = Misc.process_to_hash(values){continuous(values)}
79
+ # else
80
+ # value_colors = Misc.process_to_hash(values){distinct(values)}
81
+ # end
82
+ #
83
+ # if tsv.type == :single
84
+ # Hash[*tsv.keys.zip(value_colors.values_at(*values)).flatten]
85
+ # else
86
+ # Hash[*tsv.keys.zip(values.collect{|vs| value_colors.values_at(*vs)}).flatten]
87
+ # end
88
+ # end
89
+ #end
@@ -0,0 +1,22 @@
1
+ require 'scout/work_queue'
2
+ class RbbtProcessQueue < WorkQueue
3
+
4
+ self.define_method(:start_process, WorkQueue.instance_method(:process))
5
+
6
+ def init(&block)
7
+ @worker_proc = block
8
+ start_process(&@callback)
9
+ end
10
+
11
+ def callback(&block)
12
+ @callback = block
13
+ end
14
+
15
+ def process(obj)
16
+ self.write(obj)
17
+ end
18
+
19
+ alias add_process add_worker
20
+ alias remove_process remove_one_worker
21
+ alias remove_process remove_one_worker
22
+ end
@@ -199,10 +199,10 @@ class RbbtProcessQueue
199
199
  @queue.close_write
200
200
 
201
201
  if @callback_queue
202
- Misc.purge_pipes(@callback_queue.swrite)
202
+ Open.purge_pipes(@callback_queue.swrite)
203
203
  @callback_queue.close_read
204
204
  else
205
- Misc.purge_pipes
205
+ Open.purge_pipes
206
206
  end
207
207
 
208
208
  if respawn