rbbt-util 5.2.4 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +8 -8
  2. data/bin/rbbt +23 -10
  3. data/bin/rbbt_monitor.rb +8 -8
  4. data/lib/rbbt/annotations.rb +22 -1
  5. data/lib/rbbt/annotations/util.rb +1 -1
  6. data/lib/rbbt/entity.rb +162 -0
  7. data/lib/rbbt/fix_width_table.rb +7 -0
  8. data/lib/rbbt/persist.rb +16 -9
  9. data/lib/rbbt/persist/tsv.rb +14 -8
  10. data/lib/rbbt/resource.rb +1 -6
  11. data/lib/rbbt/resource/path.rb +23 -27
  12. data/lib/rbbt/tsv.rb +33 -4
  13. data/lib/rbbt/tsv/accessor.rb +100 -57
  14. data/lib/rbbt/tsv/attach.rb +3 -1
  15. data/lib/rbbt/tsv/attach/util.rb +34 -10
  16. data/lib/rbbt/tsv/index.rb +12 -3
  17. data/lib/rbbt/tsv/manipulate.rb +25 -1
  18. data/lib/rbbt/tsv/parser.rb +1 -0
  19. data/lib/rbbt/util/R.rb +36 -6
  20. data/lib/rbbt/util/cmd.rb +2 -1
  21. data/lib/rbbt/util/color.rb +250 -0
  22. data/lib/rbbt/util/colorize.rb +57 -0
  23. data/lib/rbbt/util/misc.rb +57 -19
  24. data/lib/rbbt/util/named_array.rb +66 -14
  25. data/lib/rbbt/util/open.rb +134 -10
  26. data/lib/rbbt/util/semaphore.rb +71 -0
  27. data/lib/rbbt/workflow.rb +34 -7
  28. data/lib/rbbt/workflow/accessor.rb +12 -8
  29. data/lib/rbbt/workflow/step.rb +44 -28
  30. data/lib/rbbt/workflow/usage.rb +3 -0
  31. data/share/lib/R/util.R +31 -0
  32. data/share/rbbt_commands/app/start +5 -4
  33. data/share/rbbt_commands/study/task +222 -0
  34. data/share/rbbt_commands/tsv/attach +13 -0
  35. data/share/rbbt_commands/tsv/change_id +15 -0
  36. data/share/rbbt_commands/tsv/info +3 -1
  37. data/share/rbbt_commands/workflow/task +14 -15
  38. data/test/rbbt/test_entity.rb +221 -0
  39. data/test/rbbt/test_tsv.rb +2 -1
  40. data/test/rbbt/test_workflow.rb +0 -2
  41. data/test/rbbt/tsv/test_accessor.rb +2 -2
  42. data/test/rbbt/util/test_R.rb +9 -2
  43. data/test/rbbt/util/test_colorize.rb +12 -0
  44. data/test/rbbt/util/test_misc.rb +0 -5
  45. data/test/rbbt/util/test_open.rb +31 -0
  46. data/test/rbbt/workflow/test_step.rb +32 -0
  47. metadata +13 -2
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- ZjIwN2RhYmRjNDg2MjMzMWNmYWU4NGQ5ZDUyYjcyNGU4MmExMzJhYw==
4
+ ZDNlMzk5N2E5ZWRkYWExZmVhNGYzZDliMjAwMDhkZWE5MDZmMjhjZA==
5
5
  data.tar.gz: !binary |-
6
- NzAyMmUyMjg3ZWRmY2I1N2EwNDk1OGVlNTBmNWUyNmIyNTUxN2QwNg==
6
+ NGUzYzZiOTc2MGY2NmNhYWFmNGQxZjVjYWY3ZjgzOGFmZTJlN2RhMg==
7
7
  !binary "U0hBNTEy":
8
8
  metadata.gz: !binary |-
9
- M2M0OTlkODVmMjY1ZWNjZWIzYjhjMTE2M2MyOGNiNWY5YjZiYjNiNDdkMTQx
10
- MjkyNTk0MWRkZGM4ZjU3OTVjOTQ3ODg3YmNhNjVmZjljZDlmZGU3YTI4YWIw
11
- MmFkZjcwMTE5OGFlNmI3MjRkZWI0NTQwZDBlZTg3ZTUwMWJhYjY=
9
+ ZmNiOTVmYWFlN2JhN2M0ZWU1YjA0M2VhZjM3OTgzZDA1OTAxZGUxYjMxNTg4
10
+ OGI3YjNjZTdiOTkzMGU5Yzc1YjM4YTliOWVhMGVmOGUwNWYwOTE1Nzk0MmRm
11
+ ZDJhYmVmZDAwZGJkNzRhNmJlNmI2MzZkODUwNTQwYTU4NTMyMTk=
12
12
  data.tar.gz: !binary |-
13
- ZTIxNTBhZGQyZjg0ZjNiYjhlZjU3MTQ4M2YzMzgyNWNlZWE4MWMxNDg2OWUw
14
- MzRlMzQyMzkyZWJiNDM1NjQ3ODljNmUwYjg1ZjJhYzYxOWE3MTZjZjc1ZDM3
15
- OTI1ZjMzMTNmZDE4Mzc4OTQ3NDJmY2UyYTQzNWRjZDg0MjhlOWI=
13
+ OGIzZmFhYmQyMWYzY2U2YTFiMWExYzJkNGI3ZDY2YzE5OTFkMmViNzM5NjM3
14
+ YWViMjBiOTVjM2I1M2Y3MDRmYjFhZWFlMjIyM2FkNzI1NjFmMzc0MGMyODBk
15
+ YmVmMGJiMWQyYzQzYTkwZWIwYjcyZmJiZTkzODYyNmI5YjZhM2M=
data/bin/rbbt CHANGED
@@ -3,7 +3,7 @@
3
3
  require 'rbbt'
4
4
  require 'rbbt/util/simpleopt'
5
5
 
6
- options = SOPT.get("--log* Log level from 0 (debug) 7 (errors):--command_dir* Directory from where to load commands")
6
+ options = SOPT.get("--log* Log level from 0 (debug) 7 (errors):--command_dir* Directory from where to load commands:--profile Profile execution")
7
7
 
8
8
  if options[:log]
9
9
  Log.severity = options[:log].to_i
@@ -15,7 +15,12 @@ else
15
15
  $rbbt_command_dir = Rbbt.share.rbbt_commands
16
16
  end
17
17
 
18
- SOPT.synopsys = "rbbt <command> <subcommand> ... -a --arg1 --arg2='value'"
18
+ if options[:profile]
19
+ require 'ruby-prof'
20
+ RubyProf.start
21
+ end
22
+
23
+ SOPT.synopsys = "rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value'"
19
24
  SOPT.summary = "Ruby bioinformatics toolkit"
20
25
  SOPT.description = <<-EOF
21
26
  This command controls many aspects of the Rbbt framework, from configuration tasks to running applications.
@@ -41,20 +46,28 @@ end
41
46
  dir = $rbbt_command_dir
42
47
  prev = []
43
48
 
44
- while ARGV.any?
49
+ begin
50
+ while ARGV.any?
45
51
  command = ARGV.shift
46
52
  case
47
53
  when File.directory?(dir[command].find)
48
- prev << command
49
- dir = dir[command]
54
+ prev << command
55
+ dir = dir[command]
50
56
  when dir[command].exists?
51
- load dir[command].find
52
- exit 0
57
+ load dir[command].find
58
+ exit 0
53
59
  else
54
- puts "Error: Command not understood"
55
- puts
56
- break
60
+ puts "Error: Command not understood"
61
+ puts
62
+ break
57
63
  end
64
+ end
65
+ ensure
66
+ if options[:profile]
67
+ result = RubyProf.stop
68
+ printer = RubyProf::FlatPrinter.new(result)
69
+ printer.print(STDOUT, :min_percent => 10)
70
+ end
58
71
  end
59
72
 
60
73
  puts SOPT.doc
data/bin/rbbt_monitor.rb CHANGED
@@ -26,7 +26,7 @@ def print_job(file, info, severity_color = nil)
26
26
  else
27
27
  info ||= {:status => :missing_info_file}
28
28
  str = [clean_file, info[:status].to_s] * " [ STATUS = " + " ]"
29
- str += " (#{running?(info)? :running : :dead} #{info[:pid]})" if info.include? :pid
29
+ str += " (#{running?(info)? :running : :dead} #{info[:pid]})" if info[:pid]
30
30
  str += " (children: #{info[:children_pids].collect{|pid| [pid, Misc.pid_exists?(pid) ? "R" : "D"] * ":"} * ", "})" if info.include? :children_pids
31
31
 
32
32
  str = "#{severity_color}" << str << "\033[0m" if severity_color
@@ -42,8 +42,8 @@ def list_jobs(options)
42
42
  clean_file = file.sub('.info','')
43
43
  begin
44
44
  next if File.exists? clean_file and $quick
45
- info = YAML.load(Open.read(file))
46
- next if File.exists? clean_file and not running? info
45
+ info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
46
+ next if (File.exists?(clean_file) or info[:status] == :done) and (info[:children_pids].nil? or info[:children_done] or info[:children_pids].select{|pid| Misc.pid_exists? pid}.empty?)
47
47
  rescue Exception
48
48
  puts "Error parsing info file: #{ file }"
49
49
  info = nil
@@ -82,13 +82,13 @@ end
82
82
  def clean_jobs(options)
83
83
  info_files.each do |file|
84
84
  clean_file = file.sub('.info','')
85
- next if File.exists? clean_file
86
85
  info = nil
86
+ next if File.exists? clean_file
87
87
  begin
88
- info = YAML.load(Open.read(file))
89
- rescue
88
+ info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
89
+ rescue Exception
90
90
  Log.debug "Error process #{ file }"
91
- raise $!
91
+ remove_job file if options[:errors]
92
92
  end
93
93
  case
94
94
  when options[:all]
@@ -115,7 +115,7 @@ when (options[:clean] and not options[:list])
115
115
  end
116
116
  else
117
117
  if options[:file]
118
- info = YAML.load(Open.read(options[:file]))
118
+ info = Step::INFO_SERIALIAZER.load(Open.read(options[:file], :mode => 'rb'))
119
119
  print_job options[:file], info
120
120
  else
121
121
  list_jobs options
@@ -106,8 +106,10 @@ module Annotated
106
106
 
107
107
  def annotate(object)
108
108
 
109
+ return object unless Array === object or String === object
110
+
109
111
  annotation_types.each do |annotation|
110
- object.extend annotation
112
+ object.extend annotation unless annotation === object
111
113
  end
112
114
 
113
115
  object.instance_variable_set(:@annotation_types, nil)
@@ -145,6 +147,25 @@ module Annotated
145
147
  end
146
148
  end
147
149
 
150
+ def self.purge(object)
151
+ case object
152
+ when String
153
+ object.respond_to?(:clean_annotations) ?
154
+ object.clean_annotations :
155
+ object
156
+ when Array
157
+ object.respond_to?(:clean_annotations) ?
158
+ object.clean_annotations :
159
+ object.collect{|e| Annotated.purge e}
160
+ when Hash
161
+ object.each do |key, value|
162
+ object[key] = Annotated.purge value
163
+ end
164
+ else
165
+ object
166
+ end
167
+ end
168
+
148
169
  def make_list
149
170
  new = [self]
150
171
  self.annotate(new)
@@ -14,7 +14,7 @@ module Annotated
14
14
 
15
15
  annotation_types.each do |mod|
16
16
  mod = Misc.string2const(mod) if String === mod
17
- object.extend mod
17
+ object.extend mod unless mod === object
18
18
  end
19
19
 
20
20
  object.instance_variable_set(:@annotation_values, info)
@@ -0,0 +1,162 @@
1
+ require 'rbbt/annotations'
2
+
3
+ module Entity
4
+
5
+ class << self
6
+ attr_accessor :formats, :entity_property_cache
7
+ end
8
+
9
+ self.entity_property_cache = "var/entity_property"
10
+ self.formats = {}
11
+
12
+ UNPERSISTED_PREFIX = "entity_unpersisted_property_"
13
+
14
+
15
+ def self.extended(base)
16
+ base.extend Annotation
17
+ Entity.formats[base.to_s] = base
18
+
19
+ base.module_eval do
20
+ attr_accessor :_ary_property_cache
21
+
22
+ attr_accessor :template, :list_template, :action_template, :list_action_template, :keep_id
23
+
24
+ def _ary_property_cache
25
+ @_ary_property_cache ||= {}
26
+ end
27
+
28
+ def self.format=(formats)
29
+ formats = [formats] unless Array === formats
30
+ formats.each do |format|
31
+ Entity.formats[format] = self
32
+ end
33
+ end
34
+
35
+ def property(*args, &block)
36
+ class << self; self; end.property(*args,&block)
37
+ end
38
+
39
+ def to_yaml(*args)
40
+ self.clean_annotations.dup.to_yaml(*args)
41
+ end
42
+
43
+
44
+ def encode_with(coder)
45
+ coder.scalar = clean_annotations
46
+ end
47
+
48
+ def marshal_dump
49
+ clean_annotations
50
+ end
51
+
52
+ def consolidate
53
+ self.inject(nil){|acc,e|
54
+ if acc.nil?
55
+ acc = e
56
+ else
57
+ acc.concat e
58
+ end
59
+ }
60
+ end
61
+
62
+ def self.property(name, &block)
63
+ case
64
+ when (Hash === name and name.size == 1)
65
+ name, type = name.collect.first
66
+ when (String === name or Symbol === name)
67
+ type = :single
68
+ else
69
+ raise "Format of name ( => type) not understood: #{name.inspect}"
70
+ end
71
+
72
+ name = name.to_s unless String === name
73
+
74
+ persisted_name = UNPERSISTED_PREFIX + name
75
+ self.remove_method persisted_name if methods.include? persisted_name
76
+
77
+ case type
78
+ when :both
79
+ define_method name, &block
80
+
81
+ when :single, :single2array
82
+ single_name = "_single_" << name
83
+ define_method single_name, &block
84
+ define_method name do |*args|
85
+ if Array === self
86
+ self.collect{|e| e.send(name, *args)}
87
+ else
88
+ self.send(single_name, *args)
89
+ end
90
+ end
91
+ when :array, :array2single
92
+ ary_name = "_ary_" << name
93
+ define_method ary_name, &block
94
+
95
+ define_method name do |*args|
96
+ #ary_name = "_ary_" << __method__.to_s
97
+ case
98
+ when Array === self
99
+ self.send(ary_name, *args)
100
+ when (Array === self.container and not self.container_index.nil? and self.container.respond_to? ary_name)
101
+ cache_code = Misc.hash2md5({:name => ary_name, :args => args})
102
+ res = (self.container._ary_property_cache[cache_code] ||= self.container.send(name, *args))
103
+ if Hash === res
104
+ res[self]
105
+ else
106
+ res[self.container_index]
107
+ end
108
+ else
109
+ res = self.make_list.send(ary_name, *args)
110
+ Hash === res ? res[self] : res[0]
111
+ end
112
+ end
113
+
114
+ end
115
+ end
116
+
117
+ def self.persist(method_name, type = nil, options = {})
118
+ type = :memory if type.nil?
119
+ options ||= {}
120
+ options = Misc.add_defaults options, :dir => Entity.entity_property_cache
121
+
122
+ orig_name = UNPERSISTED_PREFIX + method_name.to_s
123
+ alias_method orig_name, method_name unless self.instance_methods.include? orig_name.to_sym
124
+
125
+ define_method method_name do |*args|
126
+ id = self.id
127
+ persist_name = __method__.to_s << ":" << (Array === id ? Misc.hash2md5(:id => id) : id)
128
+ persist_name << ":" << Misc.hash2md5({:args => args}) unless args.nil? or args.empty?
129
+
130
+ persist_options = options
131
+ persist_options = persist_options.merge(:other => {:args => args}) if args.any?
132
+
133
+ Persist.persist(persist_name, type, persist_options) do
134
+ self.send(orig_name, *args)
135
+ end
136
+ end
137
+ end
138
+
139
+ def self.unpersist(method_name)
140
+ return unless persisted? method_name
141
+ orig_name = UNPERSISTED_PREFIX + method_name.to_s
142
+
143
+ alias_method method_name, orig_name
144
+ remove_method orig_name
145
+ end
146
+
147
+ def self.persisted?(method_name)
148
+ orig_name = UNPERSISTED_PREFIX + method_name.to_s
149
+ instance_methods.include? orig_name.to_sym
150
+ end
151
+
152
+ def self.with_persisted(method_name)
153
+ persisted = persisted? method_name
154
+ persist method_name unless persisted
155
+ res = yield
156
+ unpersist method_name unless persisted
157
+ res
158
+ end
159
+
160
+ end
161
+ end
162
+ end
@@ -244,4 +244,11 @@ class FixWidthTable
244
244
  }
245
245
  end
246
246
 
247
+ def chunked_values_at(keys, max = 5000)
248
+ Misc.ordered_divide(keys, max).inject([]) do |acc,c|
249
+ new = self.values_at(*c)
250
+ new.annotate acc if new.respond_to? :annotate and acc.empty?
251
+ acc.concat(new)
252
+ end
253
+ end
247
254
  end
data/lib/rbbt/persist.rb CHANGED
@@ -17,15 +17,16 @@ module Persist
17
17
  end
18
18
 
19
19
  MEMORY = {} unless defined? MEMORY
20
+ MAX_FILE_LENGTH = 150
20
21
 
21
22
  def self.newer?(path, file)
22
- return true if not File.exists? file
23
+ return true if not Open.exists? file
23
24
  return true if File.mtime(path) < File.mtime(file)
24
25
  return false
25
26
  end
26
27
 
27
28
  def self.is_persisted?(path, persist_options = {})
28
- return false if not File.exists? path
29
+ return false if not Open.exists? path
29
30
  return false if TrueClass === persist_options[:update]
30
31
 
31
32
  check = persist_options[:check]
@@ -66,6 +67,8 @@ module Persist
66
67
  clean_options.delete :unnamed
67
68
  clean_options.delete "unnamed"
68
69
 
70
+ filename = filename[0..MAX_FILE_LENGTH] << Misc.digest(filename[MAX_FILE_LENGTH+1..-1]) if filename.length > MAX_FILE_LENGTH + 10
71
+
69
72
  options_md5 = Misc.hash2md5 clean_options
70
73
  filename << ":" << options_md5 unless options_md5.empty?
71
74
 
@@ -248,6 +251,7 @@ module Persist
248
251
  end
249
252
 
250
253
  else
254
+
251
255
  if is_persisted?(path, persist_options)
252
256
  Log.low "Persist up-to-date: #{ path } - #{persist_options.inspect[0..100]}"
253
257
  return nil if persist_options[:no_load]
@@ -255,6 +259,7 @@ module Persist
255
259
  else
256
260
  Log.medium "Persist create: #{ path } - #{persist_options.inspect[0..100]}"
257
261
  end
262
+
258
263
  begin
259
264
  res = yield
260
265
  Misc.lock(path) do
@@ -262,8 +267,8 @@ module Persist
262
267
  end
263
268
  res
264
269
  rescue
265
- Log.high "Error in persist. #{File.exists?(path) ? "Erasing '#{ path }'" : ""}"
266
- FileUtils.rm path if File.exists? path
270
+ Log.high "Error in persist. #{Open.exists?(path) ? "Erasing '#{ path }'" : ""}"
271
+ FileUtils.rm path if Open.exists? path
267
272
  raise $!
268
273
  end
269
274
  end
@@ -279,16 +284,18 @@ module LocalPersist
279
284
  attr_accessor :local_persist_dir
280
285
 
281
286
  def local_persist_dir
282
- @local_persist_dir ||= Rbbt.var.cache.persistence.find(:lib) if defined? Rbbt
287
+ @local_persist_dir ||= Rbbt.var.cache.persistence if defined? Rbbt
283
288
  @local_persist_dir
284
289
  end
285
290
 
286
- def local_persist(name, type = nil, options= {}, &block)
287
- Persist.persist(name, type, options.merge({:dir => local_persist_dir}), &block)
291
+ def local_persist(name, type = nil, options= {}, persist_options = nil, &block)
292
+ persist_options ||= {}
293
+ persist_options = {:dir => local_persist_dir}.merge persist_options
294
+ persist_options[:other] = options
295
+ Persist.persist(name, type, persist_options, &block)
288
296
  end
289
297
 
290
298
  def local_persist_tsv(source, name, opt = {}, options= {}, &block)
291
- Persist.persist_tsv(source, name, opt, options.merge({:dir => local_persist_dir}), &block)
299
+ Persist.persist_tsv(source, name, opt, options.merge({:dir => local_persist_dir, :persist => true}), &block)
292
300
  end
293
-
294
301
  end
@@ -25,6 +25,10 @@ module Persist
25
25
  class << database
26
26
  attr_accessor :writable, :closed, :persistence_path, :tokyocabinet_class
27
27
 
28
+ def prefix(key)
29
+ range(key, 1, key + 255.chr, 1)
30
+ end
31
+
28
32
  def closed?
29
33
  @closed
30
34
  end
@@ -82,7 +86,7 @@ module Persist
82
86
  end
83
87
 
84
88
  def write_and_close
85
- lock_filename = Persist.persistence_path(persistence_path, {:dir => Rbbt.tmp.tsv_open_locks.find})
89
+ lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
86
90
  Misc.lock(lock_filename) do
87
91
  write if @closed or not write?
88
92
  res = begin
@@ -124,8 +128,11 @@ module Persist
124
128
  database.persistence_path ||= path
125
129
  database.tokyocabinet_class = tokyocabinet_class
126
130
 
127
- TSV.setup database
128
- database.serializer = serializer || database.serializer
131
+ unless serializer == :clean
132
+ TSV.setup database
133
+ database.serializer = serializer || database.serializer
134
+ database.fields
135
+ end
129
136
 
130
137
  database
131
138
  end
@@ -153,18 +160,18 @@ module Persist
153
160
 
154
161
  path = persistence_path(filename, persist_options, options)
155
162
 
156
- if is_persisted? path
163
+ if is_persisted? path and not persist_options[:update]
157
164
  Log.debug "TSV persistence up-to-date: #{ path }"
158
165
  lock_filename = Persist.persistence_path(path, {:dir => Rbbt.tmp.tsv_open_locks.find})
159
- return Misc.lock(lock_filename) do open_tokyocabinet(path, false); end
166
+ return Misc.lock(lock_filename) do open_tokyocabinet(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB); end
160
167
  else
161
168
  Log.medium "TSV persistence creating: #{ path }"
162
169
  end
163
170
 
164
171
  FileUtils.rm path if File.exists? path
165
172
 
166
- data = open_tokyocabinet(path, true, persist_options[:serializer])
167
- data.serializer = :type unless data.serializer
173
+ data = open_tokyocabinet(path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB)
174
+ data.serializer = :type if TSV === data and data.serializer.nil?
168
175
 
169
176
  data.close
170
177
 
@@ -197,5 +204,4 @@ module Persist
197
204
 
198
205
  data
199
206
  end
200
-
201
207
  end