rbbt-util 5.1.0 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. data/LICENSE +1 -1
  2. data/README.rdoc +2 -2
  3. data/bin/rbbt +45 -0
  4. data/bin/rbbt_dangling_locks.rb +9 -0
  5. data/bin/rbbt_monitor.rb +12 -10
  6. data/bin/run_workflow.rb +80 -18
  7. data/lib/rbbt.rb +1 -1
  8. data/lib/rbbt/annotations.rb +1 -19
  9. data/lib/rbbt/annotations/annotated_array.rb +23 -0
  10. data/lib/rbbt/fix_width_table.rb +2 -2
  11. data/lib/rbbt/persist.rb +13 -5
  12. data/lib/rbbt/persist/tsv.rb +2 -0
  13. data/lib/rbbt/resource.rb +4 -4
  14. data/lib/rbbt/resource/path.rb +35 -10
  15. data/lib/rbbt/resource/util.rb +54 -47
  16. data/lib/rbbt/tsv.rb +17 -15
  17. data/lib/rbbt/tsv/accessor.rb +35 -37
  18. data/lib/rbbt/tsv/excel.rb +3 -1
  19. data/lib/rbbt/tsv/manipulate.rb +27 -4
  20. data/lib/rbbt/tsv/parser.rb +13 -7
  21. data/lib/rbbt/util/R.rb +11 -1
  22. data/lib/rbbt/util/misc.rb +182 -26
  23. data/lib/rbbt/util/named_array.rb +14 -7
  24. data/lib/rbbt/util/open.rb +2 -1
  25. data/lib/rbbt/util/tmpfile.rb +16 -1
  26. data/lib/rbbt/workflow.rb +63 -101
  27. data/lib/rbbt/workflow/accessor.rb +19 -9
  28. data/lib/rbbt/workflow/annotate.rb +33 -64
  29. data/lib/rbbt/workflow/definition.rb +71 -0
  30. data/lib/rbbt/workflow/soap.rb +15 -5
  31. data/lib/rbbt/workflow/step.rb +57 -8
  32. data/lib/rbbt/workflow/usage.rb +72 -0
  33. data/share/lib/R/util.R +12 -0
  34. data/share/rbbt_commands/conf/web_user/add +26 -0
  35. data/share/rbbt_commands/conf/web_user/list +9 -0
  36. data/share/rbbt_commands/conf/web_user/remove +18 -0
  37. data/share/rbbt_commands/workflow/remote/add +11 -0
  38. data/share/rbbt_commands/workflow/remote/list +9 -0
  39. data/share/rbbt_commands/workflow/remote/remove +9 -0
  40. data/share/rbbt_commands/workflow/task +181 -0
  41. data/test/rbbt/test_resource.rb +2 -1
  42. data/test/rbbt/test_workflow.rb +13 -0
  43. data/test/rbbt/tsv/test_manipulate.rb +18 -0
  44. data/test/rbbt/util/test_misc.rb +19 -39
  45. data/test/rbbt/util/test_tmpfile.rb +8 -0
  46. data/test/rbbt/workflow/test_soap.rb +2 -0
  47. metadata +31 -2
@@ -4,7 +4,7 @@ module TSV
4
4
  name = Misc.process_options options, :name
5
5
  sort_by = Misc.process_options options, :sort_by
6
6
  sort_by_cast = Misc.process_options options, :sort_by_cast
7
- fields = Misc.process_options(options, :fields) || all_fields
7
+ fields = Misc.process_options(options, :fields) || tsv.all_fields
8
8
 
9
9
  book = Spreadsheet::Workbook.new
10
10
  sheet1 = book.create_worksheet
@@ -30,6 +30,7 @@ module TSV
30
30
  cells = []
31
31
  cells.push((name and key.respond_to?(:name)) ? key.name || key : key )
32
32
 
33
+ values = [values] unless Array === values
33
34
  values.each do |value|
34
35
  v = (name and value.respond_to?(:name)) ? value.name || value : value
35
36
  if Array === v
@@ -74,6 +75,7 @@ module TSV
74
75
  cells = []
75
76
  cells.push((name and key.respond_to?(:name)) ? key.name || key : key )
76
77
 
78
+ values = [values] unless Array === values
77
79
  values.each do |value|
78
80
  v = (name and value.respond_to?(:name)) ? value.name || value : value
79
81
  if Array === v
@@ -173,7 +173,7 @@ module TSV
173
173
  progress_monitor.tick if progress_monitor
174
174
 
175
175
  keys, value = traverser.process(key, value)
176
-
176
+
177
177
  keys = [keys].compact unless Array === keys
178
178
 
179
179
  # Annotated with Entity and NamedArray
@@ -187,10 +187,10 @@ module TSV
187
187
  if value.nil?
188
188
  nil
189
189
  else
190
- NamedArray.setup value, traverser.new_field_names, key, entity_options
190
+ NamedArray.setup value, traverser.new_field_names, key, entity_options, entity_templates
191
191
  end
192
192
  when :flat, :single
193
- Misc.prepare_entity(value, traverser.new_field_names.first, entity_options)
193
+ prepare_entity(value, traverser.new_field_names.first, entity_options)
194
194
  end
195
195
  end
196
196
  end
@@ -255,6 +255,8 @@ module TSV
255
255
  data.fields = new_field_names
256
256
  data.filename = filename
257
257
  data.namespace = namespace
258
+ data.entity_options = entity_options
259
+ data.entity_templates = entity_templates
258
260
  data.type = type
259
261
  end
260
262
  end
@@ -295,6 +297,8 @@ module TSV
295
297
  new.type = type
296
298
  new.filename = filename
297
299
  new.namespace = namespace
300
+ new.entity_options = entity_options
301
+ new.entity_templates = entity_templates
298
302
 
299
303
  case
300
304
  when (method.nil? and block_given?)
@@ -307,7 +311,7 @@ module TSV
307
311
  case type
308
312
  when :single
309
313
  through do |key, value|
310
- new[key] = values if method.include? key or method.include? value
314
+ new[key] = value if method.include? key or method.include? value
311
315
  end
312
316
  when :list, :flat
313
317
  through do |key, values|
@@ -527,4 +531,23 @@ module TSV
527
531
 
528
532
  self
529
533
  end
534
+
535
+ def transpose(key_field)
536
+ raise "Transposing only works for TSVs of type :list" unless type == :list
537
+ new_fields = keys
538
+ new = TSV.setup({}, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
539
+
540
+ through do |key, values|
541
+ fields.zip(values) do |new_key, value|
542
+ new[new_key] ||= []
543
+ new[new_key][new_fields.index key] = value
544
+ end
545
+ end
546
+
547
+ new.entity_options = entity_options
548
+ new.entity_templates = entity_templates
549
+ new.namespace = namespace
550
+
551
+ new
552
+ end
530
553
  end
@@ -84,11 +84,14 @@ module TSV
84
84
  return parts.shift, parts if field_positions.nil? and key_position.nil?
85
85
  key = parts[key_position]
86
86
 
87
- values = if field_positions.nil?
87
+ values = case
88
+ when field_positions.nil?
88
89
  parts.tap{|o| o.delete_at key_position}
89
- else
90
+ when field_positions.empty?
91
+ []
92
+ else
90
93
  parts.values_at *field_positions
91
- end
94
+ end
92
95
 
93
96
  [key, values]
94
97
  end
@@ -96,11 +99,14 @@ module TSV
96
99
  def get_values_double(parts)
97
100
  return parts.shift.split(@sep2, -1), parts.collect{|value| value.split(@sep2, -1)} if field_positions.nil? and key_position.nil?
98
101
  keys = parts[key_position].split(@sep2, -1)
99
- values = if field_positions.nil?
102
+ values = case
103
+ when field_positions.nil?
100
104
  parts.tap{|o| o.delete_at key_position}
101
- else
102
- parts.values_at *field_positions
103
- end.collect{|value| value.split(@sep2, -1)}
105
+ when field_positions.empty?
106
+ []
107
+ else
108
+ parts.values_at *field_positions
109
+ end.collect{|value| value.split(@sep2, -1)}
104
110
  [keys, values]
105
111
  end
106
112
 
data/lib/rbbt/util/R.rb CHANGED
@@ -38,6 +38,17 @@ module R
38
38
  end
39
39
  end
40
40
 
41
+ def self.ruby2R(object)
42
+ case object
43
+ when String
44
+ "'#{ object }'"
45
+ when Fixnum
46
+ object
47
+ when Array
48
+ "c(#{object.collect{|e| ruby2R(e) } * ", "})"
49
+ end
50
+ end
51
+
41
52
  end
42
53
 
43
54
  module TSV
@@ -64,4 +75,3 @@ if (! is.null(data)){ rbbt.tsv.write('#{f}', data); }
64
75
  end
65
76
  end
66
77
  end
67
-
@@ -6,11 +6,27 @@ require 'net/smtp'
6
6
  require 'narray'
7
7
  require 'digest/md5'
8
8
 
9
+ class Hash
10
+ def chunked_values_at(keys, max = 5000)
11
+ Misc.ordered_divide(keys, max).inject([]) do |acc,c|
12
+ new = self.values_at(*c)
13
+ new.annotate acc if new.respond_to? :annotate and acc.empty?
14
+ acc.concat(new)
15
+ end
16
+ end
17
+ end
18
+
9
19
  module Misc
10
20
  class FieldNotFoundError < StandardError;end
11
21
 
12
- def self.humanize(string)
13
- string.gsub(/([a-z])([A-Z])/,'\1_\2').downcase
22
+ def self.pid_exists?(pid)
23
+ return false if pid.nil?
24
+ begin
25
+ Process.getpgid(pid.to_i)
26
+ true
27
+ rescue Errno::ESRCH
28
+ false
29
+ end
14
30
  end
15
31
 
16
32
  COLOR_LIST = %w(#BC80BD #CCEBC5 #FFED6F #8DD3C7 #FFFFB3 #BEBADA #FB8072 #80B1D3 #FDB462 #B3DE69 #FCCDE5 #D9D9D9)
@@ -32,7 +48,7 @@ module Misc
32
48
  [colors, used]
33
49
  end
34
50
 
35
- def self.total_length(ranges)
51
+ def self.collapse_ranges(ranges)
36
52
  processed = []
37
53
  last = nil
38
54
  ranges.sort_by{|range| range.begin }.each do |range|
@@ -55,9 +71,14 @@ module Misc
55
71
  end
56
72
  end
57
73
 
58
- processed.inject(0) do |total,range| total += range.end - range.begin + 1 end
74
+ processed
75
+ end
76
+
77
+ def self.total_length(ranges)
78
+ Misc.collapse_ranges(ranges).inject(0) do |total,range| total += range.end - range.begin + 1 end
59
79
  end
60
80
 
81
+
61
82
  def self.random_sample_in_range(total, size)
62
83
  p = Set.new
63
84
 
@@ -144,6 +165,8 @@ module Misc
144
165
 
145
166
  def self.remove_long_items(obj)
146
167
  case
168
+ when TSV === obj
169
+ remove_long_items(obj.fields + obj.keys.sort)
147
170
  when (Array === obj and obj.length > ARRAY_MAX_LENGTH)
148
171
  remove_long_items(obj[0..ARRAY_MAX_LENGTH-2] << "TRUNCATED at #{ ARRAY_MAX_LENGTH } (#{obj.length})")
149
172
  when (Hash === obj and obj.length > ARRAY_MAX_LENGTH)
@@ -267,17 +290,12 @@ end
267
290
  end
268
291
 
269
292
  def self.counts(array)
270
- counts = Hash.new 0
293
+ counts = {}
271
294
  array.each do |e|
295
+ counts[e] ||= 0
272
296
  counts[e] += 1
273
297
  end
274
298
 
275
- class << counts; self;end.class_eval do
276
- def to_s
277
- sort{|a,b| a[1] == b[1] ? a[0] <=> b[0] : a[1] <=> b[1]}.collect{|k,c| "%3d\t%s" % [c, k]} * "\n"
278
- end
279
- end
280
-
281
299
  counts
282
300
  end
283
301
 
@@ -364,13 +382,26 @@ end
364
382
  for spos in 0..cols-1 do a[spos, 0] = spos * init_gap end
365
383
  for rpos in 0..rows-1 do a[0, rpos] = rpos * init_gap end
366
384
 
367
- for spos in 1..cols-1 do
368
- for rpos in 1..rows-1 do
385
+ #for spos in 1..cols-1 do
386
+ # for rpos in 1..rows-1 do
387
+ # match = a[spos-1,rpos-1] + (sequence[spos-1] != reference[rpos-1] ? diff : same)
388
+ # skip_sequence = a[spos-1,rpos] + gap
389
+ # skip_reference = a[spos,rpos-1] + gap
390
+ # a[spos,rpos] = [match, skip_sequence, skip_reference].max
391
+ # end
392
+ #end
393
+
394
+ spos = 1
395
+ while spos < cols do
396
+ rpos = 1
397
+ while rpos < rows do
369
398
  match = a[spos-1,rpos-1] + (sequence[spos-1] != reference[rpos-1] ? diff : same)
370
399
  skip_sequence = a[spos-1,rpos] + gap
371
400
  skip_reference = a[spos,rpos-1] + gap
372
401
  a[spos,rpos] = [match, skip_sequence, skip_reference].max
402
+ rpos += 1
373
403
  end
404
+ spos += 1
374
405
  end
375
406
 
376
407
  start = Misc.max(a[-1,0..rows-1])
@@ -417,9 +448,10 @@ end
417
448
  ref = ref + '-'
418
449
  spos -= 1
419
450
  end
420
-
451
+
421
452
  [ref.reverse + reference[start_pos..-1], seq.reverse + '-' * (rows - start_pos - 1)]
422
453
  end
454
+
423
455
  def self.IUPAC_to_base(iupac)
424
456
  IUPAC2BASE[iupac]
425
457
  end
@@ -580,7 +612,6 @@ end
580
612
  raise $!
581
613
  ensure
582
614
  result = RubyProf.stop
583
- #result.eliminate_methods!([/annotated_array_clean_/])
584
615
  printer = RubyProf::FlatPrinter.new(result)
585
616
  printer.print(STDOUT, options)
586
617
  end
@@ -604,12 +635,13 @@ end
604
635
  res
605
636
  end
606
637
 
607
- def self.insist(times = 3)
638
+ def self.insist(times = 3, sleep = nil)
608
639
  try = 0
609
640
  begin
610
641
  yield
611
642
  rescue
612
643
  Log.warn("Insisting after exception: #{$!.message}")
644
+ sleep sleep if sleep
613
645
  try += 1
614
646
  retry if try < times
615
647
  raise $!
@@ -652,11 +684,48 @@ end
652
684
  }.compact * "&"
653
685
  end
654
686
 
687
+ def self.hash_to_html_tag_attributes(hash)
688
+ return "" if hash.nil? or hash.empty?
689
+ hash.collect{|k,v|
690
+ case
691
+ when (k.nil? or v.nil? or (String === v and v.empty?))
692
+ nil
693
+ when Array === v
694
+ [k,"'" << v * " " << "'"] * "="
695
+ when String === v
696
+ [k,"'" << v << "'"] * "="
697
+ when Symbol === v
698
+ [k,"'" << v.to_s << "'"] * "="
699
+ when TrueClass === v
700
+ [k,"'" << v.to_s << "'"] * "="
701
+ when (Fixnum === v or Float === v)
702
+ [k,"'" << v.to_s << "'"] * "="
703
+ else
704
+ nil
705
+ end
706
+ }.compact * " "
707
+ end
708
+
709
+ def self.html_tag(tag, content = nil, params = {})
710
+ attr_str = hash_to_html_tag_attributes(params)
711
+ html = if content.nil?
712
+ "<#{ tag } #{attr_str} />"
713
+ else
714
+ "<#{ tag } #{attr_str} >#{ content }</#{ tag }>"
715
+ end
716
+
717
+ html
718
+ end
719
+
720
+
655
721
  def self.path_relative_to(basedir, path)
656
722
  path = File.expand_path(path)
657
723
  basedir = File.expand_path(basedir)
658
724
 
659
- if path =~ /#{Regexp.quote basedir}\/(.*)/
725
+ case
726
+ when path == basedir
727
+ "."
728
+ when path =~ /#{Regexp.quote basedir}\/(.*)/
660
729
  return $1
661
730
  else
662
731
  return nil
@@ -671,7 +740,20 @@ end
671
740
 
672
741
  lockfile = Lockfile.new(File.expand_path(file + '.lock'))
673
742
 
674
- lockfile.lock do
743
+ begin
744
+ if File.exists? lockfile and
745
+ `hostname`.strip == (info = YAML.load_file(lockfile))["host"] and
746
+ info["pid"] and not Misc.pid_exists?(info["pid"])
747
+
748
+ Log.info("Removing lockfile: #{lockfile}. This pid #{Process.pid}. Content: #{info.inspect}")
749
+ FileUtils.rm lockfile
750
+ end
751
+ rescue
752
+ Log.warn("Error checking lockfile #{lockfile}: #{$!.message}. Removing. Content: #{begin Open.read(lockfile) rescue "Could not open file" end}")
753
+ FileUtils.rm lockfile if File.exists? lockfile
754
+ end
755
+
756
+ lockfile.lock do
675
757
  res = yield file, *args
676
758
  end
677
759
 
@@ -706,37 +788,56 @@ end
706
788
  res
707
789
  end
708
790
 
791
+ def self.to_utf8(string)
792
+ string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
793
+ end
794
+
709
795
  def self.fixutf8(string)
710
796
  return string if (string.respond_to? :valid_encoding? and string.valid_encoding?) or
711
- (string.respond_to? :valid_encoding and string.valid_encoding)
797
+ (string.respond_to? :valid_encoding and string.valid_encoding)
712
798
  if string.respond_to?(:encode)
713
799
  string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
714
800
  else
801
+ require 'iconv'
715
802
  @@ic ||= Iconv.new('UTF-8//IGNORE', 'UTF-8')
716
803
  @@ic.iconv(string)
717
804
  end
718
805
  end
719
806
 
807
+ def self.fixascii(string)
808
+ if string.respond_to?(:encode)
809
+ self.fixutf8(string).encode("ASCII-8BIT")
810
+ else
811
+ string
812
+ end
813
+ end
814
+
720
815
  def self.sensiblewrite(path, content)
721
816
  Misc.lock path + '.sensible_write' do
722
- begin
723
- case
724
- when String === content
725
- File.open(path, 'w') do |f| f.write content end
726
- when (IO === content or StringIO === content)
727
- File.open(path, 'w') do |f| while l = content.gets; f.write l; end end
817
+ if not File.exists? path
818
+ begin
819
+ tmp_path = path + '.tmp'
820
+ case
821
+ when String === content
822
+ File.open(tmp_path, 'w') do |f| f.write content end
823
+ when (IO === content or StringIO === content)
824
+ File.open(tmp_path, 'w') do |f| while l = content.gets; f.write l; end end
728
825
  else
729
- File.open(path, 'w') do |f| end
826
+ File.open(tmp_path, 'w') do |f| end
730
827
  end
828
+ FileUtils.mv tmp_path, path
731
829
  rescue Interrupt
830
+ FileUtils.rm_f tmp_path if File.exists? tmp_path
732
831
  FileUtils.rm_f path if File.exists? path
733
832
  raise "Interrupted (Ctrl-c)"
734
833
  rescue Exception
834
+ FileUtils.rm_f tmp_path if File.exists? tmp_path
735
835
  FileUtils.rm_f path if File.exists? path
736
836
  raise $!
737
837
  end
738
838
  end
739
839
  end
840
+ end
740
841
 
741
842
  def self.add_defaults(options, defaults = {})
742
843
  case
@@ -800,6 +901,8 @@ end
800
901
  end
801
902
 
802
903
  end
904
+
905
+ str << "_" << hash2md5(v.info) if Annotated === v
803
906
  end
804
907
 
805
908
  if str.empty?
@@ -934,6 +1037,59 @@ end
934
1037
  array[0].zip(*array[1..-1])
935
1038
  end
936
1039
 
1040
+ def self.snake_case(string)
1041
+ return nil if string.nil?
1042
+ string.
1043
+ gsub(/([A-Z]{2,})([A-Z][a-z])/,'\1_\2').
1044
+ gsub(/([a-z])([A-Z])/,'\1_\2').
1045
+ gsub(/\s/,'_').gsub(/[^\w_]/, '').
1046
+ split("_").collect{|p| p.match(/[A-Z]{2,}/) ? p : p.downcase } * "_"
1047
+ end
1048
+
1049
+ # source: https://gist.github.com/ekdevdes/2450285
1050
+ # author: Ethan Kramer (https://github.com/ekdevdes)
1051
+ def self.humanize(value, options = {})
1052
+ if options.empty?
1053
+ options[:format] = :sentence
1054
+ end
1055
+
1056
+ values = []
1057
+ values = value.split('_')
1058
+ values.each_index do |index|
1059
+ # lower case each item in array
1060
+ # Miguel Vazquez edit: Except for acronyms
1061
+ values[index].downcase! unless values[index].match(/[a-zA-Z][A-Z]/)
1062
+ end
1063
+ if options[:format] == :allcaps
1064
+ values.each do |value|
1065
+ value.capitalize!
1066
+ end
1067
+
1068
+ if options.empty?
1069
+ options[:seperator] = " "
1070
+ end
1071
+
1072
+ return values.join " "
1073
+ end
1074
+
1075
+ if options[:format] == :class
1076
+ values.each do |value|
1077
+ value.capitalize!
1078
+ end
1079
+
1080
+ return values.join ""
1081
+ end
1082
+
1083
+ if options[:format] == :sentence
1084
+ values[0].capitalize! unless values[0].match(/[a-zA-Z][A-Z]/)
1085
+
1086
+ return values.join " "
1087
+ end
1088
+
1089
+ if options[:format] == :nocaps
1090
+ return values.join " "
1091
+ end
1092
+ end
937
1093
  end
938
1094
 
939
1095
  class RBBTError < StandardError