rbbt-util 5.1.0 → 5.2.0

data/lib/rbbt/tsv/excel.rb

@@ -4,7 +4,7 @@ module TSV
     name = Misc.process_options options, :name
     sort_by = Misc.process_options options, :sort_by
     sort_by_cast = Misc.process_options options, :sort_by_cast
-    fields = Misc.process_options(options, :fields) || all_fields
+    fields = Misc.process_options(options, :fields) || tsv.all_fields
 
     book = Spreadsheet::Workbook.new
     sheet1 = book.create_worksheet 
@@ -30,6 +30,7 @@ module TSV
       cells = []
       cells.push((name and key.respond_to?(:name)) ?  key.name || key : key )
 
+      values = [values] unless Array === values
       values.each do |value|
         v = (name and value.respond_to?(:name)) ?  value.name || value : value 
         if Array === v
@@ -74,6 +75,7 @@ module TSV
       cells = []
       cells.push((name and key.respond_to?(:name)) ?  key.name || key : key )
 
+      values = [values] unless Array === values
       values.each do |value|
         v = (name and value.respond_to?(:name)) ?  value.name || value : value 
         if Array === v

data/lib/rbbt/tsv/manipulate.rb

@@ -173,7 +173,7 @@ module TSV
       progress_monitor.tick if progress_monitor
 
       keys, value = traverser.process(key, value)
-
+      
       keys = [keys].compact unless Array === keys
 
       # Annotated with Entity and NamedArray
@@ -187,10 +187,10 @@ module TSV
             if value.nil?
               nil
             else
-              NamedArray.setup value, traverser.new_field_names, key, entity_options
+              NamedArray.setup value, traverser.new_field_names, key, entity_options, entity_templates
             end
           when :flat, :single
-            Misc.prepare_entity(value, traverser.new_field_names.first, entity_options)
+            prepare_entity(value, traverser.new_field_names.first, entity_options)
           end
         end
       end
@@ -255,6 +255,8 @@ module TSV
         data.fields = new_field_names
         data.filename = filename
         data.namespace = namespace
+        data.entity_options = entity_options
+        data.entity_templates = entity_templates
         data.type = type
       end
     end
@@ -295,6 +297,8 @@ module TSV
     new.type      = type
     new.filename  = filename
     new.namespace = namespace
+    new.entity_options = entity_options
+    new.entity_templates = entity_templates
     
    case
     when (method.nil? and block_given?)
@@ -307,7 +311,7 @@ module TSV
         case type
         when :single
           through do |key, value|
-            new[key] = values if method.include? key or method.include? value
+            new[key] = value if method.include? key or method.include? value
           end
         when :list, :flat
           through do |key, values|
@@ -527,4 +531,23 @@ module TSV
 
     self
   end
+
+  def transpose(key_field)
+    raise "Transposing only works for TSVs of type :list" unless type == :list
+    new_fields = keys
+    new = TSV.setup({}, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
+
+    through do |key, values|
+      fields.zip(values) do |new_key, value|
+        new[new_key] ||= []
+        new[new_key][new_fields.index key] = value
+      end
+    end
+
+    new.entity_options = entity_options
+    new.entity_templates = entity_templates
+    new.namespace = namespace
+
+    new
+  end
 end

data/lib/rbbt/tsv/parser.rb

@@ -84,11 +84,14 @@ module TSV
       return parts.shift, parts if field_positions.nil? and key_position.nil?
       key = parts[key_position]
 
-      values = if field_positions.nil?
+      values = case
+               when field_positions.nil?
                 parts.tap{|o| o.delete_at key_position}
-              else
+               when field_positions.empty?
+                 []
+               else
                 parts.values_at *field_positions
-              end
+               end
 
       [key, values]
     end
@@ -96,11 +99,14 @@ module TSV
     def get_values_double(parts)
       return parts.shift.split(@sep2, -1), parts.collect{|value| value.split(@sep2, -1)} if field_positions.nil? and key_position.nil?
       keys = parts[key_position].split(@sep2, -1)
-      values = if field_positions.nil?
+      values = case
+               when field_positions.nil?
                 parts.tap{|o| o.delete_at key_position}
-              else
-                parts.values_at *field_positions
-              end.collect{|value| value.split(@sep2, -1)}
+               when field_positions.empty?
+                 []
+               else
+                 parts.values_at *field_positions
+               end.collect{|value| value.split(@sep2, -1)}
       [keys, values]
     end
 

data/lib/rbbt/util/R.rb

@@ -38,6 +38,17 @@ module R
     end
   end
 
+  def self.ruby2R(object)
+    case object
+    when String
+      "'#{ object }'"
+    when Fixnum
+      object
+    when Array
+      "c(#{object.collect{|e| ruby2R(e) } * ", "})"
+    end
+  end
+
 end
 
 module TSV
@@ -64,4 +75,3 @@ if (! is.null(data)){ rbbt.tsv.write('#{f}', data); }
     end
   end
 end
-

data/lib/rbbt/util/misc.rb

@@ -6,11 +6,27 @@ require 'net/smtp'
 require 'narray'
 require 'digest/md5'
 
+class Hash
+  def chunked_values_at(keys, max = 5000)
+    Misc.ordered_divide(keys, max).inject([]) do |acc,c|
+      new = self.values_at(*c)
+      new.annotate acc if new.respond_to? :annotate and acc.empty?
+      acc.concat(new)
+    end
+  end
+end
+
 module Misc
   class FieldNotFoundError < StandardError;end
 
-  def self.humanize(string)
-    string.gsub(/([a-z])([A-Z])/,'\1_\2').downcase
+  def self.pid_exists?(pid)
+    return false if pid.nil?
+    begin
+      Process.getpgid(pid.to_i)
+      true
+    rescue Errno::ESRCH
+      false
+    end
   end
 
   COLOR_LIST = %w(#BC80BD #CCEBC5 #FFED6F #8DD3C7 #FFFFB3 #BEBADA #FB8072 #80B1D3 #FDB462 #B3DE69 #FCCDE5 #D9D9D9)
@@ -32,7 +48,7 @@ module Misc
     [colors, used]
   end
 
-  def self.total_length(ranges)
+  def self.collapse_ranges(ranges)
     processed = []
     last = nil
     ranges.sort_by{|range| range.begin }.each do |range|
@@ -55,9 +71,14 @@ module Misc
       end
     end
 
-    processed.inject(0) do |total,range| total += range.end - range.begin + 1 end
+    processed
+  end
+
+  def self.total_length(ranges)
+    Misc.collapse_ranges(ranges).inject(0) do |total,range| total += range.end - range.begin + 1 end
   end
 
+
   def self.random_sample_in_range(total, size)
     p = Set.new
 
@@ -144,6 +165,8 @@ module Misc
 
   def self.remove_long_items(obj)
     case
+    when TSV === obj
+      remove_long_items(obj.fields + obj.keys.sort)
     when (Array === obj and obj.length > ARRAY_MAX_LENGTH)
       remove_long_items(obj[0..ARRAY_MAX_LENGTH-2] << "TRUNCATED at #{ ARRAY_MAX_LENGTH } (#{obj.length})")
     when (Hash === obj and obj.length > ARRAY_MAX_LENGTH)
@@ -267,17 +290,12 @@ end
   end
 
   def self.counts(array)
-    counts = Hash.new 0
+    counts = {}
     array.each do |e|
+      counts[e] ||= 0
       counts[e] += 1
     end
 
-    class << counts; self;end.class_eval do
-      def to_s
-        sort{|a,b| a[1] == b[1] ? a[0] <=> b[0] : a[1] <=> b[1]}.collect{|k,c| "%3d\t%s" % [c, k]} * "\n"
-      end
-    end
-
     counts
   end
 
@@ -364,13 +382,26 @@ end
     for spos in 0..cols-1 do a[spos, 0] = spos * init_gap end
     for rpos in 0..rows-1 do a[0, rpos] = rpos * init_gap end
 
-    for spos in 1..cols-1 do
-      for rpos in 1..rows-1 do
+    #for spos in 1..cols-1 do
+    #  for rpos in 1..rows-1 do
+    #    match = a[spos-1,rpos-1] + (sequence[spos-1] != reference[rpos-1] ? diff : same)
+    #    skip_sequence = a[spos-1,rpos] + gap
+    #    skip_reference = a[spos,rpos-1] + gap
+    #    a[spos,rpos] = [match, skip_sequence, skip_reference].max
+    #  end
+    #end
+
+    spos = 1
+    while spos < cols do
+      rpos = 1
+      while rpos < rows do
         match = a[spos-1,rpos-1] + (sequence[spos-1] != reference[rpos-1] ? diff : same)
         skip_sequence = a[spos-1,rpos] + gap
         skip_reference = a[spos,rpos-1] + gap
         a[spos,rpos] = [match, skip_sequence, skip_reference].max
+        rpos += 1
       end
+      spos += 1
     end
 
     start = Misc.max(a[-1,0..rows-1])
@@ -417,9 +448,10 @@ end
       ref = ref + '-'
       spos -= 1
     end
-
+    
     [ref.reverse + reference[start_pos..-1], seq.reverse + '-' * (rows - start_pos - 1)]
   end
+
   def self.IUPAC_to_base(iupac)
     IUPAC2BASE[iupac]
   end
@@ -580,7 +612,6 @@ end
       raise $!
     ensure
       result = RubyProf.stop
-      #result.eliminate_methods!([/annotated_array_clean_/])
       printer = RubyProf::FlatPrinter.new(result)
       printer.print(STDOUT, options)
     end
@@ -604,12 +635,13 @@ end
     res
   end
 
-  def self.insist(times = 3)
+  def self.insist(times = 3, sleep = nil)
     try = 0
     begin
       yield
     rescue
       Log.warn("Insisting after exception: #{$!.message}")
+      sleep sleep if sleep
       try += 1
       retry if try < times
       raise $!
@@ -652,11 +684,48 @@ end
     }.compact * "&"
   end
 
+  def self.hash_to_html_tag_attributes(hash)
+    return "" if hash.nil? or hash.empty?
+    hash.collect{|k,v| 
+      case 
+      when (k.nil? or v.nil? or (String === v and v.empty?))
+        nil
+      when Array === v
+        [k,"'" << v * " " << "'"] * "="
+      when String === v
+        [k,"'" << v << "'"] * "="
+      when Symbol === v
+        [k,"'" << v.to_s << "'"] * "="
+      when TrueClass === v
+        [k,"'" << v.to_s << "'"] * "="
+      when (Fixnum === v or Float === v)
+        [k,"'" << v.to_s << "'"] * "="
+      else
+        nil
+      end
+    }.compact * " "
+  end
+
+  def self.html_tag(tag, content = nil, params = {})
+    attr_str = hash_to_html_tag_attributes(params)
+    html = if content.nil?
+      "<#{ tag } #{attr_str} />"
+    else
+      "<#{ tag } #{attr_str} >#{ content }</#{ tag }>"
+    end
+
+    html
+  end
+
+
   def self.path_relative_to(basedir, path)
     path = File.expand_path(path)
     basedir = File.expand_path(basedir)
 
-    if path =~ /#{Regexp.quote basedir}\/(.*)/
+    case
+    when path == basedir
+      "."
+    when path =~ /#{Regexp.quote basedir}\/(.*)/
       return $1
     else
       return nil
@@ -671,7 +740,20 @@ end
 
     lockfile = Lockfile.new(File.expand_path(file + '.lock'))
 
-    lockfile.lock do
+    begin
+      if File.exists? lockfile and
+        `hostname`.strip == (info = YAML.load_file(lockfile))["host"] and 
+        info["pid"] and not Misc.pid_exists?(info["pid"])
+
+        Log.info("Removing lockfile: #{lockfile}. This pid #{Process.pid}. Content: #{info.inspect}")
+        FileUtils.rm lockfile 
+      end
+    rescue
+      Log.warn("Error checking lockfile #{lockfile}: #{$!.message}. Removing. Content: #{begin Open.read(lockfile) rescue "Could not open file" end}")
+      FileUtils.rm lockfile if File.exists? lockfile 
+    end
+
+    lockfile.lock do 
       res = yield file, *args
     end
 
@@ -706,37 +788,56 @@ end
     res
   end
 
+  def self.to_utf8(string)
+    string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
+  end
+
   def self.fixutf8(string)
     return string if (string.respond_to? :valid_encoding? and string.valid_encoding?) or
-                     (string.respond_to? :valid_encoding and string.valid_encoding)
+    (string.respond_to? :valid_encoding and string.valid_encoding)
     if string.respond_to?(:encode)
       string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
     else
+      require 'iconv'
       @@ic ||= Iconv.new('UTF-8//IGNORE', 'UTF-8')
       @@ic.iconv(string)
     end
   end
 
+  def self.fixascii(string)
+    if string.respond_to?(:encode)
+      self.fixutf8(string).encode("ASCII-8BIT") 
+    else
+      string
+    end
+  end
+
   def self.sensiblewrite(path, content)
     Misc.lock path + '.sensible_write' do
-      begin
-        case
-        when String === content
-          File.open(path, 'w') do |f|  f.write content  end
-        when (IO === content or StringIO === content)
-          File.open(path, 'w') do |f|  while l = content.gets; f.write l; end  end
+      if not File.exists? path
+        begin
+          tmp_path = path + '.tmp'
+          case
+          when String === content
+            File.open(tmp_path, 'w') do |f|  f.write content  end
+          when (IO === content or StringIO === content)
+            File.open(tmp_path, 'w') do |f|  while l = content.gets; f.write l; end  end
         else
-          File.open(path, 'w') do |f|  end
+          File.open(tmp_path, 'w') do |f|  end
         end
+        FileUtils.mv tmp_path, path
       rescue Interrupt
+        FileUtils.rm_f tmp_path if File.exists? tmp_path
         FileUtils.rm_f path if File.exists? path
         raise "Interrupted (Ctrl-c)"
       rescue Exception
+        FileUtils.rm_f tmp_path if File.exists? tmp_path
         FileUtils.rm_f path if File.exists? path
         raise $!
       end
     end
   end
+  end
 
   def self.add_defaults(options, defaults = {})
     case
@@ -800,6 +901,8 @@ end
         end
 
       end
+
+      str << "_" << hash2md5(v.info) if Annotated === v
     end
 
     if str.empty?
@@ -934,6 +1037,59 @@ end
     array[0].zip(*array[1..-1])
   end
 
+  def self.snake_case(string)
+    return nil if string.nil?
+    string.
+      gsub(/([A-Z]{2,})([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z])([A-Z])/,'\1_\2').
+      gsub(/\s/,'_').gsub(/[^\w_]/, '').
+      split("_").collect{|p| p.match(/[A-Z]{2,}/) ? p : p.downcase } * "_"
+  end
+
+  # source: https://gist.github.com/ekdevdes/2450285
+  # author: Ethan Kramer (https://github.com/ekdevdes)
+  def self.humanize(value, options = {})
+    if options.empty?
+      options[:format] = :sentence
+    end
+
+    values = []
+    values = value.split('_')
+    values.each_index do |index|
+      # lower case each item in array
+      # Miguel Vazquez edit: Except for acronyms
+      values[index].downcase! unless values[index].match(/[a-zA-Z][A-Z]/)
+    end
+    if options[:format] == :allcaps
+      values.each do |value|
+        value.capitalize!
+      end
+
+      if options.empty?
+        options[:seperator] = " "
+      end
+
+      return values.join " "
+    end
+
+    if options[:format] == :class
+      values.each do |value|
+        value.capitalize!
+      end
+
+      return values.join ""
+    end
+
+    if options[:format] == :sentence
+      values[0].capitalize! unless values[0].match(/[a-zA-Z][A-Z]/)
+
+      return values.join " "
+    end
+
+    if options[:format] == :nocaps
+      return values.join " "
+    end
+  end
 end
 
 class RBBTError < StandardError