rbbt-util 4.3.0 → 4.4.0

data/lib/rbbt/tsv/parser.rb

@@ -1,7 +1,7 @@
 require 'rbbt/util/cmd'
 module TSV
   class Parser
-    attr_accessor :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight, :take_all, :zipped, :namespace
+    attr_accessor :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight, :take_all, :zipped, :namespace, :first_line
 
     class SKIP_LINE < Exception; end
     class END_PARSING < Exception; end
@@ -48,7 +48,7 @@ module TSV
 
     def process(line)
       l = line.chomp
-      raise Parser::SKIP_LINE if Proc === @select and not @select.call l
+      raise Parser::SKIP_LINE if l[0] == "#"[0] or (Proc === @select and not @select.call l)
       l = @fix.call l if Proc === @fix
       raise Parser::END_PARSING unless l
       l
@@ -75,7 +75,7 @@ module TSV
     def get_values_single(parts)
       return parts.shift, parts.first if field_positions.nil? and key_position.nil?
       key = parts[key_position]
-      value = parts[field_positions.nil? ? 0 : field_positions.first]
+      value = parts[(field_positions.nil? or field_positions.empty?) ? 0 : field_positions.first]
       [key, value]
     end
 

data/lib/rbbt/tsv/util.rb

@@ -1,29 +1,6 @@
 require 'rbbt/resource/path'
 module TSV
 
-  def self.field_match_counts(file, values)
-    fields = TSV.parse_header(Open.open(file)).all_fields
-
-    counts = {}
-    TmpFile.with_file do |tmpfile|
-      if Array === values
-        Open.write(tmpfile, values * "\n")
-      else
-        FileUtils.ln_s values, tmpfile
-      end
-
-      fields.each_with_index do |field,i|
-        counts[field] = begin
-                          CMD.cmd("cat #{ file } |grep -v ^#|cut -f #{i + 1}|tr '|' '\\n' |sort -u |grep [[:alpha:]]|grep -f #{tmpfile} -F -w").read.count("\n")
-                        rescue
-                          0
-                        end
-      end
-    end
-
-    counts
-  end
-
   def self.field_match_counts(file, values, options = {})
     options = Misc.add_defaults options, :persist_prefix => "Field_Matches"
     persist_options = Misc.pull_keys options, :persist
@@ -45,12 +22,12 @@ module TSV
     end
 
     path = Persist.persistence_path(filename, persist_options)
-    TmpFile.with_file(values * "\n") do |value_file|
-      cmd = "cat '#{ path }' | grep -w -F -f '#{ value_file }' |cut -f 2 |sort|uniq -c|sed 's/^ *//;s/ /\t/'"
+    TmpFile.with_file(values.uniq * "\n") do |value_file|
+      cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' |cut -f 2 | sed 's/HEADERNOMATCH//' | sort|uniq -c|sed 's/^ *//;s/ /\t/'"
       begin
         TSV.open(CMD.cmd(cmd), :key_field => 1, :type => :single, :cast => :to_i)
       rescue
-        TSV.setup({nil => 0}, :type => :single, :cast => :to_i)
+        TSV.setup({}, :type => :single, :cast => :to_i)
       end
     end
   end
@@ -67,12 +44,12 @@ module TSV
     filename
   end
 
-  def self.get_stream(file)
+  def self.get_stream(file, open_options = {})
     case
     when Path === file
-      file.open
+      file.open(open_options)
     when String === file
-      File.open(file)
+      Open.open(file, open_options)
     when file.respond_to?(:gets)
       file
     else

data/lib/rbbt/util/R.rb

@@ -22,9 +22,26 @@ module R
     CMD.cmd('R --vanilla --slave --quiet', options.merge(:in => cmd))
   end
 
+  def self.interactive(init_file, options = {})
+    CMD.cmd("env R_PROFILE='#{init_file}' xterm R")
+  end
+
+  def self.interactive(script, options = {})
+    TmpFile.with_file do |init_file|
+        Open.write(init_file) do |file|
+          profile = File.join(ENV["HOME"], ".Rprofile")
+          file.puts "source('#{profile}');\n" if File.exists? profile
+          file.puts "source('#{R::UTIL}');\n"
+          file.puts script
+        end
+        CMD.cmd("env R_PROFILE='#{init_file}' xterm R")
+    end
+  end
+
 end
 
 module TSV
+
   def R(script, open_options = {})
     TmpFile.with_file do |f|
       Open.write(f, self.to_s)
@@ -32,11 +49,19 @@ module TSV
       <<-EOF
 data = rbbt.tsv('#{f}');
 #{script.strip}
-rbbt.tsv.write('#{f}', data);
+if (! is.null(data)){ rbbt.tsv.write('#{f}', data); }
       EOF
       ).read)
       open_options = Misc.add_defaults open_options, :type => :list
-      TSV.open(f, open_options)
+      TSV.open(f, open_options) unless open_options[:ignore_output]
+    end
+  end
+
+  def R_interactive(open_options = {})
+    TmpFile.with_file do |f|
+      Open.write(f, self.to_s)
+      R.interactive("data_file = '#{f}';\n")
     end
   end
 end
+

data/lib/rbbt/util/chain_methods.rb

@@ -1,4 +1,3 @@
-require 'rbbt/util/log'
 require 'set'
 
 module ChainMethods
@@ -33,7 +32,7 @@ module ChainMethods
 
             class << base; self; end.module_eval do
               methods.each do |new_method|
-                original = new_method.sub(prefix.to_s + '_', '')
+                original = new_method.to_s.sub(prefix.to_s + '_', '')
                 clean_method = prefix.to_s + '_clean_' + original
 
                 original = "[]" if original == "get_brackets"

data/lib/rbbt/util/misc.rb

@@ -4,12 +4,125 @@ require 'rbbt/resource/path'
 require 'rbbt/annotations'
 require 'net/smtp'
 require 'narray'
+require 'digest/md5'
 
 module Misc
   class FieldNotFoundError < StandardError;end
 
+  COLOR_LIST = %w(red green blue black yellow pink purple)
+  def self.colors_for(list)
+    unused = COLOR_LIST.dup
+
+    used = {}
+    colors = list.collect do |elem|
+      if used.include? elem
+        used[elem]
+      else
+        color = unused.shift
+        used[elem]=color
+        color
+      end
+    end
+
+    [colors, used]
+  end
+
+  def self.total_length(ranges)
+    processed = []
+    last = nil
+    ranges.sort_by{|range| range.begin }.each do |range|
+      if last.nil? or range.begin > last
+        processed << range
+        last = range.end
+      else
+        new_processed = []
+        processed.each do |processed_range|
+          if processed_range.end < range.begin
+            new_processed << processed_range
+          else
+            eend = [range.end, processed_range.end].max
+            new_processed << (processed_range.begin..eend)
+            break
+          end
+        end
+        processed = new_processed
+        last = range.end if range.end > last
+      end
+    end
+
+    processed.inject(0) do |total,range| total += range.end - range.begin + 1 end
+  end
+
+  def self.random_sample_in_range(total, size)
+    p = Set.new
+
+    if size > total / 10
+      template = (0..total - 1).to_a
+      size.times do |i|
+        pos = (rand * (total - i)).floor
+        if pos == template.length - 1
+          v = template.pop
+        else
+          v, n = template[pos], template[-1]
+          template.pop
+          template[pos] = n 
+        end
+        p << v
+      end
+    else
+      size.times do 
+        pos = nil
+        while pos.nil? 
+          pos = (rand * total).floor
+          if p.include? pos
+            pos = nil
+          end
+        end
+        p << pos
+      end
+    end
+    p
+  end
+
+  def self.sample(ary, size, replacement = false)
+    total = ary.length
+    p = random_sample_in_range(total, size)
+    ary.values_at *p
+  end
+
+  Log2Multiplier = 1.0 / Math.log(2.0)
+  def self.log2(x)
+    Math.log(x) * Log2Multiplier
+  end
+
+  def self.prepare_entity(entity, field, options = {})
+    return entity unless String === entity or Array === entity
+    options ||= {}
+    dup_array = options.delete :dup_array
+    entity = Entity.formats[field].setup(((entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) ), options.merge({:format => field})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? field
+    entity
+  end
+
   ARRAY_MAX_LENGTH = 10000
   STRING_MAX_LENGTH = ARRAY_MAX_LENGTH * 10
+
+  def self.sanitize_filename(filename, length = 200)
+    if filename.length > length
+      if filename =~ /(\..{2,4})$/
+        extension = $1
+      else
+        extension = ''
+      end
+
+      post_fix = " TRUNCATED at #{length} (#{filename.length})" + extension
+
+      filename = filename[0..(length - post_fix.length - 1)] << post_fix
+    else
+      filename
+    end
+    filename
+  end
+
   def self.remove_long_items(obj)
     case
     when (Array === obj and obj.length > ARRAY_MAX_LENGTH)
@@ -460,11 +573,14 @@ end
   end
 
   def self.lock(file, *args)
+    return yield file, *args if file.nil?
     FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
 
     res = nil
 
-    Lockfile.new(file + '.lock') do
+    lockfile = Lockfile.new(File.expand_path(file + '.lock'))
+
+    lockfile.lock do
       res = yield file, *args
     end
 
@@ -500,16 +616,18 @@ end
   end
 
   def self.fixutf8(string)
-    if string.respond_to?(:valid_encoding?) and ! string.valid_encoding?
+    return string if (string.respond_to? :valid_encoding? and string.valid_encoding?) or
+                     (string.respond_to? :valid_encoding and string.valid_encoding)
+    if string.respond_to?(:encode)
+      string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
+    else
       @@ic ||= Iconv.new('UTF-8//IGNORE', 'UTF-8')
       @@ic.iconv(string)
-    else
-      string
     end
   end
 
   def self.sensiblewrite(path, content)
-    Misc.lock path do
+    Misc.lock path + '.sensible_write' do
       begin
         case
         when String === content
@@ -520,10 +638,10 @@ end
           File.open(path, 'w') do |f|  end
         end
       rescue Interrupt
-        FileUtils.rm_f path
+        FileUtils.rm_f path if File.exists? path
         raise "Interrupted (Ctrl-c)"
       rescue Exception
-        FileUtils.rm_f path
+        FileUtils.rm_f path if File.exists? path
         raise $!
       end
     end
@@ -551,9 +669,15 @@ end
     Digest::MD5.hexdigest(text)
   end
 
+  HASH2MD5_MAX_STRING_LENGTH = 1000
+  HASH2MD5_MAX_ARRAY_LENGTH = 100
   def self.hash2md5(hash)
     str = ""
-    hash.keys.sort_by{|k| k.to_s}.each do |k|
+    keys = hash.keys
+    keys = keys.clean_annotations if keys.respond_to? :clean_annotations
+    keys = keys.sort_by{|k| k.to_s}
+
+    keys.each do |k|
       next if k == :monitor or k == "monitor" or k == :in_situ_persistence or k == "in_situ_persistence"
       v = hash[k]
       case
@@ -565,10 +689,14 @@ end
         str << k.to_s << "=>" << hash2md5(v)
       when Symbol === v
         str << k.to_s << "=>" << v.to_s
+      when (String === v and v.length > HASH2MD5_MAX_STRING_LENGTH)
+        str << k.to_s << "=>" << v[0..HASH2MD5_MAX_STRING_LENGTH]
       when String === v
-        str << k.to_s << "=>" << v[0..10000]
+        str << k.to_s << "=>" << v
+      when (Array === v and v.length > HASH2MD5_MAX_ARRAY_LENGTH)
+        str << k.to_s << "=>[" << v[0..HASH2MD5_MAX_ARRAY_LENGTH] * "," << "]"
       when Array === v
-        str << k.to_s << "=>[" << v[0..1000] * "," << "]"
+        str << k.to_s << "=>[" << v * "," << "]"
       else
         v_ins = v.inspect
 
@@ -585,7 +713,7 @@ end
     if str.empty?
       ""
     else
-      Digest::MD5.hexdigest(str)
+      digest(str)
     end
   end
 
@@ -696,6 +824,7 @@ end
   end
 
   def self.zip_fields(array)
+    return [] if array.empty?
     array[0].zip(*array[1..-1])
   end
 
@@ -766,13 +895,13 @@ module IndiferentHash
 end
 
 module PDF2Text
-  def self.pdftotext(filename)
+  def self.pdftotext(filename, options = {})
     require 'rbbt/util/cmd'
     require 'rbbt/util/tmpfile'
     require 'rbbt/util/open'
 
 
-    TmpFile.with_file(Open.open(filename, :nocache => true).read) do |pdf_file|
+    TmpFile.with_file(Open.open(filename, options.merge(:nocache => true)).read) do |pdf_file|
       CMD.cmd("pdftotext #{pdf_file} -", :pipe => false, :stderr => true)
     end
   end

data/lib/rbbt/util/named_array.rb

@@ -7,13 +7,13 @@ module NamedArray
   self.chain_prefix = :named_array
   attr_accessor :fields
   attr_accessor :key
-  attr_accessor :namespace
+  attr_accessor :entity_options
 
-  def self.setup(array, fields, key = nil, namespace = nil)
+  def self.setup(array, fields, key = nil, entity_options = nil)
     array.extend NamedArray unless NamedArray === array
     array.fields = fields
     array.key = key
-    array.namespace = namespace
+    array.entity_options = entity_options
     array
   end
 
@@ -47,14 +47,14 @@ module NamedArray
     return elem if @fields.nil? or @fields.empty?
 
     field = NamedArray === @fields ? @fields.named_array_clean_get_brackets(pos) : @fields[pos]
-    elem = Entity.formats[field].setup((elem.frozen? ? elem.dup : elem), (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => field})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include?(field) and not field == elem
+    elem = Misc.prepare_entity(elem, field, entity_options)
     elem
   end
 
   def named_array_each(&block)
     if defined?(Entity) and not @fields.nil? and not @fields.empty?
       @fields.zip(self).each do |field,elem|
-        elem = Entity.formats[field].setup((elem.frozen? ? elem.dup : elem), (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => field})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include?(field) and not field == elem
+        elem = Misc.prepare_entity(elem, field, entity_options)
         yield(elem)
         elem
       end

data/lib/rbbt/util/open.rb

@@ -174,6 +174,8 @@ module Open
     wget_options[:cookies] = options.delete(:cookies)
 
     io = case
+         when (IO === url or StringIO === url)
+           url
          when (not remote?(url))
            file_open(url, options[:grep])
          when (options[:nocache] and options[:nocache] != :update)
@@ -189,8 +191,8 @@ module Open
            io.close
            file_open(in_cache(url, wget_options), options[:grep])
          end
-    io = unzip(io)  if (zip?(url)  and not options[:noz]) or options[:zip]
-    io = gunzip(io) if (gzip?(url) and not options[:noz]) or options[:gzip]
+    io = unzip(io)  if ((String === url and zip?(url))  and not options[:noz]) or options[:zip]
+    io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]
 
     if block_given?
       yield io 
@@ -214,12 +216,14 @@ module Open
     f = open(file, options)
 
     if block_given?
+      res = []
       while not f.eof?
         l = f.gets
-        l = fixutf8(l) if l.respond_to?(:valid_encoding?) && ! l.valid_encoding?
-        yield l
+        l = Misc.fixutf8(l) 
+        res << yield(l)
       end
       f.close
+      res
     else
       text = Misc.fixutf8(f.read)
       f.close unless f.closed?