rbbt-util 4.0.2 → 4.1.0

data/lib/rbbt/tsv/parser.rb

@@ -1,7 +1,7 @@
 require 'rbbt/util/cmd'
 module TSV
   class Parser
-    attr_accessor :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight
+    attr_accessor :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight, :take_all, :zipped
 
     class SKIP_LINE < Exception; end
     class END_PARSING < Exception; end
@@ -63,23 +63,64 @@ module TSV
     end
 
     def get_values_single(parts)
-      return parts.shift, parts.first if field_positions.nil?
+      return parts.shift, parts.first if field_positions.nil? and key_position.nil?
       key = parts[key_position]
-      value = parts[field_positions.first]
+      value = parts[field_positions.nil? ? 0 : field_positions.first]
       [key, value]
     end
 
     def get_values_list(parts)
-      return parts.shift, parts if field_positions.nil?
+      return parts.shift, parts if field_positions.nil? and key_position.nil?
       key = parts[key_position]
-      values = parts.values_at *field_positions
+
+      values = if field_positions.nil?
+                parts.tap{|o| o.delete_at key_position}
+              else
+                parts.values_at *field_positions
+              end
+
       [key, values]
     end
 
     def get_values_double(parts)
-      return parts.shift.split(@sep2, -1), parts.collect{|value| value.split(@sep2, -1)} if field_positions.nil?
+      return parts.shift.split(@sep2, -1), parts.collect{|value| value.split(@sep2, -1)} if field_positions.nil? and key_position.nil?
+      keys = parts[key_position].split(@sep2, -1)
+      values = if field_positions.nil?
+                parts.tap{|o| o.delete_at key_position}
+              else
+                parts.values_at *field_positions
+              end.collect{|value| value.split(@sep2, -1)}
+      [keys, values]
+    end
+
+    def get_values_flat_inverse(parts)
+      value = parts.shift
+      keys = parts
+      [keys, [value]]
+    end
+
+    def get_values_flat(parts)
+      if key_position and key_position != 0 and field_positions.nil?
+        value = parts.shift
+        keys = parts
+        return [keys, [value]]
+      end
+
+      return parts.shift.split(@sep2, -1), parts.collect{|value| value.split(@sep2, -1)} if 
+        field_positions.nil? and (key_position.nil? or key_position == 0)
+
       keys = parts[key_position].split(@sep2, -1)
-      values = parts.values_at(*field_positions).collect{|value| value.split(@sep2, -1)}
+
+      if @take_all
+        values = parts.collect{|value| value.split(@sep2, -1)}
+      else
+
+        values = if field_positions.nil?
+                   parts.tap{|o| o.delete_at key_position}
+                 else
+                   parts.values_at *field_positions
+                 end.collect{|value| value.split(@sep2, -1)}
+      end
       [keys, values]
     end
 
@@ -110,18 +151,52 @@ module TSV
     end
 
     def add_to_data_merge(data, keys, values)
-      keys.each do |key|
+      keys.uniq.each do |key|
+        if data.include? key
+          #data[key] = data[key].zip(values).collect do |old, new|
+          #  old.concat new
+          #  old
+          #end
+          new = data[key]
+          new.each_with_index do |old, i|
+            old.concat values[i]
+          end
+          data[key] = new
+        else
+          data[key] = values
+        end
+      end
+    end
+
+    def add_to_data_merge_zipped(data, keys, values)
+      num = keys.length
+      values = values.collect{|v| v.length != num ? [v.first] * num : v}
+      all = values.unshift keys
+      Misc.zip_fields(all).each do |values|
+        key = values.shift
         if data.include? key
           data[key] = data[key].zip(values).collect do |old, new|
-            old.concat new
+            old.push new
             old
           end
         else
-          data[key] = values
+          data[key] = values.collect{|v| [v]}
         end
       end
     end
 
+    def add_to_data_zipped(data, keys, values)
+      num = keys.length
+      values = values.collect{|v| v.length != num ? [v.first] * num : v}
+      all = values.unshift keys
+      Misc.zip_fields(all).each do |values|
+        key = values.shift
+        next if data.include? key
+        data[key] = values.collect{|v| [v]}
+      end
+    end
+
+
     def cast_values_single(value)
       case
       when Symbol === cast
@@ -157,6 +232,7 @@ module TSV
       key_field = Misc.process_options options, :key_field
       fields    = Misc.process_options options, :fields
 
+
       if (key_field.nil? or key_field == 0 or key_field == :key) and
         (fields.nil? or fields == @fields or (not @fields.nil? and fields == (1..@fields.length).to_a))
 
@@ -166,7 +242,7 @@ module TSV
         @straight = false
 
         case
-        when (key_field.nil? or key_field == @key_field or key_field == 0)
+        when (key_field.nil? or (not Integer === key_field and @key_field.nil?) or key_field == @key_field or key_field == 0)
           @key_position = 0
         when Integer === key_field
           @key_position = key_field
@@ -176,9 +252,11 @@ module TSV
           raise "Format of key_field not understood: #{key_field.inspect}"
         end
 
-        if (fields.nil? or fields == @fields or (not @fields.nil? and fields == (1..@fields.length).to_a))
-          @field_positions = (0..@fields.length).to_a
-          @field_positions.delete @key_position
+        if (fields.nil? or (not (Array === fields and Integer === fields.first) and @fields.nil?) or fields == @fields or (not @fields.nil? and fields == (1..@fields.length).to_a))
+          if not @fields.nil? and type != :flat
+            @field_positions = (0..@fields.length).to_a
+            @field_positions.delete @key_position
+          end
         else
           fields = [fields] if not Array === fields
           @field_positions = fields.collect{|field|
@@ -196,8 +274,12 @@ module TSV
         end
 
         new_key_field = @fields.dup.unshift(@key_field)[@key_position] if not @fields.nil?
-        @fields = @fields.dup.unshift(@key_field).values_at *@field_positions if not @fields.nil?
-        @key_field = new_key_field
+        @fields = @fields.dup.unshift(@key_field).values_at *@field_positions if not @fields.nil? and not @field_positions.nil?
+        @fields ||= fields if Array === fields and String === fields.first
+        @fields = [@key_field] if new_key_field != @key_field and type == :flat and @field_positions.nil?
+        @key_field = new_key_field 
+        @key_field ||= key_field if String === key_field
+
       end
     end
 
@@ -215,13 +297,19 @@ module TSV
       @type ||= Misc.process_options options, :type
       @fix = Misc.process_options(options, :fix) 
       @select= Misc.process_options options, :select
+      @zipped = Misc.process_options options, :zipped
 
       case @type
       when :double 
         self.instance_eval do alias get_values get_values_double end
         self.instance_eval do alias cast_values cast_values_double end
-        if merge
-          self.instance_eval do alias add_to_data add_to_data_merge end
+        case
+        when (merge and not zipped)
+            self.instance_eval do alias add_to_data add_to_data_merge end
+        when (merge and zipped)
+            self.instance_eval do alias add_to_data add_to_data_merge_zipped end
+        when zipped
+            self.instance_eval do alias add_to_data add_to_data_zipped end
         else
           self.instance_eval do alias add_to_data add_to_data_no_merge_double end
         end
@@ -234,7 +322,8 @@ module TSV
         self.instance_eval do alias cast_values cast_values_list end
         self.instance_eval do alias add_to_data add_to_data_no_merge_list end
       when :flat
-        self.instance_eval do alias get_values get_values_double end
+        @take_all = true if options[:fields].nil?
+        self.instance_eval do alias get_values get_values_flat end
         self.instance_eval do alias cast_values cast_values_double end
         if merge
           self.instance_eval do alias add_to_data add_to_data_flat_merge end
@@ -243,9 +332,10 @@ module TSV
         end
       end
 
+      fields = options[:fields]
       fix_fields(options)
 
-      @straight = false if @sep != "\t" or not @cast.nil? or merge
+      @straight = false if @sep != "\t" or not @cast.nil? or merge or (@type == :flat and fields)
     end
 
     def setup(data)

data/lib/rbbt/tsv/serializers.rb

@@ -14,6 +14,11 @@ module TSV
     def self.load(str); str.unpack("l*"); end
   end
 
+  class FloatArraySerializer
+    def self.dump(a); a.pack("d*"); end
+    def self.load(str); str.unpack("d*"); end
+  end
+
   class StringSerializer
     def self.dump(str); str.to_s; end
     def self.load(str); str; end
@@ -66,6 +71,7 @@ module TSV
     :integer => IntegerSerializer, 
     :float => FloatSerializer, 
     :integer_array => IntegerArraySerializer,
+    :float_array => FloatArraySerializer,
     :marshal => Marshal,
     :single => StringSerializer,
     :string => StringSerializer,

data/lib/rbbt/tsv/util.rb

@@ -23,6 +23,38 @@ module TSV
 
     counts
   end
+
+  def self.field_match_counts(file, values, options = {})
+    options = Misc.add_defaults options, :persist_prefix => "Field_Matches"
+    persist_options = Misc.pull_keys options, :persist
+
+    filename = TSV === file ? file.filename : file
+    text = Persist.persist filename, :string, persist_options do
+      tsv = TSV === file ? file : TSV.open(file)
+
+      text = ""
+      fields = nil
+      tsv.tap{|e| e.unnamed =  true; fields = e.fields}.through do |gene, names|
+        names.zip(fields).each do |list, format|
+          list.delete_if do |name| name.empty? end
+          next if list.empty?
+          text << list.collect{|name| [name, format] * "\t"} * "\n" << "\n"
+        end
+      end
+      text
+    end
+
+    path = Persist.persistence_path(filename, persist_options)
+    TmpFile.with_file(values * "\n") do |value_file|
+      cmd = "cat '#{ path }' | grep -w -F -f '#{ value_file }' |cut -f 2 |sort|uniq -c|sed 's/^ *//;s/ /\\t/'"
+      begin
+        TSV.open(CMD.cmd(cmd), :key_field => 1, :type => :single, :cast => :to_i)
+      rescue
+        TSV.setup({nil => 0}, :type => :single, :cast => :to_i)
+      end
+    end
+  end
+
   def self.get_filename(file)
     case
     when String === file
@@ -55,7 +87,9 @@ module TSV
     when (field.nil? or field == :key or key_field == field)
       :key
     when String === field
-      fields.index field
+      pos = fields.index field
+      Log.medium "Field #{ field } was not found. Options: #{fields * ", "}" if pos.nil?
+      pos
     end
   end
 

data/lib/rbbt/util/chain_methods.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/log'
+require 'set'
 
 module ChainMethods
   def self.chain_methods_extended(base)
@@ -19,18 +20,31 @@ module ChainMethods
 
         def setup_chains(base)
           raise "No prefix specified for #{self.to_s}" if self.chain_prefix.nil? or (String === self.chain_prefix and self.chain_prefix.empty?)
-          #methods = self.instance_methods.select{|method| method =~ /^#{self.chain_prefix}/}
           methods = self.chained_methods
 
           return if methods.empty?
 
           prefix = self.chain_prefix
 
-          new_method = methods.first
-          original = new_method.sub(prefix.to_s + '_', '')
-          first_clean_method = prefix.to_s + '_clean_' + original
+          #do_chain = true
+          #methods.collect{|new_method|
+          #  original = new_method.sub(prefix.to_s + '_', '')
+          #  clean = prefix.to_s + '_clean_' + original
+          #  if base.respond_to? clean
+          #    do_chain = false
+          #    break
+          #  end
+          #}
+
+
+          if not base.respond_to?(:processed_chains) or base.processed_chains.nil? or not base.processed_chains.include? prefix
+            class << base
+              attr_accessor :processed_chains 
+            end if not base.respond_to? :processed_chains
+
+            base.processed_chains = Set.new if base.processed_chains.nil?
+            base.processed_chains << prefix
 
-          if not base.respond_to? first_clean_method
             class << base; self; end.module_eval do
               methods.each do |new_method|
                 original = new_method.sub(prefix.to_s + '_', '')
@@ -39,6 +53,10 @@ module ChainMethods
                 original = "[]" if original == "get_brackets"
                 original = "[]=" if original == "set_brackets"
 
+                if base.respond_to? clean_method
+                  raise "Method already defined: #{clean_method}. #{ prefix }"
+                end
+
                 begin
                   alias_method clean_method, original 
                 rescue
@@ -52,10 +70,10 @@ module ChainMethods
 
       if not metaclass.respond_to? :extended
         metaclass.module_eval do
-          alias prev_chain_methods_extended extended
+          alias prev_chain_methods_extended extended if methods.include? "extended"
 
           def extended(base)
-            prev_chain_methods_extended(base)
+            prev_chain_methods_extended(base) if methods.include? "prev_chain_methods_extended"
             setup_chains(base)
           end
         end
@@ -68,7 +86,4 @@ module ChainMethods
   def self.extended(base)
     chain_methods_extended(base)
   end
-
-
-
 end

data/lib/rbbt/util/misc.rb

@@ -2,10 +2,54 @@ require 'lockfile'
 require 'rbbt/util/chain_methods'
 require 'rbbt/resource/path'
 require 'rbbt/annotations'
+require 'net/smtp'
 
 module Misc
   class FieldNotFoundError < StandardError;end
 
+  def self.consolidate(list)
+    list.inject(nil){|acc,e|
+      if acc.nil?
+        acc = e
+      else
+        acc.concat e
+        acc
+      end
+    }
+  end
+
+  def self.positional2hash(keys, *values)
+    if Hash === values.last
+      extra = values.pop
+      inputs = Misc.zip2hash(keys, values)
+      inputs.delete_if{|k,v| v.nil?}
+      inputs = Misc.add_defaults inputs, extra
+      inputs.delete_if{|k,v| not keys.include? k}
+    else
+      Misc.zip2hash(keys, values)
+    end
+  end
+
+  def self.send_email(from, to, subject, message, options = {})
+    IndiferentHash.setup(options)
+    options = Misc.add_defaults options, :from_alias => nil, :to_alias => nil, :server => 'localhost', :port => 25, :user => nil, :pass => nil, :auth => :login
+    IndiferentHash.setup(options)
+    
+    server, port, user, pass, from_alias, to_alias, auth = Misc.process_options options, :server, :port, :user, :pass, :from_alias, :to_alias, :auth
+
+    msg = <<-END_OF_MESSAGE
+From: #{from_alias} <#{from}>
+To: #{to_alias} <#{to}>
+Subject: #{subject}
+
+#{message}
+    END_OF_MESSAGE
+
+    Net::SMTP.start(server, port, server, user, pass, auth) do |smtp|
+      smtp.send_message msg, from, to
+    end
+  end
+
   def self.counts(array)
     counts = Hash.new 0
     array.each do |e|
@@ -13,7 +57,7 @@ module Misc
     end
     counts
   end
-  
+
   IUPAC2BASE = {
     "A" => ["A"],
     "C" => ["C"],
@@ -55,19 +99,16 @@ module Misc
     e1, e2 = a1.shift, a2.shift
     intersect = []
     while true
-      case
-      when (e1 and e2)
-        case e1 <=> e2
-        when 0
-          intersect << e1
-          e1, e2 = a1.shift, a2.shift
-        when -1
-          e1 = a1.shift
-        when 1
-          e2 = a2.shift
-        end
-      else
-        break
+      break if e1.nil? or e2.nil?
+      case e1 <=> e2
+      when 0
+        intersect << e1
+        e1, e2 = a1.shift, a2.shift
+      when -1
+        e1 = a1.shift while not e1.nil? and e1 < e2
+      when 1
+        e2 = a2.shift
+        e2 = a2.shift while not e2.nil? and e2 < e1
       end
     end
     intersect
@@ -167,6 +208,22 @@ module Misc
     res
   end
 
+  def self.memprof
+    require 'memprof'
+    Memprof.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      Memprof.stop
+      print Memprof.stats
+    end
+
+    res
+  end
+
   def self.insist(times = 3)
     try = 0
     begin
@@ -228,6 +285,8 @@ module Misc
       FileUtils.mkdir_p dir unless File.exists? dir
       FileUtils.cd dir
       res = yield
+    rescue
+      raise $!
     ensure
       FileUtils.cd old_pwd
     end
@@ -271,8 +330,9 @@ module Misc
     else
       raise "Format of '#{options.inspect}' not understood. It should be a hash"
     end
+
     defaults.each do |key, value|
-      next unless new_options[key].nil?
+      next if options.include? key
 
       new_options[key] = value 
     end
@@ -419,12 +479,15 @@ end
 
 module NamedArray
   extend ChainMethods
+
   self.chain_prefix = :named_array
   attr_accessor :fields
+  attr_accessor :key
 
-  def self.setup(array, fields)
+  def self.setup(array, fields, key = nil)
     array.extend NamedArray
     array.fields = fields
+    array.key = key
     array
   end
 
@@ -455,11 +518,13 @@ module NamedArray
     if defined? Entity
       entity = (defined?(Entity) and Entity.respond_to?(:formats)) ? Entity.formats[key] : nil
       if entity
-        if entity.annotations.first == :format
-          entity.setup(named_array_clean_get_brackets(Misc.field_position(fields, key)), key)
-        else
-          entity.setup(named_array_clean_get_brackets(Misc.field_position(fields, key)))
-        end
+        elem = if entity.annotations.first == :format
+                 entity.setup(named_array_clean_get_brackets(Misc.field_position(fields, key)), key)
+               else
+                 entity.setup(named_array_clean_get_brackets(Misc.field_position(fields, key)))
+               end
+        elem.context = self
+        elem
       else
         named_array_clean_get_brackets(Misc.field_position(fields, key))
       end
@@ -469,18 +534,21 @@ module NamedArray
   end
 
   def named_array_each(&block)
-    if defined?(Entity) and not fields.nil? and not fields.empty?
-      fields.zip(self) do |field,elem|
+    if defined?(Entity) and not @fields.nil? and not @fields.empty?
+      @fields.zip(self).each do |field,elem|
         entity = (defined?(Entity) and Entity.respond_to?(:formats)) ? Entity.formats[field] : nil
+
         if entity
           elem = elem.dup if elem.frozen?
           if entity.annotations.first == :format
-            elem = entity.setup(elem, field) 
+            entity.setup(elem, field) 
           else
-            elem = entity.setup(elem)
+            entity.setup(elem)
           end
-        else
+          elem.context = self
+          elem
         end
+
         yield(elem)
         elem
       end
@@ -489,6 +557,20 @@ module NamedArray
     end
   end
 
+  def named_array_collect
+    res = []
+
+    named_array_each do |elem|
+      if block_given?
+        res << yield(elem)
+      else
+        res << elem
+      end
+    end
+
+    res
+  end
+
   def named_array_set_brackets(key,value)
     named_array_clean_set_brackets(Misc.field_position(fields, key), value)
   end
@@ -517,7 +599,7 @@ module NamedArray
 
   def report
     fields.zip(self).collect do |field,value|
-      "\nAttributes:\n* #{ field }: #{ Array === value ? value * "|" : value }"
+      "#{ field }: #{ Array === value ? value * "|" : value }"
     end * "\n"
   end