rbbt-util 4.3.0 → 4.4.0

data/lib/rbbt/resource/util.rb

@@ -2,7 +2,7 @@ require 'rbbt/resource/rake'
 
 module Path
 
-  def self.caller_lib_dir(file = nil)
+  def self.caller_lib_dir(file = nil, relative_to = 'lib')
     file = caller.reject{|l| 
       l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
       l =~ /rbbt\/resource\/path\.rb/ or
@@ -11,11 +11,11 @@ module Path
     }.first.sub(/\.rb.*/,'.rb') if file.nil?
 
     file = File.expand_path file
-    return Path.setup(file) if File.exists? File.join(file, 'lib')
+    return Path.setup(file) if File.exists? File.join(file, relative_to)
 
     while file != '/'
       dir = File.dirname file
-      return Path.setup(dir) if File.exists? File.join(dir, 'lib')
+      return Path.setup(dir) if File.exists? File.join(dir, relative_to)
       file = File.dirname file
     end
 

data/lib/rbbt/tsv.rb

@@ -49,7 +49,9 @@ module TSV
         data.serializer = serializer
       end
 
-      stream = get_stream source
+      open_options = Misc.pull_keys options, :open
+
+      stream = get_stream source, open_options
       parse stream, data, options
 
       data.filename = filename.to_s unless filename.nil?
@@ -71,11 +73,15 @@ module TSV
 
   def self.parse(stream, data, options = {})
     monitor, grep = Misc.process_options options, :monitor, :grep
+
+    parser = Parser.new stream, options
+
     if grep
+      stream.rewind
       stream = Open.grep(stream, grep)
+      parser.first_line = stream.gets
     end
 
-    parser = Parser.new stream, options
     line = parser.rescue_first_line
 
     if TokyoCabinet::HDB === data and parser.straight

data/lib/rbbt/tsv/accessor.rb

@@ -1,10 +1,18 @@
 require 'rbbt/util/chain_methods'
-
 module TSV
   extend ChainMethods
   self.chain_prefix = :tsv
 
-  attr_accessor :unnamed, :serializer_module
+  attr_accessor :unnamed, :serializer_module, :entity_options
+
+  def entity_options
+    options = namespace ? {:namespace => namespace, :organism => namespace} : {}
+    if @entity_options
+      options.merge(@entity_options)
+    else
+      options
+    end
+  end
 
   def with_unnamed
     saved_unnamed = @unnamed 
@@ -72,10 +80,11 @@ module TSV
 
     case type
     when :double, :list
-      NamedArray.setup value, fields, key, namespace
+      NamedArray.setup value, fields, key, entity_options
     when :flat, :single
       value = value.dup if value.frozen?
-      Entity.formats[fields.first].setup(value, (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => fields.first})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? fields.first
+
+      value = Misc.prepare_entity(value, fields.first, entity_options)
     end
     value
   end
@@ -88,11 +97,7 @@ module TSV
     keys = tsv_clean_keys - ENTRY_KEYS
     return keys if @unnamed or key_field.nil?
 
-    if defined?(Entity) and  Entity.respond_to?(:formats) and Entity.formats.include? key_field
-      Entity.formats[key_field].setup(keys.collect{|k| k.dup}, (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => key_field}) )
-    else
-      keys
-    end
+    Misc.prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
   end
 
   def tsv_values
@@ -101,11 +106,9 @@ module TSV
 
     case type
     when :double, :list
-      values.each{|value| NamedArray.setup value, fields, nil, namespace }
+      values.each{|value| NamedArray.setup value, fields, nil, entity_options}
     when :flat, :single
-      values.each{|value| 
-        Entity.formats[fields.first].setup(value, (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => fields.first}))
-      } if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? fields.first
+      values = values.collect{|v| Misc.prepare_entity(v, fields.first, entity_options)}
     end
       
     values
@@ -127,14 +130,12 @@ module TSV
         if not fields.nil? 
           case type
           when :double, :list
-            NamedArray.setup value, fields, key, namespace if Array === value 
+            NamedArray.setup value, fields, key, entity_options if Array === value
           when :flat, :single
-            Entity.formats[fields.first].setup(value, (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => fields.first})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? fields.first
+            Misc.prepare_entity(value, fields.first, entity_options)
           end
         end
-        if defined?(Entity) and not key_field.nil? and Entity.respond_to?(:formats) and Entity.formats.include? key_field
-          key = Entity.formats[key_field].setup(key.dup, (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => key_field})) 
-        end
+        key = Misc.prepare_entity(key, key_field, entity_options)
       end
 
       yield key, value if block_given?
@@ -156,14 +157,12 @@ module TSV
         if not fields.nil? 
           case type
           when :double, :list
-            NamedArray.setup value, fields, key, namespace if Array === value 
+            NamedArray.setup value, fields, key, entity_options if Array === value 
           when :flat, :single
-            Entity.formats[fields.first].setup(value, (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => fields.first})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? fields.first
+            value = Misc.prepare_entity(value, fields.first, entity_options)
           end
         end
-        if defined?(Entity) and not key_field.nil? and Entity.respond_to?(:formats) and Entity.formats.include? key_field
-          key = Entity.formats[key_field].setup(key.dup, (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => key_field})) 
-        end
+        key = Misc.prepare_entity(key, key_field, entity_options)
       end
 
 
@@ -217,14 +216,14 @@ module TSV
       if fields == :all
         if just_keys
           keys = elems.sort_by{|key, value| key }.collect{|key, values| key}
-          Entity.formats[key_field].setup(keys, (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => key_field})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? key_field
+          keys = Misc.prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
         else
           elems.sort_by{|key, value| key }
         end
       else
         if just_keys
           keys = elems.sort_by{|key, value| value }.collect{|key, value| key}
-          Entity.formats[key_field].setup(keys, (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => fields.first})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? key_field
+          keys = Misc.prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
           keys
         else
           elems.sort_by{|key, value| value }.collect{|key, value| [key, self[key]]}
@@ -346,7 +345,7 @@ end
     if @fields.nil? or @unnamed
       @fields
     else
-      NamedArray.setup @fields, @fields, nil, namespace
+      NamedArray.setup @fields, @fields, nil, entity_options
     end
   end
 
@@ -366,7 +365,7 @@ end
       case
       when (TSV === identifiers.first or identifiers.empty?)
         identifiers
-      when
+      else
         identifiers.collect{|f| Path === f ? f : Path.setup(f, nil, namespace)}
       end
     when identifiers
@@ -413,7 +412,9 @@ end
     end
 
     if keys == :sort
-      keys = self.keys.sort
+      with_unnamed do
+        keys = self.keys.sort
+      end
     end
 
     str = ""

data/lib/rbbt/tsv/attach.rb

@@ -50,7 +50,7 @@ module TSV
   # Merge two files with the same keys and different fields
   def self.merge_different_fields(file1, file2, output, sep = "\t")
     case
-    when (String === file1 and not file1.index("\n") and file1.length < 250 and File.exists?(file1))
+    when (String === file1 and not file1 =~ /\n/ and file1.length < 250 and File.exists?(file1))
       file1 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file1 } | grep -v '^#{sep}' ", :pipe => true)
     when (String === file1 or StringIO === file1)
       file1 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file1, :pipe => true)
@@ -59,7 +59,7 @@ module TSV
     end
 
     case
-    when (String === file2 and not file2.index("\n") and file2.length < 250 and File.exists?(file2))
+    when (String === file2 and not file2 =~ /\n/ and file2.length < 250 and File.exists?(file2))
       file2 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file2 } | grep -v '^#{sep}' ", :pipe => true)
     when (String === file2 or StringIO === file2)
       file2 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file2, :pipe => true)
@@ -146,7 +146,12 @@ module TSV
 
     Log.medium("Attaching fields:#{fields.inspect} from #{other.filename.inspect}.")
 
-    other = other.tsv(:persist => options[:persist_input] == true) unless TSV === other 
+    unless TSV === other
+      other_identifier_files = other.identifier_files
+      other = other.tsv(:persist => options[:persist_input] == true) unless TSV === other 
+      other.identifiers = other_identifier_files 
+    end
+
     case
     when key_field == other.key_field
       attach_same_key other, fields

data/lib/rbbt/tsv/attach/util.rb

@@ -3,6 +3,8 @@ module TSV
   def attach_same_key(other, fields = nil)
     fields = other.fields - [key_field].concat(self.fields) if fields.nil?
 
+    fields = [fields].compact unless Array === fields
+
     field_positions = fields.collect{|field| other.identify_field field}
     other.with_unnamed do
       with_unnamed do
@@ -114,6 +116,7 @@ module TSV
         with_unnamed do
           through do |key, values|
             source_keys = index[key]
+            source_keys = [source_keys] unless Array === source_keys
             if source_keys.nil? or source_keys.empty?
               all_new_values = []
             else

data/lib/rbbt/tsv/excel.rb

@@ -0,0 +1,91 @@
+require 'spreadsheet'
+module TSV
+  def self.excel(tsv, filename, options ={})
+    name = Misc.process_options options, :name
+    sort_by = Misc.process_options options, :sort_by
+    sort_by_cast = Misc.process_options options, :sort_by_cast
+    fields = Misc.process_options(options, :fields) || all_fields
+
+    book = Spreadsheet::Workbook.new
+    sheet1 = book.create_worksheet 
+    sheet1.row(0).concat fields
+    i = 1
+    if sort_by
+      if sort_by_cast
+        data = tsv.sort_by sort_by do |k, v| 
+          if Array === v
+            v.first.send(sort_by_cast)
+          else
+            v.send(sort_by_cast)
+          end
+        end
+      else
+        data = tsv.sort_by sort_by
+      end
+    else
+      data = tsv
+    end
+
+    data.each do |key, values|
+      cells = []
+      cells.push((name and key.respond_to?(:name)) ?  key.name || key : key )
+
+      values.each do |value|
+        v = (name and value.respond_to?(:name)) ?  value.name || value : value 
+        if Array === v
+          cells.push v * ", "
+        else
+          cells.push v
+        end
+      end
+
+      sheet1.row(i).concat cells
+      i += 1
+    end
+    book.write filename
+  end
+
+  def excel(filename, options ={})
+    name = Misc.process_options options, :name
+    sort_by = Misc.process_options options, :sort_by
+    sort_by_cast = Misc.process_options options, :sort_by_cast
+
+    book = Spreadsheet::Workbook.new
+    sheet1 = book.create_worksheet 
+    sheet1.row(0).concat all_fields
+    i = 1
+    if sort_by
+      if sort_by_cast
+        data = self.sort_by sort_by do |k, v| 
+          if Array === v
+            v.first.send(sort_by_cast)
+          else
+            v.send(sort_by_cast)
+          end
+        end
+      else
+        data = self.sort_by sort_by
+      end
+    else
+      data = self
+    end
+
+    data.each do |key, values|
+      cells = []
+      cells.push((name and key.respond_to?(:name)) ?  key.name || key : key )
+
+      values.each do |value|
+        v = (name and value.respond_to?(:name)) ?  value.name || value : value 
+        if Array === v
+          cells.push v * ", "
+        else
+          cells.push v
+        end
+      end
+
+      sheet1.row(i).concat cells
+      i += 1
+    end
+    book.write filename
+  end
+end

data/lib/rbbt/tsv/filter.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/misc'
+require 'set'
 module Filtered
 
   class FilterArray
@@ -37,12 +38,21 @@ module Filtered
 
       @list = nil
       case
+      when @match == :key
+        @value = Set.new(@value)
+        class << self
+          self
+        end.class_eval <<-EOC
+          def match_entry(key, entry)
+            key == @value or (Set === @value and @value.include? key)
+          end
+        EOC
       when @match.match(/field:(.*)/)
         @fieldnum = data.identify_field $1
         class << self
           self
         end.class_eval <<-EOC
-          def match_entry(entry)
+          def match_entry(key, entry)
             value = entry[@fieldnum] 
             value == @value or (Array === value and value.include? @value)
           end
@@ -78,7 +88,7 @@ module Filtered
 
       data.with_unnamed do
         data.unfiltered_each do |key, entry|
-          ids << key if match_entry(entry)
+          ids << key if match_entry(key, entry)
         end
       end
 
@@ -173,7 +183,7 @@ module Filtered
     if filters.empty?
       unfiltered_filename
     else
-      unfiltered_filename + ":Filtered[#{filters.collect{|f| [f.match, f.value] * "="} * ", "}]"
+      unfiltered_filename + ":Filtered[#{filters.collect{|f| [f.match, Array === f.value ? Misc.hash2md5(:values => f.value) : f.value] * "="} * ", "}]"
     end
   end
 
@@ -182,7 +192,7 @@ module Filtered
       self.send(:unfiltered_set, key, value)
     else
       filters.each do |filter| 
-        filter.add key if filter.match_entry value
+        filter.add key if filter.match_entry key, value
       end
       self.send(:unfiltered_set, key, value)
     end
@@ -269,12 +279,12 @@ module TSV
   end
 
   def reset_filters
-    if filter_dir.nil? or filter_dir.empty?
-      filters.each do |filter| filter.reset end
+    if @filter_dir.nil? or @filter_dir.empty?
+      @filters.each do |filter| filter.reset end if Array === @filters
       return
     end
 
-    Dir.glob(File.join(filter_dir, '*.filter')).each do |f|
+    Dir.glob(File.join(@filter_dir, '*.filter')).each do |f|
       FileUtils.rm f
     end
   end

data/lib/rbbt/tsv/manipulate.rb

@@ -174,14 +174,16 @@ module TSV
 
       keys, value = traverser.process(key, value)
 
+      keys = [keys].compact unless Array === keys
+
       # Annotated with Entity and NamedArray
       if not @unnamed
         if not traverser.new_field_names.nil? 
           case type
           when :double, :list
-            NamedArray.setup value, traverser.new_field_names 
+            NamedArray.setup value, traverser.new_field_names, key, entity_options
           when :flat, :single
-            Entity.formats[traverser.new_field_names.first].setup(value, :format => traverser.new_field_names.first) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? traverser.new_field_names
+            Misc.prepare_entity(value, traverser.new_field_names.first, entity_options)
           end
         end
       end
@@ -197,8 +199,8 @@ module TSV
             r
           }
 
-          if not @unnamed and defined?(Entity) and not traverser.new_key_field_name.nil? and Entity.respond_to?(:formats) and Entity.formats.include? traverser.new_key_field_name
-            k = Entity.formats[traverser.new_key_field_name].setup(k.dup, :format => traverser.new_key_field_name) 
+          if not @unnamed 
+            k = Misc.prepare_entity(k, traverser.new_key_field_name, entity_options)
           end
           v.key = k if NamedArray === v
           yield k, v
@@ -207,8 +209,8 @@ module TSV
 
       else
         keys.each do |key|
-          if not @unnamed and defined?(Entity) and not traverser.new_key_field_name.nil? and Entity.respond_to?(:formats) and Entity.formats.include? traverser.new_key_field_name
-            key = Entity.formats[traverser.new_key_field_name].setup(key.dup, :format => traverser.new_key_field_name) 
+          if not @unnamed
+            k = Misc.prepare_entity(k, traverser.new_key_field_name, entity_options)
           end
           value.key = key if NamedArray === value
           yield key, value
@@ -318,11 +320,23 @@ module TSV
       end
     when String === method
       if block_given?
-        pos = identify_field method
         with_unnamed do
-          through do |key, values|
-            new[key] = values if yield((method == key_field or method == :key)? key : values[pos])
+          case
+          when (method == key_field or method == :key)
+            through do |key, values|
+              new[key] = values if yield(key)
+            end
+          when (type == :single or type == :flat)
+            through do |key, value|
+              new[key] = value if yield(value)
+            end
+          else
+            pos = identify_field method
+            through do |key, values|
+              new[key] = values if yield(values[pos])
+            end
           end
+
         end
       else
         with_unnamed do
@@ -351,7 +365,7 @@ module TSV
             end
           when :list
             through :key, key do |key, values|
-              new[key] = self[key] if method.include? value.first
+              new[key] = self[key] if method.include? values.first
             end
           when :flat #untested
             through :key, key do |key, values|
@@ -359,7 +373,7 @@ module TSV
             end
           else
             through :key, key do |key, values|
-              new[key] = self[key] if (method & values.first).any?
+              new[key] = self[key] if (method & values.flatten).any?
             end
           end
         end
@@ -448,6 +462,7 @@ module TSV
       else
         values << new_values
       end
+
       self[key] = values
     end
     @monitor = old_monitor