rbbt-util 5.14.25 → 5.14.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88885c1c0256441286016e2dcce5064c4bd62c75
-  data.tar.gz: 73044f407bdd1fb7c936841074211291cc13a327
+  metadata.gz: e2e5f1318aa0869985d9387dc3ed40c68ea0f6f6
+  data.tar.gz: 521b91ab2b83d9665c18e02c79e5685b6e4290ad
 SHA512:
-  metadata.gz: c61172f11902cc5eb070d82a764a469741f9899efbf7fda38c011a18cde6306bbfcfc6e6251372af29f8a5565dd7ffaccf7d091b98cc5ce3715bc6508ef41131
-  data.tar.gz: f34f9a02776870e4dcc253877f953ea48d278a5a60bec8481b04eb084c97d53aa65034133c336ea53e8edfa2c65b20e3cb8de9cb0af3a42d4861ae09771839f5
+  metadata.gz: 7599d8f628c59747e0aead2b7bd2583d2a3800cf73d27b7b1bea7811726713a4dc77c878280814889ef87ce616f4e1dff4725bd0e0590394483c306d45c1274a
+  data.tar.gz: 0457ae9ef9d490080271f1749b1e97eb68fc74cb0762a8e4944b01fd5f52725f6b16ef1d1f4c44487720d9491c9ae0cab8382fd6b96f0ba94fac8e296897cd4a

data/lib/rbbt/association.rb

@@ -76,7 +76,7 @@ module Association
       :fields => fields.collect{|f| String === f ? all_fields.index(f): f },
       #:type => (options[:type] and options[:type].to_sym == :flat) ? :flat : nil,
       :unnamed => true,
-      :merge => (options[:type] and options[:type].to_sym == :flat) ? false : true
+      :merge => (options[:merge] or (options[:type] and options[:type].to_sym == :flat) ? false : true)
     })
 
     open_options["header_hash"] = "#" if options["header_hash"] == ""
@@ -259,10 +259,10 @@ module Association
   #  Persist.open_tokyocabinet(index_file, write, :list, TokyoCabinet::BDB).tap{|r| r.unnamed = true; Association::Index.setup r }
   #end
 
-  def self.index(file, options = {}, persist_options = {})
+  def self.index(file, options = {}, persist_options = nil)
     options = {} if options.nil?
     options = Misc.add_defaults options, :persist => true
-    persist_options = {} if persist_options.nil?
+    persist_options = Misc.pull_keys options, persist_options if persist_options.nil?
 
     Persist.persist_tsv(file, nil, options, {:persist => true, :prefix => "Association Index"}.merge(persist_options).merge(:engine => TokyoCabinet::BDB, :serializer => :clean)) do |assocs|
       undirected = options[:undirected]

data/lib/rbbt/knowledge_base.rb

@@ -163,7 +163,7 @@ class KnowledgeBase
           file, registered_options = registry[name]
 
           options = Misc.add_defaults options, :persist_file => persist_file, :namespace => namespace, :format => format
-          options = Misc.add_defaults options, registered_options
+          options = Misc.add_defaults options, registered_options if registered_options
 
           persist_options = Misc.pull_keys options, :persist
 

data/lib/rbbt/tsv.rb

@@ -16,6 +16,7 @@ require 'rbbt/tsv/filter'
 require 'rbbt/tsv/field_index'
 require 'rbbt/tsv/parallel'
 require 'rbbt/tsv/stream'
+require 'rbbt/tsv/melt'
 
 module TSV
   class << self
@@ -58,7 +59,7 @@ module TSV
     unnamed = Misc.process_options options, :unnamed
     entity_options = Misc.process_options options, :entity_options
 
-    Log.debug "TSV open: #{ filename } - #{options.inspect}.#{unnamed ? " [unnamed]" : "[not unnamed]"}"
+    Log.debug "TSV open: #{ filename } - #{Misc.fingerprint options }.#{unnamed ? " [unnamed]" : "[not unnamed]"}"
 
     data = nil
 
@@ -118,6 +119,7 @@ module TSV
       Parser.new stream, options
     end
   end
+
   def self.parse(stream, data, options = {})
 
     parser = TSV::Parser.new stream, options

data/lib/rbbt/tsv/attach.rb

@@ -200,10 +200,10 @@ module TSV
       Log.debug "Attachment with same key: #{other.key_field}"
       attach_same_key other, fields
     when (not in_namespace and self.fields.include?(other.key_field))
-      Log.debug "Found other's key field: #{other.key_field}"
+      Log.debug "Found other key field: #{other.key_field}"
       attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
     when (in_namespace and self.fields_in_namespace.include?(other.key_field))
-      Log.debug "Found other's key field in #{in_namespace}: #{other.key_field}"
+      Log.debug "Found other key field in #{in_namespace}: #{other.key_field}"
       attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
     else
       index = TSV.find_traversal(self, other, options)

data/lib/rbbt/tsv/melt.rb

@@ -0,0 +1,24 @@
+module TSV
+  def self.melt(tsv, key_field, header_field, fields, *info_fields, &block)
+    info_fields.unshift header_field
+    TSV.traverse tsv, :into => :dumper, :key_field => key_field, :fields => info_fields do |k,values|
+      res = fields.zip(values).collect do |field, value|
+        info_values = if block_given?
+                        new = block.call value
+                        next if new.nil?
+                        new
+                      else
+                        [value]
+                      end
+        info_values.unshift field
+        [field, info_values]
+      end
+      res.extend MultipleResult
+      res
+    end
+  end
+
+  def melt(header_field, *info_fields, &block)
+    TSV.melt self, key_field, header_field, fields, *info_fields, &block
+  end
+end

data/lib/rbbt/tsv/parser.rb

@@ -128,16 +128,18 @@ module TSV
       [keys, [value]]
     end
 
-    def get_values_flat(parts)
+
+    def get_values_flat_merge(parts)
       begin
         orig = parts
-      if key_position and key_position != 0 and field_positions.nil?
-        value = parts.shift
-        keys = parts.dup
-        return [keys, [value]]
-      end
 
-        return parts.shift.split(@sep2, -1).first, parts.collect{|value| value.split(@sep2, -1)}.flatten if 
+        if key_position and key_position != 0 and field_positions.nil?
+          value = parts.shift.split(@sep2, -1)
+          keys = parts.collect{|p| p.split(@sep2, -1) }.flatten
+          return [keys, value]
+        end
+
+        return parts.shift.split(@sep2, -1), parts.collect{|value| value.split(@sep2, -1)}.flatten if 
         field_positions.nil? and (key_position.nil? or key_position == 0)
       rescue
         raise $!
@@ -159,18 +161,22 @@ module TSV
         values = values.split(@sep2, -1)
       end
 
+      [keys, values]
+    end
+
+    def get_values_flat(parts)
+      keys, values = get_values_flat_merge(parts)
       [keys.first, values]
     end
 
+
     def add_to_data_no_merge_list(data, key, values)
       data[key] = values unless data.include? key
       nil
     end
 
-    def add_to_data_flat_keys(data, keys, values)
-      keys.each do |key|
-        data[key] = values unless data.include? key
-      end
+    def add_to_data_flat_keys(data, key, values)
+      data[key] = values unless data.include? key
       nil
     end
 
@@ -188,6 +194,17 @@ module TSV
       nil
     end
 
+    def add_to_data_flat_merge_double(data, keys, values)
+      keys.each do |key|
+        if data.include? key
+          data[key] = data[key].concat values
+        else
+          data[key] = values
+        end
+      end
+      nil
+    end
+
     def add_to_data_flat_merge_keys(data, keys, values)
       keys.each do |key|
         if data.include? key
@@ -346,7 +363,7 @@ module TSV
               raise "Field not identified: #{ field }" if pos.nil?
               pos
             else
-              raise "Format of fields not understood: #{fields.inspect}"
+              raise "Format of fields not understood: #{field.inspect}"
             end
           }
         end
@@ -423,15 +440,17 @@ module TSV
         self.instance_eval do alias add_to_data add_to_data_no_merge_list end
       when :flat
         @take_all = true if field_positions.nil?
-        self.instance_eval do alias get_values get_values_flat end
         self.instance_eval do alias cast_values cast_values_flat end
-        if merge
+        merge = true if key_position and key_position != 0 and field_positions.nil?
+        if merge 
+          self.instance_eval do alias get_values get_values_flat_merge end
           if key_position and key_position != 0 and field_positions.nil?
             self.instance_eval do alias add_to_data add_to_data_flat_merge_keys end
           else
-            self.instance_eval do alias add_to_data add_to_data_flat_merge end
+            self.instance_eval do alias add_to_data add_to_data_flat_merge_double end
           end
         else
+          self.instance_eval do alias get_values get_values_flat end
           if key_position and key_position != 0 and field_positions.nil?
             self.instance_eval do alias add_to_data add_to_data_flat_keys end
           else

data/lib/rbbt/tsv/util.rb

@@ -86,6 +86,10 @@ module TSV
       file
     when (defined? Bgzf and Bgzf)
       file
+    when TSV
+      file
+    when TSV::Dumper
+      file.stream
     when TSV::Parser
       file.stream
     when Path
@@ -117,8 +121,6 @@ module TSV
         file.join
         get_stream(file.path)
       end
-    when TSV::Dumper
-      file.stream
     when Array
       Misc.open_pipe do |sin|
         file.each do |l|

data/lib/rbbt/util/misc/bgzf.rb

@@ -33,7 +33,7 @@ module Bgzf
 
   def _index
     @_index ||= begin
-                  index = Persist.persist("BGZF index" + filename.sub(/.bgz$/,''), :marshal, :dir => Rbbt.var.bgzf_index) do
+                  index = Persist.persist("BGZF index" + (filename || "").sub(/.bgz$/,''), :marshal, :dir => Rbbt.var.bgzf_index) do
                     index = []
                     pos = 0
                     while true do

data/lib/rbbt/util/misc/development.rb

@@ -1,5 +1,9 @@
 module Misc
 
+  def self.add_libdir(dir)
+    $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include? dir
+  end
+
   def self.pre_fork
     Persist::CONNECTIONS.values.each do |db| db.close if db.write? end
     ObjectSpace.each_object(Mutex) do |m| 

data/lib/rbbt/util/misc/inspect.rb

@@ -33,7 +33,7 @@ module Misc
       ":" << obj.to_s
     when String
       if obj.length > 100
-        "'" << obj.slice(0,20) << "<...#{obj.length}...>" << obj.slice(-10,10) << " " << "'"
+        "'" << obj.slice(0,30) << "<...#{obj.length}...>" << obj.slice(-10,30) << " " << "'"
       else 
         "'" << obj << "'"
       end

data/test/rbbt/test_tsv.rb

@@ -5,7 +5,7 @@ require 'test/unit'
 
 class TestTSV < Test::Unit::TestCase
 
-  def test_extend
+  def _test_extend
     a = {
       "one" => "1", 
       "two" => "2"
@@ -18,7 +18,7 @@ class TestTSV < Test::Unit::TestCase
     assert_equal "1", a["one"]
   end
   
-  def test_tsv
+  def _test_tsv
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -41,7 +41,7 @@ row2    A    B    Id3
     end
   end
 
-  def test_headerless
+  def _test_headerless
     content =<<-EOF
 row1    a|aa|aaa    b    Id1|Id2
 row2    A    B    Id3
@@ -54,7 +54,7 @@ row2    A    B    Id3
     end
   end
 
-  def test_headerless_fields
+  def _test_headerless_fields
     content =<<-EOF
 row1    a|aa|aaa    b    Id1|Id2
 row2    A    B    Id3
@@ -68,7 +68,7 @@ row2    A    B    Id3
     end
   end
 
-  def test_tsv_persistence
+  def _test_tsv_persistence
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -94,7 +94,7 @@ row2    A    B    Id3
     end
   end
 
-  def test_tsv_field_selection
+  def _test_tsv_field_selection
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -136,7 +136,7 @@ row2    A    B    Id3
     end
   end
 
-  def test_tsv_cast
+  def _test_tsv_cast
     content =<<-EOF
 #Id    Value
 row1    1|2|3
@@ -151,7 +151,7 @@ row2    4
     end
   end
 
-  def test_tsv_single
+  def _test_tsv_single
     content =<<-EOF
 #Id    Value
 row1    1
@@ -168,7 +168,7 @@ row2    4
     end
   end
 
-  def test_tsv_single_from_flat
+  def _test_tsv_single_from_flat
     content =<<-EOF
 #: :type=:flat
 #Id    Value
@@ -183,7 +183,7 @@ row2    4
   end
 
 
-  def test_tsv_serializer
+  def _test_tsv_serializer
     content =<<-EOF
 #Id    Value
 row1    1
@@ -197,7 +197,7 @@ row2    4
     end
   end
 
-  def test_tsv_header_options
+  def _test_tsv_header_options
     content =<<-EOF
 #: :sep=/\\s+/
 #Id Value
@@ -211,7 +211,7 @@ row2    4
     end
   end
 
-  def test_tsv_fastimport
+  def _test_tsv_fastimport
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -228,7 +228,7 @@ row3    AA    B    Id3
     end
   end
 
-  def test_header_type
+  def _test_header_type
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single
 #Id Value
@@ -244,7 +244,7 @@ row3    AA    B    Id3
  
   end
 
-  def test_single_cast
+  def _test_single_cast
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single#:cast=:to_i
 #Id Value
@@ -259,7 +259,7 @@ b 2
     end
   end
 
-  def test_key_field
+  def _test_key_field
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single
 #Id Value
@@ -275,7 +275,7 @@ b 2
     end
   end
 
-  def test_fix
+  def _test_fix
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single
 #Id Value
@@ -289,7 +289,7 @@ b 2
     end
   end
 
-  def test_select
+  def _test_select
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single
 #Id Value
@@ -303,7 +303,7 @@ b 2
     end
   end
 
-  def test_grep
+  def _test_grep
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single
 #Id Value
@@ -317,7 +317,7 @@ b 2
     end
   end
 
-  def test_grep_invert
+  def _test_grep_invert
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single
 #Id Value
@@ -331,7 +331,7 @@ b 2
     end
   end
 
-  def test_grep_header
+  def _test_grep_header
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single#:namespace=Test
 #Id Value
@@ -346,7 +346,7 @@ b 2
     end
   end
 
-  def test_json
+  def _test_json
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single
 #Id Value
@@ -361,7 +361,7 @@ b 2
  
   end
 
-  def test_flat_no_merge
+  def _test_flat_no_merge
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -377,7 +377,7 @@ row2    A    B    Id3
     end
   end
 
-  def test_flat_merge
+  def _test_flat_merge
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -399,12 +399,12 @@ row2   b  bbb bbbb bb
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :flat, :fields => ["ValueA"])
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => false, :type => :flat, :fields => ["ValueA"])
       assert_equal ["a", "aa", "aaa"], tsv["row1"]
     end
   end
 
-  def test_flat_key
+  def _test_flat_key
     content =<<-EOF
 #Id    ValueA 
 row1   a   aa   aaa
@@ -420,7 +420,7 @@ row2   b  bbb bbbb bb aa
 
 
 
-  def test_zipped
+  def _test_zipped
     content =<<-EOF
 #Id    ValueA    ValueB
 row1    a|aa|aaa    b|bb|bbb
@@ -433,7 +433,7 @@ row2    a|aa|aaa    c|cc|ccc
     end
   end
 
-  def test_named_array_key
+  def _test_named_array_key
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -448,7 +448,7 @@ row2    A    B    Id3
  
   end
 
-  def test_unnamed_key
+  def _test_unnamed_key
     content =<<-EOF
 row1    a|aa|aaa    b    Id1|Id2
 row2    A    B    Id3
@@ -461,7 +461,7 @@ row2    A    B    Id3
  
   end
 
-  def test_float_array
+  def _test_float_array
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1   0.2   0.3 0
@@ -476,7 +476,7 @@ row2    0.1  4.5 0
  
   end
 
-  def test_flat_field_select
+  def _test_flat_field_select
     content =<<-EOF
 #: :type=:flat
 #Id    Value
@@ -485,11 +485,12 @@ row2    b  bb bbb
     EOF
 
     TmpFile.with_file(content) do |filename|
+      puts TSV.open(filename, :sep => /\s+/, :key_field => "Value").to_s
       assert TSV.open(filename, :sep => /\s+/, :key_field => "Value").include? "aa"
     end
   end
 
-  def test_flat2
+  def _test_flat2
     content =<<-EOF
 #: :type=:flat
 #Id    Value
@@ -503,10 +504,25 @@ row2    A|AA|AAA
       assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id")["row1"].include? "a"
       assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id", :fields => ["Value"])["row1"].include? "a"
     end
+  end
+
+
+  def _test_tsv_flat_double
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "ValueA", :fields => ["OtherID"], :merge => true)
+      assert tsv["aaa"].include? "Id1"
+      assert tsv["aaa"].include? "Id2"
  
+    end
   end
 
-  def test_flat2single
+  def _test_flat2single
     content =<<-EOF
 #: :type=:flat
 #Id    Value
@@ -520,7 +536,7 @@ row2    A AA AAA
  
   end
 
-  def test_shard
+  def _test_shard
     shard_function = Proc.new do |key|
       key[-1]
     end

data/test/rbbt/util/test_misc.rb

@@ -451,13 +451,4 @@ eum fugiat quo voluptas nulla pariatur?"
     puts `ls -l /proc/#{ Process.pid }/fd`
   end
 
-  def test_bootstrap
-    Misc.bootstrap (0..10).to_a do
-      Misc.bootstrap (0..10).to_a do
-        Misc.bootstrap (0..10).to_a do
-          sleep 0.1
-        end
-      end
-    end
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.14.25
+  version: 5.14.26
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-07 00:00:00.000000000 Z
+date: 2014-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -176,6 +176,7 @@ files:
 - lib/rbbt/tsv/filter.rb
 - lib/rbbt/tsv/index.rb
 - lib/rbbt/tsv/manipulate.rb
+- lib/rbbt/tsv/melt.rb
 - lib/rbbt/tsv/parallel.rb
 - lib/rbbt/tsv/parallel/through.rb
 - lib/rbbt/tsv/parallel/traverse.rb