rbbt-util 5.14.34 → 5.14.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec2d34290a61ca4f3f32cd875a07a15c9c7c06fd
-  data.tar.gz: 08ce21aa7a885530dcba3859231d94874f99bba2
+  metadata.gz: 14a5356d56a5794296f8708a848957a4efd82264
+  data.tar.gz: b4e41917f06c7d28736b6ed235c61271eb25ec47
 SHA512:
-  metadata.gz: d532ef379f2238e8dbaf311f485a4e18ac774177624e327d7b1d432a6cd0304f17f945448523407bb0cbb8f14a1067b50d36585b6d554d646dd5a01124c5b4f2
-  data.tar.gz: b15a2660fd8be0a782fb0a139bc44b8670adf3629446be6c0b8dc3e6d27f017a2c71db0cb6d76602b306917c9909f03013226566d963c0247506a4723968ead8
+  metadata.gz: 66d1db7e9f85049030c34a153f114c864b239bca644aade3f2f9803e7764b3c97a9ec342416045a1bb4ae4f40119749ae58c2d572c00e920eb9d91273366bb88
+  data.tar.gz: e9432f28095748fd4d25e945d1a9cb4c2b100708e200216ee589a9f57eb19d999d55ac3556ae040a388775faba145212da99ff3ffbe0001cc218821ffbf53597

data/lib/rbbt/association/database.rb

@@ -11,7 +11,9 @@ module Association
         tsv.through do |source, values|
           Misc.zip_fields(values).each do |info|
             target, *rest = info
-            new.zip_new target, [source] + rest
+            next if target == source
+            rest.unshift source
+            new.zip_new target, rest
           end
         end
       else
@@ -60,7 +62,7 @@ module Association
   end
 
   def self.reorder_tsv(tsv, options = {})
-    fields, undirected, persist = Misc.process_options options, :fields, :undirected, :persist 
+    fields, persist = Misc.process_options options, :fields, :persist 
     all_fields = tsv.all_fields
 
     source_pos, field_pos, source_header, field_headers, source_format, target_format = headers(all_fields, fields, options)
@@ -69,20 +71,18 @@ module Association
     info_fields = field_pos.collect{|f| f == :key ? :key : all_fields[f]}
     options = options.merge({:key_field => source_field, :fields =>  info_fields})
 
-    tsv = tsv.reorder source_field, fields, :zipped => true
+    tsv = tsv.reorder source_field, fields if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
 
     tsv.key_field = source_header
     tsv.fields = field_headers
 
     tsv = translate tsv, source_format, target_format, :persist => persist if source_format or target_format
 
-    tsv = add_reciprocal tsv if undirected
-
     tsv
   end
 
   def self.open_stream(stream, options = {})
-    fields, undirected, persist = Misc.process_options options, :fields, :undirected, :persist
+    fields, persist = Misc.process_options options, :fields, :persist
 
     parser = TSV::Parser.new stream, options.merge(:fields => nil, :key_field => nil)
 
@@ -132,8 +132,6 @@ module Association
 
     tsv = translate tsv, source_format, target_format, :persist => persist if source_format or target_format
 
-    tsv = add_reciprocal tsv if undirected
-
     tsv
   end
 

data/lib/rbbt/association/index.rb

@@ -11,12 +11,14 @@ module Association
 
     file = version_file(file, options[:namespace]) if options[:namespace] and String === file
 
-    undirected = options[:undirected]
     Persist.persist_tsv(file, "Association Index", options, persist_options.dup) do |data|
       recycle = options[:recycle]
+      undirected = options[:undirected]
 
       persist_options[:file] = persist_options[:file] + '.database' if persist_options[:file]
-      database = open(file, options, persist_options.dup)
+      database = open(file, options, persist_options.dup.merge(:engine => "HDB"))
+
+      undirected = true if undirected.nil? and database.key_field == database.fields.first
 
       fields = database.fields
       source_field = database.key_field
@@ -68,6 +70,10 @@ module Association
               info = old_info.zip(info).collect{|p| p * ";;" }
               data[key] = info
             end
+            if undirected
+              reverse_key = [target,source] * "~"
+              data[reverse_key] = info unless data.include? reverse_key
+            end
           end
         end
       end

data/lib/rbbt/association/util.rb

@@ -123,8 +123,8 @@ module Association
 
 
     if format = options[:format]
-      source_format = process_formats(specs[:source][1] || specs[:source][0], format) || source_format
-      target_format = process_formats(specs[:target][1] || specs[:target][0], format) || target_format
+      source_format = process_formats(specs[:source][1] || specs[:source][0], format) || source_format unless source_format
+      target_format = process_formats(specs[:target][1] || specs[:target][0], format) || target_format unless target_format
     end
 
     Log.low "Headers -- #{[source_pos, field_pos, source_header, field_headers, source_format, target_format]}"

data/lib/rbbt/entity.rb

@@ -7,13 +7,11 @@ module Entity
     attr_accessor :formats, :entity_property_cache
   end
 
-
   self.entity_property_cache = "var/entity_property"
   self.formats = {}
   
   UNPERSISTED_PREFIX = "entity_unpersisted_property_"
 
-
   attr_accessor :all_formats
   def self.extended(base)
     base.extend Annotation

data/lib/rbbt/entity/identifiers.rb

@@ -29,6 +29,7 @@ module Entity
             source ||= self.respond_to?(:format)? self.format : nil
 
             index = TSV.translation_index(identifier_files, format, source, :persist => true)
+            raise "No index from #{ source } to #{ format }: #{Misc.fingerprint identifier_files}" if index.nil?
             index.unnamed = true
             index
           end
@@ -98,6 +99,7 @@ module Entity
 
     @identifier_files ||= []
     @identifier_files << file
+    @identifier_files.uniq!
 
 
     self.include Entity::Identified unless Entity::Identified === self

data/lib/rbbt/knowledge_base/enrichment.rb

@@ -0,0 +1,9 @@
+require 'rbbt/knowledge_base/registry'
+require 'rbbt/statistics/hypergeometric'
+class KnowledgeBase
+  def enrichment(name, entities, options = {})
+    database = get_database(name, options)
+    entities = identify_source name, entities
+    database.enrichment entities, database.fields.first, :persist => false
+  end
+end

data/lib/rbbt/knowledge_base/entity.rb

@@ -88,8 +88,8 @@ class KnowledgeBase
   def identify_source(name, entity)
     return :all if entity == :all
     index = source_index(name)
-    return nil if index.nil?
-    index.values_at *entity
+    return entity if index.nil?
+    Array === entity ? index.values_at(*entity) : index[entity]
   end
 
   
@@ -97,7 +97,7 @@ class KnowledgeBase
     return :all if entity == :all
     index = target_index(name)
     return nil if index.nil?
-    index.values_at *entity
+    Array === entity ? index.values_at(*entity) : index[entity]
   end
 
   def identify(name, entity)

data/lib/rbbt/knowledge_base/registry.rb

@@ -45,8 +45,8 @@ class KnowledgeBase
           persist_file = persist_dir[key]
           file, registered_options = registry[name]
 
-          options = Misc.add_defaults options, :persist_file => persist_file, :persist_dir => persist_dir, :namespace => namespace, :format => format, :persist => true
           options = Misc.add_defaults options, registered_options if registered_options and registered_options.any?
+          options = Misc.add_defaults options, :persist_file => persist_file, :persist_dir => persist_dir, :namespace => namespace, :format => format, :persist => true
 
           persist_options = Misc.pull_keys options, :persist
 
@@ -77,12 +77,13 @@ class KnowledgeBase
           persist_file = dir.indices[key]
           file, registered_options = registry[name]
 
-          options = Misc.add_defaults options, :persist_file => persist_file, :namespace => namespace, :format => format, :persist => true
+
           options = Misc.add_defaults options, registered_options if registered_options and registered_options.any?
+          options = Misc.add_defaults options, :persist_file => persist_file, :namespace => namespace, :format => format, :persist => true
 
           persist_options = Misc.pull_keys options, :persist
 
-          database = if persist_file.exists?
+          database = if persist_file.exists? and persist_options[:persist] and not persist_options[:update]
                     Log.low "Re-opening database #{ name } from #{ Misc.fingerprint persist_file }. #{options}"
                     Association.open(nil, options, persist_options)
                   else

data/lib/rbbt/tsv/accessor.rb

@@ -602,6 +602,16 @@ module TSV
     peek
   end
 
+  def head(times=10)
+    stream = dumper_stream
+    str = ""
+    times.times do |i|
+      break if stream.eof?
+      str << stream.gets
+    end
+    str
+  end
+
   def summary
 
     key = nil

data/lib/rbbt/tsv/change_id.rb

@@ -94,11 +94,16 @@ module TSV
       if TSV === file
         all_fields = file.all_fields
         target = file.fields.first if target.nil?
-        return file.index(options.merge(:target => target, :fields => fields, :order => true)) if (source.nil? or all_fields.include? source) and all_fields.include? target
+        if (source.nil? or all_fields.include? source) and all_fields.include? target
+          return file.index(options.merge(:target => target, :fields => fields, :order => true)) 
+        end
       else
         all_fields = TSV.parse_header(file).all_fields
         target = all_fields[1] if target.nil?
-        return TSV.index(file, options.merge(:target => target, :fields => fields, :order => true)) if (source.nil? or all_fields.include? source) and all_fields.include? target
+        if (source.nil? or all_fields.include? source) and all_fields.include? target
+          index = TSV.index(file, options.merge(:target => target, :fields => fields, :order => true)) 
+          return index
+        end
       end
     end
 
@@ -112,10 +117,10 @@ module TSV
 
         common_field = (all_fields & other_all_fields).first
 
-        if common_field and (source.nil? or fields.include? source) and all_fields.include? common_field and 
-          other_all_fields.include? common_field and other_all_fields.include? target 
+        if common_field and (source.nil? or all_fields.include? source) and other_all_fields.include? target 
+
+          index = Persist.persist_tsv(nil, Misc.fingerprint(files), {:files => files, :source => source, :target => target}, :prefix => "Translation index", :persist => options[:persist]) do |data|
 
-          return Persist.persist_tsv(nil, Misc.fingerprint(files), {:files => files, :source => source, :target => target}, :prefix => "Translation index", :persist => options[:persist]) do |data|
             index = TSV === file ? 
               file.index(options.merge(:target => common_field, :fields => fields)) :
               TSV.index(file, options.merge(:target => common_field, :fields => fields))
@@ -125,10 +130,11 @@ module TSV
               TSV.index(other_file, options.merge(:target => target, :fields => [common_field]))
 
             data.serializer = :clean
+            
+            # ToDo: remove the need to to the `to_list` transformation
             data.merge! index.to_list.attach(other_index.to_list).slice([target]).to_single
-
-            data
           end
+          return index
         end
       end
     end

data/lib/rbbt/util/misc/inspect.rb

@@ -116,6 +116,8 @@ module Misc
   HASH2MD5_MAX_STRING_LENGTH = 1000
   HASH2MD5_MAX_ARRAY_LENGTH = 100
   def self.hash2md5(hash)
+    return "" if hash.nil? or hash.empty?
+
     str = ""
     keys = hash.keys
     keys = keys.clean_annotations if keys.respond_to? :clean_annotations
@@ -161,7 +163,10 @@ module Misc
 
       end
 
-      str << "_" << hash2md5(v.info) if defined? Annotated and Annotated === v and not (defined? AssociationItem and AssociationItem === v)
+      if defined? Annotated and Annotated === v and not (defined? AssociationItem and AssociationItem === v)
+        info = Annotated.purge(v.info)
+        str << "_" << hash2md5(info) 
+      end
     end
     hash.unnamed = unnamed if hash.respond_to? :unnamed
 

data/share/rbbt_commands/tsv/head

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Subset entries in a tsv
+
+$ rbbt tsv head [options] file.tsv 
+
+Subsets entries from a TSV file from a given list. Works with Tokyocabinet HDB and BDB as well.
+
+-tch--tokyocabinet File is a TC HDB
+-tcb--tokyocabinet_bd File is a TC BDB
+-hh--header_hash* Change the character used to mark the header line (defaults to #)
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+file = ARGV.shift
+
+file = STDIN if file == '-'
+
+raise ParameterException, "Please specify the tsv file as argument" if file.nil?
+
+
+options[:fields] = options[:fields].split(/,\|/) if options[:fields]
+options[:header_hash] = options["header_hash"]
+
+case
+when options[:tokyocabinet]
+  tsv = Persist.open_tokyocabinet(file, false)
+when options[:tokyocabinet_bd]
+  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
+else
+  tsv = TSV.open(file, options)
+end
+
+puts tsv.head

data/test/rbbt/association/test_open.rb

@@ -28,11 +28,6 @@ TP53 NFKB1|GLI1 activation|activation true|true
     assert_equal ["false"], tsv["MDM2"]["directed?"]
   end
 
-  def test_open_no_persist_undirected
-    tsv = Association.open(EFFECT_TSV, :source => "SG=~Gene", :target => "TG=~Gene", :undirected => true, :persist => false)
-    assert_equal "TP53", tsv["MDM2"]["Gene"].first
-    assert tsv["TP53"]["Gene"].include? "MDM2"
-  end
 
   def test_open_persist
     tsv = Association.open(EFFECT_TSV, EFFECT_OPTIONS, :persist => true, :update => true)

data/test/rbbt/knowledge_base/test_enrichment.rb

@@ -0,0 +1,42 @@
+require File.expand_path(File.dirname(__FILE__) + '../../../test_helper')
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+require 'rbbt/knowledge_base'
+require 'rbbt/knowledge_base/enrichment'
+
+
+class TestKnowledgeBaseEnrichment < Test::Unit::TestCase
+
+  EFFECT =StringIO.new <<-END
+#: :sep=" "#:type=:double
+#SG TG Effect
+MDM2 TP53 inhibition
+TP53 NFKB1|GLI1 activation|activation true|true
+  END
+
+  EFFECT_OPTIONS = {
+    :source => "SG=~Associated Gene Name",
+    :target => "TG=~Associated Gene Name=>Ensembl Gene ID",
+    :persist => false,
+    :identifiers => datafile_test('identifiers'),
+    :undirected => true,
+    :namespace => "Hsa"
+  }
+
+  EFFECT_TSV = TSV.open EFFECT, EFFECT_OPTIONS.dup 
+
+  KNOWLEDGE_BASE = KnowledgeBase.new '/tmp/kb.foo', "Hsa/feb2014"
+  KNOWLEDGE_BASE.format = {"Gene" => "Ensembl Gene ID"}
+
+  KNOWLEDGE_BASE.register :effects, EFFECT_TSV, EFFECT_OPTIONS.dup
+  KNOWLEDGE_BASE.register :gene_ages, datafile_test('gene_ages')
+  KNOWLEDGE_BASE.register :nature, datafile_test('nature'), :source => "UniProt/SwissProt Accession", :target => "NCI Nature Pathway ID"
+  
+  def test_enrichment
+    genes = %w(P17706-2 LMAN1 P17706-1 P29353-2 JAK3 Q8NFM1 EIF2AK2 JAK1 SRC PIAS1 KPNB1 KPNA2 STAT3 PTPN1 ATR CREBBP PTPRA SGK1 P46108-1 NCK2 O00145 PTPN1 CBL SORBS1 IRS1 SHC1 AKT2 GRB2 SOS1 RPS6KB1 AKT1 DOK1 RASA1 NCK1 FOXO3 RAPGEF1 TRIP10 EIF4EBP1 PDPK1 GRB14 PTPN11 INS GRB10 CAV1 EIF4B RPS6KB1 EEF2 EEF2K AKT1 TSC1 TSC2 CLIP1 DDIT4 SGK1 PDPK1 DEPTOR SREBF1 CYCS IRS1 RPS6KA1 BNIP3 RRN3 RICTOR IKBKB AKT1S1 PXN PML EIF4A1 PPARGC1A YY1 PRKCA RPTOR PDCD4 SIK1 P10636-8 BRSK1 MYC SMARCD3 STK11 ETV4 MARK4 MAP2 MARK2 CRTC2 PSEN2 MST4 CTSD BRSK2 SIK2 ESR1 CAB39 STK11IP SMAD4 CREB1 PRKACA EZR TP53 GSK3B SIK3 CDC37 HSP90AA1 )
+    assert KNOWLEDGE_BASE.enrichment(:nature, genes).any?
+  end
+
+end
+
+

data/test/rbbt/knowledge_base/test_registry.rb

@@ -29,6 +29,8 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   KNOWLEDGE_BASE.register :effects, EFFECT_TSV, EFFECT_OPTIONS.dup
 
+  KNOWLEDGE_BASE.register :pina, datafile_test('pina'), :source => "UniProt/SwissProt Accession", :target => "Interactor UniProt/SwissProt Accession=~UniProt/SwissProt Accession", :undirected => true
+
   def test_database
     assert_equal "Associated Gene Name", KNOWLEDGE_BASE.get_database(:effects, :source_format => "Associated Gene Name").key_field
   end
@@ -41,12 +43,39 @@ TP53 NFKB1|GLI1 activation|activation true|true
     assert KNOWLEDGE_BASE.get_index(:effects, :source_format => "Associated Gene Name", :target_format => "Ensembl Gene ID", :persist => true).include? "MDM2~ENSG00000141510"
   end
 
-
   def test_index_flat
     require 'rbbt/sources/tfacts'
     file = TFacts.regulators
     KNOWLEDGE_BASE.register :tfacts, file,  :type => :flat, :source => "Transcription Factor Associated Gene Name=~Associated Gene Name", :merge => true
     assert KNOWLEDGE_BASE.subset(:tfacts, :source => ["TP53"], :target => :all).length > 10
   end
+
+  def test_pina
+    index = KNOWLEDGE_BASE.get_index(:pina, :persist => false, :source_format => "Associated Gene Name", :target_format => "Associated Gene Name")
+    assert index["TP53~ARID1A"]
+    assert index["ARID1A~TP53"]
+    assert_equal index["ARID1A~TP53"], index["TP53~ARID1A"]
+
+    index = KNOWLEDGE_BASE.get_index(:pina, :persist => false, :source_format => "Associated Gene Name", :target_format => "Associated Gene Name", :undirected => false)
+    count = 0
+    index.through do |k,values|
+      split_values = values.collect{|v| v.split ";;" }
+      count += 1 if Misc.zip_fields(split_values).uniq != Misc.zip_fields(split_values)
+    end
+
+    index = KNOWLEDGE_BASE.get_index(:pina, :persist => false, :source_format => "Associated Gene Name", :target_format => "Associated Gene Name", :undirected => true)
+    count2 = 0
+    index.through do |k,values|
+      split_values = values.collect{|v| v.split ";;" }
+      count2 += 1 if Misc.zip_fields(split_values).uniq != Misc.zip_fields(split_values)
+    end
+
+    assert_equal count, count2
+  end
+
+
+  def test_pina2
+    index = KNOWLEDGE_BASE.get_index(:pina, :persist => true, :source_format => "Ensembl Gene ID", :target_format => "Ensembl Gene ID", :undirected => true)
+  end
 end
 

data/test/rbbt/tsv/test_change_id.rb

@@ -1,4 +1,5 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt-util'
 require 'rbbt/tsv'
 require 'rbbt/tsv/change_id'
 

data/test/rbbt/tsv/test_manipulate.rb

@@ -290,8 +290,5 @@ row3    a    C    Id4
 
 
     end
- 
   end
-
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.14.34
+  version: 5.14.35
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-13 00:00:00.000000000 Z
+date: 2014-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -151,6 +151,7 @@ files:
 - lib/rbbt/entity/identifiers.rb
 - lib/rbbt/fix_width_table.rb
 - lib/rbbt/knowledge_base.rb
+- lib/rbbt/knowledge_base/enrichment.rb
 - lib/rbbt/knowledge_base/entity.rb
 - lib/rbbt/knowledge_base/query.rb
 - lib/rbbt/knowledge_base/registry.rb
@@ -290,6 +291,7 @@ files:
 - share/rbbt_commands/tsv/attach
 - share/rbbt_commands/tsv/change_id
 - share/rbbt_commands/tsv/get
+- share/rbbt_commands/tsv/head
 - share/rbbt_commands/tsv/info
 - share/rbbt_commands/tsv/json
 - share/rbbt_commands/tsv/read
@@ -320,6 +322,7 @@ files:
 - test/rbbt/association/test_open.rb
 - test/rbbt/association/test_util.rb
 - test/rbbt/entity/test_identifiers.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
 - test/rbbt/knowledge_base/test_entity.rb
 - test/rbbt/knowledge_base/test_query.rb
 - test/rbbt/knowledge_base/test_registry.rb
@@ -444,6 +447,7 @@ test_files:
 - test/rbbt/test_association.rb
 - test/rbbt/knowledge_base/test_registry.rb
 - test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
 - test/rbbt/knowledge_base/test_query.rb
 - test/rbbt/test_resource.rb
 - test/rbbt/test_entity.rb