rbbt-sources 3.0.0 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 00cd4a9602b9ea2637a620b30cd3d48a6d63a9fe
-  data.tar.gz: c282f8c86de5148343e5a83ea524cdc09435b9fb
+  metadata.gz: 4b76cc735fd8fd3ff8f93d84517a8c628fc6c8ac
+  data.tar.gz: 1abc7bb9a8414311f014eed4f1eacfabacf5559f
 SHA512:
-  metadata.gz: 701a67455ca18d9c705e2c409628cd5463f7449d2ee40ba4d26cce6f203018db21b9c6ee6f233cf8d80e44e28d3ffcfa08e474678b538b8db7cb80c44e5eac5a
-  data.tar.gz: 4bbcf6f222c01c5f3314617ed7c2458b3cebb9d8b3293ac631305ea2c610c935792fe0e5d6a7402f041aab4304e5586a7989f1e2a097b5dc620f0cb7a208250c
+  metadata.gz: d511a3b4aa38e9b090e57fdbc10888c41a0b98c727993e099ed0bab590fb3375735e57d3ba2744ac32cb186b741236478922b7fa7eb4506122ff70ef106ced62
+  data.tar.gz: f5e962266840d656270a1efe5be92edc6cd56a856cef88909d55f523e258e598c285f85574ed652128111459d4a36d13c3345b4aec3c51eafdef0877164d8a9a

data/lib/rbbt/sources/HPRD.rb

@@ -1,7 +1,5 @@
-require 'rbbt-util'
-require 'rbbt/entity/gene'
-require 'rbbt/tsv'
-require 'rbbt/sources/organism'
+require 'rbbt'
+require 'rbbt/resource'
 
 module HPRD
   extend Resource

data/lib/rbbt/sources/InterPro.rb

@@ -1,6 +1,4 @@
 require 'rbbt-util'
-require 'rbbt/entity/gene'
-require 'rbbt/tsv'
 require 'rbbt/sources/organism'
 
 module InterPro

data/lib/rbbt/sources/biomart.rb

@@ -46,6 +46,15 @@ module BioMart
     @archive_url = nil
   end
 
+  def self.with_archive(data)
+    begin
+      set_archive(data)
+      yield
+    ensure
+      unset_archive
+    end
+  end
+
   def self.get(database, main, attrs = nil, filters = nil, data = nil, open_options = {})
     open_options = Misc.add_defaults :wget_options => {"--read-timeout=" => 9000, "--tries=" => 1}
     repeats = true

data/lib/rbbt/sources/ensembl_ftp.rb

@@ -85,7 +85,3 @@ module Ensembl
     end
   end
 end
-
-if __FILE__ == $0
-  ddd Ensembl::FTP.ensembl_tsv("Hsa/may2012", 'exon')
-end

data/lib/rbbt/sources/entrez.rb

@@ -96,14 +96,16 @@ module Entrez
       end
 
       values.each do |xml|
-        geneid = gene.match(/<Gene-track_geneid>(\d+)/)[1]
+        geneid = xml.match(/<Gene-track_geneid>(\d+)/)[1]
         
         result[geneid] = xml
       end
+
+      result
     end
 
     genes = {}
-    geneids.each{|id| genes[id] = Gene.new(Open.read(result_files[id])) }
+    geneids.each{|id| next if id.nil? or result_files[id].nil?; genes[id] = Gene.new(Open.read(result_files[id])) }
 
     if _array
       genes
@@ -123,7 +125,7 @@ def self.gene_text_similarity(gene, text)
   when String === gene || Fixnum === gene
     begin
       gene_text =  get_gene(gene).text
-    rescue CMD::CMDError
+    rescue NoMethodError, CMD::CMDError
       return 0
     end
   else

data/lib/rbbt/sources/matador.rb

@@ -1,8 +1,8 @@
-require 'phgx'
+require 'rbbt'
+require 'rbbt/resource'
 
 module Matador
   extend Resource
-  self.pkgdir = "phgx"
   self.subdir = "share/matador"
 
   Matador.claim Matador.root, :rake, Rbbt.share.install.Matador.Rakefile.find(:lib)

data/lib/rbbt/sources/pharmagkb.rb

@@ -1,8 +1,8 @@
-require 'phgx'
+require 'rbbt'
+require 'rbbt/resource'
 
 module PharmaGKB
   extend Resource
-  self.pkgdir = "phgx"
   self.subdir = "share/pharmagkb"
 
   PharmaGKB.claim PharmaGKB.root, :rake, Rbbt.share.install.PharmaGKB.Rakefile.find(:lib)

data/lib/rbbt/sources/pina.rb

@@ -1,15 +1,14 @@
-require 'phgx'
+require 'rbbt'
+require 'rbbt/resource'
 
 module Pina
   extend Resource
-  self.pkgdir = "phgx"
   self.subdir = "share/pina"
 
   Pina.claim Pina.root, :rake, Rbbt.share.install.Pina.Rakefile.find(:lib)
 end
 
 if defined? Entity and defined? Gene and Entity === Gene
-  require 'rbbt/entity/gene'
   require 'rbbt/entity/interactor'
   require 'rbbt/sources/PSI_MI'
 

data/lib/rbbt/sources/pubmed.rb

@@ -1,45 +1,24 @@
-require 'rbbt'
+require 'rbbt-util'
 require 'libxml'
 require 'rbbt/sources/gscholar'
+require 'rbbt/util/filecache'
 
 # This module offers an interface with PubMed, to perform queries, and
 # retrieve simple information from articles. It uses the caching
 # services of Rbbt.
 module PubMed
 
-  private
   @@pubmed_lag = 1
-  def self.get_online(pmids)
 
-    pmids_complete =  pmids.is_a?(Array) ? pmids : [pmids]
-
-    url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" 
-    articles = []
-
-    Misc.divide(pmids.sort_by{|v| v.nil? ? 0 : v.to_i}, (pmids.length / 1000) + 1).each do |pmid_list|
-      postdata = "db=pubmed&retmode=xml&id=#{pmid_list* ","}"
-      xml = TmpFile.with_file(postdata) do |postfile|
-        Open.read(url, :quiet => true, :nocache => true, :nice => @@pubmed_lag, :nice_key => "PubMed", "--post-file=" => postfile)
-      end
-
-      articles += xml.scan(/(<PubmedArticle>.*?<\/PubmedArticle>)/smu).flatten
-    end
-
-    if pmids.is_a? Array
-      list = {}
-      articles.each{|article|
-        pmid = article.scan(/<PMID[^>]*?>(.*?)<\/PMID>/).flatten.first
-        list[pmid] = article
-      }
-      return list
-    else
-      return articles.first
-    end
+  # Performs the specified query and returns an array with the PubMed
+  # Ids returned. +retmax+ can be used to limit the number of ids
+  # returned, if is not specified 30000 is used.
+  def self.query(query, retmax=nil)
+    retmax ||= 30000
 
+    Open.read("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?retmax=#{retmax}&db=pubmed&term=#{query}",:quiet => true, :nocache => true).scan(/<Id>(\d+)<\/Id>/).flatten
   end
 
-  public
-
   # Processes the xml with an articles as served by MedLine and extracts
   # the abstract, title and journal information
   class Article
@@ -239,12 +218,48 @@ module PubMed
     end
   end
 
-  # Performs the specified query and returns an array with the PubMed
-  # Ids returned. +retmax+ can be used to limit the number of ids
-  # returned, if is not specified 30000 is used.
-  def self.query(query, retmax=nil)
-    retmax ||= 30000
+  def self.get_article(pmids)
+    _array = Array === pmids
 
-    Open.read("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?retmax=#{retmax}&db=pubmed&term=#{query}",:quiet => true, :nocache => true).scan(/<Id>(\d+)<\/Id>/).flatten
+    pmids = [pmids] unless Array === pmids
+    pmids = pmids.compact.collect{|id| id}
+
+    result_files = FileCache.cache_online_elements(pmids, 'pubmed-{ID}.xml') do |ids|
+      result = {}
+      values = []
+      Misc.divide(ids, (ids.length / 100) + 1).each do |list|
+        begin
+          Misc.try3times do
+            url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" 
+
+            postdata = "db=pubmed&retmode=xml&id=#{list* ","}"
+            xml = TmpFile.with_file(postdata) do |postfile|
+              Open.read(url, :quiet => true, :nocache => true, :nice => @@pubmed_lag, :nice_key => "PubMed", "--post-file=" => postfile)
+            end
+
+            values += xml.scan(/(<PubmedArticle>.*?<\/PubmedArticle>)/smu).flatten
+          end
+        rescue
+          Log.exception $!
+        end
+      end
+
+      values.each do |xml|
+        pmid = xml.scan(/<PMID[^>]*?>(.*?)<\/PMID>/).flatten.first
+        
+        result[pmid] = xml
+      end
+
+      result
+    end
+
+    articles = {}
+    pmids.each{|id| next if id.nil? or result_files[id].nil?; articles[id] = Article.new(Open.read(result_files[id])) }
+
+    if _array
+      articles
+    else
+      articles.values.first
+    end
   end
 end

data/lib/rbbt/sources/stitch.rb

@@ -1,8 +1,8 @@
-require 'phgx'
+require 'rbbt'
+require 'rbbt/resource'
 
 module STITCH
   extend Resource
-  self.pkgdir = "phgx"
   self.subdir = "share/stitch"
 
   STITCH.claim STITCH.root, :rake, Rbbt.share.install.STITCH.Rakefile.find(:lib)

data/lib/rbbt/sources/string.rb

@@ -1,8 +1,8 @@
-require 'phgx'
+require 'rbbt'
+require 'rbbt/resource'
 
 module STRING
   extend Resource
-  self.pkgdir = "phgx"
   self.subdir = "share/string"
 
   STRING.claim STRING.root, :rake, Rbbt.share.install.STRING.Rakefile.find(:lib)

data/lib/rbbt/sources/tfacts.rb

@@ -1,6 +1,6 @@
 require 'rbbt'
+require 'rbbt/tsv'
 require 'rbbt/resource'
-require 'spreadsheet'
 
 module TFacts
   extend Resource
@@ -9,6 +9,7 @@ module TFacts
   TFacts.claim TFacts.source["Catalogues.xls"], :url, "http://www.tfacts.org/TFactS-new/TFactS-v2/tfacts/data/Catalogues.xls"
 
   TFacts.claim TFacts.targets, :proc do
+    require 'spreadsheet'
     book = Spreadsheet.open TFacts.source["Catalogues.xls"].produce.find
     sheet = book.worksheet 0
 
@@ -23,6 +24,7 @@ module TFacts
   end
 
   TFacts.claim TFacts.targets_signed, :proc do
+    require 'spreadsheet'
     book = Spreadsheet.open TFacts.source["Catalogues.xls"].produce.find
     sheet = book.worksheet 0
 

data/share/install/Matador/Rakefile

@@ -0,0 +1,14 @@
+require File.join(File.dirname(__FILE__),'../lib/rake_helper')
+
+define_source_tasks "matador" => "http://matador.embl.de/media/download/matador.tsv.gz"
+
+process_tsv :protein_drug, 'matador',
+  :key_field  => 3, 
+  :fix     => lambda{|l| l.sub(/9606./,'')}, 
+  :fields   => [1,0,7,8,9,10,11,12], 
+  :header_hash => "",
+  :keep_empty => true do 
+
+  headers ['Ensembl Protein ID', 'Chemical', 'Matador ID', 'Score', 'Annotation', 'Mesh_Score', 'Mesh_Annotation', 'Matador_Score', 'Matador_Annotation']
+end
+

data/share/install/PharmaGKB/Rakefile

@@ -115,23 +115,6 @@ file :pathways => 'source/pathways' do |t|
   end
 end
 
-file :gene_pathway => 'source/pathways' do |t|
-  pathways = {}
-  last_pathway = nil
-
-  Open.read(t.prerequisites.first).split(/\n/).each do |line|
-    if line =~ /(P.*):(.*)/
-      last_pathway = $1
-      pathways[last_pathway] = {:name => $2}
-    else
-      type, code, name = line.split(/\t/)
-      next unless type =='Gene'
-      pathways[last_pathway][:genes] ||= []
-      pathways[last_pathway][:genes] << name
-    end
-  end
-end
-
 file :gene_pathway => 'source/pathways' do |t|
   pathways = {}
   last_pathway = nil

data/share/install/STITCH/Rakefile

@@ -1,7 +1,7 @@
 require File.join(File.dirname(__FILE__),'../lib/rake_helper')
 
-define_source_tasks "protein_chemicals" => "http://stitch.embl.de:8080/download/protein_chemical.links.v2.0.tsv.gz",
-  "chemicals" => "http://stitch.embl.de:8080/download/chemical.aliases.v2.0.tsv.gz"
+define_source_tasks "protein_chemicals" => "http://stitch.embl.de/download/protein_chemical.links.v3.1.tsv.gz",
+  "chemicals" => "http://stitch.embl.de/download/chemicals.v3.1.tsv.gz"
 
 process_tsv :protein_chemical, 'protein_chemicals',
   :key => 1,

data/share/install/lib/rake_helper.rb

@@ -0,0 +1,91 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '../../../lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rbbt/tsv'
+require 'rbbt/util/open'
+require 'rbbt/util/log'
+
+SOURCE_DIR = 'source'
+def define_source_tasks(sources)
+  sources.each do |name, url|
+    file File.join(SOURCE_DIR, name) do |t|
+      FileUtils.mkdir SOURCE_DIR unless File.exists? SOURCE_DIR
+      Log.log "Retrieving file '#{name}' into '#{t.name}': '#{url}'", Log::LOW
+      Open.write(t.name, Open.open(url, :cache => false, :wget_options => {"--no-check-certificate" => true, "--quiet" => false, :pipe => true}))
+    end
+  end
+end
+
+$__headers = nil
+def headers(values)
+  $__headers = values
+end
+
+$__data = nil
+def data(&block)
+  $__data = block
+end
+
+$__tsv_tasks = []
+def tsv_tasks
+  $__tsv_tasks 
+end
+
+$__files = []
+def add_to_defaults(list)
+  $__files = list
+end
+
+def process_tsv(file, source, options = {}, &block)
+
+  $__tsv_tasks << file
+
+  file file => File.join(SOURCE_DIR, source) do |t|
+    block.call
+
+    d = TSV.open(t.prerequisites.first, options)
+
+    if d.fields != nil
+      data_fields = d.fields.dup.unshift d.key_field
+      if $__headers.nil?
+        $__headers = data_fields
+      end
+    end
+
+    if d.fields
+      headers = d.fields.dup.unshift d.key_field
+    else
+      headers = nil
+    end
+
+    File.open(t.name.to_s, 'w') do |f|
+      f.puts "#" + $__headers * "\t" if $__headers != nil
+      d.each do |key, values|
+        if $__data.nil?
+          line = values.unshift key
+        else
+          line = $__data.call key, values
+        end
+
+        if Array === line
+          key   = line.shift
+          fields = line.collect{|elem| Array === elem ? elem * "|" : elem }
+          fields.unshift key
+          f.puts fields * "\t" 
+        else
+          f.puts line
+        end
+      end
+    end
+  end
+end
+
+task :default do |t|
+  ($__tsv_tasks + $__files).each do |file| Rake::Task[file].invoke end
+end
+
+task :all => :default 
+
+task :clean do
+  ($__tsv_tasks + $__files).each do |file| FileUtils.rm file.to_s if File.exists?(file.to_s) end
+end

data/test/rbbt/sources/test_HPRD.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+require 'rbbt/sources/HPRD'
+require 'test/unit'
+
+class TestPina < Test::Unit::TestCase
+  def __test_HPRD
+    ddd HPRD.protein_protein.find.produce
+  end
+end
+
+

data/test/rbbt/sources/test_biomart.rb

@@ -8,6 +8,14 @@ require 'test/unit'
 
 class TestBioMart < Test::Unit::TestCase
 
+  def setup
+    BioMart.set_archive "jun2011"
+  end
+
+  def teardown
+    BioMart.unset_archive
+  end
+
   def test_get
     assert_raise BioMart::QueryError do 
       BioMart.get('scerevisiae_gene_ensembl','entrezgene', ['protein_id'],['with_unknownattr'])
@@ -41,6 +49,8 @@ class TestBioMart < Test::Unit::TestCase
 
     TmpFile.with_file do |f|
       filename = BioMart.tsv('scerevisiae_gene_ensembl',['Entrez Gene', 'entrezgene'], [['Protein ID', 'protein_id'],['RefSeq Peptide','refseq_peptide']], [], nil, :nocache => false, :wget_options => { :quiet => false}, :filename => f)
+      return
+      puts Open.read(filename)
       data = TSV.open Open.open(filename, :merge => true)
       assert(data['852236']['Protein ID'].include? 'CAA84864')
       assert_equal 'Entrez Gene', data.key_field

data/test/rbbt/sources/test_gscholar.rb

@@ -5,8 +5,8 @@ require 'test/unit'
 
 class TestGScholar < Test::Unit::TestCase
   def test_citation
-    assert_match GoogleScholar.citation_link("Ten Years of Pathway Analysis: Current Approaches and Outstanding Challenges").to_s, /cites/
-    assert_match GoogleScholar.number_cites("Ten Years of Pathway Analysis: Current Approaches and Outstanding Challenges").to_s, /\d+/
+    assert_match /cites/, GoogleScholar.citation_link("Ten Years of Pathway Analysis: Current Approaches and Outstanding Challenges").to_s
+    assert_match /\d+/, GoogleScholar.number_cites("Ten Years of Pathway Analysis: Current Approaches and Outstanding Challenges").to_s
   end
 
 end

data/test/rbbt/sources/test_kegg.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+require 'rbbt/sources/kegg'
+require 'test/unit'
+
+class TestKEGG < Test::Unit::TestCase
+  def test_kegg
+    assert KEGG.gene_pathway.exists?
+  end
+end
+
+

data/test/rbbt/sources/test_matador.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+require 'rbbt/sources/matador'
+require 'test/unit'
+
+class TestMatador < Test::Unit::TestCase
+  def test_matador
+    assert Matador.protein_drug.exists?
+  end
+end
+
+

data/test/rbbt/sources/test_pharmagkb.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+require 'rbbt/sources/pharmagkb'
+require 'test/unit'
+
+class TestPGKB < Test::Unit::TestCase
+  def test_pharmagkb
+    assert PharmaGKB.gene_pathway.exists?
+  end
+end
+
+

data/test/rbbt/sources/test_pina.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+require 'rbbt/sources/pina'
+require 'test/unit'
+
+class TestPina < Test::Unit::TestCase
+  def test_pina
+    assert Pina.protein_protein.exists?
+  end
+end
+
+

data/test/rbbt/sources/test_pubmed.rb

@@ -5,14 +5,6 @@ require 'test/unit'
 
 class TestPubMed < Test::Unit::TestCase
 
-  def test_get_online
-    pmid = '16438716'
-    assert(PubMed.get_online(pmid) =~ /Discovering semantic features in the literature: a foundation for building functional associations./)
-
-    pmids = ['16438716', 17204154]
-    assert(PubMed.get_online(pmids)[pmid] =~ /Discovering semantic features in the literature: a foundation for building functional associations./)
-  end
-
   def test_get_article
     pmid = '16438716'
     assert(PubMed.get_article(pmid).title == "Discovering semantic features in the literature: a foundation for building functional associations.")

data/test/rbbt/sources/test_stitch.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+require 'rbbt/sources/stitch'
+require 'test/unit'
+
+class TestSTITCH < Test::Unit::TestCase
+  def test_stitch
+    assert STITCH.protein_chemical.exists?
+  end
+end
+
+

data/test/rbbt/sources/test_string.rb

@@ -0,0 +1,12 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+require 'rbbt/sources/string'
+require 'test/unit'
+
+class TestSTRING < Test::Unit::TestCase
+  def test_string
+    assert STRING.protein_protein.exists?
+  end
+end
+
+

data/test/rbbt/sources/test_tfacts.rb

@@ -0,0 +1,14 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+require 'rbbt/sources/tfacts'
+require 'test/unit'
+
+class TestTFacts < Test::Unit::TestCase
+  def test_tfacts
+    assert TFacts.targets.exists?
+    assert TFacts.targets_signed.exists?
+    assert TFacts.regulators.exists?
+  end
+end
+
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-sources
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.0.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-25 00:00:00.000000000 Z
+date: 2014-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -123,6 +123,7 @@ files:
 - share/install/Genomes1000/Rakefile
 - share/install/JoChem/Rakefile
 - share/install/KEGG/Rakefile
+- share/install/Matador/Rakefile
 - share/install/NCI/Rakefile
 - share/install/Organism/Hsa/Rakefile
 - share/install/Organism/Mmu/Rakefile
@@ -134,12 +135,21 @@ files:
 - share/install/STITCH/Rakefile
 - share/install/STRING/Rakefile
 - share/install/lib/helpers.rb
+- share/install/lib/rake_helper.rb
+- test/rbbt/sources/test_HPRD.rb
 - test/rbbt/sources/test_biomart.rb
 - test/rbbt/sources/test_entrez.rb
 - test/rbbt/sources/test_go.rb
 - test/rbbt/sources/test_gscholar.rb
+- test/rbbt/sources/test_kegg.rb
+- test/rbbt/sources/test_matador.rb
 - test/rbbt/sources/test_organism.rb
+- test/rbbt/sources/test_pharmagkb.rb
+- test/rbbt/sources/test_pina.rb
 - test/rbbt/sources/test_pubmed.rb
+- test/rbbt/sources/test_stitch.rb
+- test/rbbt/sources/test_string.rb
+- test/rbbt/sources/test_tfacts.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-sources
 licenses: []
@@ -166,9 +176,17 @@ specification_version: 4
 summary: Data sources for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
 - test/rbbt/sources/test_pubmed.rb
+- test/rbbt/sources/test_pharmagkb.rb
 - test/rbbt/sources/test_biomart.rb
 - test/rbbt/sources/test_gscholar.rb
+- test/rbbt/sources/test_kegg.rb
+- test/rbbt/sources/test_pina.rb
 - test/rbbt/sources/test_entrez.rb
+- test/rbbt/sources/test_matador.rb
+- test/rbbt/sources/test_HPRD.rb
 - test/rbbt/sources/test_go.rb
+- test/rbbt/sources/test_stitch.rb
 - test/rbbt/sources/test_organism.rb
+- test/rbbt/sources/test_string.rb
+- test/rbbt/sources/test_tfacts.rb
 - test/test_helper.rb