rbbt-sources 3.1.41 → 3.1.47

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0a35626b221e15867cb54ca9e8bbeb007762b5a99afed75d57d6b473385c287
-  data.tar.gz: 59d067abbfdcd686f39b3604282a280ca9bf4a0e8f3284698202dae8d9d50d55
+  metadata.gz: 6cecde427390fc444e5ef5b6bef42072bf87714b404647c6c086da92a1eaa13a
+  data.tar.gz: 673429c8eab433ec9854c81d52450f8885c57e4b3f8b6779ded44585e6f88343
 SHA512:
-  metadata.gz: 1c9c6a2dc5184319e94af671762227bba331b49784cf3e4ac0550b6c023c7b072920d75175b0547c2d8cb4034dbca53dda9514238773cf98f103cc6be978c4ec
-  data.tar.gz: df52691bf65b891a906217f5e2647577bc74940bbd3fa1752115f624a313472a70a43fb96b786e30a4612bc71badb7b3e68c539e0024fe0f0958ee5f33d910af
+  metadata.gz: e39a3a6418bfb514ec786da0128f6fb8437da9728224dacd41b837300680c6d34e821f08954820cb5b7fd064546fee21fd0557d179c469eda969485ffd5cd965
+  data.tar.gz: 9f332f35fcd9c66db7506875efcc7c5276b1f588dd3b70ca299aaa2a7f25c33a650e496ba207af4e12e3bb2a6dfbc9ec8b1f0729e3acedc354145572dd268836

data/etc/biomart/missing_in_archive

@@ -1,3 +1,7 @@
+">oct2014":
+    - rgd~rgd_id
+">jun2019":
+    - entrezgene~entrezgene_id
 ">dec2017":
     - unigene
 ">dec2016":

data/lib/rbbt/sources/biomart.rb

@@ -134,11 +134,14 @@ module BioMart
   # cause an error if the BioMart WS does not allow filtering with that
   # attribute.
   def self.query(database, main, attrs = nil, filters = nil, data = nil, open_options = {})
+    IndiferentHash.setup(open_options)
     open_options = Misc.add_defaults open_options, :nocache => false, :filename => nil, :field_names => nil, :by_chr => false
     filename, field_names, by_chr = Misc.process_options open_options, :filename, :field_names, :by_chr
     attrs   ||= []
     open_options = Misc.add_defaults open_options, :keep_empty => false, :merge => true
 
+    IndiferentHash.setup(open_options)
+
     Log.low "BioMart query: '#{main}' [#{(attrs || []) * ', '}] [#{(filters || []) * ', '}] #{open_options.inspect}"
 
     max_items = 2
@@ -167,6 +170,7 @@ module BioMart
     end
 
     open_options[:filename] = "BioMart[#{main}+#{attrs.length}]"
+
     if filename.nil?
       results = TSV.open data, open_options
       results.key_field = main
@@ -200,6 +204,7 @@ module BioMart
         missing+=v if Organism.compare_archives(current_archive, t) == -1
       elsif k=~ /^>(.*)/ 
         t = $1.strip
+        iii [current_archive, t, Organism.compare_archives(current_archive, t)]
         missing+=v if Organism.compare_archives(current_archive, t) == 1
       end
     end

data/lib/rbbt/sources/entrez.rb

@@ -14,7 +14,7 @@ module Entrez
     options = Misc.add_defaults options, :key_field => 1, :fields => [5], :persist => true, :merge => true
 
     taxs = [taxs] unless Array === taxs
-    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
+    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}, :fixed_grep => false
     
     tsv = Rbbt.share.databases.entrez.gene_info.tsv :flat, options
     tsv.key_field = "Entrez Gene ID"
@@ -26,7 +26,7 @@ module Entrez
     options = Misc.add_defaults options, :key_field => 1, :fields => [2], :persist => true, :merge => true
 
     taxs = [taxs] unless Array === taxs
-    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
+    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}, :fixed_grep => false
     
     tsv = Rbbt.share.databases.entrez.gene_info.tsv :flat, options
     tsv.key_field = "Entrez Gene ID"
@@ -39,7 +39,7 @@ module Entrez
     options = {:key_field => 1, :fields => [2], :persist => true, :merge => true}
 
     taxs = [taxs] unless taxs.is_a?(Array)
-    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
+    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}, :fixed_grep => false
 
     Rbbt.share.databases.entrez.gene2pubmed.tsv :flat, options
   end

data/lib/rbbt/sources/go.rb

@@ -27,7 +27,7 @@ module GO
   def self.init
     Persist.persist_tsv(nil, 'gene_ontology', {}, :persist => true) do |info|
       info.serializer = :marshal if info.respond_to? :serializer
-      Rbbt.share.databases.GO.gene_ontology.read.split(/\[Term\]/).each{|term| 
+      Rbbt.share.databases.GO.gene_ontology.produce.read.split(/\[Term\]/).each{|term| 
         term_info = {}
 
         term.split(/\n/). select{|l| l =~ /:/}.each{|l| 

data/lib/rbbt/sources/pina.rb

@@ -7,7 +7,7 @@ module Pina
   Pina.claim Pina.protein_protein, :proc do
     require 'rbbt/sources/organism'
 
-    url = "http://cbg.garvan.unsw.edu.au/pina/download/Homo%20sapiens-20121210.txt"
+    url = "https://omics.bjcancer.org/pina/download/Homo%20sapiens-20140521.tsv"
 
     dumper = TSV::Dumper.new :type => :double,
       :key_field => 'UniProt/SwissProt Accession', :namespace => Organism.default_code("Hsa"), 

data/lib/rbbt/sources/pubmed.rb

@@ -109,6 +109,14 @@ module PubMed
       end
     end
 
+    def pmc_full_xml
+      begin
+        Open.read("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pmc&id=#{pmid}")
+      rescue
+        nil
+      end
+    end
+
     def pdf_url
       return pmc_pdf if pmc_pdf
       @gscholar_pdf ||= begin
@@ -121,18 +129,22 @@ module PubMed
     end
 
     def full_text
-      return nil if pdf_url.nil?
-
-      text = nil
-      TmpFile.with_file do |pdf|
-
-        # Change user-agent, oh well...
-        `wget --user-agent=firefox #{ pdf_url } -O #{ pdf } -t 3`
-        TmpFile.with_file do |txt|
-          `pdftotext #{ pdf } #{ txt }`
-          text = Open.read(txt) if File.exists? txt
-        end
-      end
+      text = if pdf_url
+               text = nil
+               TmpFile.with_file do |pdf|
+                 # Change user-agent, oh well...
+                 `wget --user-agent=firefox #{ pdf_url } -O #{ pdf } -t 3`
+                 TmpFile.with_file do |txt|
+                   `pdftotext #{ pdf } #{ txt }`
+                   text = Open.read(txt) if File.exists? txt
+                 end
+               end
+               text
+             elsif pmc_full_xml
+               pmc_full_xml
+             else
+               nil
+             end
 
       Misc.fixutf8(text)
     end
@@ -177,46 +189,46 @@ module PubMed
   # ID specified as an argument. If +pmid+ is an array instead of a single
   # identifier it returns an hash with the Article object for each id.
   # It uses the Rbbt cache to save the articles xml.
-  def self.get_article(pmid)
-
-    if pmid.is_a? Array
-      missing = []
-      list = {}
-
-      pmid.each{|p|
-        filename = p.to_s + '.xml'
-        if File.exists? FileCache.path(filename)
-          list[p] = Article.new(Open.read(FileCache.path(filename)))
-        else
-          missing << p
-        end
-      }
-
-      return list unless missing.any?
-
-      articles = get_online(missing)
-
-      articles.each{|p, xml|
-        filename = p + '.xml'
-        FileCache.add(filename,xml)
-        list[p] =  Article.new(xml)
-      }
-
-      return list
-
-    else
-      filename = pmid.to_s + '.xml'
-
-      if File.exists? FileCache.path(filename)
-        return Article.new(Open.read(FileCache.path(filename)))
-      else
-        xml = get_online(pmid)
-        FileCache.add(filename,xml)
-
-        return Article.new(xml)
-      end
-    end
-  end
+  #def self.get_article(pmid)
+
+  #  if pmid.is_a? Array
+  #    missing = []
+  #    list = {}
+
+  #    pmid.each{|p|
+  #      filename = p.to_s + '.xml'
+  #      if File.exists? FileCache.path(filename)
+  #        list[p] = Article.new(Open.read(FileCache.path(filename)))
+  #      else
+  #        missing << p
+  #      end
+  #    }
+
+  #    return list unless missing.any?
+
+  #    articles = get_online(missing)
+
+  #    articles.each{|p, xml|
+  #      filename = p + '.xml'
+  #      FileCache.add(filename,xml)
+  #      list[p] =  Article.new(xml)
+  #    }
+
+  #    return list
+
+  #  else
+  #    filename = pmid.to_s + '.xml'
+
+  #    if File.exists? FileCache.path(filename)
+  #      return Article.new(Open.read(FileCache.path(filename)))
+  #    else
+  #      xml = get_online(pmid)
+  #      FileCache.add(filename,xml)
+
+  #      return Article.new(xml)
+  #    end
+  #  end
+  #end
 
   def self.get_article(pmids)
     _array = Array === pmids

data/lib/rbbt/sources/tfacts.rb

@@ -2,15 +2,15 @@ require 'rbbt'
 require 'rbbt/tsv'
 require 'rbbt/resource'
 
-module TFacts
+module TFactS
   extend Resource
-  self.subdir = "share/databases/TFacts"
+  self.subdir = "share/databases/TFactS"
 
-  TFacts.claim TFacts[".source"]["Catalogues.xls"], :url, "http://www.tfacts.org/TFactS-new/TFactS-v2/tfacts/data/Catalogues.xls"
+  TFactS.claim TFactS[".source"]["Catalogues.xls"], :url, "http://www.tfacts.org/TFactS-new/TFactS-v2/tfacts/data/Catalogues.xls"
 
-  TFacts.claim TFacts.targets, :proc do
+  TFactS.claim TFactS.targets, :proc do
     require 'spreadsheet'
-    book = Spreadsheet.open TFacts[".source"]["Catalogues.xls"].produce.find
+    book = Spreadsheet.open TFactS[".source"]["Catalogues.xls"].produce.find
     sheet = book.worksheet 0
 
     tsv = TSV.setup({}, :key_field => "Target Gene (Associated Gene Name)", :fields => ["Transcription Factor (Associated Gene Name)"], :namespace => "Hsa", :type => :flat)
@@ -24,9 +24,9 @@ module TFacts
     tsv.to_s
   end
 
-  TFacts.claim TFacts.targets_signed, :proc do
+  TFactS.claim TFactS.targets_signed, :proc do
     require 'spreadsheet'
-    book = Spreadsheet.open TFacts[".source"]["Catalogues.xls"].produce.find
+    book = Spreadsheet.open TFactS[".source"]["Catalogues.xls"].produce.find
     sheet = book.worksheet 1
 
     tsv = TSV.setup({}, :key_field => "Target Gene (Associated Gene Name)", :fields => ["Transcription Factor (Associated Gene Name)", "Sign", "PMID"], :namespace => "Hsa", :type => :double)
@@ -43,13 +43,13 @@ module TFacts
     tsv.to_s
   end
 
-  TFacts.claim TFacts.regulators, :proc do
-    TFacts.targets.tsv.reorder("Transcription Factor (Associated Gene Name)").to_s
+  TFactS.claim TFactS.regulators, :proc do
+    TFactS.targets.tsv.reorder("Transcription Factor (Associated Gene Name)").to_s
   end
 
-  TFacts.claim TFacts.tf_tg, :proc do
+  TFactS.claim TFactS.tf_tg, :proc do
     require 'spreadsheet'
-    book = Spreadsheet.open TFacts[".source"]["Catalogues.xls"].produce.find
+    book = Spreadsheet.open TFactS[".source"]["Catalogues.xls"].produce.find
 
     tsv = TSV.setup({}, :key_field => "Transcription Factor (Associated Gene Name)", :fields => ["Target Gene (Associated Gene Name)", "Sign", "Species", "Source", "PMID"], :namespace => "Hsa", :type => :double)
 
@@ -128,16 +128,16 @@ if defined? Entity and defined? Gene and Entity === Gene
 
   module Gene
     property :is_transcription_factor? => :array2single do
-      tfs = TFacts.targets.keys
+      tfs = TFactS.targets.keys
       self.name.collect{|gene| tfs.include? gene}
     end
 
     property :transcription_regulators => :array2single do
-      Gene.setup(TFacts.regulators.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
+      Gene.setup(TFactS.regulators.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
     end
 
     property :transcription_targets => :array2single do
-      Gene.setup(TFacts.targets.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
+      Gene.setup(TFactS.targets.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
     end
   end
 end

data/share/install/Organism/Hsa/Rakefile

@@ -5,7 +5,7 @@ require File.join(File.dirname(__FILE__), '../../lib/helpers')
 
 $taxs = [9606]
 $scientific_name = "Homo sapiens"
-$ortholog_key = "human_ensembl_gene"
+$ortholog_key = "hsapiens_homolog_ensembl_gene"
 
 $biomart_db = 'hsapiens_gene_ensembl'
 $biomart_db_germline_variation = 'hsapiens_snp'

data/share/install/Organism/Mmu/Rakefile

@@ -5,7 +5,7 @@ require File.join(File.dirname(__FILE__), '../../lib/helpers')
 
 $taxs = [10090]
 $scientific_name = "Mus musculus"
-$ortholog_key = "mouse_ensembl_gene"
+$ortholog_key = "mmusculus_homolog_ensembl_gene"
 
 $biomart_db = 'mmusculus_gene_ensembl'
 $biomart_db_germline_variation = 'mmusculus_snp'

data/share/install/Organism/Rno/Rakefile

@@ -9,7 +9,7 @@ $scientific_name = "Rattus norvegicus"
 $biomart_db = 'rnorvegicus_gene_ensembl'
 $biomart_db_germline_variation = 'rnorvegicus_snp'
 $biomart_db_somatic_variation = 'rnorvegicus_snp_som'
-$ortholog_key = "rat_ensembl_gene"
+$ortholog_key = "rnorvegicus_homolog_ensembl_gene"
 
 $biomart_lexicon = [ 
   [ 'Associated Gene Name' , "external_gene_id"], 

data/share/install/Organism/organism_helpers.rb

@@ -547,13 +547,13 @@ end
 rule /^possible_ortholog_(.*)/ do |t|
   other = t.name.match(/ortholog_(.*)/)[1]
   other_key = Organism.ortholog_key(other).produce.read
-  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", "inter_paralog_" + other_key]], [], nil, :keep_empty => false, :type => :flat, :filename => t.name, :namespace => Thread.current['namespace'])
+  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", "inter_paralog_" + other_key]], [], nil, :keep_empty => false, :type => :double, :filename => t.name, :namespace => Thread.current['namespace'])
 end
 
 rule /^ortholog_(.*)/ do |t|
   other = t.name.match(/ortholog_(.*)/)[1]
   other_key = Organism.ortholog_key(other).produce.read
-  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", other_key]], [], nil, :keep_empty => false, :type => :flat, :filename => t.name, :namespace => Thread.current['namespace'])
+  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", other_key]], [], nil, :keep_empty => false, :type => :double, :filename => t.name, :namespace => Thread.current['namespace'])
 end
 
 rule /[a-z]{3}[0-9]{4}\/.*/i do |t|

data/test/rbbt/sources/test_entrez.rb

@@ -17,7 +17,10 @@ class TestEntrez < Test::Unit::TestCase
   def test_entrez2pubmed
     tax   = $yeast_tax
 
+    Log.severity = 0
     data = Entrez.entrez2pubmed(tax)
+    data.read
+    Log.tsv data
     assert(data['850320'].include? '1574125') 
   end
 

data/test/rbbt/sources/test_pubmed.rb

@@ -17,6 +17,12 @@ class TestPubMed < Test::Unit::TestCase
     pmid = '16438716'
     assert(PubMed.get_article(pmid).full_text =~ /Discovering/)
   end
+
+  def test_pmc_full_xml
+    pmid = '4304705'
+    assert PubMed.get_article(pmid).pmc_full_xml.include?("HBV antigen")
+  end
+ 
  
   def test_query
     assert(PubMed.query('chagoyen[All Fields] AND ("loattrfull text"[sb] AND hasabstract[text])').include? '16438716')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-sources
 version: !ruby/object:Gem::Version
-  version: 3.1.41
+  version: 3.1.47
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-15 00:00:00.000000000 Z
+date: 2020-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.0
-- !ruby/object:Gem::Dependency
-  name: rbbt-text
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement