rbbt-sources 3.1.43 → 3.1.49

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb929c25537c93976e03cac4193b7bcbde8c6b8fa1975506779290cc9579d2cb
-  data.tar.gz: d316a40f9ac9c00997dbecf35246290ee5c87704bf42fab5b687b1e595f803c3
+  metadata.gz: e5e2516cf7cd0ae996164a41f940843369c5ab8c5092a76fcdb81ca80b43b1b6
+  data.tar.gz: 5d90eb802ea522d2c760910ce1e19a6c197b7afef6a183cb5ed8335adf8b083b
 SHA512:
-  metadata.gz: 93b9260a58ed0bfdfa2be1d63160fae4a70a71df3f9c8510e42b99ac740d2c90b1433c8b2aae28e2978ddccfb8abb822bc106341bf8dcef31092db768b5caf19
-  data.tar.gz: 7931ac20e016779659aca2bf6966abdefe21eec7dc9ca6bf843ddd09004f6fda3c022a74ed48e6d269d2b702aa4bf77cfb22ae760ea65d80db916efeb6998ebd
+  metadata.gz: 5451de956158d1f9e40c216cdf4107fede82080423acb32924831168d5f60ea21590a1e996a08bb68c1951cb8da42c793fffa84ea873cc889073fda2cfec07b7
+  data.tar.gz: 80a79bada240deec85485f4f831a0e1aaa520a10889ac28d7665424d61173938c95638753cd2eb190d744e0a978e04eb891ee7bd88bd5a4442792c9beba45ad1

data/etc/biomart/missing_in_archive

@@ -1,3 +1,7 @@
+">oct2014":
+    - rgd~rgd_id
+">jun2019":
+    - entrezgene~entrezgene_id
 ">dec2017":
     - unigene
 ">dec2016":

data/lib/rbbt/sources/biomart.rb

@@ -134,11 +134,14 @@ module BioMart
   # cause an error if the BioMart WS does not allow filtering with that
   # attribute.
   def self.query(database, main, attrs = nil, filters = nil, data = nil, open_options = {})
+    IndiferentHash.setup(open_options)
     open_options = Misc.add_defaults open_options, :nocache => false, :filename => nil, :field_names => nil, :by_chr => false
     filename, field_names, by_chr = Misc.process_options open_options, :filename, :field_names, :by_chr
     attrs   ||= []
     open_options = Misc.add_defaults open_options, :keep_empty => false, :merge => true
 
+    IndiferentHash.setup(open_options)
+
     Log.low "BioMart query: '#{main}' [#{(attrs || []) * ', '}] [#{(filters || []) * ', '}] #{open_options.inspect}"
 
     max_items = 2
@@ -167,6 +170,7 @@ module BioMart
     end
 
     open_options[:filename] = "BioMart[#{main}+#{attrs.length}]"
+
     if filename.nil?
       results = TSV.open data, open_options
       results.key_field = main

data/lib/rbbt/sources/ensembl_ftp.rb

@@ -52,24 +52,35 @@ module Ensembl
       File.join("ftp://" + SERVER, ftp_directory_for(organism) )
     end
 
-    def self.url_for(organism, table)
-      "#{base_url(organism)}/#{table}.txt.gz.bz2"
+    def self.url_for(organism, table, extension)
+      File.join(base_url(organism), table) + ".#{extension}.gz"
+    end
+
+    def self._get_gz(url)
+      begin
+        CMD.cmd("wget '#{url}' -O  - | gunzip").read
+      rescue
+        CMD.cmd("wget '#{url}.bz2' -O  - | bunzip2 | gunzip").read
+      end
+    end
+
+    def self._get_file(organism, table, extension)
+      url = url_for(organism, table, extension)
+      self._get_gz(url)
     end
 
     def self.has_table?(organism, table)
-      sql_file = CMD.cmd("wget '#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz.bz2' -O  -| bunzip2| gunzip").read
+      sql_file = _get_file(organism, File.basename(base_url(organism)), 'sql')
       ! sql_file.match(/^CREATE TABLE .#{table}. \((.*?)^\)/sm).nil?
     end
 
     def self.fields_for(organism, table)
-      sql_file = CMD.cmd("wget '#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz.bz2' -O  -| bunzip2| gunzip").read
-      
+      sql_file = _get_file(organism, File.basename(base_url(organism)), 'sql')
       chunk = sql_file.match(/^CREATE TABLE .#{table}. \((.*?)^\)/sm)[1]
       chunk.scan(/^\s+`(.*?)`/).flatten
     end
 
     def self.ensembl_tsv(organism, table, key_field = nil, fields = nil, options = {})
-      url = url_for(organism, table)
       if key_field and fields
         all_fields = fields_for(organism, table)
         key_pos = all_fields.index key_field
@@ -78,7 +89,8 @@ module Ensembl
         options[:key_field] = key_pos
         options[:fields]    = field_pos
       end
-      tsv = TSV.open(CMD.cmd("wget '#{url}' -O - |bunzip2|gunzip", :pipe => true), options)
+
+      tsv = TSV.open(StringIO.new(_get_file(organism, table, "txt")), options)
       tsv.key_field = key_field
       tsv.fields = fields
       tsv

data/lib/rbbt/sources/entrez.rb

@@ -14,7 +14,7 @@ module Entrez
     options = Misc.add_defaults options, :key_field => 1, :fields => [5], :persist => true, :merge => true
 
     taxs = [taxs] unless Array === taxs
-    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
+    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}, :fixed_grep => false
     
     tsv = Rbbt.share.databases.entrez.gene_info.tsv :flat, options
     tsv.key_field = "Entrez Gene ID"
@@ -26,7 +26,7 @@ module Entrez
     options = Misc.add_defaults options, :key_field => 1, :fields => [2], :persist => true, :merge => true
 
     taxs = [taxs] unless Array === taxs
-    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
+    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}, :fixed_grep => false
     
     tsv = Rbbt.share.databases.entrez.gene_info.tsv :flat, options
     tsv.key_field = "Entrez Gene ID"
@@ -39,7 +39,7 @@ module Entrez
     options = {:key_field => 1, :fields => [2], :persist => true, :merge => true}
 
     taxs = [taxs] unless taxs.is_a?(Array)
-    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
+    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}, :fixed_grep => false
 
     Rbbt.share.databases.entrez.gene2pubmed.tsv :flat, options
   end

data/lib/rbbt/sources/go.rb

@@ -27,7 +27,7 @@ module GO
   def self.init
     Persist.persist_tsv(nil, 'gene_ontology', {}, :persist => true) do |info|
       info.serializer = :marshal if info.respond_to? :serializer
-      Rbbt.share.databases.GO.gene_ontology.read.split(/\[Term\]/).each{|term| 
+      Rbbt.share.databases.GO.gene_ontology.produce.read.split(/\[Term\]/).each{|term| 
         term_info = {}
 
         term.split(/\n/). select{|l| l =~ /:/}.each{|l| 

data/lib/rbbt/sources/organism.rb

@@ -88,8 +88,10 @@ module Organism
       end
     when "Mmu"
       "mm10"
+    when "Rno"
+      "rn6"
     else
-      raise "Only organism 'Hsa' (Homo sapiens) and Mmu (Mus musculus) supported" 
+      raise "Only organism 'Hsa' (Homo sapiens), 'Rno' (Rattus norvegicus), and Mmu (Mus musculus) supported" 
     end
   end
 
@@ -117,6 +119,8 @@ module Organism
       end
     when "Mmu"
       "GRCm38"
+    when "Rno"
+      "Rnor_6.0"
     else
       raise "Only organism 'Hsa' (Homo sapiens) and Mmu (Mus musculus) supported" 
     end

data/lib/rbbt/sources/pubmed.rb

@@ -109,6 +109,14 @@ module PubMed
       end
     end
 
+    def pmc_full_xml
+      begin
+        Open.read("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pmc&id=#{pmid}")
+      rescue
+        nil
+      end
+    end
+
     def pdf_url
       return pmc_pdf if pmc_pdf
       @gscholar_pdf ||= begin
@@ -121,18 +129,22 @@ module PubMed
     end
 
     def full_text
-      return nil if pdf_url.nil?
-
-      text = nil
-      TmpFile.with_file do |pdf|
-
-        # Change user-agent, oh well...
-        `wget --user-agent=firefox #{ pdf_url } -O #{ pdf } -t 3`
-        TmpFile.with_file do |txt|
-          `pdftotext #{ pdf } #{ txt }`
-          text = Open.read(txt) if File.exists? txt
-        end
-      end
+      text = if pdf_url
+               text = nil
+               TmpFile.with_file do |pdf|
+                 # Change user-agent, oh well...
+                 `wget --user-agent=firefox #{ pdf_url } -O #{ pdf } -t 3`
+                 TmpFile.with_file do |txt|
+                   `pdftotext #{ pdf } #{ txt }`
+                   text = Open.read(txt) if File.exists? txt
+                 end
+               end
+               text
+             elsif pmc_full_xml
+               pmc_full_xml
+             else
+               nil
+             end
 
       Misc.fixutf8(text)
     end

data/lib/rbbt/sources/tfacts.rb

@@ -2,15 +2,15 @@ require 'rbbt'
 require 'rbbt/tsv'
 require 'rbbt/resource'
 
-module TFacts
+module TFactS
   extend Resource
-  self.subdir = "share/databases/TFacts"
+  self.subdir = "share/databases/TFactS"
 
-  TFacts.claim TFacts[".source"]["Catalogues.xls"], :url, "http://www.tfacts.org/TFactS-new/TFactS-v2/tfacts/data/Catalogues.xls"
+  TFactS.claim TFactS[".source"]["Catalogues.xls"], :url, "http://www.tfacts.org/TFactS-new/TFactS-v2/tfacts/data/Catalogues.xls"
 
-  TFacts.claim TFacts.targets, :proc do
+  TFactS.claim TFactS.targets, :proc do
     require 'spreadsheet'
-    book = Spreadsheet.open TFacts[".source"]["Catalogues.xls"].produce.find
+    book = Spreadsheet.open TFactS[".source"]["Catalogues.xls"].produce.find
     sheet = book.worksheet 0
 
     tsv = TSV.setup({}, :key_field => "Target Gene (Associated Gene Name)", :fields => ["Transcription Factor (Associated Gene Name)"], :namespace => "Hsa", :type => :flat)
@@ -24,9 +24,9 @@ module TFacts
     tsv.to_s
   end
 
-  TFacts.claim TFacts.targets_signed, :proc do
+  TFactS.claim TFactS.targets_signed, :proc do
     require 'spreadsheet'
-    book = Spreadsheet.open TFacts[".source"]["Catalogues.xls"].produce.find
+    book = Spreadsheet.open TFactS[".source"]["Catalogues.xls"].produce.find
     sheet = book.worksheet 1
 
     tsv = TSV.setup({}, :key_field => "Target Gene (Associated Gene Name)", :fields => ["Transcription Factor (Associated Gene Name)", "Sign", "PMID"], :namespace => "Hsa", :type => :double)
@@ -43,13 +43,13 @@ module TFacts
     tsv.to_s
   end
 
-  TFacts.claim TFacts.regulators, :proc do
-    TFacts.targets.tsv.reorder("Transcription Factor (Associated Gene Name)").to_s
+  TFactS.claim TFactS.regulators, :proc do
+    TFactS.targets.tsv.reorder("Transcription Factor (Associated Gene Name)").to_s
   end
 
-  TFacts.claim TFacts.tf_tg, :proc do
+  TFactS.claim TFactS.tf_tg, :proc do
     require 'spreadsheet'
-    book = Spreadsheet.open TFacts[".source"]["Catalogues.xls"].produce.find
+    book = Spreadsheet.open TFactS[".source"]["Catalogues.xls"].produce.find
 
     tsv = TSV.setup({}, :key_field => "Transcription Factor (Associated Gene Name)", :fields => ["Target Gene (Associated Gene Name)", "Sign", "Species", "Source", "PMID"], :namespace => "Hsa", :type => :double)
 
@@ -128,16 +128,16 @@ if defined? Entity and defined? Gene and Entity === Gene
 
   module Gene
     property :is_transcription_factor? => :array2single do
-      tfs = TFacts.targets.keys
+      tfs = TFactS.targets.keys
       self.name.collect{|gene| tfs.include? gene}
     end
 
     property :transcription_regulators => :array2single do
-      Gene.setup(TFacts.regulators.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
+      Gene.setup(TFactS.regulators.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
     end
 
     property :transcription_targets => :array2single do
-      Gene.setup(TFacts.targets.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
+      Gene.setup(TFactS.targets.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
     end
   end
 end

data/share/Ensembl/release_dates

@@ -1,5 +1,10 @@
 #: :type=:single
 #Release	build
+release-100	apr2020
+release-99	jan2020
+release-98	sep2019
+release-97	jul2019
+release-96	apr2019
 release-95	jan2019
 release-94	oct2018
 release-93	jul2018

data/share/install/Organism/Hsa/Rakefile

@@ -5,7 +5,7 @@ require File.join(File.dirname(__FILE__), '../../lib/helpers')
 
 $taxs = [9606]
 $scientific_name = "Homo sapiens"
-$ortholog_key = "human_ensembl_gene"
+$ortholog_key = "hsapiens_homolog_ensembl_gene"
 
 $biomart_db = 'hsapiens_gene_ensembl'
 $biomart_db_germline_variation = 'hsapiens_snp'

data/share/install/Organism/Mmu/Rakefile

@@ -5,7 +5,7 @@ require File.join(File.dirname(__FILE__), '../../lib/helpers')
 
 $taxs = [10090]
 $scientific_name = "Mus musculus"
-$ortholog_key = "mouse_ensembl_gene"
+$ortholog_key = "mmusculus_homolog_ensembl_gene"
 
 $biomart_db = 'mmusculus_gene_ensembl'
 $biomart_db_germline_variation = 'mmusculus_snp'

data/share/install/Organism/Rno/Rakefile

@@ -9,7 +9,7 @@ $scientific_name = "Rattus norvegicus"
 $biomart_db = 'rnorvegicus_gene_ensembl'
 $biomart_db_germline_variation = 'rnorvegicus_snp'
 $biomart_db_somatic_variation = 'rnorvegicus_snp_som'
-$ortholog_key = "rat_ensembl_gene"
+$ortholog_key = "rnorvegicus_homolog_ensembl_gene"
 
 $biomart_lexicon = [ 
   [ 'Associated Gene Name' , "external_gene_id"], 

data/share/install/Organism/organism_helpers.rb

@@ -547,13 +547,13 @@ end
 rule /^possible_ortholog_(.*)/ do |t|
   other = t.name.match(/ortholog_(.*)/)[1]
   other_key = Organism.ortholog_key(other).produce.read
-  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", "inter_paralog_" + other_key]], [], nil, :keep_empty => false, :type => :flat, :filename => t.name, :namespace => Thread.current['namespace'])
+  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", "inter_paralog_" + other_key]], [], nil, :keep_empty => false, :type => :double, :filename => t.name, :namespace => Thread.current['namespace'])
 end
 
 rule /^ortholog_(.*)/ do |t|
   other = t.name.match(/ortholog_(.*)/)[1]
   other_key = Organism.ortholog_key(other).produce.read
-  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", other_key]], [], nil, :keep_empty => false, :type => :flat, :filename => t.name, :namespace => Thread.current['namespace'])
+  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", other_key]], [], nil, :keep_empty => false, :type => :double, :filename => t.name, :namespace => Thread.current['namespace'])
 end
 
 rule /[a-z]{3}[0-9]{4}\/.*/i do |t|
@@ -728,13 +728,18 @@ file 'protein_sequence' => ["transcripts", "transcript_5utr", "transcript_3utr",
   transcript_sequence.through do |transcript, sequence|
     protein = transcript_protein[transcript]
     next if protein.nil? or protein.empty?
+
     utr5 = transcript_5utr[transcript]
     utr3 = transcript_3utr[transcript]
     phase = transcript_phase[transcript] || 0
+
     if phase < 0
-      utr5 = - phase if utr5 == 0
+      if utr5.nil? || utr5 == 0 || utr5 == "0"
+        utr5 = 0
+      end
       phase = 0
     end
+
     psequence = Bio::Sequence::NA.new(("N" * phase) << sequence[utr5..sequence.length-utr3-1]).translate
     protein_sequence[protein]=psequence
   end

data/test/rbbt/sources/test_entrez.rb

@@ -17,7 +17,10 @@ class TestEntrez < Test::Unit::TestCase
   def test_entrez2pubmed
     tax   = $yeast_tax
 
+    Log.severity = 0
     data = Entrez.entrez2pubmed(tax)
+    data.read
+    Log.tsv data
     assert(data['850320'].include? '1574125') 
   end
 

data/test/rbbt/sources/test_pubmed.rb

@@ -17,6 +17,12 @@ class TestPubMed < Test::Unit::TestCase
     pmid = '16438716'
     assert(PubMed.get_article(pmid).full_text =~ /Discovering/)
   end
+
+  def test_pmc_full_xml
+    pmid = '4304705'
+    assert PubMed.get_article(pmid).pmc_full_xml.include?("HBV antigen")
+  end
+ 
  
   def test_query
     assert(PubMed.query('chagoyen[All Fields] AND ("loattrfull text"[sb] AND hasabstract[text])').include? '16438716')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-sources
 version: !ruby/object:Gem::Version
-  version: 3.1.43
+  version: 3.1.49
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-01 00:00:00.000000000 Z
+date: 2020-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util