rbbt-sources 3.1.42 → 3.1.48

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d58670542b15951452edfc4607511ae6d3a4e2795bd13a1b207489c75cb869c9
-  data.tar.gz: b53a205a5e9b0b857a39f9a3dfa0a7da9db88c8ed3f734af9b0310320504993d
+  metadata.gz: 92bb6e2852266c73666e0b07221484822441d19603b2ce9d4fe9ceb6faa44d37
+  data.tar.gz: 9eda4d907bb34ae5b5b95dbdb972b6c659111b042c5b68661d41cf3ccd93e3b8
 SHA512:
-  metadata.gz: c84dfab9e1f914b0ecfaede54839ee90644f6bbb9777044ede0cc0358f18e0e78ff3e757583049e699ba6ff7700c2dfc74816998c74a78ac0a82580a042e6f29
-  data.tar.gz: bb649a7250d617e9fef26068ed1c7624828ff2c4a2a0e8696a621e7b6b821918d9ddcff1ccae4729c8e26a1fe79f6d9723936f85519e57b677ff405b30e3f5ea
+  metadata.gz: 4ac47f700cfc54144bfbc2dc7b3eeaf2e9dce885ccfc1e47c26d978a79dd02d99802a83a888bb1bd80de96ecf9bf023ae548041f3b1867308228c319777dabd4
+  data.tar.gz: 05b07ac77a194f0ad87a36b050eb97d2285490974d0bca039c2163d762c11461f4ccf6c4585df62b24298524bb9f02e4d1d65dae51418e491644b4725fbd864f

data/etc/biomart/missing_in_archive

@@ -1,3 +1,7 @@
+">oct2014":
+    - rgd~rgd_id
+">jun2019":
+    - entrezgene~entrezgene_id
 ">dec2017":
     - unigene
 ">dec2016":

data/lib/rbbt/sources/biomart.rb

@@ -134,11 +134,14 @@ module BioMart
   # cause an error if the BioMart WS does not allow filtering with that
   # attribute.
   def self.query(database, main, attrs = nil, filters = nil, data = nil, open_options = {})
+    IndiferentHash.setup(open_options)
     open_options = Misc.add_defaults open_options, :nocache => false, :filename => nil, :field_names => nil, :by_chr => false
     filename, field_names, by_chr = Misc.process_options open_options, :filename, :field_names, :by_chr
     attrs   ||= []
     open_options = Misc.add_defaults open_options, :keep_empty => false, :merge => true
 
+    IndiferentHash.setup(open_options)
+
     Log.low "BioMart query: '#{main}' [#{(attrs || []) * ', '}] [#{(filters || []) * ', '}] #{open_options.inspect}"
 
     max_items = 2
@@ -167,6 +170,7 @@ module BioMart
     end
 
     open_options[:filename] = "BioMart[#{main}+#{attrs.length}]"
+
     if filename.nil?
       results = TSV.open data, open_options
       results.key_field = main

data/lib/rbbt/sources/ensembl_ftp.rb

@@ -52,24 +52,35 @@ module Ensembl
       File.join("ftp://" + SERVER, ftp_directory_for(organism) )
     end
 
-    def self.url_for(organism, table)
-      "#{base_url(organism)}/#{table}.txt.gz.bz2"
+    def self.url_for(organism, table, extension)
+      File.join(base_url(organism), table) + ".#{extension}.gz"
+    end
+
+    def self._get_gz(url)
+      begin
+        CMD.cmd("wget '#{url}' -O  - | gunzip").read
+      rescue
+        CMD.cmd("wget '#{url}.bz2' -O  - | bunzip2 | gunzip").read
+      end
+    end
+
+    def self._get_file(organism, table, extension)
+      url = url_for(organism, table, extension)
+      self._get_gz(url)
     end
 
     def self.has_table?(organism, table)
-      sql_file = CMD.cmd("wget '#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz.bz2' -O  -| bunzip2| gunzip").read
+      sql_file = _get_file(organism, File.basename(base_url(organism)), 'sql')
       ! sql_file.match(/^CREATE TABLE .#{table}. \((.*?)^\)/sm).nil?
     end
 
     def self.fields_for(organism, table)
-      sql_file = CMD.cmd("wget '#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz.bz2' -O  -| bunzip2| gunzip").read
-      
+      sql_file = _get_file(organism, File.basename(base_url(organism)), 'sql')
       chunk = sql_file.match(/^CREATE TABLE .#{table}. \((.*?)^\)/sm)[1]
       chunk.scan(/^\s+`(.*?)`/).flatten
     end
 
     def self.ensembl_tsv(organism, table, key_field = nil, fields = nil, options = {})
-      url = url_for(organism, table)
       if key_field and fields
         all_fields = fields_for(organism, table)
         key_pos = all_fields.index key_field
@@ -78,7 +89,8 @@ module Ensembl
         options[:key_field] = key_pos
         options[:fields]    = field_pos
       end
-      tsv = TSV.open(CMD.cmd("wget '#{url}' -O - |bunzip2|gunzip", :pipe => true), options)
+
+      tsv = TSV.open(StringIO.new(_get_file(organism, table, "txt")), options)
       tsv.key_field = key_field
       tsv.fields = fields
       tsv

data/lib/rbbt/sources/entrez.rb

@@ -14,7 +14,7 @@ module Entrez
     options = Misc.add_defaults options, :key_field => 1, :fields => [5], :persist => true, :merge => true
 
     taxs = [taxs] unless Array === taxs
-    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
+    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}, :fixed_grep => false
     
     tsv = Rbbt.share.databases.entrez.gene_info.tsv :flat, options
     tsv.key_field = "Entrez Gene ID"
@@ -26,7 +26,7 @@ module Entrez
     options = Misc.add_defaults options, :key_field => 1, :fields => [2], :persist => true, :merge => true
 
     taxs = [taxs] unless Array === taxs
-    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
+    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}, :fixed_grep => false
     
     tsv = Rbbt.share.databases.entrez.gene_info.tsv :flat, options
     tsv.key_field = "Entrez Gene ID"
@@ -39,7 +39,7 @@ module Entrez
     options = {:key_field => 1, :fields => [2], :persist => true, :merge => true}
 
     taxs = [taxs] unless taxs.is_a?(Array)
-    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
+    options.merge! :grep => taxs.collect{|t| "^" + t.to_s}, :fixed_grep => false
 
     Rbbt.share.databases.entrez.gene2pubmed.tsv :flat, options
   end

data/lib/rbbt/sources/go.rb

@@ -27,7 +27,7 @@ module GO
   def self.init
     Persist.persist_tsv(nil, 'gene_ontology', {}, :persist => true) do |info|
       info.serializer = :marshal if info.respond_to? :serializer
-      Rbbt.share.databases.GO.gene_ontology.read.split(/\[Term\]/).each{|term| 
+      Rbbt.share.databases.GO.gene_ontology.produce.read.split(/\[Term\]/).each{|term| 
         term_info = {}
 
         term.split(/\n/). select{|l| l =~ /:/}.each{|l| 

data/lib/rbbt/sources/pina.rb

@@ -7,7 +7,7 @@ module Pina
   Pina.claim Pina.protein_protein, :proc do
     require 'rbbt/sources/organism'
 
-    url = "http://cbg.garvan.unsw.edu.au/pina/download/Homo%20sapiens-20121210.txt"
+    url = "https://omics.bjcancer.org/pina/download/Homo%20sapiens-20140521.tsv"
 
     dumper = TSV::Dumper.new :type => :double,
       :key_field => 'UniProt/SwissProt Accession', :namespace => Organism.default_code("Hsa"), 

data/lib/rbbt/sources/pubmed.rb

@@ -109,6 +109,14 @@ module PubMed
       end
     end
 
+    def pmc_full_xml
+      begin
+        Open.read("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pmc&id=#{pmid}")
+      rescue
+        nil
+      end
+    end
+
     def pdf_url
       return pmc_pdf if pmc_pdf
       @gscholar_pdf ||= begin
@@ -121,18 +129,22 @@ module PubMed
     end
 
     def full_text
-      return nil if pdf_url.nil?
-
-      text = nil
-      TmpFile.with_file do |pdf|
-
-        # Change user-agent, oh well...
-        `wget --user-agent=firefox #{ pdf_url } -O #{ pdf } -t 3`
-        TmpFile.with_file do |txt|
-          `pdftotext #{ pdf } #{ txt }`
-          text = Open.read(txt) if File.exists? txt
-        end
-      end
+      text = if pdf_url
+               text = nil
+               TmpFile.with_file do |pdf|
+                 # Change user-agent, oh well...
+                 `wget --user-agent=firefox #{ pdf_url } -O #{ pdf } -t 3`
+                 TmpFile.with_file do |txt|
+                   `pdftotext #{ pdf } #{ txt }`
+                   text = Open.read(txt) if File.exists? txt
+                 end
+               end
+               text
+             elsif pmc_full_xml
+               pmc_full_xml
+             else
+               nil
+             end
 
       Misc.fixutf8(text)
     end

data/lib/rbbt/sources/tfacts.rb

@@ -2,15 +2,15 @@ require 'rbbt'
 require 'rbbt/tsv'
 require 'rbbt/resource'
 
-module TFacts
+module TFactS
   extend Resource
-  self.subdir = "share/databases/TFacts"
+  self.subdir = "share/databases/TFactS"
 
-  TFacts.claim TFacts[".source"]["Catalogues.xls"], :url, "http://www.tfacts.org/TFactS-new/TFactS-v2/tfacts/data/Catalogues.xls"
+  TFactS.claim TFactS[".source"]["Catalogues.xls"], :url, "http://www.tfacts.org/TFactS-new/TFactS-v2/tfacts/data/Catalogues.xls"
 
-  TFacts.claim TFacts.targets, :proc do
+  TFactS.claim TFactS.targets, :proc do
     require 'spreadsheet'
-    book = Spreadsheet.open TFacts[".source"]["Catalogues.xls"].produce.find
+    book = Spreadsheet.open TFactS[".source"]["Catalogues.xls"].produce.find
     sheet = book.worksheet 0
 
     tsv = TSV.setup({}, :key_field => "Target Gene (Associated Gene Name)", :fields => ["Transcription Factor (Associated Gene Name)"], :namespace => "Hsa", :type => :flat)
@@ -24,9 +24,9 @@ module TFacts
     tsv.to_s
   end
 
-  TFacts.claim TFacts.targets_signed, :proc do
+  TFactS.claim TFactS.targets_signed, :proc do
     require 'spreadsheet'
-    book = Spreadsheet.open TFacts[".source"]["Catalogues.xls"].produce.find
+    book = Spreadsheet.open TFactS[".source"]["Catalogues.xls"].produce.find
     sheet = book.worksheet 1
 
     tsv = TSV.setup({}, :key_field => "Target Gene (Associated Gene Name)", :fields => ["Transcription Factor (Associated Gene Name)", "Sign", "PMID"], :namespace => "Hsa", :type => :double)
@@ -43,13 +43,13 @@ module TFacts
     tsv.to_s
   end
 
-  TFacts.claim TFacts.regulators, :proc do
-    TFacts.targets.tsv.reorder("Transcription Factor (Associated Gene Name)").to_s
+  TFactS.claim TFactS.regulators, :proc do
+    TFactS.targets.tsv.reorder("Transcription Factor (Associated Gene Name)").to_s
   end
 
-  TFacts.claim TFacts.tf_tg, :proc do
+  TFactS.claim TFactS.tf_tg, :proc do
     require 'spreadsheet'
-    book = Spreadsheet.open TFacts[".source"]["Catalogues.xls"].produce.find
+    book = Spreadsheet.open TFactS[".source"]["Catalogues.xls"].produce.find
 
     tsv = TSV.setup({}, :key_field => "Transcription Factor (Associated Gene Name)", :fields => ["Target Gene (Associated Gene Name)", "Sign", "Species", "Source", "PMID"], :namespace => "Hsa", :type => :double)
 
@@ -128,16 +128,16 @@ if defined? Entity and defined? Gene and Entity === Gene
 
   module Gene
     property :is_transcription_factor? => :array2single do
-      tfs = TFacts.targets.keys
+      tfs = TFactS.targets.keys
       self.name.collect{|gene| tfs.include? gene}
     end
 
     property :transcription_regulators => :array2single do
-      Gene.setup(TFacts.regulators.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
+      Gene.setup(TFactS.regulators.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
     end
 
     property :transcription_targets => :array2single do
-      Gene.setup(TFacts.targets.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
+      Gene.setup(TFactS.targets.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
     end
   end
 end

data/share/install/Organism/Hsa/Rakefile

@@ -5,7 +5,7 @@ require File.join(File.dirname(__FILE__), '../../lib/helpers')
 
 $taxs = [9606]
 $scientific_name = "Homo sapiens"
-$ortholog_key = "human_ensembl_gene"
+$ortholog_key = "hsapiens_homolog_ensembl_gene"
 
 $biomart_db = 'hsapiens_gene_ensembl'
 $biomart_db_germline_variation = 'hsapiens_snp'

data/share/install/Organism/Mmu/Rakefile

@@ -5,7 +5,7 @@ require File.join(File.dirname(__FILE__), '../../lib/helpers')
 
 $taxs = [10090]
 $scientific_name = "Mus musculus"
-$ortholog_key = "mouse_ensembl_gene"
+$ortholog_key = "mmusculus_homolog_ensembl_gene"
 
 $biomart_db = 'mmusculus_gene_ensembl'
 $biomart_db_germline_variation = 'mmusculus_snp'

data/share/install/Organism/Rno/Rakefile

@@ -9,7 +9,7 @@ $scientific_name = "Rattus norvegicus"
 $biomart_db = 'rnorvegicus_gene_ensembl'
 $biomart_db_germline_variation = 'rnorvegicus_snp'
 $biomart_db_somatic_variation = 'rnorvegicus_snp_som'
-$ortholog_key = "rat_ensembl_gene"
+$ortholog_key = "rnorvegicus_homolog_ensembl_gene"
 
 $biomart_lexicon = [ 
   [ 'Associated Gene Name' , "external_gene_id"], 

data/share/install/Organism/organism_helpers.rb

@@ -547,13 +547,13 @@ end
 rule /^possible_ortholog_(.*)/ do |t|
   other = t.name.match(/ortholog_(.*)/)[1]
   other_key = Organism.ortholog_key(other).produce.read
-  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", "inter_paralog_" + other_key]], [], nil, :keep_empty => false, :type => :flat, :filename => t.name, :namespace => Thread.current['namespace'])
+  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", "inter_paralog_" + other_key]], [], nil, :keep_empty => false, :type => :double, :filename => t.name, :namespace => Thread.current['namespace'])
 end
 
 rule /^ortholog_(.*)/ do |t|
   other = t.name.match(/ortholog_(.*)/)[1]
   other_key = Organism.ortholog_key(other).produce.read
-  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", other_key]], [], nil, :keep_empty => false, :type => :flat, :filename => t.name, :namespace => Thread.current['namespace'])
+  BioMart.tsv($biomart_db, $biomart_ensembl_gene, [["Ortholog Ensembl Gene ID", other_key]], [], nil, :keep_empty => false, :type => :double, :filename => t.name, :namespace => Thread.current['namespace'])
 end
 
 rule /[a-z]{3}[0-9]{4}\/.*/i do |t|
@@ -728,13 +728,18 @@ file 'protein_sequence' => ["transcripts", "transcript_5utr", "transcript_3utr",
   transcript_sequence.through do |transcript, sequence|
     protein = transcript_protein[transcript]
     next if protein.nil? or protein.empty?
+
     utr5 = transcript_5utr[transcript]
     utr3 = transcript_3utr[transcript]
     phase = transcript_phase[transcript] || 0
+
     if phase < 0
-      utr5 = - phase if utr5 == 0
+      if utr5.nil? || utr5 == 0 || utr5 == "0"
+        utr5 = 0
+      end
       phase = 0
     end
+
     psequence = Bio::Sequence::NA.new(("N" * phase) << sequence[utr5..sequence.length-utr3-1]).translate
     protein_sequence[protein]=psequence
   end

data/test/rbbt/sources/test_entrez.rb

@@ -17,7 +17,10 @@ class TestEntrez < Test::Unit::TestCase
   def test_entrez2pubmed
     tax   = $yeast_tax
 
+    Log.severity = 0
     data = Entrez.entrez2pubmed(tax)
+    data.read
+    Log.tsv data
     assert(data['850320'].include? '1574125') 
   end
 

data/test/rbbt/sources/test_pubmed.rb

@@ -17,6 +17,12 @@ class TestPubMed < Test::Unit::TestCase
     pmid = '16438716'
     assert(PubMed.get_article(pmid).full_text =~ /Discovering/)
   end
+
+  def test_pmc_full_xml
+    pmid = '4304705'
+    assert PubMed.get_article(pmid).pmc_full_xml.include?("HBV antigen")
+  end
+ 
  
   def test_query
     assert(PubMed.query('chagoyen[All Fields] AND ("loattrfull text"[sb] AND hasabstract[text])').include? '16438716')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-sources
 version: !ruby/object:Gem::Version
-  version: 3.1.42
+  version: 3.1.48
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-31 00:00:00.000000000 Z
+date: 2020-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util