rbbt-sources 3.1.38 → 3.1.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0b45df8e4ef43f18147f001595817df05dddc6ef3b43dcea3b7c17a1d379163
-  data.tar.gz: 3b1ac66222ca9dd9d0342f97917cc3d969fc566b2c068a319101ec6fe7133f58
+  metadata.gz: 160b144b03f1da99d829748d36b74f96d31cf2ee04fb8751b7a9fa826e5f35ad
+  data.tar.gz: 5228bffd99260248fc9a46ee773d17f7aaec2753f939814d5f9a35530f228276
 SHA512:
-  metadata.gz: b75e51b2338a0b3c55545ba788db502156126327ed920f86e448fbe239c2bb2f5a053b1009dd8e42bd7e6f425614e504c83a8ea894b070a07e9275fb1e5ea399
-  data.tar.gz: 35b7852629a3dc69df65d8e3c226c4995051a2dd7e0a4435f485c947014e88ee9a97455f7270de902d0058db0a4d55315731e29ce2195cb0b79ceef390d92676
+  metadata.gz: e2c0702241dd63d5ee9858f3bab1c90c02de911f459ea2fb6c61e14e82f1e6f17ebb897f8ed9058eb002c89af415b16c54ed713127e154f3b8d87e8bfbe4cdde
+  data.tar.gz: 6544095d31861d13dfd89fed3289c8d2877e5e96ec35d7dbee4db84512f5b0a47bca7a6a99a3b9a7abfb8c106fbd2ec7ab61340290a78a1165b9793cdd50dc6c

data/etc/build_organism

@@ -0,0 +1,8 @@
+#Build	Organism
+hg18	Hsa/may2008
+hg19	Hsa/feb2014
+b37	Hsa/feb2014
+hg38	Hsa/may2017
+GRCh38	Hsa/may2017
+mm10	Mmu/may2017
+GRCm38	Mmu/may2017

data/etc/organisms

@@ -1 +1,2 @@
 Hsa
+Mmu

data/lib/rbbt/sources/ensembl_ftp.rb

@@ -33,7 +33,7 @@ module Ensembl
         ftp.passive = true
         ftp.login
         ftp.chdir(File.join('pub', release, 'mysql'))
-        file = ftp.list(name.downcase.gsub(" ",'_') + "_core_*").collect{|l| l.split(" ").last}.last
+        file = ftp.list(name.downcase.gsub(" ",'_') + "_core_*").reject{|f| f =~ /\.gz$/}.collect{|l| l.split(" ").last}.last
         ftp.close
       end
       [release, file]
@@ -53,17 +53,17 @@ module Ensembl
     end
 
     def self.url_for(organism, table)
-      "#{base_url(organism)}/#{table}.txt.gz"
+      "#{base_url(organism)}/#{table}.txt.gz.bz2"
     end
 
     def self.has_table?(organism, table)
-      sql_file = Open.read("#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz")
+      sql_file = CMD.cmd("wget '#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz.bz2' -O  -| bunzip2| gunzip").read
       ! sql_file.match(/^CREATE TABLE .#{table}. \((.*?)^\)/sm).nil?
     end
 
     def self.fields_for(organism, table)
-      sql_file = Open.read("#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz")
-
+      sql_file = CMD.cmd("wget '#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz.bz2' -O  -| bunzip2| gunzip").read
+      
       chunk = sql_file.match(/^CREATE TABLE .#{table}. \((.*?)^\)/sm)[1]
       chunk.scan(/^\s+`(.*?)`/).flatten
     end
@@ -78,7 +78,7 @@ module Ensembl
         options[:key_field] = key_pos
         options[:fields]    = field_pos
       end
-      tsv = TSV.open(url, options)
+      tsv = TSV.open(CMD.cmd("wget '#{url}' -O - |bunzip2|gunzip", :pipe => true), options)
       tsv.key_field = key_field
       tsv.fields = fields
       tsv

data/lib/rbbt/sources/organism.rb

@@ -86,24 +86,50 @@ module Organism
       else
         'hg38'
       end
+    when "Mmu"
+      "mm10"
     else
-      raise "Only organism 'Hsa' (Homo sapiens) supported" unless organism =~ /^Hsa/
+      raise "Only organism 'Hsa' (Homo sapiens) and Mmu (Mus musculus) supported" 
     end
   end
 
-  def self.organism_for_build(build)
-    case build.to_s
-    when 'hg18'
-      "Hsa/may2008"
-    when 'hg19', 'b37'
-      "Hsa/feb2014"
-    when 'hg38'
-      "Hsa/may2017"
+  def self.GRC_build(organism)
+    require 'rbbt/sources/ensembl_ftp'
+    return organism if organism =~ /^hg\d\d$/
+
+    return 'hg19' unless organism =~ /\//
+
+    species, date = organism.split("/")
+
+    case species
+    when "Hsa"
+      date = organism.split("/")[1] 
+
+      release = Ensembl.releases[date]
+
+      release_number = release.sub(/.*-/,'').to_i
+      if release_number <= 54 
+        'GRCh36'
+      elsif release_number <= 75
+        'GRCh37'
+      else
+        'GRCh38'
+      end
+    when "Mmu"
+      "GRCm38"
     else
-      raise RbbtException, "Unknown organism build #{build}"
+      raise "Only organism 'Hsa' (Homo sapiens) and Mmu (Mus musculus) supported" 
     end
   end
 
+  def self.organism_for_build(build)
+    build = build.sub('_noalt', '')
+
+    build_organism = Rbbt.etc.build_organism.tsv :type => :single
+
+    build_organism[build]
+  end
+
   def self.liftOver(positions, source, target)
 
     source_hg = hg_build(source)

data/share/install/Organism/organism_helpers.rb

@@ -648,12 +648,12 @@ end
 file 'transcript_5utr' => ["exons", "transcript_exons", "transcripts"] do |t|
   path = File.expand_path(t.name)
   dirname = File.dirname(path)
-  organism = File.basename(dirname)
 
-  if organism =~ /[a-z]{3}20[0-9]{2}/
-    build = organism
+  organism = File.basename(dirname)
+  if organism =~ /^[a-z]{3}20[0-9]{2}/
+    archive = organism
     organism = File.basename(File.dirname(dirname))
-    organism = File.join(organism, build)
+    organism = File.join(organism, archive)
   end
 
   translation        = Ensembl::FTP.ensembl_tsv(organism, 'translation', 'transcript_id', %w(seq_start start_exon_id seq_end end_exon_id), :type => :list, :unmamed => true)
@@ -788,12 +788,19 @@ end
 file 'gene_set' do |t|
   path = File.expand_path(t.name)
   dirname = File.dirname(path)
+
   organism = File.basename(dirname)
+  if organism =~ /^[a-z]{3}20[0-9]{2}/
+    archive = organism
+    organism = File.basename(File.dirname(dirname))
+    organism = File.join(organism, archive)
+  end
 
   release = Ensembl.org2release(organism)
   num = release.split("-").last
-  build_code = num.to_i > 75 ? "GRCh38" : "GRCh37" 
-  url = "ftp://ftp.ensembl.org/pub/release-#{num}/gtf/homo_sapiens/Homo_sapiens.#{build_code}.#{num}.gtf.gz"
+  build_code = Organism.GRC_build(organism)
+  scientific_name = $scientific_name
+  url = "ftp://ftp.ensembl.org/pub/release-#{num}/gtf/#{scientific_name.downcase.sub(" ", '_')}/#{scientific_name.sub(" ", '_')}.#{build_code}.#{num}.gtf.gz"
   CMD.cmd("wget '#{url}' -O #{t.name}.gz")
   nil
 end
@@ -801,12 +808,19 @@ end
 file 'cdna_fasta' do |t|
   path = File.expand_path(t.name)
   dirname = File.dirname(path)
+
   organism = File.basename(dirname)
+  if organism =~ /^[a-z]{3}20[0-9]{2}/
+    archive = organism
+    organism = File.basename(File.dirname(dirname))
+    organism = File.join(organism, archive)
+  end
 
   release = Ensembl.org2release(organism)
   num = release.split("-").last
-  build_code = num.to_i > 75 ? "GRCh38" : "GRCh37" 
-  url = "ftp://ftp.ensembl.org/pub/release-#{num}/fasta/homo_sapiens/cdna/Homo_sapiens.#{build_code}.#{num}.cdna.all.fa.gz"
+  build_code = Organism.GRC_build(organism)
+  scientific_name = Organism.scientific_name(organism)
+  url = "ftp://ftp.ensembl.org/pub/release-#{num}/fasta/#{scientific_name.downcase.sub(" ", '_')}/cdna/#{scientific_name.sub(" ", '_')}.#{build_code}.cdna.all.fa.gz"
   CMD.cmd("wget '#{url}' -O #{t.name}.gz")
   nil
 end

data/test/rbbt/sources/test_organism.rb

@@ -14,7 +14,7 @@ class TestOrganism < Test::Unit::TestCase
   end
 
   def test_identifiers
-    assert Organism.identifiers('Hsa').tsv(:key_field => "Entrez Gene ID", :persist => true)['1020']["Associated Gene Name"].include?('CDK5')
+    assert Organism.identifiers('Hsa/feb2014').tsv(:key_field => "Entrez Gene ID", :persist => true)['1020']["Associated Gene Name"].include?('CDK5')
     assert Organism.identifiers('Sce').tsv(:persist => true)['S000006120']["Ensembl Gene ID"].include?('YPL199C')
     assert Organism.identifiers("Sce").tsv(:persist => true)['S000006120']["Ensembl Gene ID"].include?('YPL199C')
   end
@@ -70,6 +70,12 @@ class TestOrganism < Test::Unit::TestCase
     assert Organism.chromosome_sizes["2"].to_i > 10_000_000
   end
 
+  def test_build_organism
+    assert_equal 'Hsa/may2017', Organism.organism_for_build('hg38')
+    assert_equal 'Hsa/feb2014', Organism.organism_for_build('b37')
+    assert_equal 'Mmu/may2017', Organism.organism_for_build('mm10')
+  end
+
   #def test_genes_at_chromosome
   #  pos = [12, 117799500]
   #  assert_equal "ENSG00000089250", Organism::Hsa.genes_at_chromosome_positions(pos.first, pos.last)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-sources
 version: !ruby/object:Gem::Version
-  version: 3.1.38
+  version: 3.1.39
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-22 00:00:00.000000000 Z
+date: 2019-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -88,6 +88,7 @@ extra_rdoc_files: []
 files:
 - etc/allowed_biomart_archives
 - etc/biomart/missing_in_archive
+- etc/build_organism
 - etc/organisms
 - etc/xena_hubs
 - lib/rbbt/sources/CASCADE.rb