rbbt-sources 1.2.0 → 2.0.0

data/lib/rbbt/sources/ensembl_ftp.rb

@@ -0,0 +1,79 @@
+require 'rbbt/util/open'
+require 'rbbt/sources/organism'
+require 'rbbt/tsv'
+require 'net/ftp'
+
+module Ensembl
+
+  
+  def self.releases
+    @releases ||= Rbbt.share.Ensembl.release_dates.find.tsv :key_field => "build"
+  end
+
+  module FTP
+
+    SERVER = "ftp.ensembl.org"
+
+    def self.ftp_name_for(organism)
+      code, build = organism.split "/"
+      build ||= "current"
+
+      if build.to_s == "current"
+      else
+        release = Ensembl.releases[build]
+        name = Organism.scientific_name(organism)
+        ftp = Net::FTP.new(Ensembl::FTP::SERVER)
+        ftp.login
+        ftp.chdir(File.join('pub', release, 'mysql'))
+        file = ftp.list(name.downcase.gsub(" ",'_') + "_core_*").collect{|l| l.split(" ").last}.last
+        ftp.close
+      end
+      [release, file]
+    end
+
+    def self.ftp_directory_for(organism)
+      release, ftp_name = ftp_name_for(organism)
+      File.join('/pub/', release, 'mysql', ftp_name)
+    end
+
+    def self.base_url(organism)
+      File.join("ftp://" + SERVER, ftp_directory_for(organism) )
+    end
+
+    def self.url_for(organism, table)
+      "#{base_url(organism)}/#{table}.txt.gz"
+    end
+
+    def self.has_table?(organism, table)
+      sql_file = Open.read("#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz")
+      ! sql_file.match(/^CREATE TABLE .#{table}. \((.*?)^\)/sm).nil?
+    end
+
+    def self.fields_for(organism, table)
+      sql_file = Open.read("#{base_url(organism)}/#{File.basename(base_url(organism))}.sql.gz")
+
+      chunk = sql_file.match(/^CREATE TABLE .#{table}. \((.*?)^\)/sm)[1]
+      chunk.scan(/^\s+`(.*?)`/).flatten
+    end
+
+    def self.ensembl_tsv(organism, table, key_field = nil, fields = nil, options = {})
+      url = url_for(organism, table)
+      if key_field and fields
+        all_fields = fields_for(organism, table)
+        key_pos = all_fields.index key_field
+        field_pos = fields.collect{|f| all_fields.index f}
+
+        options[:key_field] = key_pos
+        options[:fields]    = field_pos
+      end
+      tsv = TSV.open(url, options)
+      tsv.key_field = key_field
+      tsv.fields = fields
+      tsv
+    end
+  end
+end
+
+if __FILE__ == $0
+  ddd Ensembl::FTP.ensembl_tsv("Hsa/may2012", 'exon')
+end

data/lib/rbbt/sources/entrez.rb

@@ -10,7 +10,7 @@ module Entrez
   Rbbt.claim Rbbt.share.databases.entrez.gene2pubmed, :url, 'ftp://ftp.ncbi.nih.gov/gene/DATA/gene2pubmed.gz'
 
   def self.entrez2native(taxs, options = {})
-    options = Misc.add_defaults options, :key_field => 1, :fields => 5, :persist => true, :merge => true
+    options = Misc.add_defaults options, :key_field => 1, :fields => [5], :persist => true, :merge => true
 
     taxs = [taxs] unless Array === taxs
     options.merge! :grep => taxs.collect{|t| "^" + t.to_s}
@@ -22,7 +22,7 @@ module Entrez
   end
 
   def self.entrez2name(taxs, options = {})
-    options = Misc.add_defaults options, :key_field => 1, :fields => 2, :persist => true, :merge => true
+    options = Misc.add_defaults options, :key_field => 1, :fields => [2], :persist => true, :merge => true
 
     taxs = [taxs] unless Array === taxs
     options.merge! :grep => taxs.collect{|t| "^" + t.to_s}

data/lib/rbbt/sources/genomes1000.rb

@@ -0,0 +1,45 @@
+require 'rbbt'
+require 'rbbt/util/open'
+require 'rbbt/resource'
+
+module Genomes1000
+  extend Resource
+  self.subdir = "share/databases/genomes_1000"
+
+  RELEASE_URL = "ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20110521/ALL.wgs.phase1_release_v3.20101123.snps_indels_sv.sites.vcf.gz"
+
+  Genomes1000.claim Genomes1000.mutations, :proc do
+
+    tsv = TSV.setup({}, :key_field => "Variant ID", :fields => ["Genomic Mutation"], :type => :single)
+    Open.read(RELEASE_URL) do |line|
+      next if line[0] == "#"[0]
+
+      chromosome, position, id, references, alternative, quality, filter, info = line.split("\t")
+
+      tsv[id] = [chromosome, position, alternative] * ":"
+    end
+
+    tsv.namespace = "Hsa"
+
+    tsv.to_s
+  end
+
+  Genomes1000.claim Genomes1000.mutations_hg18, :proc do
+    require 'rbbt/sources/organism'
+
+    hg19_tsv = Genomes1000.mutations.tsv :unnamed => true
+
+    mutations = hg19_tsv.values
+
+    translations = Misc.process_to_hash(mutations){|mutations| Organism.liftOver(mutations, "Hsa/jun2011", "Hsa/may2009")}
+
+    tsv = hg19_tsv.process "Genomic Mutation" do |mutation|
+      translations[mutation]
+    end
+
+    tsv.namespace = "Hsa/may2009"
+
+    tsv.to_s
+  end
+
+end

data/lib/rbbt/sources/go.rb

@@ -97,9 +97,12 @@ if defined? Entity
       @name ||= GO.id2name(self)
     end
 
-    property :genes => :array2single do |organism|
+    property :genes => :array2single do |*args|
+      organism = args.first
       organism ||= self.organism
-      @genes ||= Organism.gene_go(organism).tsv(:persist => true, :key_field => "GO ID", :fields => ["Ensembl Gene ID"], :type => :flat, :merge => true).values_at *self
+      res = Organism.gene_go(organism).tsv(:persist => true, :key_field => "GO ID", :fields => ["Ensembl Gene ID"], :type => :flat, :merge => true).values_at *self
+      res.collect{|r| r.organism = organism if r and r.respond_to? :organism}
+      res
     end
 
     property :description => :single2array do
@@ -114,12 +117,21 @@ if defined? Entity
   if defined? Gene and Entity === Gene
     module Gene
       property :go_terms => :array2single do 
-        @go_terms ||= Organism.gene_go(organism).tsv(:persist => true, :key_field => "Ensembl Gene ID", :fields => ["GO ID"], :type => :flat, :merge => true).values_at *self.ensembl
+        @go_terms ||= Organism.gene_go(organism).tsv(:persist => true, :key_field => "Ensembl Gene ID", :fields => ["GO ID"], :type => :flat, :merge => true, :namespace => organism).values_at *self.ensembl
       end
 
       property :go_bp_terms => :array2single do 
-        @go_bp_terms ||= Organism.gene_go_bp(organism).tsv(:persist => true, :key_field => "Ensembl Gene ID", :fields => ["GO ID"], :type => :flat, :merge => true).values_at *self.ensembl
+        @go_bp_terms ||= Organism.gene_go_bp(organism).tsv(:persist => true, :key_field => "Ensembl Gene ID", :fields => ["GO ID"], :type => :flat, :merge => true, :namespace => organism).values_at *self.ensembl
       end
+
+      property :go_cc_terms => :array2single do 
+        @go_cc_terms ||= Organism.gene_go_cc(organism).tsv(:persist => true, :key_field => "Ensembl Gene ID", :fields => ["GO ID"], :type => :flat, :merge => true, :namespace => organism).values_at *self.ensembl
+      end
+
+      property :go_mf_terms => :array2single do 
+        @go_mf_terms ||= Organism.gene_go_mf(organism).tsv(:persist => true, :key_field => "Ensembl Gene ID", :fields => ["GO ID"], :type => :flat, :merge => true, :namespace => organism).values_at *self.ensembl
+      end
+ 
     end
   end
 end

data/lib/rbbt/sources/organism.rb

@@ -1,18 +1,80 @@
 require 'rbbt'
 require 'rbbt/resource'
-require 'rbbt/resource/with_key'
 
 module Organism
   extend Resource
   self.pkgdir = "rbbt"
   self.subdir = "share/organisms"
 
-  ["Hsa", "Mmu", "Rno", "Sce"].each do |organism|
+  def self.installable_organisms
+    Rbbt.share.install.Organism.find.glob('???').collect{|f| File.basename(f)}
+  end
+
+
+  Organism.installable_organisms.each do |organism|
     claim Organism[organism], :rake, Rbbt.share.install.Organism[organism].Rakefile.find
 
     module_eval "#{ organism } = with_key '#{organism}'"
   end
 
+  Rbbt.claim Rbbt.software.opt.bin.liftOver, :url, "http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/liftOver"
+
+  def self.hg_build(organism)
+    require 'rbbt/sources/ensembl_ftp'
+
+    raise "Only organism 'Hsa' (Homo sapiens) supported" unless organism =~ /^Hsa/
+
+    return 'hg19' unless organism =~ /\//
+    date = organism.split("/")[1] 
+
+    release = Ensembl.releases[date]
+
+    release.sub(/.*-/,'').to_i > 54 ? 'hg19' : 'hg18'
+  end
+
+  def self.liftOver(positions, source, target)
+
+    source_hg = hg_build(source)
+    target_hg = hg_build(target)
+
+    case
+    when (source_hg == 'hg19' and target_hg == 'hg18')
+      map_url = "http://hgdownload.cse.ucsc.edu/goldenPath/hg19/liftOver/hg19ToHg18.over.chain.gz" 
+    when (source_hg == 'hg18' and target_hg == 'hg19')
+      map_url = "http://hgdownload.cse.ucsc.edu/goldenPath/hg18/liftOver/hg18ToHg19.over.chain.gz" 
+    else
+      return positions
+    end
+
+    positions_bed = positions.collect{|position| chr, pos = position.split(":").values_at(0,1); ["chr" << chr, pos.to_i-1, pos, position] * "\t"} * "\n" + "\n"
+    new_positions = {}
+
+    TmpFile.with_file(positions_bed) do |source_bed|
+      TmpFile.with_file() do |unmapped_file|
+        TmpFile.with_file() do |map_file|
+
+
+          Open.write(map_file, Open.read(map_url))
+          new_mutations = TmpFile.with_file() do |target_bed|
+            FileUtils.chmod(755, Rbbt.software.opt.bin.liftOver.produce.find)
+            CMD.cmd("#{Rbbt.software.opt.bin.liftOver.find} '#{source_bed}' '#{map_file}' '#{target_bed}' '#{unmapped_file}'").read
+            Open.read(target_bed) do |line|
+              chr, position_alt, position, name = line.chomp.split("\t")
+              chr.sub! /chr/, ''
+
+              old_chr, old_position, *rest = name.split(":")
+              new_positions[name] = ([chr, position].concat rest) * ":"
+            end
+          end
+        end
+      end
+    end
+
+    positions.collect do |position|
+      new_positions[position]
+    end
+  end
+
   class OrganismNotProcessedError < StandardError; end
 
   def self.attach_translations(org, tsv, target = nil, fields = nil, options = {})
@@ -53,30 +115,36 @@ module Organism
     end
   end
 
-  def self.guess_id(org, values, identifiers = nil)
-    identifiers ||= TSV.setup(Organism.identifiers(org), :persist => true)
-    field_matches = identifiers.field_matches(values)
-    field_matches.sort_by{|field, matches| matches.uniq.length}.last
-  end
-
   def self.guess_id(org, values)
     field_matches = TSV.field_match_counts(Organism.identifiers(org).find, values)
     field_matches.sort_by{|field, count| count.to_i}.last
   end
 
-
   def self.organisms
     Dir.glob(File.join(Organism.root.find, '*')).collect{|f| File.basename(f)}
   end
 
-  def self.name(organism)
-    Organism.scientific_name(organism).read.strip
+  def self.scientific_name(organism)
+    Organism[organism]["scientific_name"].produce.read.strip
   end
 
   def self.organism(name)
     organisms.select{|organism|
-      organism == name or Organism.name(organism) =~ /#{ name }/i
+      organism == name or Organism.scientific_name(organism) =~ /#{ name }/i
     }.first
   end
 
+  def self.known_ids(name)
+    TSV::Parser.new(Organism.identifiers(name).open).all_fields
+  end
+  
+  def self.entrez_taxid_organism(taxid)
+    all_organisms = Organism.installable_organisms
+
+    all_organisms.each do |organism|
+      return organism if Organism.entrez_taxids(organism).read.split("\n").include? taxid.to_s
+    end
+
+    raise "No organism identified for taxid #{taxid}. Supported organism are: #{all_organisms * ", "}"
+  end
 end

data/lib/rbbt/sources/pfam.rb

@@ -1,6 +1,8 @@
 require 'rbbt'
 require 'rbbt/tsv'
 require 'rbbt/resource'
+require 'rbbt/entity'
+require 'rbbt/sources/InterPro'
 
 module Pfam
   extend Resource
@@ -12,24 +14,82 @@ module Pfam
     tsv.to_s
   end
 
-  NAMES_FILE = Rbbt.share.databases.InterPro.pfam_names.find
+  NAMES_FILE = InterPro.pfam_names.find
 
   def self.name_index
-    @name_index ||= TSV.open NAMES_FILE, :single
+    @name_index ||= TSV.open NAMES_FILE, :single, :unnamed => true
   end
 
   def self.name(id)
-    name_index[id]
+    name_index[id] || id
   end
 end
 
+module InterPro
+  def self.pfam_index
+    @@pfam_index ||= InterPro.pfam_equivalences.tsv(:persist => true, :key_field => "InterPro ID", :fields => ["Pfam Domain"])
+  end
+end
+
+InterPro.claim InterPro.pfam_names.find, :proc do
+  pfam_domains = Pfam.domains.read.split("\n").collect{|l| l.split("\t").first}.compact.flatten
+  tsv = nil
+  TmpFile.with_file(pfam_domains * "\n") do |tmpfile|
+    tsv = TSV.open(CMD.cmd("cut -f 4,3 | sort -u |grep -w -f #{ tmpfile }", :in => InterPro.source.protein2ipr.open, :pipe => true), :key_field => 1, :fields => [0], :type => :single)
+  end
+  tsv.key_field = "InterPro ID"
+  tsv.fields = ["Domain Name"]
+  tsv.to_s
+end
+
+InterPro.claim InterPro.pfam_equivalences.find, :proc do
+  pfam_domains = Pfam.domains.read.split("\n").collect{|l| l.split("\t").first}.compact.flatten
+  tsv = nil
+  TmpFile.with_file(pfam_domains * "\n") do |tmpfile|
+    tsv = TSV.open(CMD.cmd("cut -f 2,4 | sort -u |grep -w -f #{ tmpfile }", :in => InterPro.source.protein2ipr.open, :pipe => true), :key_field => 0, :fields => [1], :type => :single)
+  end
+  tsv.key_field = "InterPro ID"
+  tsv.fields = ["Pfam Domain"]
+  tsv.to_s
+end
+
+
 if defined? Entity
   module PfamDomain
     extend Entity
     self.format = "Pfam Domain"
+    self.format = "Pfam Domain ID"
+
+    self.annotation :organism
     
     property :name => :array2single do
       self.collect{|id| Pfam.name(id)}
     end
+
+    property :genes => :array2single do
+      @genes ||= Organism.gene_pfam(organism).tsv(:key_field => "Pfam Domain", :fields => ["Ensembl Gene ID"], :persist => true, :merge => true, :type => :flat, :namespace => organism).values_at *self
+    end
+  end
+
+
+  module InterProDomain
+    property :pfam => :array2single do
+      InterPro.pfam_index.values_at(*self).
+        each{|domain| domain.organism = organism if domain.respond_to? :organism }
+    end
+  end
+
+  if defined? Gene and Entity === Gene
+    module Gene
+      INDEX_CACHE = {}
+
+      property :pfam_domains => :array2single do
+        index = INDEX_CACHE[organism] ||= Organism.gene_pfam(organism).tsv(:persist => true, :type => :flat, :fields => ["Pfam Domain"], :key_field => "Ensembl Gene ID", :namespace => organism)
+        @pfam_domains ||= index.values_at *self.ensembl
+      end
+ 
+    end
   end
 end
+
+

data/lib/rbbt/sources/pubmed.rb

@@ -15,7 +15,8 @@ module PubMed
 
     url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" 
     articles = []
-    Misc.divide(pmids.sort, (pmids.length / 1000) + 1) do |pmid_list|
+
+    Misc.divide(pmids.sort_by{|v| v.nil? ? 0 : v.to_i}, (pmids.length / 1000) + 1).each do |pmid_list|
       postdata = "db=pubmed&retmode=xml&id=#{pmid_list* ","}"
       xml = TmpFile.with_file(postdata) do |postfile|
         Open.read(url, :quiet => true, :nocache => true, :nice => @@pubmed_lag, :nice_key => "PubMed", "--post-file=" => postfile)
@@ -130,7 +131,13 @@ module PubMed
 
     def pdf_url
       return pmc_pdf if pmc_pdf
-      @gscholar_pdf ||= GoogleScholar::full_text_url title
+      @gscholar_pdf ||= begin
+                          GoogleScholar::full_text_url title
+                        rescue
+                          Log.medium "GoogleScholar#full_text failed: #{title}"
+                          sleep 0.1
+                          nil
+                        end
     end
 
     def full_text
@@ -140,7 +147,7 @@ module PubMed
       TmpFile.with_file do |pdf|
 
         # Change user-agent, oh well...
-        `wget --user-agent=firefox #{ pdf_url } -O #{ pdf }`
+        `wget --user-agent=firefox #{ pdf_url } -O #{ pdf } -t 3`
         TmpFile.with_file do |txt|
           `pdftotext #{ pdf } #{ txt }`
           text = Open.read(txt) if File.exists? txt

data/lib/rbbt/sources/reactome.rb

@@ -0,0 +1,82 @@
+require 'rbbt'
+require 'rbbt/resource'
+
+module Reactome
+  extend Resource
+  self.subdir = "share/databases/Reactome"
+
+  Reactome.claim Reactome.protein_pathways, :proc  do
+    url = "http://www.reactome.org/download/current/uniprot_2_pathways.stid.txt"
+    tsv = TSV.open(Open.open(url), :key_field => 0, :fields => [1], :merge => true, :type => :double)
+    tsv.key_field = "UniProt/SwissProt Accession"
+    tsv.fields = ["Reactome Pathway ID"]
+    tsv.namespace = "Hsa"
+    tsv.to_s
+  end
+
+  Reactome.claim Reactome.pathway_names, :proc  do
+    url = "http://www.reactome.org/download/current/uniprot_2_pathways.stid.txt"
+    tsv = TSV.open(Open.open(url), :key_field => 1, :fields => [2], :type => :single)
+    tsv.key_field = "Reactome Pathway ID"
+    tsv.fields = ["Pathway Name"]
+    tsv.namespace = "Hsa"
+    tsv.to_s
+  end
+
+  Reactome.claim Reactome.protein_protein, :proc  do
+    url = "http://www.reactome.org/download/current/homo_sapiens.interactions.txt.gz"
+    tsv = TSV.open(CMD.cmd('cut -f 1,4,7,8,9|sed "s/UniProt://g;s/,/;/g"', :in => Open.open(url), :pipe => true), :type => :double, :merge => true)
+    tsv.key_field = "UniProt/SwissProt Accession"
+    tsv.fields = ["Interactor UniProt/SwissProt Accession", "Interaction type", "Reactions", "PMID"]
+    tsv.namespace = "Hsa"
+    tsv.to_s
+  end
+
+end
+
+if defined? Entity 
+  module ReactomePathway
+    extend Entity
+    self.format = "Reactome Pathway ID"
+    
+    self.annotation :organism
+
+    def self.name_index
+      @name_index ||= Reactome.pathway_names.tsv(:persist => true, :key_field => "Reactome Pathway ID", :fields => ["Pathway Name"], :type => :single)
+    end
+
+    def self.gene_index
+      @gene_index ||= Reactome.protein_pathways.tsv(:persist => true, :key_field => "Reactome Pathway ID", :fields => ["UniProt/SwissProt Accession"], :type => :flat, :merge => true)
+    end
+
+    def self.filter(query, field = nil, options = nil, entity = nil)
+      return true if query == entity
+
+      return true if self.setup(entity.dup, options.merge(:format => field)).name.index query
+
+      false
+    end
+
+    property :name => :array2single do
+      @name ||= ReactomePathway.name_index.values_at *self
+    end
+
+    property :genes => :array2single do
+      @genes ||= ReactomePathway.gene_index.values_at(*self).
+        each{|gene| gene.organism = organism if gene.respond_to? :organism }
+    end
+
+    property :url => :single do
+      "http://www.reactome.org/cgi-bin/eventbrowser_st_id?ST_ID=#{ self }"
+    end
+  end
+
+  if defined? Gene and Entity === Gene
+    module Gene
+      property :reactome_pathways => :array2single do
+        @reactome_pathways ||= Reactome.protein_pathways.tsv(:persist => true, :key_field => "UniProt/SwissProt Accession", :fields => ["Reactome Pathway ID"], :type => :flat, :merge => true).values_at(*self.to("UniProt/SwissProt Accession")).
+          each{|pth| pth.organism = organism if pth.respond_to? :organism }.tap{|o| ReactomePathway.setup(o, organism)}
+      end
+    end
+  end
+end