rbbt-sources 1.2.0 → 2.0.0

data/etc/biomart/missing_in_archive

@@ -1,9 +1,19 @@
+jun2011:
+    - refseq_mrna
+may2010:
+    - refseq_mrna
+    - agilent_wholegenome
+    - agilent_cgh_44b
+    - illumina_humanwg_6_v2
+    - illumina_humanwg_6_v3
 may2009:
+    - refseq_mrna
     - agilent_wholegenome
     - agilent_cgh_44b
     - illumina_humanwg_6_v2
     - illumina_humanwg_6_v3
 dec2007:
+    - refseq_mrna
     - protein_id
     - affy_hc_g110
     - affy_hg_u133a_2
@@ -14,6 +24,7 @@ dec2007:
     - illumina_humanwg_6_v2
     - illumina_humanwg_6_v3
 aug2007:
+    - refseq_mrna
     - protein_id
     - affy_hc_g110
     - affy_hg_u133a_2

data/lib/rbbt/sources/COSMIC.rb

@@ -5,10 +5,53 @@ module COSMIC
   self.subdir = "share/databases/COSMIC"
 
   COSMIC.claim COSMIC.Mutations, :proc do 
-    url = "ftp://ftp.sanger.ac.uk/pub/CGP/cosmic/data_export/CosmicMutantExport_v54_120711.tsv"
+    url = "ftp://ftp.sanger.ac.uk/pub/CGP/wgs/data_export/CosmicWGS_MutantExport_v61_260912.tsv.gz"
 
-    TSV.open(Open.open(url), :header_hash => "", :key_field => "Mutation GRCh37 genome position", :merge => true).to_s
+    tsv = TSV.open(Open.open(url), :type => :list, :header_hash => "", :key_field => "Mutation ID", :namespace => "Hsa/jun2011")
+    tsv.fields = tsv.fields.collect{|f| f == "Gene name" ? "Associated Gene Name" : f}
+    tsv.add_field "Genomic Mutation" do |mid, values|
+      position = values["Mutation GRCh37 genome position"]
+      cds = values["Mutation CDS"]
+      if position.nil? or position.empty?
+        nil
+      else
+        position = position.split("-").first
+        if cds.nil?
+          position
+        else
+          change = case
+                   when cds =~ />/
+                     cds.split(">").last
+                   when cds =~ /del/
+                     deletion = cds.split("del").last
+                     case
+                     when deletion =~ /^\d+$/
+                       "-" * deletion.to_i 
+                     when deletion =~ /^[ACTG]+$/i
+                       "-" * deletion.length
+                     else
+                       Log.debug "Unknown deletion: #{ deletion }"
+                       deletion
+                     end
+                   when cds =~ /ins/
+                     insertion = cds.split("ins").last
+                     case
+                     when insertion =~ /^\d+$/
+                       "+" + "N" * insertion.to_i 
+                     when insertion =~ /^[NACTG]+$/i
+                       "+" + insertion
+                     else
+                       Log.debug "Unknown insertion: #{insertion }"
+                       insertion
+                     end
+                   else
+                     Log.debug "Unknown change: #{cds}"
+                     "?(" << cds << ")"
+                   end
+          position + ":" + change
+        end
+      end
+    end
+    tsv.to_s.gsub(/^(\d)/m,'COSM\1').gsub(/(\d)-(\d)/,'\1:\2')
   end
 end
-
-puts COSMIC.Mutations.produce

data/lib/rbbt/sources/HPRD.rb

@@ -0,0 +1,23 @@
+require 'rbbt-util'
+require 'rbbt/entity/gene'
+require 'rbbt/tsv'
+require 'rbbt/sources/organism'
+
+module HPRD
+  extend Resource
+  self.subdir = "share/databases/HPRD"
+
+  HPRD.claim HPRD["BINARY_PROTEIN_PROTEIN_INTERACTIONS.txt"], :proc do
+    raise "File BINARY_PROTEIN_PROTEIN_INTERACTIONS.txt not found in '#{HPRD["BINARY_PROTEIN_PROTEIN_INTERACTIONS.txt"].find}', download manually from http://www.hprd.org/"
+  end
+
+  HPRD.claim HPRD.protein_protein, :proc do
+    tsv = HPRD["BINARY_PROTEIN_PROTEIN_INTERACTIONS.txt"].tsv
+
+    tsv.key_field = "Associated Gene Name 1"
+    tsv.fields = ["HPRD id 1","RefSeq Protein ID 1","Associated Gene Name 2","HPRD id 2","RefSeq Protein ID 2", "Experiment type", "PMID"]
+    tsv.namespace = "Hsa"
+
+    tsv.to_s
+  end
+end  

data/lib/rbbt/sources/InterPro.rb

@@ -1,17 +1,107 @@
 require 'rbbt-util'
+require 'rbbt/entity/gene'
+require 'rbbt/tsv'
+require 'rbbt/sources/organism'
+
 module InterPro
   extend Resource
   self.subdir = "share/databases/InterPro"
 
-  InterPro.claim InterPro.root.find, :rake, Rbbt.share.install.InterPro.Rakefile.find(:lib)
+  InterPro.claim InterPro.source.protein2ipr.find, :url, "ftp://ftp.ebi.ac.uk/pub/databases/interpro/protein2ipr.dat.gz"
+
+  InterPro.claim InterPro.protein_domains.find, :proc do
+    organism = "Hsa"
+    uniprot_colum = TSV::Parser.new(Organism.protein_identifiers(organism).open).all_fields.index("UniProt/SwissProt Accession")
+    uniprots = CMD.cmd("grep -v  '^#'|cut -f #{uniprot_colum+1}", :in => Organism.protein_identifiers(organism).open).read.split("\n").collect{|l| l.split("|")}.flatten.uniq.reject{|l| l.empty?}
+   
+    tsv = nil
+    TmpFile.with_file(uniprots * "\n") do |tmpfile|
+        tsv = TSV.open(CMD.cmd("cut -f 1,2,5,6 | sort -u |grep -w -f #{ tmpfile }", :in => InterPro.source.protein2ipr.open, :pipe => true), :merge => true, :type => :double)
+    end
+ 
+    tsv.key_field = "UniProt/SwissProt Accession"
+    tsv.fields = ["InterPro ID", "Domain Start AA", "Domain End AA"]
+    tsv.to_s
+  end
+
+  InterPro.claim InterPro.domain_names.find, :proc do
+    #tsv = InterPro.source.protein2ipr.tsv :key_field => 1, :fields => [2], :type => :single
+    tsv = TSV.open(CMD.cmd("cut -f 2,3 | sort -u", :in => InterPro.source.protein2ipr.open, :pipe => true), :merge => true, :type => :single)
+ 
+    tsv.key_field = "InterPro ID"
+    tsv.fields = ["Domain Name"]
+    tsv.to_s
+  end
+
+  def self.name_index
+    @@name_index ||= InterPro.domain_names.tsv(:persist => true, :unnamed => true)
+  end
+
+  def self.gene_index
+    @@gene_index ||= InterPro.protein_domains.tsv(:persist => true, :key_field => "InterPro ID", :fields => ["UniProt/SwissProt Accession"], :type => :flat, :merge => true, :unnamed => true)
+  end
+
+  def self.domain_index
+    @@domain_index ||= InterPro.protein_domains.tsv(:persist => true, :unnamed => true, :key_field => "UniProt/SwissProt Accession", :fields => ["InterPro ID"], :merge => true)
+  end
+
+  def self.domain_position_index
+    @@domain_position_index ||= InterPro.protein_domains.tsv(:persist => true, :unnamed => true, :key_field => "UniProt/SwissProt Accession", :fields => ["InterPro ID", "Domain Start AA", "Domain End AA"], :type => :double, :merge => true)
+  end
+
+  def self.ens2uniprot(organism)
+    @@ens2uniprot_index ||= {}
+    @@ens2uniprot_index[organism] ||= Organism.protein_identifiers(organism).tsv(:persist => true, :unnamed => true, :fields => ["UniProt/SwissProt Accession"], :key_field => "Ensembl Protein ID", :type => :double, :merge => true)
+  end
+
+end
+
+if defined? Entity 
+  module InterProDomain
+    extend Entity
+    self.format = "InterPro ID"
+
+    self.annotation :organism
+    property :description => :array2single do
+      InterPro.name_index.values_at *self
+    end
+
+    property :name => :array2single do
+      InterPro.name_index.values_at *self
+    end
+
+    property :proteins => :array2single do
+      InterPro.gene_index.values_at(*self).
+        collect{|genes| genes = genes.uniq;  genes.organism = organism if genes.respond_to? :organism; genes }.tap{|o| Protein.setup(o, "UniProt/SwissProt Accession", organism)}
+    end
+
+    property :genes => :array2single do
+      InterPro.gene_index.values_at(*self).
+        collect{|genes| genes = genes.uniq;  genes.organism = organism if genes.respond_to? :organism; genes }.tap{|o| Gene.setup(o, "UniProt/SwissProt Accession", organism)}
+    end
+  end
+
+  if defined? Protein and Entity === Protein
+    module Protein
+      property :interpro_domains => :array2single do
+        self.collect do |protein|
+          uniprot = (InterPro.ens2uniprot(protein.organism)[protein] || []).flatten
+          uniprot.empty? ? nil : 
+            InterPro.domain_index.values_at(*uniprot).compact.flatten.  each{|pth| pth.organism = organism if pth.respond_to? :organism }.uniq.tap{|o| InterProDomain.setup(o, organism)}
+        end
+      end
 
-  def self.tsv(*args)
-    old_url = BioMart::BIOMART_URL
-    begin
-      BioMart::BIOMART_URL.replace "http://www.ebi.ac.uk/interpro/biomart/martservice?query="
-      BioMart.tsv(*args)
-    ensure
-      BioMart::BIOMART_URL.replace old_url
+      property :interpro_domain_positions => :array2single do
+        self.collect do |protein|
+          if protein.nil?
+            [].tap{|o| InterProDomain.setup(o, organism)}
+          else
+            uniprot = (InterPro.ens2uniprot(protein.organism)[protein] || []).flatten
+            uniprot.empty? ? nil : 
+              InterPro.domain_position_index.values_at(*uniprot).compact.flatten(1).tap{|o| InterProDomain.setup(o, organism)}
+          end
+        end
+      end
     end
   end
 end

data/lib/rbbt/sources/NCI.rb

@@ -34,7 +34,8 @@ if defined? Entity
     end
 
     property :genes => :array2single do
-      @genes ||= NCINaturePathway.gene_index.values_at *self
+      @genes ||= NCINaturePathway.gene_index.values_at(*self).
+        each{|gene| gene.organism = organism if gene.respond_to? :organism }
     end
   end
 
@@ -65,7 +66,8 @@ if defined? Entity
     end
 
     property :genes => :array2single do
-      @genes ||= NCIReactomePathway.gene_index.values_at *self
+      @genes ||= NCIReactomePathway.gene_index.values_at(*self).
+        each{|gene| gene.organism = organism if gene.respond_to? :organism }
     end
   end
 
@@ -97,7 +99,7 @@ if defined? Entity
 
     property :genes => :array2single do
       @genes ||= NCIBioCartaPathway.gene_index.values_at(*self).
-        each{|pth| pth.organism = organism if pth.respond_to? :organism }
+        each{|gene| gene.organism = organism if gene.respond_to? :organism }
     end
   end
 
@@ -109,8 +111,8 @@ if defined? Entity
           each{|pth| pth.organism = organism if pth.respond_to? :organism }.tap{|o| NCINaturePathway.setup(o, organism)}
       end
 
-      property :reactome_pathways => :array2single do
-        @reactome_pathways ||= NCI.reactome_pathways.tsv(:persist => true, :key_field => "UniProt/SwissProt Accession", :fields => ["NCI Reactome Pathway ID"], :type => :flat, :merge => true).values_at(*self.to("UniProt/SwissProt Accession")).
+      property :nci_reactome_pathways => :array2single do
+        @nci_reactome_pathways ||= NCI.reactome_pathways.tsv(:persist => true, :key_field => "UniProt/SwissProt Accession", :fields => ["NCI Reactome Pathway ID"], :type => :flat, :merge => true).values_at(*self.to("UniProt/SwissProt Accession")).
           each{|pth| pth.organism = organism if pth.respond_to? :organism }.tap{|o| NCIReactomePathway.setup(o, organism)}
       end
  

data/lib/rbbt/sources/PSI_MI.rb

@@ -0,0 +1,41 @@
+require 'rbbt'
+require 'rbbt/resource'
+require 'rbbt/tsv'
+
+module PSI_MI
+  extend Resource
+  self.subdir = Rbbt.share.databases.PSI_MI
+
+  URL="http://psidev.cvs.sourceforge.net/viewvc/psidev/psi/mi/rel25/data/psi-mi25.obo"
+
+  PSI_MI.claim PSI_MI.identifiers, :proc do
+    tsv = TSV.setup({}, :type => :list, :key_field => "PSI-MI Term", :fields => ["Name", "Description"])
+    Open.open(URL).read.split("[Term]").each do |chunk|
+      id = chunk.scan(/id: ([^\n]*)/)[0]
+      name = chunk.scan(/name: ([^\n]*)/)[0]
+      description = chunk.scan(/def: "([^\n]*)"/)[0]
+      tsv[id] = [name, description]
+    end
+    tsv.to_s
+  end
+end
+
+
+if defined? Entity 
+  require 'rbbt/entity/gene'
+  require 'rbbt/entity/interactor'
+  require 'rbbt/sources/PSI_MI'
+
+  module PSI_MITerm
+    extend Entity
+
+    self.format = "PSI-MI Term"
+
+    property :name => :array2single do
+      @@index ||= PSI_MI.identifiers.tsv(:persist => true, :fields => ["Name"], :type => :single)
+      @@index.values_at(*self)
+    end
+    
+  end
+end
+

data/lib/rbbt/sources/STITCH.rb

@@ -0,0 +1,92 @@
+require 'rbbt/resource'
+require 'rbbt/sources/organism'
+
+module STITCH
+  extend Resource
+  self.subdir = "share/databases/STITCH"
+
+  STITCH.claim STITCH.source.chemical_chemical.find, :url, "http://stitch.embl.de/download/chemical_chemical.links.detailed.v3.1.tsv.gz"
+  STITCH.claim STITCH.source.protein_chemical.find, :url, "http://stitch.embl.de/download/protein_chemical.links.detailed.v3.1.tsv.gz"
+  STITCH.claim STITCH.source.actions.find, :url, "http://stitch.embl.de/download/actions.v3.1.tsv.gz"
+  STITCH.claim STITCH.source.aliases.find, :url, "http://stitch.embl.de/download/chemical.aliases.v3.1.tsv.gz"
+  STITCH.claim STITCH.source.sources.find, :url, "http://stitch.embl.de/download/chemical.sources.v3.1.tsv.gz"
+
+  Organism.installable_organisms.each do |organism|
+    STITCH.claim STITCH.chemical_protein(organism), :proc do
+      taxids = Organism.entrez_taxids(organism).read.split("\n")
+      tsv = TSV.open(CMD.cmd("grep '\t\\(#{ taxids * '\|' }\\)\\.' | sed 's/\\(#{taxids * "|"}\\)\\.//'", :in =>  STITCH.source.protein_chemical.open, :pipe => true), :type => :double, :merge => true)
+      tsv.key_field =  "Chemical CID"
+      tsv.fields = ["Ensembl Gene ID", "experimental", "database", "textmining", "combined_score"]
+      tsv.to_s
+    end
+  end
+
+
+  STITCH.claim STITCH.identifiers, :proc do
+    identifiers = {}
+    fields = []
+    first_line = true
+    i = 0
+    STITCH.source.aliases.read do |line|
+      i += 1
+      next if i == 1
+      puts i
+      cid, code, source = line.split("\t")
+
+      pos = fields.index source
+      if pos.nil?
+        fields << source
+        pos = fields.length - 1
+      end
+      identifiers[cid] ||= []
+      identifiers[cid][pos] = code
+    end
+
+    TSV.setup(identifiers, :key_field => ["Chemical CID"], :fields => fields, :type => :double)
+
+    identifiers.to_s
+  end
+
+  STITCH.claim STITCH.iupac, :proc do
+    tsv = STITCH.source.aliases.tsv :type => :double, :merge => true, :grep => "IUPAC", :fields => [1]
+    tsv.key_field = "Chemical CID"
+    tsv.fields = ["IUPAC"]
+
+    tsv.to_s
+  end
+
+  STITCH.claim STITCH.drug_bank, :proc do
+    tsv = STITCH.source.aliases.tsv :type => :double, :merge => true, :grep => "DrugBank", :fields => [1]
+    tsv.key_field = "Chemical CID"
+    tsv.fields = ["DrugBank ID"]
+
+    tsv.to_s
+  end
+
+end
+
+if defined? Entity
+  module Compound
+    extend Entity
+
+    self.annotation :format
+    self.format = "Chemical CID"
+
+    property :iupac => :array2single do
+    end
+
+
+  end
+end
+
+if __FILE__ ==      $0
+  STITCH.source.chemical_chemical.produce
+  STITCH.source.protein_chemical.produce
+  STITCH.source.actions.produce
+  STITCH.source.aliases.produce
+  STITCH.source.sources.produce
+  STITCH.chemical_protein("Hsa").produce
+  STITCH.iupac.produce
+  STITCH.drug_bank.produce
+  STITCH.identifiers.produce
+end

data/lib/rbbt/sources/barcode.rb

@@ -18,6 +18,3 @@ module Barcode
   end
 
 end
-
-
-Barcode.transcriptome.produce

data/lib/rbbt/sources/biomart.rb

@@ -21,12 +21,12 @@ module BioMart
 
   @@biomart_query_xml = <<-EOT
 <?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE Query>        
+<!DOCTYPE Query>
 <Query completionStamp="1" virtualSchemaName = "default" formatter = "TSV" header = "0" uniqueRows = "1" count = "" datasetConfigVersion = "0.6" >
 <Dataset name = "<!--DATABASE-->" interface = "default" >
 <!--FILTERS-->
-<!--MAIN-->           
-<!--ATTRIBUTES-->     
+<!--MAIN-->
+<!--ATTRIBUTES-->
 </Dataset>
 </Query>
   EOT

data/lib/rbbt/sources/dbSNP.rb

@@ -0,0 +1,100 @@
+require 'rbbt'
+require 'rbbt/util/open'
+require 'rbbt/resource'
+require 'net/ftp'
+
+module DbSNP
+  extend Resource
+  self.subdir = "share/databases/dbSNP"
+
+  URL = "ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606/VCF/common_all.vcf.gz"
+
+  DbSNP.claim DbSNP.mutations, :proc do
+    tsv = TSV.setup({}, :key_field => "RS ID", :fields => ["Genomic Mutation"], :type => :single)
+    file = Open.open(URL, :nocache => true) 
+    while line = file.gets do
+      next if line[0] == "#"[0]
+      chr, position, id, ref, alt = line.split "\t"
+      alt = alt.split(",").first
+      if alt[0] == ref[0]
+        alt[0] = '+'[0]
+      end
+      mutation = [chr, position, alt] * ":"
+
+      tsv.namespace = "Hsa/may2012"
+      tsv[id] = mutation
+    end
+
+    tsv.to_s
+  end
+
+  DbSNP.claim DbSNP.mutations_gatk, :proc do
+    ftp = Net::FTP.new('ftp.broadinstitute.org')
+    ftp.login('gsapubftp-anonymous', 'devnull@nomail.org')
+    ftp.chdir('/bundle/2.3/hg19')
+
+    tmpfile = TmpFile.tmp_file + '.gz'
+    ftp.getbinaryfile('dbsnp_137.hg19.vcf.gz', tmpfile, 1024)
+
+    tsv = TSV.setup({}, :key_field => "RS ID", :fields => ["Genomic Mutation", "GMAF", "G5", "G5A", "dbSNP Build ID"], :type => :list)
+    file = Open.open(tmpfile, :nocache => true) 
+    while line = file.gets do
+      next if line[0] == "#"[0]
+
+      chr, position, id, ref, mut, qual, filter, info = line.split "\t"
+
+      chr.sub!('chr', '')
+
+      mut = mut.split(",").first
+      case
+      when ref == '-'
+        mut = "+" << mut
+      when mut == '-'
+        mut = "-" * ref.length
+      when (mut.length > 1 and ref.length > 1)
+        mut = '-' * ref.length << mut
+      when (mut.length > 1 and ref.length == 1 and mut.index(ref) == 0)
+        mut = '+' << mut[1..-1]
+      when (mut.length == 1 and ref.length > 1 and ref.index(mut) == 0)
+        mut = '-' * (ref.length - 1)
+      else
+        mut = mut
+      end
+
+      g5 = g5a = dbsnp_build_id = gmaf = nil
+     
+      gmaf = $1 if info =~ /GMAF=([0-9.]+)/
+      g5 = true if info =~ /\bG5\b/
+      g5a = true if info =~ /\bG5A\b/
+      dbsnp_build_id = $1 if info =~ /dbSNPBuildID=(\d+)/
+
+      mutation = [chr, position, mut] * ":"
+
+      tsv.namespace = "Hsa/may2012"
+
+      tsv[id] = [mutation, gmaf, g5, g5a, dbsnp_build_id]
+    end
+
+    FileUtils.rm tmpfile
+
+    tsv.to_s
+  end
+
+  DbSNP.claim DbSNP.mutations_hg18, :proc do
+    require 'rbbt/sources/organism'
+
+    hg19_tsv = DbSNP.mutations.tsv :unnamed => true
+
+    mutations = hg19_tsv.values
+
+    translations = Misc.process_to_hash(mutations){|mutations| Organism.liftOver(mutations, "Hsa/jun2011", "Hsa/may2009")}
+
+    tsv = hg19_tsv.process "Genomic Mutation" do |mutation|
+      translations[mutation]
+    end
+
+    tsv.namespace = "Hsa/may2009"
+
+    tsv.to_s
+  end
+end