rbbt-sources 2.1.5 → 2.1.7

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MTZiNDcxMzVmMzdkZjM3MzdiNzE5YTFmN2JiMjlkMmVhZWMzODliOA==
-  data.tar.gz: !binary |-
-    YWJlYTE4Y2M2YWM0ZjIxYTAxZTE4ZjExZmExNjQwYTJjNTg3NGVmZg==
+SHA1:
+  metadata.gz: daf367338fb6e78d2cb7b76440e67712d27f34ab
+  data.tar.gz: 5b7a7308779ec4441fa2eb997d6f9b7f0dd37e3a
 SHA512:
-  metadata.gz: !binary |-
-    NDVjY2ViNTE4ZDliY2FiMzMwYjk1YzUxMWI4ODk2NWI2OTdiNmU1YTQ0MzMx
-    ZTM0MjA5ZWFmZjlkMzAzMmRjYTBhOGVhMjM4Y2JhMmM2OThjMjQ1MDRkY2Vi
-    YTFiOWYyNmEwMGZmMzg5MDFiNjQwMWNlNDVhODEwM2VjNTg0MTc=
-  data.tar.gz: !binary |-
-    ZTY3ZGFjM2E3ZmY0OThmZjZiNzI2OTAwNWNmZWZlYmI5ODRkMTEyY2IzODNm
-    YmZkNjY3NTI2MjQzNjMzMTc4YjgzYjVkM2IwZjc0OTA0NWM0YzM1ZDUzMjU5
-    ZTliYTNjZWY4YWMwMjUxMDFkMTRiMGRmNWRkNWQyNjBjYjgwYzE=
+  metadata.gz: bb568b0d788284e82d0ac0d9cdbd14db7c0e59b4977ddce57e2701f25ca18bbef93d43424179a188f73daaacc87963d039a17aaf0916872945f2d384e6441552
+  data.tar.gz: b24f422176f10f518f692a7878c2389df0276df27a074feb5918bae1993f860fae4558d330f95de66a7857da712b5c811e487c0b117f553106215d1065f856af

data/lib/rbbt/sources/entrez.rb

@@ -1,6 +1,7 @@
-require 'rbbt'
+require 'rbbt-util'
 require 'rbbt/tsv'
 require 'rbbt/resource'
+require 'rbbt/util/filecache'
 require 'rbbt/bow/bow'
 require 'set'
 
@@ -70,85 +71,44 @@ module Entrez
 
   private 
 
-  def self.get_online(geneids)
 
-    genes_complete =  geneids.is_a?(Array) ? geneids : [geneids]
+  def self.get_gene(geneids)
+    _array = Array === geneids
 
-    genes = []
-    Misc.divide(genes_complete, (genes_complete.length / 100) + 1).each do |geneids_list|
-      begin
-        Misc.try3times do
-          url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=gene&retmode=xml&id=#{geneids_list * ","}" 
+    geneids = [geneids] unless Array === geneids
+    geneids = geneids.compact.collect{|id| id}
 
-          xml = Open.read(url, :wget_options => {:quiet => true}, :nocache => true)
+    result_files = FileCache.cache_online_elements(geneids, 'gene-{ID}.xml') do |ids|
+      result = {}
+      values = []
+      Misc.divide(ids, (ids.length / 100) + 1).each do |list|
+        begin
+          Misc.try3times do
+            url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=gene&retmode=xml&id=#{list * ","}" 
 
-          genes += xml.scan(/(<Entrezgene>.*?<\/Entrezgene>)/sm).flatten
+            xml = Open.read(url, :wget_options => {:quiet => true}, :nocache => true)
+
+            values += xml.scan(/(<Entrezgene>.*?<\/Entrezgene>)/sm).flatten
+          end
+        rescue
+          Log.error $!.message
         end
-      rescue
-        puts $!.message
-        genes += geneids_list.collect{|g| nil}
       end
-    end
 
-    if geneids.is_a? Array
-      list = Hash[*genes_complete.zip([nil]).flatten]
-      genes.each{|gene|
+      values.each do |xml|
         geneid = gene.match(/<Gene-track_geneid>(\d+)/)[1]
-        geneid = geneid.to_i unless list.include? geneid
-        list[geneid] = gene
-      }
-      return list
-    else
-      return genes.first
+        
+        result[geneid] = xml
+      end
     end
-  end
 
-  public
+    genes = {}
+    geneids.each{|id| genes[id] = Gene.new(Open.read(result_files[id])) }
 
-  def self.gene_filename(id)
-    'gene-' + id.to_s + '.xml'
-  end
-
-  def self.get_gene(geneid)
-    return nil if geneid.nil?
-
-    if Array === geneid
-      missing = []
-      list = {}
-
-      geneid.each{|p|
-        next if p.nil?
-        if FileCache.found(gene_filename p)
-          list[p] = Gene.new(Open.read(FileCache.path(gene_filename p)))
-        else
-          missing << p 
-        end
-      }
-
-
-      return list unless missing.any?
-      genes = get_online(missing)
-
-      genes.each{|p, xml|
-        filename = gene_filename p    
-        FileCache.add(filename,xml) unless FileCache.found(filename)
-        list[p] =  Gene.new(xml)
-      }
-
-      return list
+    if _array
+      genes
     else
-      filename = gene_filename geneid    
-
-
-      if FileCache.found(filename)
-        return Gene.new(Open.read(FileCache.path(filename)))
-      else
-        xml = get_online(geneid)
-
-        FileCache.add(filename, xml) unless FileCache.found(filename)
-
-        return Gene.new(xml)
-      end
+      genes.values.first
     end
   end
 

data/lib/rbbt/sources/organism.rb

@@ -144,6 +144,12 @@ module Organism
     }.first
   end
 
+  def self.organism_code(name)
+    organisms.select{|organism|
+      organism == name or Organism.scientific_name(organism) =~ /#{ name }/i
+    }.first
+  end
+
   def self.known_ids(name)
     TSV::Parser.new(Organism.identifiers(name).open).all_fields
   end

data/lib/rbbt/sources/uniprot.rb

@@ -1,5 +1,6 @@
-require 'rbbt'
+require 'rbbt-util'
 require 'rbbt/util/open'
+require 'rbbt/util/filecache'
 require 'rbbt/resource'
 require 'rbbt/sources/cath'
 require 'rbbt/sources/uniprot'
@@ -32,12 +33,78 @@ module UniProt
     tsv.to_s
   end
 
-
   UNIPROT_TEXT="http://www.uniprot.org/uniprot/[PROTEIN].txt"
   UNIPROT_FASTA="http://www.uniprot.org/uniprot/[PROTEIN].fasta"
+
+  def self.get_uniprot_entry(uniprotids)
+    _array = Array === uniprotids
+
+    uniprotids = [uniprotids] unless Array === uniprotids
+    uniprotids = uniprotids.compact.collect{|id| id}
+
+    result_files = FileCache.cache_online_elements(uniprotids, 'uniprot-{ID}.xml') do |ids|
+      result = {}
+      ids.each do |id|
+        begin
+          Misc.try3times do
+
+            content = Open.read(UNIPROT_TEXT.sub("[PROTEIN]", id), :wget_options => {:quiet => true}, :nocache => true)
+
+            result[id] = content
+          end
+        rescue
+          Log.error $!.message
+        end
+      end
+      result
+    end
+
+    uniprots = {}
+    uniprotids.each{|id| uniprots[id] = Open.read(result_files[id]) }
+
+    if _array
+      uniprots
+    else
+      uniprots.values.first
+    end
+  end
+
+  def self.get_uniprot_sequence(uniprotids)
+    _array = Array === uniprotids
+
+    uniprotids = [uniprotids] unless Array === uniprotids
+    uniprotids = uniprotids.compact.collect{|id| id}
+
+    result_files = FileCache.cache_online_elements(uniprotids, 'uniprot-sequence-{ID}') do |ids|
+      result = {}
+      ids.each do |id|
+        begin
+          Misc.try3times do
+
+            url = UNIPROT_FASTA.sub "[PROTEIN]", id
+            text = Open.read(url, :nocache => true)
+
+            result[id] = text.split(/\n/).select{|line| line !~ /^>/} * ""
+          end
+        rescue
+          Log.error $!.message
+        end
+      end
+      result
+    end
+
+    uniprots = {}
+    uniprotids.each{|id| uniprots[id] = Open.read(result_files[id]) }
+
+    if _array
+      uniprots
+    else
+      uniprots.values.first
+    end
+  end
+
   def self.pdbs(protein)
-    url = UNIPROT_TEXT.sub "[PROTEIN]", protein
-    text = Open.read(url)
+    text = get_uniprot_entry(protein)
 
     pdb = {}
 
@@ -59,15 +126,11 @@ module UniProt
   end
 
   def self.sequence(protein)
-    url = UNIPROT_FASTA.sub "[PROTEIN]", protein
-    text = Open.read(url)
-
-    text.split(/\n/).select{|line| line !~ /^>/} * ""
+    get_uniprot_sequence(protein)
   end
 
   def self.features(protein)
-    url = UNIPROT_TEXT.sub "[PROTEIN]", protein
-    text = Open.read(url)
+    text = get_uniprot_entry(protein)
 
     text = text.split(/\n/).select{|line| line =~ /^FT/} * "\n"
 
@@ -78,7 +141,6 @@ module UniProt
 
     type = nil
     parts.each do |part|
-      parts
       if part[0..1] == "FT"
         type = part.gsub(/FT\s+/,'')
         next
@@ -111,8 +173,7 @@ module UniProt
 
 
   def self.variants(protein)
-    url = UNIPROT_TEXT.sub "[PROTEIN]", protein
-    text = Open.read(url)
+    text = get_uniprot_entry(protein)
 
     text = text.split(/\n/).select{|line| line =~ /^FT/} * "\n"
 
@@ -157,8 +218,7 @@ module UniProt
   end
 
   def self.cath(protein)
-    url = UNIPROT_TEXT.sub "[PROTEIN]", protein
-    text = Open.read(url)
+    text = get_uniprot_entry(protein)
 
     cath = {}
     text.split(/\n/).each{|l| 

data/test/rbbt/sources/test_entrez.rb

@@ -21,21 +21,12 @@ class TestEntrez < Test::Unit::TestCase
     assert(data['850320'].include? '1574125') 
   end
 
-  def test_getonline
-    geneids = 9129
-
-    assert_match(/PRP3 pre-mRNA processing factor/s, Entrez.get_online(geneids))
-
-    geneids = [9129,9]
-    assert_match(/PRP3 pre-mRNA processing factor/s, Entrez.get_online(geneids)[9129])
-  end
-
   def test_getgene
     geneids = 9129
-    assert_equal([["PRP3 pre-mRNA processing factor 3 homolog (S. cerevisiae)"]], Entrez.get_gene(geneids).description)
+    assert_equal([["pre-mRNA processing factor 3"]], Entrez.get_gene(geneids).description)
 
     geneids = [9129, 728049]
-    assert_equal([["PRP3 pre-mRNA processing factor 3 homolog (S. cerevisiae)"]], Entrez.get_gene(geneids)[9129].description)
+    assert_equal([["pre-mRNA processing factor 3"]], Entrez.get_gene(geneids)[9129].description)
   end
 
   def test_similarity

data/test/rbbt/sources/test_organism.rb

@@ -5,37 +5,37 @@ require 'rbbt/sources/ensembl_ftp'
 
 class TestOrganism < Test::Unit::TestCase
 
-  def test_known_ids
+  def _test_known_ids
     assert Organism.known_ids("Hsa").include?("Associated Gene Name")
   end
 
-  def test_location
+  def _test_location
     assert_equal "share/organisms/Sce/identifiers", Organism.identifiers('Sce')
   end
 
-  def test_identifiers
+  def _test_identifiers
     assert Organism.identifiers('Hsa').tsv(:key_field => "Entrez Gene ID", :persist => true)['1020']["Associated Gene Name"].include?('CDK5')
     assert Organism.identifiers('Sce').tsv(:persist => true)['S000006120']["Ensembl Gene ID"].include?('YPL199C')
     assert Organism.identifiers("Sce").tsv(:persist => true)['S000006120']["Ensembl Gene ID"].include?('YPL199C')
   end
 
-  def test_lexicon
+  def _test_lexicon
     assert TSV.open(Organism.lexicon('Sce'))['S000006120'].flatten.include?('YPL199C')
   end
 
-  def test_guess_id
+  def _test_guess_id
     ensembl = %w(YOL044W YDR289C YAL034C YGR246C ARS519 tH(GUG)E2 YDR218C YLR002C YGL224C)
     gene_name = %w(SNR64 MIP1 MRPS18 TFB2 JEN1 IVY1 TRS33 GAS3)
     assert_equal "Associated Gene Name", Organism.guess_id("Sce", gene_name).first
     assert_equal "Ensembl Gene ID", Organism.guess_id("Sce", ensembl).first
   end
 
-  def test_organisms
+  def _test_organisms
     assert Organism.organisms.include? "Hsa"
     assert_equal "Hsa", Organism.organism("Homo sapiens")
   end
 
-  def test_attach_translations
+  def _test_attach_translations
     tsv = TSV.setup({"1020" => []}, :type => :list)
     tsv.key_field = "Entrez Gene ID"
     tsv.fields = []
@@ -47,7 +47,7 @@ class TestOrganism < Test::Unit::TestCase
     assert_equal "CDK5", tsv["1020"]["Associated Gene Name"]
   end
 
-  def test_entrez_taxids
+  def _test_entrez_taxids
     assert_equal "Hsa", Organism.entrez_taxid_organism('9606')
   end
 
@@ -61,22 +61,22 @@ class TestOrganism < Test::Unit::TestCase
     assert_equal mutation_19, Organism.liftOver([mutation_18], target_build, source_build).first
   end
 
-  def test_orhtolog
+  def _test_orhtolog
     require 'rbbt/entity/gene'
     assert_equal ["ENSG00000133703"], Gene.setup("Kras", "Associated Gene Name", "Mmu/jun2011").ensembl.ortholog("Hsa/jun2011")
   end
 
-  #def test_genes_at_chromosome
+  #def _test_genes_at_chromosome
   #  pos = [12, 117799500]
   #  assert_equal "ENSG00000089250", Organism::Hsa.genes_at_chromosome_positions(pos.first, pos.last)
   #end
 
-  #def test_genes_at_chromosome_array
+  #def _test_genes_at_chromosome_array
   #  pos = [12, [117799500, 106903900]]
   #  assert_equal ["ENSG00000089250", "ENSG00000013503"], Organism::Hsa.genes_at_chromosome_positions(pos.first, pos.last)
   #end
  
-  #def test_genes_at_genomic_positions
+  #def _test_genes_at_genomic_positions
   #  pos = [[12, 117799500], [12, 106903900], [1, 115259500]]
   #  assert_equal ["ENSG00000089250", "ENSG00000013503", "ENSG00000213281"], Organism::Hsa.genes_at_genomic_positions(pos)
   #end

metadata

@@ -1,83 +1,83 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-sources
 version: !ruby/object:Gem::Version
-  version: 2.1.5
+  version: 2.1.7
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-25 00:00:00.000000000 Z
+date: 2014-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: rbbt-text
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: libxml-ruby
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bio
+  name: libxml-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: bio
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Data sources like PubMed, Entrez Gene, or Gene Ontology
@@ -88,7 +88,6 @@ extra_rdoc_files: []
 files:
 - etc/allowed_biomart_archives
 - etc/biomart/missing_in_archive
-- lib/rbbt/sources/COSMIC.rb
 - lib/rbbt/sources/COSTART.rb
 - lib/rbbt/sources/CTCAE.rb
 - lib/rbbt/sources/HPRD.rb
@@ -100,11 +99,9 @@ files:
 - lib/rbbt/sources/bibtex.rb
 - lib/rbbt/sources/biomart.rb
 - lib/rbbt/sources/cath.rb
-- lib/rbbt/sources/dbSNP.rb
 - lib/rbbt/sources/ensembl.rb
 - lib/rbbt/sources/ensembl_ftp.rb
 - lib/rbbt/sources/entrez.rb
-- lib/rbbt/sources/genomes1000.rb
 - lib/rbbt/sources/go.rb
 - lib/rbbt/sources/gscholar.rb
 - lib/rbbt/sources/jochem.rb
@@ -143,25 +140,25 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
 summary: Data sources for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/rbbt/sources/test_go.rb
-- test/rbbt/sources/test_entrez.rb
+- test/rbbt/sources/test_pubmed.rb
 - test/rbbt/sources/test_biomart.rb
 - test/rbbt/sources/test_gscholar.rb
+- test/rbbt/sources/test_entrez.rb
+- test/rbbt/sources/test_go.rb
 - test/rbbt/sources/test_organism.rb
-- test/rbbt/sources/test_pubmed.rb
 - test/test_helper.rb

data/lib/rbbt/sources/COSMIC.rb

@@ -1,153 +0,0 @@
-require 'rbbt'
-require 'rbbt/resource'
-
-module COSMIC
-  extend Resource
-  self.subdir = "share/databases/COSMIC"
-
-  COSMIC.claim COSMIC.mutations, :proc do 
-    url = "ftp://ftp.sanger.ac.uk/pub/CGP/cosmic/data_export/CosmicCompleteExport_v67_241013.tsv.gz"
-
-    stream = CMD.cmd('awk \'BEGIN{FS="\t"} { if ($12 != "" && $12 != "Mutation ID") { sub($12, "COSM" $12 ":" $4)}; print}\'', :in => Open.open(url), :pipe => true)
-    tsv = TSV.open(stream, :type => :list, :header_hash => "", :key_field => "Mutation ID", :namespace => "Hsa/jun2011")
-    tsv.fields = tsv.fields.collect{|f| f == "Gene name" ? "Associated Gene Name" : f}
-    tsv.add_field "Genomic Mutation" do |mid, values|
-      position = values["Mutation GRCh37 genome position"]
-      cds = values["Mutation CDS"]
-
-      if position.nil? or position.empty?
-        nil
-      else
-        position = position.split("-").first
-
-        chr, pos = position.split(":")
-        chr = "X" if chr == "23"
-        chr = "Y" if chr == "24"
-        chr = "M" if chr == "25"
-        position = [chr, pos ] * ":"
-
-        if cds.nil?
-          position
-        else
-          change = case
-                   when cds =~ />/
-                     cds.split(">").last
-                   when cds =~ /del/
-                     deletion = cds.split("del").last
-                     case
-                     when deletion =~ /^\d+$/
-                       "-" * deletion.to_i 
-                     when deletion =~ /^[ACTG]+$/i
-                       "-" * deletion.length
-                     else
-                       Log.debug "Unknown deletion: #{ deletion }"
-                       deletion
-                     end
-                   when cds =~ /ins/
-                     insertion = cds.split("ins").last
-                     case
-                     when insertion =~ /^\d+$/
-                       "+" + "N" * insertion.to_i 
-                     when insertion =~ /^[NACTG]+$/i
-                       "+" + insertion
-                     else
-                       Log.debug "Unknown insertion: #{insertion }"
-                       insertion
-                     end
-                   else
-                     Log.debug "Unknown change: #{cds}"
-                     "?(" << cds << ")"
-                   end
-          position + ":" + change
-        end
-      end
-    end
-
-    tsv.to_s.gsub(/(\d)-(\d)/,'\1:\2')
-  end
-
-  COSMIC.claim COSMIC.mutations_hg18, :proc do |filename|
-    require 'rbbt/sources/organism'
-    file = COSMIC.mutations.open
-    begin
-
-      while (line = file.gets) !~ /Genomic Mutation/; end
-      fields = line[1..-2].split("\t")
-      mutation_pos = fields.index "Genomic Mutation"
-
-      mutations = CMD.cmd("grep -v '^#'|cut -f #{mutation_pos + 1}|sort -u", :in => COSMIC.mutations.open).read.split("\n").select{|m| m.include? ":" }
-
-      translations = Misc.process_to_hash(mutations){|mutations| Organism.liftOver(mutations, "Hsa/jun2011", "Hsa/may2009")}
-
-      File.open(filename, 'w') do |f|
-        f.puts "#: :type=:list#:namespace=Hsa/may2009"
-        f.puts "#" + fields * "\t"
-        while line = file.gets do
-          next if line[0] == "#"[0]
-          line.strip!
-          parts = line.split("\t")
-          parts[mutation_pos] = translations[parts[mutation_pos]]
-          f.puts parts * "\t"
-        end
-      end
-    rescue Exception
-      FileUtils.rm filename if File.exists? filename
-      raise $!
-    ensure
-      file.close
-    end
-
-    nil
-  end
-
-
-  def self.rsid_index(organism, chromosome = nil)
-    build = Organism.hg_build(organism)
-
-    tag = [build, chromosome] * ":"
-    fwt = nil
-    Persist.persist("StaticPosIndex for COSMIC [#{ tag }]", :fwt, :persist => true) do
-      value_size = 0
-      file = COSMIC[build == "hg19" ? "mutations" : "mutations_hg18"]
-      chr_positions = []
-      begin
-        Open.read(CMD.cmd("grep '\t#{chromosome}:'", :in => file.open, :pipe => true)) do |line|
-          next if line[0] == "#"[0]
-          rsid, mutation = line.split("\t").values_at 0, 25
-          next if mutation.nil? or mutation.empty?
-          chr, pos = mutation.split(":")
-          next if chr != chromosome or pos.nil? or pos.empty?
-          chr_positions << [rsid, pos.to_i]
-          value_size = rsid.length if rsid.length > value_size
-        end
-      rescue
-      end
-      fwt = FixWidthTable.new :memory, value_size
-      fwt.add_point(chr_positions)
-      fwt
-    end
-  end
-
-  def self.mutation_index(organism)
-    build = Organism.hg_build(organism)
-    file = COSMIC[build == "hg19" ? "mutations" : "mutations_hg18"]
-    @mutation_index ||= {}
-    @mutation_index[build] ||= file.tsv :persist => true, :fields => ["Genomic Mutation"], :type => :single, :persist => true
-  end
-
-
-end
-
-if defined? Entity
-  if defined? Gene and Entity === Gene
-    module Gene
-      property :COSMIC_rsids => :single2array do
-        COSMIC.rsid_index(organism, chromosome)[self.chr_range]
-      end
-
-      property :COSMIC_mutations => :single2array do
-        GenomicMutation.setup(COSMIC.mutation_index(organism).values_at(*self.COSMIC_rsids).uniq, "COSMIC mutations over #{self.name || self}", organism, false)
-      end
-    end
-  end
-end

data/lib/rbbt/sources/dbSNP.rb

@@ -1,194 +0,0 @@
-require 'rbbt'
-require 'rbbt/util/open'
-require 'rbbt/resource'
-require 'net/ftp'
-
-module DbSNP
-  extend Resource
-  self.subdir = "share/databases/dbSNP"
-
-  URL = "ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606/VCF/common_all.vcf.gz"
-
-  DbSNP.claim DbSNP.mutations_ncbi, :proc do
-    tsv = TSV.setup({}, :key_field => "RS ID", :fields => ["Genomic Mutation"], :type => :flat)
-    file = Open.open(URL, :nocache => true) 
-    while line = file.gets do
-      next if line[0] == "#"[0]
-      chr, position, id, ref, alt = line.split "\t"
-
-      mutations = alt.split(",").collect do |a|
-        if alt[0] == ref[0]
-          alt[0] = '+'[0]
-        end
-        [chr, position, alt] * ":"
-      end
-
-      tsv.namespace = "Hsa/may2012"
-      tsv[id] = mutations
-    end
-
-    tsv.to_s
-  end
-
-  DbSNP.claim DbSNP.rsids, :proc do |filename|
-    ftp = Net::FTP.new('ftp.broadinstitute.org')
-    ftp.passive = true
-    ftp.login('gsapubftp-anonymous', 'devnull@nomail.org')
-    ftp.chdir('/bundle/2.3/hg19')
-
-    tmpfile = TmpFile.tmp_file + '.gz'
-    ftp.getbinaryfile('dbsnp_137.hg19.vcf.gz', tmpfile, 1024)
-
-    file = Open.open(tmpfile, :nocache => true) 
-    begin
-      File.open(filename, 'w') do |f|
-        f.puts "#: :type=:list#:namespace=Hsa/may2012"
-        f.puts "#" + ["RS ID", "GMAF", "G5", "G5A", "dbSNP Build ID"] * "\t"
-        while line = file.gets do
-          next if line[0] == "#"[0]
-
-          chr, position, id, ref, muts, qual, filter, info = line.split "\t"
-
-          g5 = g5a = dbsnp_build_id = gmaf = nil
-
-          gmaf = $1 if info =~ /GMAF=([0-9.]+)/
-          g5 = true if info =~ /\bG5\b/
-          g5a = true if info =~ /\bG5A\b/
-          dbsnp_build_id = $1 if info =~ /dbSNPBuildID=(\d+)/
-
-          f.puts [id, gmaf, g5, g5a, dbsnp_build_id] * "\t"
-        end
-      end
-    rescue Exception
-      FileUtils.rm filename if File.exists? filename
-      raise $!
-    ensure
-      file.close
-      FileUtils.rm tmpfile
-    end
-
-    nil
-  end
-
-  DbSNP.claim DbSNP.mutations, :proc do |filename|
-    ftp = Net::FTP.new('ftp.broadinstitute.org')
-    ftp.passive = true
-    ftp.login('gsapubftp-anonymous', 'devnull@nomail.org')
-    ftp.chdir('/bundle/2.3/hg19')
-
-    tmpfile = TmpFile.tmp_file + '.gz'
-    ftp.getbinaryfile('dbsnp_137.hg19.vcf.gz', tmpfile, 1024)
-
-    file = Open.open(tmpfile, :nocache => true) 
-    begin
-      File.open(filename, 'w') do |f|
-        f.puts "#: :type=:flat#:namespace=Hsa/may2012"
-        f.puts "#" + ["RS ID", "Genomic Mutation"] * "\t"
-        while line = file.gets do
-          next if line[0] == "#"[0]
-
-          chr, position, id, ref, muts, qual, filter, info = line.split "\t"
-
-          chr.sub!('chr', '')
-
-          position, muts = Misc.correct_vcf_mutation(position.to_i, ref, muts)
-
-          mutations = muts.collect{|mut| [chr, position, mut] * ":" }
-
-          f.puts ([id] + mutations) * "\t"
-        end
-      end
-    rescue Exception
-      FileUtils.rm filename if File.exists? filename
-      raise $!
-    ensure
-      file.close
-      FileUtils.rm tmpfile
-    end
-
-    nil
-  end
-
-  DbSNP.claim DbSNP.mutations_hg18, :proc do |filename|
-    require 'rbbt/sources/organism'
-
-    mutations = CMD.cmd("grep -v '^#'|cut -f 2|sort -u", :in => DbSNP.mutations.open).read.split("\n").collect{|l| l.split("|")}.flatten
-
-    translations = Misc.process_to_hash(mutations){|mutations| Organism.liftOver(mutations, "Hsa/jun2011", "Hsa/may2009")}
-    begin
-      file = Open.open(DbSNP.mutations.find, :nocache => true) 
-      File.open(filename, 'w') do |f|
-        f.puts "#: :type=:flat#:namespace=Hsa/may2009"
-        f.puts "#" + ["RS ID", "Genomic Mutation"] * "\t"
-        while line = file.gets do
-          next if line[0] == "#"[0]
-          parts = line.split("\t")
-          parts[1..-1] = parts[1..-1].collect{|p| translations[p]} * "|"
-          f.puts parts * "\t"
-        end
-      end
-    rescue Exception
-      FileUtils.rm filename if File.exists? filename
-      raise $!
-    ensure
-      file.close
-    end
-
-    nil
-  end
-
-  def self.rsid_index(organism, chromosome = nil)
-    build = Organism.hg_build(organism)
-
-    tag = [build, chromosome] * ":"
-    Persist.persist("StaticPosIndex for dbSNP [#{ tag }]", :fwt, :persist => true) do
-      value_size = 0
-      file = DbSNP[build == "hg19" ? "mutations" : "mutations_hg18"]
-      chr_positions = []
-      Open.read(CMD.cmd("grep '\t#{chromosome}:'", :in => file.open, :pipe => true)) do |line|
-        next if line[0] == "#"[0]
-        rsid, mutation = line.split("\t")
-        next if mutation.nil? or mutation.empty?
-        chr, pos = mutation.split(":")
-        next if chr != chromosome or pos.nil? or pos.empty?
-        chr_positions << [rsid, pos.to_i]
-        value_size = rsid.length if rsid.length > value_size
-      end
-      fwt = FixWidthTable.new :memory, value_size
-      fwt.add_point(chr_positions)
-      fwt
-    end
-  end
-
-  def self.mutation_index(organism)
-    build = Organism.hg_build(organism)
-    file = DbSNP[build == "hg19" ? "mutations" : "mutations_hg18"]
-    @mutation_index ||= {}
-    @mutation_index[build] ||= file.tsv :persist => true, :fields => ["Genomic Mutation"], :type => :single, :persist => true
-  end
-
-end
-
-if defined? Entity
-  if defined? Gene and Entity === Gene
-    module Gene
-      property :dbSNP_rsids => :single2array do
-        DbSNP.rsid_index(organism, chromosome)[self.chr_range]
-      end
-
-      property :dbSNP_mutations => :single2array do
-        GenomicMutation.setup(DbSNP.mutation_index(organism).values_at(*self.dbSNP_rsids).compact.flatten.uniq, "dbSNP mutations over #{self.name || self}", organism, true)
-      end
-    end
-  end
-
-  if defined? GenomicMutation and Entity === GenomicMutation
-    module GenomicMutation
-      property :dbSNP => :array2single do
-        dbSNP.mutations.tsv(:persist => true, :key_field => "Genomic Mutation", :fields => ["RS ID"], :type => :single).values_at *self
-      end
-    end
-
-  end
-end
-

data/lib/rbbt/sources/genomes1000.rb

@@ -1,109 +0,0 @@
-require 'rbbt'
-require 'rbbt/util/open'
-require 'rbbt/resource'
-require 'rbbt/entity/gene'
-
-module Genomes1000
-  extend Resource
-  self.subdir = "share/databases/genomes_1000"
-
-  RELEASE_URL = "ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20110521/ALL.wgs.phase1_release_v3.20101123.snps_indels_sv.sites.vcf.gz"
-
-  Genomes1000.claim Genomes1000.mutations, :proc do |filename|
-
-    begin
-      Open.write(filename) do |file|
-        file.puts "#: :type=:single#:namespace=Hsa"
-        file.puts "#Variant ID\tGenomic Mutation"
-
-        Open.read(RELEASE_URL) do |line|
-          next if line[0] == "#"[0]
-
-          chromosome, position, id, references, alternative, quality, filter, info = line.split("\t")
-
-          file.puts [id, [chromosome, position, alternative] * ":"] * "\t"
-        end
-      end
-    rescue
-      FileUtils.rm filename if File.exists? filename
-      raise $!
-    end
-    nil
-  end
-
-
-  Genomes1000.claim Genomes1000.mutations_hg18, :proc do
-    require 'rbbt/sources/organism'
-
-    hg19_tsv = Genomes1000.mutations.tsv :unnamed => true
-
-    mutations = hg19_tsv.values
-
-    translations = Misc.process_to_hash(mutations){|mutations| Organism.liftOver(mutations, "Hsa/jun2011", "Hsa/may2009")}
-
-    tsv = hg19_tsv.process "Genomic Mutation" do |mutation|
-      translations[mutation]
-    end
-
-    tsv.namespace = "Hsa/may2009"
-
-    tsv.to_s
-  end
-
-  def self.rsid_index(organism, chromosome = nil)
-    build = Organism.hg_build(organism)
-
-    tag = [build, chromosome] * ":"
-    Persist.persist("StaticPosIndex for Genomes1000 [#{ tag }]", :fwt, :persist => true) do
-      value_size = 0
-      file = Genomes1000[build == "hg19" ? "mutations" : "mutations_hg18"]
-      chr_positions = []
-      Open.read(CMD.cmd("grep '\t#{chromosome}:'", :in => file.open, :pipe => true)) do |line|
-        next if line[0] == "#"[0]
-        rsid, mutation = line.split("\t")
-        next if mutation.nil? or mutation.empty?
-        chr, pos = mutation.split(":")
-        next if chr != chromosome or pos.nil? or pos.empty?
-        chr_positions << [rsid, pos.to_i]
-        value_size = rsid.length if rsid.length > value_size
-      end
-      fwt = FixWidthTable.new :memory, value_size
-      fwt.add_point(chr_positions)
-      fwt
-    end
-  end
-
-  def self.mutation_index(organism)
-    build = Organism.hg_build(organism)
-    file = Genomes1000[build == "hg19" ? "mutations" : "mutations_hg18"]
-    @mutation_index ||= {}
-    @mutation_index[build] ||= file.tsv :persist => true, :fields => ["Genomic Mutation"], :type => :single, :persist => true
-  end
-
-
-end
-
-
-if defined? Entity
-  if defined? Gene and Entity === Gene
-    module Gene
-      property :genomes_1000_rsids => :single2array do
-        Genomes1000.rsid_index(organism, chromosome)[self.chr_range]
-      end
-
-      property :genomes_1000_mutations => :single2array do
-        GenomicMutation.setup(Genomes1000.mutation_index(organism).values_at(*self.genomes_1000_rsids).uniq, "1000 Genomes mutations over #{self.name || self}", organism, true)
-      end
-    end
-  end
-
-  if defined? GenomicMutation and Entity === GenomicMutation
-    module GenomicMutation
-      property :genomes_1000 => :array2single do
-        Genomes1000.mutations.tsv(:persist => true, :key_field => "Genomic Mutation", :fields => ["Variant ID"], :type => :single).values_at *self
-      end
-    end
-  end
-end
-
-