rbbt-sources 1.0.1 → 1.1.0

data/lib/rbbt/sources/NCI.rb

@@ -5,3 +5,61 @@ module NCI
 
   NCI.claim NCI.root.find, :rake, Rbbt.share.install.NCI.Rakefile.find(:lib)
 end
+
+if defined? Entity 
+
+  module NCINaturePathways
+    extend Entity
+    self.format = "NCI Nature Pathway ID"
+
+    property :name => :array2single do
+      @name ||= NCI.nature_pathways.tsv(:persist => true, :key_field => "NCI Nature Pathway ID", :fields => ["Pathway Name"], :type => :flat, :merge => true).values_at *self
+    end
+
+    property :genes => :array2single do
+      @genes ||= NCI.nature_pathways.tsv(:persist => true, :key_field => "NCI Nature Pathway ID", :fields => ["UniProt/SwissProt Accession"], :type => :flat, :merge => true).values_at *self
+    end
+  end
+
+  module NCIReactomePathways
+    extend Entity
+    self.format = "NCI Reactome Pathway ID"
+
+    property :name => :array2single do
+      @name ||= NCI.reactome_pathways.tsv(:persist => true, :key_field => "NCI Reactome Pathway ID", :fields => ["Pathway Name"], :type => :flat, :merge => true).values_at *self
+    end
+
+    property :genes => :array2single do
+      @genes ||= NCI.reactome_pathways.tsv(:persist => true, :key_field => "NCI Reactome Pathway ID", :fields => ["UniProt/SwissProt Accession"], :type => :flat, :merge => true).values_at *self
+    end
+  end
+
+  module NCIBioCartaPathways
+    extend Entity
+    self.format = "NCI BioCarta Pathway ID"
+
+    property :name => :array2single do
+      @name ||= NCI.biocarta_pathways.tsv(:persist => true, :key_field => "NCI BioCarta Pathway ID", :fields => ["Pathway Name"], :type => :flat, :merge => true).values_at *self
+    end
+
+    property :genes => :array2single do
+      @genes ||= NCI.biocarta_pathways.tsv(:persist => true, :key_field => "NCI BioCarta Pathway ID", :fields => ["Entrez Gene ID"], :type => :flat, :merge => true).values_at *self
+    end
+  end
+
+  if defined? Gene and Entity === Gene
+    module Gene
+      property :nature_pathways => :array2single do
+        @nature_pathways ||= NCI.nature_pathways.tsv(:persist => true, :key_field => "UniProt/SwissProt Accession", :fields => ["NCI Nature Pathway ID"], :type => :flat, :merge => true).values_at *self.to("UniProt/SwissProt Accession")
+      end
+
+      property :reactome_pathways => :array2single do
+        @reactome_pathways ||= NCI.reactome_pathways.tsv(:persist => true, :key_field => "UniProt/SwissProt Accession", :fields => ["NCI Reactome Pathway ID"], :type => :flat, :merge => true).values_at *self.to("UniProt/SwissProt Accession")
+      end
+ 
+      property :biocarta_pathways => :array2single do
+        @biocarta_pathways ||= NCI.biocarta_pathways.tsv(:persist => true, :key_field => "Entrez Gene ID", :fields => ["NCI BioCarta Pathway ID"], :type => :flat, :merge => true).values_at *self.entrez
+      end
+    end
+  end
+end

data/lib/rbbt/sources/barcode.rb

@@ -0,0 +1,23 @@
+require 'rbbt-util'
+
+module Barcode
+  extend Resource
+  self.subdir = "share/databases/Barcode"
+
+  Barcode.claim Barcode.transcriptome, :proc do
+    tsv = TSV.open(Open.open("http://rafalab.jhsph.edu/barcode/abc.ntc.GPL570.csv"), 
+                   :fix => Proc.new{|l| l.gsub('"', '').gsub(',', "\t")}, :header_hash => "",
+                  :type => :list, :cast => :to_f)
+    io = Open.open("http://rafalab.jhsph.edu/barcode/abc.ntc.GPL570.csv")
+    fields = io.gets.chomp.gsub('"','').split(',')
+    io.close
+    fields.shift
+    tsv.fields = fields
+    tsv.key_field = "AFFY HG U133-PLUS-2"
+    tsv.to_s
+  end
+
+end
+
+
+Barcode.transcriptome.produce

data/lib/rbbt/sources/biomart.rb

@@ -22,7 +22,7 @@ module BioMart
   @@biomart_query_xml = <<-EOT
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE Query>        
-<Query  virtualSchemaName = "default" formatter = "TSV" header = "0" uniqueRows = "1" count = "" datasetConfigVersion = "0.6" >
+<Query completionStamp="1" virtualSchemaName = "default" formatter = "TSV" header = "0" uniqueRows = "1" count = "" datasetConfigVersion = "0.6" >
 <Dataset name = "<!--DATABASE-->" interface = "default" >
 <!--FILTERS-->
 <!--MAIN-->           
@@ -47,26 +47,50 @@ module BioMart
   end
 
   def self.get(database, main, attrs = nil, filters = nil, data = nil, open_options = {})
+    open_options = Misc.add_defaults :wget_options => {"--read-timeout=" => 9000, "--tries=" => 1}
     repeats = true
     attrs   ||= []
     filters ||= ["with_#{main}"]
 
+    if chunk_filter = open_options.delete(:chunk_filter)
+      filter, values = chunk_filter
+      merged_file = TmpFile.tmp_file
+      f = File.open(merged_file, 'w')
+      values.each do |value|
+        data = get(database, main, attrs, filters + [[filter, value]], data, open_options)
+        f.write Open.read(data)
+      end
+      f.close
+      return merged_file
+    end
+
     query = @@biomart_query_xml.dup
     query.sub!(/<!--DATABASE-->/,database)
     query.sub!(/<!--FILTERS-->/, filters.collect{|name, v| v.nil? ? "<Filter name = \"#{ name }\" excluded = \"0\"/>" : "<Filter name = \"#{ name }\" value = \"#{Array === v ? v * "," : v}\"/>" }.join("\n") )
     query.sub!(/<!--MAIN-->/,"<Attribute name = \"#{main}\" />")
     query.sub!(/<!--ATTRIBUTES-->/, attrs.collect{|name| "<Attribute name = \"#{ name }\"/>"}.join("\n") )
 
-    if @archive_url
-      response = Open.read(@archive_url + query.gsub(/\n/,' '), open_options)
-    else
-      response = Open.read(BIOMART_URL + query.gsub(/\n/,' '), open_options)
+    url = @archive_url ? @archive_url + query.gsub(/\n/,' ') : BIOMART_URL + query.gsub(/\n/,' ')
+
+    begin
+      response = Open.read(url, open_options.dup)
+    rescue
+      Open.remove_from_cache url, open_options
+      raise $!
     end
 
-    if response.empty? or response =~ /Query ERROR:/
+    if response.empty? or response =~ /Query ERROR:/ 
+      Open.remove_from_cache url, open_options
       raise BioMart::QueryError, response
     end
 
+    if not response =~ /\[success\]$/sm
+      Open.remove_from_cache url, open_options
+      raise BioMart::QueryError, "Uncomplete result"
+    end
+
+    response.sub!(/\n\[success\]$/sm,'')
+
     result_file = TmpFile.tmp_file
     Open.write(result_file, response)
 
@@ -101,10 +125,10 @@ module BioMart
   # cause an error if the BioMart WS does not allow filtering with that
   # attribute.
   def self.query(database, main, attrs = nil, filters = nil, data = nil, open_options = {})
-    open_options = Misc.add_defaults open_options, :nocache => false, :filename => nil, :field_names => nil
-    filename, field_names = Misc.process_options open_options, :filename, :field_names
+    open_options = Misc.add_defaults open_options, :nocache => false, :filename => nil, :field_names => nil, :by_chr => false
+    filename, field_names, by_chr = Misc.process_options open_options, :filename, :field_names, :by_chr
     attrs   ||= []
-      
+
     open_options = Misc.add_defaults open_options, :keep_empty => false, :merge => true
 
     Log.low "BioMart query: '#{main}' [#{(attrs || []) * ', '}] [#{(filters || []) * ', '}] #{open_options.inspect}"
@@ -122,11 +146,17 @@ module BioMart
 
     chunks << chunk if chunk.any?
 
+    chunks << [] if chunks.empty?
+
     Log.low "Chunks: #{chunks.length}"
-    chunks.each_with_index{|chunk,i|
-      Log.low "Chunk #{ i + 1 } / #{chunks.length}: [#{chunk * ", "}]"
-      data = get(database, main, chunk, filters, data, open_options)
-    }
+    if chunks.any?
+      chunks.each_with_index{|chunk,i|
+        Log.low "Chunk #{ i + 1 } / #{chunks.length}: [#{chunk * ", "}]"
+        data = get(database, main, chunk, filters, data, open_options)
+      }
+    else
+      data = get(database, main, [], filters, data, open_options)
+    end
 
     open_options[:filename] ||= "BioMart[#{main}+#{attrs.length}]"
     if filename.nil?
@@ -150,10 +180,13 @@ module BioMart
   end
 
   def self.tsv(database, main, attrs = nil, filters = nil, data = nil, open_options = {})
+    attrs ||= []
+
     if @archive_url 
       attrs = attrs.reject{|attr| (MISSING_IN_ARCHIVE[@archive] || []).include? attr[1]}
     end
 
+
     codes = attrs.collect{|attr| attr[1]}
     if open_options[:filename].nil?
       tsv = query(database, main.last, codes, filters, data, open_options)

data/lib/rbbt/sources/entrez.rb

@@ -35,7 +35,7 @@ module Entrez
 
 
   def self.entrez2pubmed(taxs)
-    options = {:key_field => 1, :fields => 2, :persist => true, :merge => true}
+    options = {:key_field => 1, :fields => [2], :persist => true, :merge => true}
 
     taxs = [taxs] unless taxs.is_a?(Array)
     options.merge! :grep => taxs.collect{|t| "^" + t.to_s}

data/lib/rbbt/sources/go.rb

@@ -84,3 +84,33 @@ module GO
     end
   end
 end
+
+if defined? Entity 
+
+  module GOTerm
+    extend Entity
+    self.format = "GO ID"
+
+    property :name => :array2single do
+      @name ||= GO.id2name(self)
+    end
+
+    property :genes => :array2single do |organism|
+      @genes ||= Organism.gene_go(organism).tsv(:persist => true, :key_field => "GO ID", :fields => ["Ensembl Gene ID"], :type => :flat, :merge => true).values_at *self
+    end
+  end
+
+  if defined? Gene and Entity === Gene
+    module Gene
+      property :go_terms => :array2single do |organism|
+        @go_terms ||= Organism.gene_go(organism).tsv(:persist => true, :key_field => "Ensembl Gene ID", :fields => ["GO ID"], :type => :flat, :merge => true).values_at *self.ensembl
+      end
+ 
+      property :go_bp_terms => :array2single do |organism|
+        @go_bp_terms ||= Organism.gene_go_bp(organism).tsv(:persist => true, :key_field => "Ensembl Gene ID", :fields => ["GO ID"], :type => :flat, :merge => true).values_at *self.ensembl
+      end
+    end
+  end
+end
+
+

data/lib/rbbt/sources/tfacts.rb

@@ -0,0 +1,65 @@
+require 'rbbt'
+require 'rbbt/resource'
+require 'nokogiri'
+
+module TFacts
+  extend Resource
+  self.subdir = "share/databases/TF"
+
+  def self.targets_for_gene_unsigned(gene_name)
+    doc = Nokogiri::HTML(Open.read("http://www.tfacts.org/source/tfsResultsns.php", :post => "TFS_ID=#{ gene_name }"))
+
+    doc.css("td a").collect{|link| link.content.strip}
+  end
+
+  def self.targets_for_gene_signed(gene_name)
+    doc = Nokogiri::HTML(Open.read("http://www.tfacts.org/source/tfsResults.php", :post => "TFS_ID=#{ gene_name }"))
+
+    rows = doc.css("tr")
+    rows.shift
+    targets = {}
+    rows.each{|row| gene, sign = row.css("td"); targets[gene.css("a").first.content.strip] = sign.content.strip}
+    targets
+  end
+
+  def self.known_transcription_factors_signed
+    Open.read("http://www.tfacts.org/source/tfs.php").scan(/OPTION VALUE=([^\s]+)/).flatten
+  end
+
+  def self.known_transcription_factors_unsigned
+    Open.read("http://www.tfacts.org/source/tfsns.php").scan(/OPTION VALUE=([^\s]+)/).flatten
+  end
+
+  TFacts.claim TFacts.targets, :proc do
+    tsv = Misc.process_to_hash(TFacts.known_transcription_factors_unsigned){|list| list.collect{|tf| TFacts.targets_for_gene_unsigned(tf)}}
+    TSV.setup tsv, :key_field => "Associated Gene Name", :fields => ["Target Associated Gene Name"], :type => :flat
+    tsv.to_s
+  end
+
+  TFacts.claim TFacts.targets_signed, :proc do
+    tsv = TSV.setup({}, :key_field => "Associated Gene Name", :fields => ["Target Associated Gene Name", "Target Sign"], :type => :double)
+    Misc.process_to_hash(TFacts.known_transcription_factors_signed){|list| list.collect{|tf| TFacts.targets_for_gene_signed(tf)}}.each do |tf, targets|
+      tsv[tf] = [targets.keys, targets.values]
+    end
+    tsv.to_s
+  end
+end
+
+
+if defined? Entity and defined? Gene and Entity === Gene
+
+  module Gene
+    property :is_transcription_factor? => :array2single do
+      @is_trasncription_factor ||= begin
+                                     tfs = TFacts.targets.keys
+                                     self.name.collect{|gene| tfs.include? gene}
+                                   end
+    end
+
+   property :transcription_targets => :array2single do
+     @transcription_targets ||= begin
+                                  Gene.setup(TFacts.targets.tsv(:persist => true).values_at(*self.name), "Associated Gene Name", self.organism)
+                                end
+   end
+  end
+end

data/share/install/NCI/Rakefile

@@ -51,7 +51,7 @@ file 'nature_pathways' do |t|
 
   pathways = NCI.get_pathways(xml)
 
-  Open.write(t.name, TSV.setup(pathways, :type => :double, :key_field => "NCI Pathway ID", :fields => ["Pathway Name","UniProt/SwissProt Accession"]).to_s)
+  Open.write(t.name, TSV.setup(pathways, :type => :double, :key_field => "NCI Nature Pathway ID", :fields => ["Pathway Name","UniProt/SwissProt Accession"]).to_s)
 end
 
 file 'biocarta_pathways' do |t|
@@ -62,7 +62,7 @@ file 'biocarta_pathways' do |t|
 
   pathways = NCI.get_pathways(xml, "LL")
 
-  Open.write(t.name, TSV.setup(pathways, :type => :double, :key_field => "NCI Pathway ID", :fields => ["Pathway Name","Entrez Gene ID"]).to_s)
+  Open.write(t.name, TSV.setup(pathways, :type => :double, :key_field => "NCI BioCarta Pathway ID", :fields => ["Pathway Name","Entrez Gene ID"]).to_s)
 end
 
 file 'reactome_pathways' do |t|
@@ -73,7 +73,7 @@ file 'reactome_pathways' do |t|
 
   pathways = NCI.get_pathways(xml, "UP")
 
-  Open.write(t.name, TSV.setup(pathways, :type => :double, :key_field => "NCI Pathway ID", :fields => ["Pathway Name","UniProt/SwissProt Accession"]).to_s)
+  Open.write(t.name, TSV.setup(pathways, :type => :double, :key_field => "NCI Reactome Pathway ID", :fields => ["Pathway Name","UniProt/SwissProt Accession"]).to_s)
 end
 
 

data/share/install/Organism/Hsa/Rakefile

@@ -91,6 +91,12 @@ $biomart_go= [
   ["GO Namespace", 'namespace_1003'],
 ]
 
+$biomart_go_2009= [
+  ["GO BP ID", 'go_biological_process_id'],
+  ["GO MF ID", 'go_molecular_function_id'],
+  ["GO CC ID", 'go_cellular_component_id'],
+]
+
 $biomart_pfam= [
   ["Pfam Domain", 'pfam'],
 ]

data/share/install/Organism/organism_helpers.rb

@@ -51,6 +51,13 @@ $biomart_transcript_exons = [
   ['Exon Rank in Transcript','rank'],
 ]
 
+$biomart_exon_phase = [
+  $biomart_ensembl_transcript,
+  ['Phase','phase'],
+]
+
+
+
 $biomart_exons = [
   $biomart_ensembl_gene,
   ['Exon Strand','strand'],
@@ -66,6 +73,7 @@ end
 
 file 'identifiers' do |t|
   identifiers = BioMart.tsv($biomart_db, $biomart_ensembl_gene, $biomart_identifiers, [], nil, :namespace => $namespace)
+  identifiers.unnamed =  true
 
   $biomart_identifiers.each do |name, key, prefix|
     next unless identifiers.all_fields.include? name
@@ -95,11 +103,14 @@ file 'identifiers' do |t|
   entrez_synonyms.fields = ["Entrez Gene Name Synonyms"]
 
   identifiers.attach entrez_synonyms
+  
 
-  identifiers.each do |key, values|
-    values.each do |list|
-      list.reject!{|v| v.nil? or v.empty?}
-      list.uniq!
+  identifiers.with_unnamed do
+    identifiers.each do |key, values|
+      values.each do |list|
+        list.reject!{|v| v.nil? or v.empty?}
+        list.uniq!
+      end
     end
   end
 
@@ -129,7 +140,7 @@ file 'protein_identifiers' do |t|
   File.open(t.name, 'w') do |f| f.puts identifiers end
 end
 
-file 'probe_transcripts' do |t|
+file 'transcript_probes' do |t|
   identifiers = BioMart.tsv($biomart_db, $biomart_ensembl_transcript, $biomart_probe_identifiers, [], nil, :namespace => $namespace)
   $biomart_probe_identifiers.each do |name, key, prefix|
     if prefix
@@ -148,7 +159,7 @@ end
 
 file 'transcripts' => 'gene_positions' do |t|
   transcripts = BioMart.tsv($biomart_db, $biomart_ensembl_transcript, $biomart_transcript, [], nil, :type => :list, :namespace => $namespace)
-  transcripts.attach TSV.open('gene_positions'), "Chromosome Name"
+  transcripts.attach TSV.open('gene_positions'), :fields => ["Chromosome Name"]
 
   File.open(t.name, 'w') do |f| f.puts transcripts end
 end
@@ -161,13 +172,12 @@ file 'transcript_3utr' do |t|
     f.puts "#Ensembl Transcript ID\t3' UTR Length"
     utrs.each do |seq,trans|
       trans.each do |tran|
-        f.puts [tran, seq.length] * "\t"
+        f.puts [tran, seq.length] * "\t" if seq =~ /^[ACTG]+$/
       end
     end
   end
 end
 
-
 file 'transcript_5utr' do |t|
   utrs = BioMart.tsv($biomart_db, $biomart_ensembl_transcript, $biomart_transcript_5utr, [], nil, :type => :flat, :namespace => $namespace)
 
@@ -176,7 +186,7 @@ file 'transcript_5utr' do |t|
     f.puts "#Ensembl Transcript ID\t5' UTR Length"
     utrs.each do |seq,trans|
       trans.each do |tran|
-        f.puts [tran, seq.length] * "\t"
+        f.puts [tran, seq.length] * "\t" if seq =~ /^[ACTG]+$/
       end
     end
   end
@@ -193,7 +203,7 @@ file 'gene_sequence' do |t|
 
   File.open(t.name, 'w') do |f| 
     f.puts "#: :type=:single"
-    f.puts "#Ensembl Gene ID\tProtein Sequence"
+    f.puts "#Ensembl Gene ID\tGene Sequence"
     sequences.each do |seq, genes|
       genes.each do |gene|
         f.write gene 
@@ -205,7 +215,9 @@ file 'gene_sequence' do |t|
   end
 end
 
-file 'protein_sequence' do |t|
+file 'protein_sequence' => 'chromosomes' do |t|
+  #chromosomes = TSV.open(t.prerequisites.first).keys
+  #sequences = BioMart.tsv($biomart_db, $biomart_ensembl_protein, $biomart_protein_sequence, [], nil, :type => :flat, :namespace => $namespace, :chunk_filter => ['chromosome_name', chromosomes])
   sequences = BioMart.tsv($biomart_db, $biomart_ensembl_protein, $biomart_protein_sequence, [], nil, :type => :flat, :namespace => $namespace)
 
   File.open(t.name, 'w') do |f| 
@@ -220,12 +232,11 @@ file 'protein_sequence' do |t|
       end
     end
   end
-
 end
 
 file 'exons' => 'gene_positions' do |t|
   exons = BioMart.tsv($biomart_db, $biomart_ensembl_exon, $biomart_exons, [], nil, :merge => false, :type => :list, :namespace => $namespace)
-  exons.attach TSV.open('gene_positions'), "Chromosome Name"
+  exons.attach TSV.open('gene_positions'), :fields => ["Chromosome Name"]
 
   File.open(t.name, 'w') do |f| f.puts exons end
 end
@@ -236,12 +247,61 @@ file 'transcript_exons' do |t|
   File.open(t.name, 'w') do |f| f.puts exons end
 end
 
+file 'exon_phase' do |t|
+  exons = BioMart.tsv($biomart_db, $biomart_ensembl_exon, $biomart_exon_phase, [], nil, :keep_empty => true, :namespace => $namespace)
+
+  File.open(t.name, 'w') do |f| f.puts exons end
+end
+
+
+#file 'transcript_phase' do |t|
+#  tsv = TSV.setup({}, :key_field => "Ensembl Transcript ID", :fields => ["Phase"], :type => :single, :cast => :to_i)
+#
+#  transcript_cds_start = BioMart.tsv($biomart_db, $biomart_ensembl_transcript, [['CDNA Start','cds_start']], [], nil, :type => :flat, :namespace => $namespace)
+#  transcript_cds_start.through do |transcript, values|
+#    phase = values.compact.reject{|p| p.empty?}.select{|p| p == "1" or p == "2"}.first
+#    tsv[transcript] = phase.to_i unless phase.nil?
+#  end
+#
+#  File.open(t.name, 'w') do |f| f.puts tsv end
+#end
+
+file 'transcript_phase' => ['exon_phase', 'transcript_exons'] do |t|
+  tsv = TSV.setup({}, :key_field => "Ensembl Transcript ID", :fields => ["phase"], :type => :single, :cast => :to_i)
+
+  transcript_exons = TSV.open(t.prerequisites.last)
+  transcript_exons.unnamed = true
+
+  exon_is_first_for_transcripts = {}
+
+  transcript_exons.through do |transcript, value|
+    exon = Misc.zip_fields(value).select{|exon, rank| rank == "1" }.first[0]
+    exon_is_first_for_transcripts[exon] ||=  []
+    exon_is_first_for_transcripts[exon] << transcript
+  end
+
+  exon_phase = TSV.open(t.prerequisites.first)
+  exon_phase.unnamed = true
+  exon_phase.monitor = true
+
+  exon_phase.through do |exon, value|
+    Misc.zip_fields(value).each{|transcript, phase| 
+      next unless exon_is_first_for_transcripts.include? exon
+      next unless exon_is_first_for_transcripts[exon].include? transcript
+      tsv[transcript] = phase  
+    }
+  end
+
+  File.open(t.name, 'w') do |f| f.puts tsv end
+end
+
+
 file 'transcript_sequence' do |t|
   sequences = BioMart.tsv($biomart_db, $biomart_ensembl_transcript, $biomart_transcript_sequence, [], nil, :type => :flat, :namespace => $namespace)
 
   File.open(t.name, 'w') do |f| 
     f.puts "#: :type=:single"
-    f.puts "#Ensembl Transcript ID\tProtein Sequence"
+    f.puts "#Ensembl Transcript ID\tTranscript Sequence"
     sequences.each do |seq, genes|
       genes.each do |gene|
         f.write gene 
@@ -272,7 +332,7 @@ end
 
 file 'gene_pmids' do |t|
   tsv =  Entrez.entrez2pubmed($taxs)
-  text = "#: :namespace=#{$namespace}"
+  text = "#: :namespace=#{$namespace}\n"
   text += "#Entrez Gene ID\tPMID"
   tsv.each do |gene, pmids|
     text << "\n" << gene << "\t" << pmids * "|"
@@ -319,7 +379,7 @@ file 'exon_offsets' => %w(exons transcript_exons gene_transcripts transcripts tr
   transcript_info  = TSV.open('transcripts', :list, :fields => ["Ensembl Protein ID"])
   transcript_exons = TSV.open('transcript_exons', :double, :fields => ["Ensembl Exon ID","Exon Rank in Transcript"])
 
-  string = "#: :namespace=#{$namespace}"
+  string = "#: :namespace=#{$namespace}\n"
   string += "#Ensembl Exon ID\tEnsembl Transcript ID\tOffset\n"
 
   exons.unnamed = true
@@ -329,29 +389,43 @@ file 'exon_offsets' => %w(exons transcript_exons gene_transcripts transcripts tr
   transcript_exons.unnamed = true
 
   exons.monitor = true
-  Misc.profile do
-    exons.through do |exon, info|
-      gene, start, finish, strand, chr = info
-
-      transcripts = coding_transcripts_for_exon(exon, exon_transcripts, transcript_info)
+  exons.through do |exon, info|
+    gene, start, finish, strand, chr = info
 
-      transcript_offsets = {}
-      transcripts.each do |transcript|
-        offset = exon_offset_in_transcript( exon, transcript, exons, transcript_exons)
-        transcript_offsets[transcript] = offset unless offset.nil?
-      end
+    transcripts = coding_transcripts_for_exon(exon, exon_transcripts, transcript_info)
 
-      string << exon << "\t" << transcript_offsets.keys * "|" << "\t" << transcript_offsets.values * "|" << "\n"
+    transcript_offsets = {}
+    transcripts.each do |transcript|
+      offset = exon_offset_in_transcript( exon, transcript, exons, transcript_exons)
+      transcript_offsets[transcript] = offset unless offset.nil?
     end
+
+    string << exon << "\t" << transcript_offsets.keys * "|" << "\t" << transcript_offsets.values * "|" << "\n"
   end
 
   Open.write(t.name, string)
 end
 
 file 'gene_go' do |t|
-  goterms = BioMart.tsv($biomart_db, $biomart_ensembl_gene, $biomart_go, [], nil, :type => :double, :namespace => $namespace)
+  if File.basename(FileUtils.pwd) =~ /^[a-z]{3}([0-9]{4})$/i and $1.to_i <= 2009
+    goterms = BioMart.tsv($biomart_db, $biomart_ensembl_gene, $biomart_go_2009, [], nil, :type => :double, :namespace => $namespace)
 
-  File.open(t.name, 'w') do |f| f.puts goterms end
+    goterms.add_field "GO ID" do |key, values|
+      values.flatten.compact.reject{|go| go.empty?}
+    end
+
+    goterms.add_field "GO Namespace" do |key, values|
+      ["biological_process"] * values["GO BP ID"].reject{|go| go.empty?}.length +
+        ["cellular_component"] * values["GO CC ID"].reject{|go| go.empty?}.length +
+        ["molecular_function"] * values["GO MF ID"].reject{|go| go.empty?}.length 
+    end
+
+    File.open(t.name, 'w') do |f| f.puts goterms.slice(["GO ID", "GO Namespace"]) end
+  else
+    goterms = BioMart.tsv($biomart_db, $biomart_ensembl_gene, $biomart_go, [], nil, :type => :double, :namespace => $namespace)
+
+    File.open(t.name, 'w') do |f| f.puts goterms end
+  end
 end
 
 file 'gene_go_bp' => 'gene_go' do |t|
@@ -370,13 +444,18 @@ file 'gene_go_bp' => 'gene_go' do |t|
 end
 
 
-
 file 'gene_pfam' do |t|
   goterms = BioMart.tsv($biomart_db, $biomart_ensembl_gene, $biomart_pfam, [], nil, :type => :double, :namespace => $namespace)
 
   File.open(t.name, 'w') do |f| f.puts goterms end
 end
 
+file 'chromosomes' do |t|
+  goterms = BioMart.tsv($biomart_db, ['Chromosome Name', "chromosome_name"] , [] , [], nil, :type => :double, :namespace => $namespace)
+
+  File.open(t.name, 'w') do |f| f.puts goterms end
+end
+
 
 rule /[a-z]{3}[0-9]{4}\/.*/i do |t|
   t.name =~ /([a-z]{3}[0-9]{4})\/(.*)/i

data/test/rbbt/sources/test_biomart.rb

@@ -47,6 +47,18 @@ class TestBioMart < Test::Unit::TestCase
       assert_equal ['Protein ID', 'RefSeq Peptide'], data.fields
     end
   end
+
+  def __test_chunk
+    chrs = %w(I II III IV V VI VII VIII IX X XI XII XIII XIV XV XVI MT 2-micron)
+    data = BioMart.query('scerevisiae_gene_ensembl','entrezgene', ['protein_id','refseq_peptide','external_gene_id','ensembl_gene_id'], [], nil, :chunk_filter => ['chromosome_name', chrs], :nocache => false, :wget_options => { :quiet => false})
+    assert(data['852236']['external_gene_id'].include? 'YBL044W')
+
+    TmpFile.with_file do |f|
+      filename = BioMart.query('scerevisiae_gene_ensembl','entrezgene', ['protein_id','refseq_peptide','external_gene_id','ensembl_gene_id'], [], nil, :nocache => false, :wget_options => { :quiet => false}, :filename => f)
+      data = TSV.open Open.open(filename)
+      assert(data['852236']['external_gene_id'].include? 'YBL044W')
+    end
+  end
 end
 
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rbbt-sources
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: 
   segments: 
   - 1
-  - 0
   - 1
-  version: 1.0.1
+  - 0
+  version: 1.1.0
 platform: ruby
 authors: 
 - Miguel Vazquez
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-03 00:00:00 +02:00
+date: 2011-11-17 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -105,6 +105,7 @@ files:
 - lib/rbbt/sources/CTCAE.rb
 - lib/rbbt/sources/InterPro.rb
 - lib/rbbt/sources/NCI.rb
+- lib/rbbt/sources/barcode.rb
 - lib/rbbt/sources/bibtex.rb
 - lib/rbbt/sources/biomart.rb
 - lib/rbbt/sources/entrez.rb
@@ -115,6 +116,7 @@ files:
 - lib/rbbt/sources/organism/sequence.rb
 - lib/rbbt/sources/polysearch.rb
 - lib/rbbt/sources/pubmed.rb
+- lib/rbbt/sources/tfacts.rb
 - lib/rbbt/sources/wgEncodeBroadHmm.rb
 - share/install/InterPro/Rakefile
 - share/install/JoChem/Rakefile