bio-ucsc-api 0.0.1 → 0.0.3

data/README.rdoc

@@ -23,16 +23,18 @@ http://github.com/misshie/bioruby-ucsc-api/issues
 == Features
 
 * Designed as a BioRuby plugin
+* Using ActiveRecord as an O/R mapping framework
 * Using the Bin index system to improve query performance (see https://github.com/misshie/UCSCBin )
 * Automatic conversion of "1-based full-closed intervals" to internal "0-based half-closed intervals" (see also bioruby-genomic-interval)
 * Supporting non-official MySql hosts (e.g. local servers)
+* NEW: Supporting locally-stored '2bit' files, which can be downloaded from the UCSC site, to retrieve referencial sequence. Now supporting unknown "N" nucleotide blocks, however, "mask-blocks", which are shown in lower-case in UCSC's DNA function, are not supported yet. 
 * Using Rspec for the testing framework
 * Trying to support whole hg19 and hg18 databases.
 
-
 == Install
 
- (to be uploaded into rubygems.org)
+ $ gem install bio-ucsc-api
+ (you may need to be root or use "sudo")
 
 == How to Use
 
@@ -48,6 +50,11 @@ http://github.com/misshie/bioruby-ucsc-api/issues
 
  p Bio::Ucsc::Hg19::Snp131.find_by_name("rs56289060")
 
+ # retrieve reference sequence from a locally-stored 2bit file
+ Bio::Ucsc::Hg19::Reference.load("hg19.2bit")
+ itv = Bio::GenomicInterval.parse("chr1:9,500-10,999")
+ p Reference.find_by_interval(itv)
+
 == Copyright
 Copyright:: (c) 2011 MISHIMA, Hiroyuki (missy at be.to / hmishima at nagasaki-u.ac.jp)
 

data/Rakefile

@@ -28,7 +28,7 @@ Jeweler::Tasks.new do |gem|
 
   gem.add_runtime_dependency 'activerecord',         '>= 3.0.0'
   gem.add_runtime_dependency 'activesupport',        '>= 3.0.0'
-  gem.add_runtime_dependency 'bio-genomic-interval', '>= 0.1.1'
+  gem.add_runtime_dependency 'bio-genomic-interval', '>= 0.1.2'
 end
 
 Jeweler::RubygemsDotOrgTasks.new

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.3

data/bio-ucsc-api.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bio-ucsc-api}
-  s.version = "0.0.1"
+  s.version = "0.0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["MISHIMA, Hiroyuki", "Francesco Strozzi", "Jan Aerts"]
-  s.date = %q{2011-04-22}
+  s.date = %q{2011-04-25}
   s.description = %q{A Bioruby plugin: an API for UCSC Genome Browser (experimental)}
   s.email = %q{missy@be.to}
   s.extra_rdoc_files = [
@@ -36,6 +36,7 @@ Gem::Specification.new do |s|
     "lib/bio-ucsc/hg18/cnpsharp2.rb",
     "lib/bio-ucsc/hg18/db_connection.rb",
     "lib/bio-ucsc/hg18/dgv.rb",
+    "lib/bio-ucsc/hg18/reference.rb",
     "lib/bio-ucsc/hg18/refgene.rb",
     "lib/bio-ucsc/hg18/rmsk.rb",
     "lib/bio-ucsc/hg18/tables.rb",
@@ -44,8 +45,10 @@ Gem::Specification.new do |s|
     "lib/bio-ucsc/hg19/ccdsgene.rb",
     "lib/bio-ucsc/hg19/cytoband.rb",
     "lib/bio-ucsc/hg19/db_connection.rb",
+    "lib/bio-ucsc/hg19/description.rb",
     "lib/bio-ucsc/hg19/dgv.rb",
     "lib/bio-ucsc/hg19/ensgene.rb",
+    "lib/bio-ucsc/hg19/gbcdnainfo.rb",
     "lib/bio-ucsc/hg19/gwascatalog.rb",
     "lib/bio-ucsc/hg19/hapmapalleleschimp.rb",
     "lib/bio-ucsc/hg19/hapmapallelesmacaque.rb",
@@ -60,11 +63,14 @@ Gem::Specification.new do |s|
     "lib/bio-ucsc/hg19/hapmapsnpsmkk.rb",
     "lib/bio-ucsc/hg19/hapmapsnpstsi.rb",
     "lib/bio-ucsc/hg19/hapmapsnpsyri.rb",
+    "lib/bio-ucsc/hg19/kgxref.rb",
     "lib/bio-ucsc/hg19/knowngene.rb",
     "lib/bio-ucsc/hg19/omimgene.rb",
     "lib/bio-ucsc/hg19/phastconselements46wayprimates.rb",
     "lib/bio-ucsc/hg19/phylop46wayprimates.rb",
+    "lib/bio-ucsc/hg19/reference.rb",
     "lib/bio-ucsc/hg19/refgene.rb",
+    "lib/bio-ucsc/hg19/refseqsummary.rb",
     "lib/bio-ucsc/hg19/rmsk.rb",
     "lib/bio-ucsc/hg19/snp131.rb",
     "lib/bio-ucsc/hg19/snp132.rb",
@@ -72,9 +78,12 @@ Gem::Specification.new do |s|
     "lib/bio-ucsc/hg19/snp132common.rb",
     "lib/bio-ucsc/hg19/snp132flagged.rb",
     "lib/bio-ucsc/hg19/snp132mult.rb",
+    "lib/bio-ucsc/hg19/trnas.rb",
     "lib/bio-ucsc/hg19/wgrna.rb",
     "lib/bio-ucsc/ucsc_bin.rb",
+    "samples/hg19-2bit-retrieve.rb",
     "samples/hg19-sample.rb",
+    "samples/symbol2summary.rb",
     "spec/hg18/cnpiafrate2_spec.rb",
     "spec/hg18/cnplocke_spec.rb",
     "spec/hg18/cnpredon_spec.rb",
@@ -82,13 +91,16 @@ Gem::Specification.new do |s|
     "spec/hg18/cnpsharp2_spec.rb",
     "spec/hg18/db_connection_spec.rb",
     "spec/hg18/dgv_spec.rb",
+    "spec/hg18/reference_spec.rb",
     "spec/hg18/refgene_spec.rb",
     "spec/hg18/rmsk_spec.rb",
     "spec/hg19/ccdsgene_spec.rb",
     "spec/hg19/cytoband_spec.rb",
     "spec/hg19/db_connection_spec.rb",
+    "spec/hg19/description_spec.rb",
     "spec/hg19/dgv_spec.rb",
     "spec/hg19/ensgene_spec.rb",
+    "spec/hg19/gbcdnainfo_spec.rb",
     "spec/hg19/gwascatalog_spec.rb",
     "spec/hg19/hapmapalleleschimp_spec.rb",
     "spec/hg19/hapmapallelesmacaque_spec.rb",
@@ -103,24 +115,28 @@ Gem::Specification.new do |s|
     "spec/hg19/hapmapsnpsmkk_spec.rb",
     "spec/hg19/hapmapsnpstsi_spec.rb",
     "spec/hg19/hapmapsnpsyri_spec.rb",
+    "spec/hg19/kgxref_spec.rb",
     "spec/hg19/knowngene_spec.rb",
     "spec/hg19/omimGene_spec.rb",
     "spec/hg19/phastconselements46wayprimates_spec.rb",
     "spec/hg19/phyloP46wayPrimates_spec.rb",
+    "spec/hg19/reference_spec.rb",
     "spec/hg19/refgene_spec.rb",
+    "spec/hg19/refseqsummary_spec.rb",
     "spec/hg19/rmsk_spec.rb",
     "spec/hg19/snp132Flagged_spec.rb",
     "spec/hg19/snp132_spec.rb",
     "spec/hg19/snp132codingdbsnp_spec.rb",
     "spec/hg19/snp132common_spec.rb",
     "spec/hg19/snp132mult_spec.rb",
+    "spec/hg19/trnas_spec.rb",
     "spec/hg19/wgrna_spec.rb",
     "spec/spec_helper.rb"
   ]
   s.homepage = %q{http://github.com/misshie/bioruby-ucsc-api}
   s.licenses = ["Ruby (Ruby's/GPLv2 dual)"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.7.2}
   s.summary = %q{A Bioruby plugin: an API for UCSC Genome Browser (experimental)}
   s.test_files = [
     "spec/hg18/cnpiafrate2_spec.rb",
@@ -130,13 +146,16 @@ Gem::Specification.new do |s|
     "spec/hg18/cnpsharp2_spec.rb",
     "spec/hg18/db_connection_spec.rb",
     "spec/hg18/dgv_spec.rb",
+    "spec/hg18/reference_spec.rb",
     "spec/hg18/refgene_spec.rb",
     "spec/hg18/rmsk_spec.rb",
     "spec/hg19/ccdsgene_spec.rb",
     "spec/hg19/cytoband_spec.rb",
     "spec/hg19/db_connection_spec.rb",
+    "spec/hg19/description_spec.rb",
     "spec/hg19/dgv_spec.rb",
     "spec/hg19/ensgene_spec.rb",
+    "spec/hg19/gbcdnainfo_spec.rb",
     "spec/hg19/gwascatalog_spec.rb",
     "spec/hg19/hapmapalleleschimp_spec.rb",
     "spec/hg19/hapmapallelesmacaque_spec.rb",
@@ -151,23 +170,26 @@ Gem::Specification.new do |s|
     "spec/hg19/hapmapsnpsmkk_spec.rb",
     "spec/hg19/hapmapsnpstsi_spec.rb",
     "spec/hg19/hapmapsnpsyri_spec.rb",
+    "spec/hg19/kgxref_spec.rb",
     "spec/hg19/knowngene_spec.rb",
     "spec/hg19/omimGene_spec.rb",
     "spec/hg19/phastconselements46wayprimates_spec.rb",
     "spec/hg19/phyloP46wayPrimates_spec.rb",
+    "spec/hg19/reference_spec.rb",
     "spec/hg19/refgene_spec.rb",
+    "spec/hg19/refseqsummary_spec.rb",
     "spec/hg19/rmsk_spec.rb",
     "spec/hg19/snp132Flagged_spec.rb",
     "spec/hg19/snp132_spec.rb",
     "spec/hg19/snp132codingdbsnp_spec.rb",
     "spec/hg19/snp132common_spec.rb",
     "spec/hg19/snp132mult_spec.rb",
+    "spec/hg19/trnas_spec.rb",
     "spec/hg19/wgrna_spec.rb",
     "spec/spec_helper.rb"
   ]
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
@@ -181,7 +203,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<bio>, [">= 1.4.1"])
       s.add_runtime_dependency(%q<activerecord>, [">= 3.0.0"])
       s.add_runtime_dependency(%q<activesupport>, [">= 3.0.0"])
-      s.add_runtime_dependency(%q<bio-genomic-interval>, [">= 0.1.1"])
+      s.add_runtime_dependency(%q<bio-genomic-interval>, [">= 0.1.2"])
     else
       s.add_dependency(%q<activerecord>, [">= 3.0.7"])
       s.add_dependency(%q<mysql>, [">= 2.8.1"])
@@ -193,7 +215,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<bio>, [">= 1.4.1"])
       s.add_dependency(%q<activerecord>, [">= 3.0.0"])
       s.add_dependency(%q<activesupport>, [">= 3.0.0"])
-      s.add_dependency(%q<bio-genomic-interval>, [">= 0.1.1"])
+      s.add_dependency(%q<bio-genomic-interval>, [">= 0.1.2"])
     end
   else
     s.add_dependency(%q<activerecord>, [">= 3.0.7"])
@@ -206,7 +228,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<bio>, [">= 1.4.1"])
     s.add_dependency(%q<activerecord>, [">= 3.0.0"])
     s.add_dependency(%q<activesupport>, [">= 3.0.0"])
-    s.add_dependency(%q<bio-genomic-interval>, [">= 0.1.1"])
+    s.add_dependency(%q<bio-genomic-interval>, [">= 0.1.2"])
   end
 end
 

data/lib/bio-ucsc.rb

@@ -11,8 +11,8 @@ require "bio-genomic-interval"
 module Bio
   module Ucsc
     base = File.dirname(__FILE__)
-    autoload :Hg18,  "#{base}/bio-ucsc/hg18"
-    autoload :Hg19,  "#{base}/bio-ucsc/hg19"
+    autoload :Hg18,      "#{base}/bio-ucsc/hg18"
+    autoload :Hg19,      "#{base}/bio-ucsc/hg19"
 
     # ToDo
     # autoload :Mm9,   "#{base}/bio-ucsc/mm9"

data/lib/bio-ucsc/hg18.rb

@@ -14,6 +14,9 @@ module Bio
   module Ucsc
     module Hg18
       base = File.dirname(__FILE__)
+      # Reference sequence retrieval via the 2bit file
+      autoload :Reference,    "#{base}/hg18/reference"
+
       autoload :RefGene,      "#{base}/hg18/refgene"
       autoload :Dgv,          "#{base}/hg18/dgv"
       autoload :CnpIafrate2,  "#{base}/hg18/cnpiafrate2"

data/lib/bio-ucsc/hg18/reference.rb

@@ -0,0 +1,171 @@
+#
+# = reference.rb
+# handle UCSC's 2bit file (locally stored) to retrieve the reference sequence
+#
+# Copyright::   Cioyrught (C) 2011
+#               MISHIMA, Hiroyuki <missy at be.to / hmishima at nagasaki-u.ac.jp> 
+# License::     Ruby license (Ryby's / GPLv2 dual)
+
+# require 'bio'
+
+module Bio
+  module Ucsc
+    module Hg18
+      TwoBitHeader = 
+        Struct.new(:signature, :version, :sequence_count, :reserved)
+      TwoBitRecord =
+        Struct.new(:dna_size,
+                   :n_block_intervals, :mask_block_intervals,
+                   :reserved, :packed_dna_offset)
+
+      class ByteQueue
+        def initialize(str)
+          @str = str
+          @index = 0
+        end
+
+        attr_accessor :index
+
+        def next(n)
+          result = @str[@index, n]
+          @index += n
+          result
+        end
+      end # class ByteQueue
+
+      class Reference
+        BINCODE = {0b00 => "T", 0b01 => "C", 0b10 => "A", 0b11 => "G"}
+
+        cattr_reader :filename, :header, :offsets
+
+        def self.load(filename)
+          two_bit = nil
+          open(filename, 'rb') {|f| two_bit = f.read}
+          @@tbq = ByteQueue.new(two_bit)
+          @@filename = filename
+
+          twobit_header = TwoBitHeader.new
+          twobit_header.signature      = @@tbq.next(4).unpack('L').first
+          twobit_header.version        = @@tbq.next(4).unpack('L').first
+          twobit_header.sequence_count = @@tbq.next(4).unpack('L').first
+          twobit_header.reserved       = @@tbq.next(4).unpack('L').first
+          @@header = twobit_header
+
+          @@offsets = Hash.new
+          @@header.sequence_count.times do
+            name_length = @@tbq.next(1).unpack('C').first
+            @@offsets[@@tbq.next(name_length).unpack('a*').first] =
+              @@tbq.next(4).unpack('L').first
+          end
+          @@records = Hash.new
+        end
+
+        def self.records(chrom)
+          return @@records[chrom] if @@records[chrom]
+          
+          @@tbq.index = @@offsets[chrom]
+          @@records[chrom] = TwoBitRecord.new
+          @@records[chrom].dna_size = @@tbq.next(4).unpack('L').first
+ 
+          n_block_count = @@tbq.next(4).unpack('L').first
+          n_block_starts = Array.new
+          n_block_count.times do
+            n_block_starts << @@tbq.next(4).unpack('L').first
+          end
+          n_block_sizes = Array.new
+          n_block_count.times do
+            n_block_sizes << @@tbq.next(4).unpack('L').first
+          end
+          @@records[chrom].n_block_intervals = Array.new
+          n_block_count.times do |idx|
+            @@records[chrom].n_block_intervals << 
+              Bio::GenomicInterval.zero_based(chrom,
+                                             n_block_starts[idx],
+                                             n_block_starts[idx]+n_block_sizes[idx])
+          end
+          
+          mask_block_count = @@tbq.next(4).unpack('L').first
+          mask_block_starts = Array.new
+          mask_block_count.times do
+            mask_block_starts << @@tbq.next(4).unpack('L').first
+          end
+          mask_block_sizes = Array.new
+          mask_block_count.times do
+            mask_block_sizes << @@tbq.next(4).unpack('L').first
+          end
+          @@records[chrom].mask_block_intervals = Array.new
+          mask_block_count.times do |idx|
+            @@records[chrom].mask_block_intervals << 
+              Bio::GenomicInterval.zero_based(chrom,
+                                             mask_block_starts[idx],
+                                             mask_block_starts[idx]+mask_block_sizes[idx])
+          end
+  
+          @@records[chrom].reserved = @@tbq.next(4).unpack('L').first
+          @@records[chrom].packed_dna_offset = @@tbq.index
+
+          @@records[chrom]
+        end
+
+        def self.find_by_interval(interval)
+          seq = self.find_by_interval_raw(interval)
+          @@records[interval.chrom].n_block_intervals.map do |nb|
+            if interval.overlapped?(nb)
+              case interval.compare(nb)
+              when :equal,:contained_by
+                seq = 'N' * interval.overlap(nb)
+              when :contains
+                left_len  = nb.chr_start - interval.chr_start + 1
+                right_len = interval.chr_end - nb.chr_end + 1
+                seq[0, left_len] = 'N' * left_len
+                seq[-right_len, right_len] = 'N' * right_len
+              when :left_overlapped
+                left_len = nb.chr_end - interval.chr_start + 1
+                seq[0, left_len] = 'N' * left_len
+              when :right_overlapped
+                right_len = interval.chr_end - nb.chr_start + 1
+                seq[-right_len, right_len] = 'N' * right_len
+              when :right_adjacent, :right_off
+                # expecting that N-blocks are sorted
+                # return Bio::Sequence::NA.new(seq) 
+                seq
+              end
+            end
+          end
+          #Bio::Sequence::NA.new(seq)
+          seq
+        end
+
+        def self.find_by_interval_raw(interval)
+          byte_count, byte_mod = interval.zero_start.divmod 4
+          chrom_top = self.records(interval.chrom).packed_dna_offset
+          div_start, mod_start = interval.zero_start.divmod 4
+          div_end, mod_end     = interval.zero_end.divmod 4
+          div_len, mod_len     = interval.length.divmod 4
+
+          byte_length = div_end - div_start + 1
+          @@tbq.index = chrom_top + div_start
+          bytes = @@tbq.next(byte_length).unpack('C*')
+          seq = Bio::Ucsc::Hg18::Reference.bytes_to_nucleotides(bytes)
+          seq[mod_start..(-1-(4-mod_end))]
+        end
+
+        def self.bytes_to_nucleotides(bytes) 
+          results = ""
+          bytes.each do |byte|
+            results << Bio::Ucsc::Hg18::Reference.byte_to_nucleotides(byte)
+          end
+          results
+        end
+
+        def self.byte_to_nucleotides(byte)
+          BINCODE[byte >> 6] +
+            BINCODE[(byte >> 4) & 0b11] +
+            BINCODE[(byte >> 2) & 0b11] +
+            BINCODE[byte & 0b11]
+        end
+      end # class Reference
+
+    end # module Hg18
+  end # module Ucsc
+end # module Bio

data/lib/bio-ucsc/hg19.rb

@@ -13,24 +13,42 @@ module Bio
   module Ucsc
     module Hg19
       base = File.dirname(__FILE__)
-      autoload :Dgv,                  "#{base}/hg19/dgv"
 
+      # Reference sequence retrieval via the 2bit file
+      autoload :Reference,            "#{base}/hg19/reference"
+
+      # group: Mapping and Sequencing Tracks
+      ## track: Chromosome Band
+      autoload :CytoBand,             "#{base}/hg19/cytoband"
+
+      # group: Genes and Gene Prediction Tracks
+      ## track: UCSC Genes
+      autoload :KnownGene,            "#{base}/hg19/knowngene"
+
+      ## track: RefSeq Genes
+      autoload :RefGene,              "#{base}/hg19/refgene"      
+
+      ## track: Ensemble Genes
+      autoload :EnsGene,              "#{base}/hg19/ensgene"
+
+      ## track: sno/miRNA
+      autoload :WgRna,                "#{base}/hg19/wgrna"
+
+      ## track: tRNA Genes             
+      autoload :TRNAs,                "#{base}/hg19/trnas"
+
+      # group: Variation and Repeats
+      ## track: SNPs(131)
       autoload :Snp131,               "#{base}/hg19/snp131"
 
+      ## tracks: All/Common/Flagged/Mult SNPs(132)
       autoload :Snp132,               "#{base}/hg19/snp132"
       autoload :Snp132Common,         "#{base}/hg19/snp132common"
       autoload :Snp132Flagged,        "#{base}/hg19/snp132flagged"
       autoload :Snp132Mult,           "#{base}/hg19/snp132mult"
       autoload :Snp132CodingDbSnp,    "#{base}/hg19/snp132codingdbsnp"
 
-      autoload :KnownGene,            "#{base}/hg19/knowngene"
-      autoload :RefGene,              "#{base}/hg19/refgene"      
-      autoload :GwasCatalog,          "#{base}/hg19/gwascatalog"
-      autoload :CytoBand,             "#{base}/hg19/cytoband"
-      autoload :OmimGene,             "#{base}/hg19/omimgene"
-      autoload :WgRna,                "#{base}/hg19/wgrna"
-      autoload :EnsGene,              "#{base}/hg19/ensgene"
-
+      ## track: HapMap SNPs
       autoload :HapMapSnpsASW,        "#{base}/hg19/hapmapsnpsasw"
       autoload :HapMapSnpsCEU,        "#{base}/hg19/hapmapsnpsceu"
       autoload :HapMapSnpsCHB,        "#{base}/hg19/hapmapsnpschb"
@@ -45,10 +63,29 @@ module Bio
       autoload :HapMapAllelesChimp,   "#{base}/hg19/hapmapalleleschimp"
       autoload :HapMapAllelesMacaque, "#{base}/hg19/hapmapallelesmacaque"
 
+      ## track: RepeatMasker
       autoload :Rmsk,                 "#{base}/hg19/rmsk"
 
+      ## track: DGV Struct Var
+      autoload :Dgv,                  "#{base}/hg19/dgv"
+
+      # group: Comparative Genomics
+      ## track: Conservation
       autoload :PhyloP46wayPrimates,  "#{base}/hg19/phylop46wayprimates"
       autoload :PhastConsElements46wayPrimates, "#{base}/hg19/phastconselements46wayprimates"
+
+      # group: Phenotype and Desease Association
+      ## track: OMIM Genes
+      autoload :OmimGene,             "#{base}/hg19/omimgene"
+
+      ## track: :Gwas Catalog
+      autoload :GwasCatalog,          "#{base}/hg19/gwascatalog"
+
+      # information tables behind tracks
+      autoload :Description,          "#{base}/hg19/description"
+      autoload :GbCdnaInfo,           "#{base}/hg19/gbcdnainfo"
+      autoload :KgXref,               "#{base}/hg19/kgxref"
+      autoload :RefSeqSummary,        "#{base}/hg19/refseqsummary"
     end
   end
 end