mgnu 2.1.1

data/lib/mgnu/parser/kegg_ontology_index.rb

@@ -0,0 +1,163 @@
+module MgNu
+  module Parser
+    class KeggOntologyIndex
+      include TokyoCabinet
+      include Enumerable
+
+      attr_reader :filename, :db_name, :db
+      alias :ontologies :db
+
+      # create a new KeggOntologyIndex
+      def initialize(filename="/work/blastdb/kegg/ko")
+        @filename = filename
+        @db_name = @filename + ".tch"
+        @db = HDB.new
+        if File.exists?(@filename) and File.readable?(@filename)
+          if File.exists?(@db_name) and File.readable?(@db_name)
+            if ! @db.open(@db_name, HDB::OREADER) # open the database read-only
+              ecode = hdb.ecode
+              $stderr.puts "ERROR: could not open #{@db_name} (code: #{hdb.errmsg(ecode)})"
+              exit(1)
+            end
+          else
+            if ! @db.open(@db_name, HDB::OWRITER | HDB::OCREAT | HDB::OLCKNB | HDB::OTSYNC ) # create and open the database rw
+              ecode = hdb.ecode
+              $stderr.puts "ERROR: could not open #{@db_name} (code: #{hdb.errmsg(ecode)})"
+              exit(1)
+            end
+            parse
+          end
+        else
+          raise "\n\n ERROR -- No file by name (#{@filename}).  Exiting.\n\n"
+          exit(1)
+        end
+      end
+
+      def each
+        @db.keys.each do |k|
+          yield MgNu::Kegg::Ontology.from_json(@db[k])
+        end
+      end
+
+      def [](k)
+        ko = nil
+        if @db[k]
+          ko = MgNu::Kegg::Ontology.from_json(@db[k])
+        else
+          $stderr.puts "warning - #{k} wasn't in the file, ko is nil!"
+        end
+        ko
+      end
+
+      # setup parse method for creating TC
+      def parse
+        buffer = Array.new
+        File.new(@filename).each do |line|
+          line.chomp!
+          if line =~ /\/\/\//
+            ko = parse_ko_buffer(buffer)
+            @db[ko.kegg_id] = ko.to_json
+            buffer.clear
+          else
+            buffer << line
+          end
+        end
+
+        if buffer.length > 0
+          ko = parse_ko_buffer(buffer)
+          @db[ko.kegg_id] = ko.to_json
+        end
+      end # end parse method
+
+      def parse_ko_buffer(buffer)
+        ko = MgNu::Kegg::Ontology.new
+        while buffer.length > 0
+          line = buffer.shift
+          if line =~ /^ENTRY\s+(\S+)\s/
+            ko.kegg_id = $1
+          elsif line =~ /^NAME\s+(.+)/
+            ko.name = $1
+          elsif line =~ /^DEFINITION\s+(.+)/
+            ko.definition = $1
+            while buffer.length > 0
+              dline = buffer.shift
+              if dline =~ /^(?:CLASS|DBLINKS|GENES)/
+                buffer.unshift(dline)
+                break
+              else
+                ko.definition += dline
+              end
+            end
+          elsif line =~ /^CLASS\s+(.+)/
+            class_str = $1 + " "
+            while buffer.length > 0
+              cline = buffer.shift
+              if cline =~ /^(?:DBLINKS|GENES)/
+                buffer.unshift(cline)
+                break
+              else
+                class_str += cline + " "
+              end
+            end
+
+            re = /\s*(.+?)\[PATH:(ko\d+)\]\s*/
+            re.global_match(class_str) do |m|
+              ko.classes << MgNu::Kegg::Ontology::KeggClass.new(:pathway => m[2], :description => m[1])
+            end
+            if ko.classes.length == 0
+              ko.classes << MgNu::Kegg::Ontology::KeggClass.new(:pathway => "unknown", :description => class_str)
+            end
+          elsif line =~ /^DBLINKS\s+(.+):\s(.+)/
+            database = $1
+            names = $2.split(/\s+/)
+            while buffer.length > 0
+              dline = buffer.shift
+              if dline =~ /^GENES\s+(.+):\s(.+)/
+                buffer.unshift(dline)
+                break
+              elsif dline =~ /\s+(.+):\s(.+)/ # new db
+                names.flatten.each do |n|
+                  next if n == ""
+                  ko.dblinks << MgNu::Kegg::Ontology::Dblink.new(:name => n, :database => database)
+                end
+                database = $1
+                names = $2.split(/\s+/)
+              else
+                names << dline.split(/\s+/)
+              end
+            end
+            names.flatten.each do |n|
+              next if n == ""
+              ko.dblinks << MgNu::Kegg::Ontology::Dblink.new(:name => n, :database => database)
+            end
+          elsif line =~ /^GENES\s+(.+):\s(.+)/
+            org = $1
+            names = $2.split(/\s+/)
+            while buffer.length > 0
+              gline = buffer.shift
+              if gline =~ /\s+(.+):\s(.+)/
+                names.flatten.each do |n|
+                  next if n == ""
+                  ko.genes << MgNu::Kegg::Ontology::Gene.new(:name => n, :organism => org)
+                end
+                org = $1
+                names = $2.split(/\s+/)
+              else
+                names << gline.split(/\s+/)
+              end
+            end
+            names.flatten.each do |n|
+              next if n == ""
+              ko.genes << MgNu::Kegg::Ontology::Gene.new(:name => n, :organism => org)
+            end
+          end # end if /ENTRY/
+        end # end buffer.each line
+        ko # return the ko object
+      end # end of #parse_ko_buffer(buffer)
+
+      def close
+        @db.close unless @db.nil?
+      end
+    end # end of MgNu::Parser::KeggOntologyIndex class
+  end # end of MgNu::Parser module
+end # end of MgNu module

data/lib/mgnu/parser/pilercr.rb

@@ -0,0 +1,102 @@
+module MgNu
+  module Parser
+    class Pilercr
+      include Enumerable
+
+      attr_reader :file, :filename
+
+      # create a new Pilercr parser
+      def initialize(filename = nil)
+        @filename = filename
+        if filename
+          if File.exists?(filename) and File.readable?(filename)
+            @file = File.open(filename)
+          else
+            raise "\n\n -- No file by that name (#{filename}).  Exiting\n\n"
+            exit(1)
+          end
+        else
+          $stderr.puts "MgNu::Parser::Pilercr.new(): need a filename or an existing file"
+          exit(1)
+        end
+      end
+
+      # override enumerables
+      def each
+        # read by blocks, starting with "Array " lines
+        array_count = 0
+        processed = 0
+        @file.each_line("Array ") do |block|
+          if array_count == processed and array_count > 0 # processed all the blocks in this file
+            break
+          else
+            if block =~ /^pilercr/
+              if block =~ /(\d+) putative/
+                array_count= $1.to_i
+              end
+              next
+            else
+              yield process_buffer(block.split(/\n/))
+              processed += 1
+            end
+          end
+        end
+      end # end of File#each
+
+      def process_buffer(buffer)
+        pilercr = MgNu::Pilercr.new
+        buffer.each do |line|
+          if line =~ /SUMMARY/
+            buffer.clear
+            next
+          end
+          next if line =~ /^\s*$/
+          next if line =~ /^\d+$/
+          next if line =~ /^\s+Pos/
+          next if line =~ /^Array\s*$/
+          next if line =~ /^=+/
+          if line =~ />(.+)/
+            temp = $1.split(/\s+/)
+            if temp.length > 1
+              pilercr.header_name = temp.shift
+              pilercr.header = temp.join(" ")
+            else
+              pilercr.header_name = temp[0]
+              pilercr.header = temp[0]
+            end
+          else
+            temp = line.split(/\s+/)
+            temp.shift # drop empty space
+            if temp.length == 4        # final line with repeat sequence
+              pilercr.total_repeats = temp[0].to_i
+              pilercr.repeat_length = temp[1].to_i
+              pilercr.total_spacers = temp[2].to_i
+              pilercr.repeat_sequence = temp[3]
+            elsif temp.length == 6     # line with unknown spacer length
+              pilercr.repeats << MgNu::Pilercr::Repeat.new(:position      => temp[0].to_i,
+                                                            :length        => temp[1].to_i,
+                                                            :identity      => temp[2].to_f,
+                                                            :spacer_length => temp[3].to_i,
+                                                            :match_line    => temp[4],
+                                                            :spacer        => temp[5])
+            elsif temp.length == 7     # normal repeat line
+              pilercr.repeats << MgNu::Pilercr::Repeat.new(:position      => temp[0].to_i,
+                                                            :length        => temp[1].to_i,
+                                                            :identity      => temp[2].to_f,
+                                                            :spacer_length => temp[3].to_i,
+                                                            :left_flank    => temp[4],
+                                                            :match_line    => temp[5],
+                                                            :spacer        => temp[6])
+
+            else
+              $stderr.puts "WARN: Unknown line format"
+              $stderr.puts line
+            end
+          end # end if/else
+        end # end buffer.each do |line|
+        pilercr
+      end # end process_buffer method
+
+    end # end of MgNu::Parser::Pilercr class
+  end # end of MgNu::Parser module
+end # end of MgNu module

data/lib/mgnu/parser/prodigal.rb

@@ -0,0 +1,170 @@
+# this is a hack of MgNu::Parser::Genbank to deal specifically with
+# prodigal's limited GFF support
+require 'mgnu/genbank/feature'
+require 'mgnu/genbank/location'
+require 'mgnu/genbank/qualifier'
+
+module MgNu
+  module Parser
+    class Prodigal
+
+      attr_reader :file
+      attr_accessor :name, :length, :definition, :features
+
+      include MgNu::Loggable
+      include MgNu::Parser
+      include Enumerable
+
+      # create a new prodigal parser
+      def initialize(filename = nil,debug=false)
+        @debug = debug
+        if filename
+          if File.exists?(filename) and File.readable?(filename)
+            @file = File.open(filename)
+          else
+            error("MgNu::Parser::Prodigal.new(): problems with filename")
+            raise "File doesn't exist or is not readable!"
+          end
+        else
+          error("MgNu::Parser::Prodigal.new(): need a filename")
+          raise "no filename given!"
+        end
+      end
+
+      def each
+        buffer = parse_until(@file,/^\/\//,false)
+        while (buffer.length > 0) do
+          buffer.shift if buffer[0] =~ /^\/\//
+          yield parse(buffer)
+          buffer = parse_until(@file,/^\/\//,false)
+        end
+      end
+
+      def parse(buffer)
+        return if buffer.nil?
+        return if buffer.length == 0
+        if buffer[0] =~ /^DEFINITION\s+(.+)$/
+          pseq = MgNu::Parser::Prodigal::Sequence.new(:definition => $1)
+          if buffer[0] =~ /^DEFINITION\s+seqnum=(\d+);seqlen=(\d+);seqhdr="(.+)\s*";gc_cont=([0-9\.]+);transl_table=(\d+).*$/
+            buffer.shift
+            pseq.seqnum = $1.to_i
+            pseq.length = $2.to_i
+            pseq.seqhdr = $3
+            pseq.gc_cont = $4.to_f
+            pseq.transl_table = $5.to_i
+            pseq.name = pseq.seqhdr.split(/\s+/)[0]
+            #pseq.features = parse_features(buffer)
+            pseq.parse_features(buffer)
+            return pseq 
+          else
+            $stderr.puts "ERROR: unknown format for DEFINITION line"
+            $stderr.puts buffer[0]
+            exit(1)
+          end # end if /DEFINITION/
+        else
+          $stderr.puts "ERROR: buffer didn't begin with DEFINITION"
+          $stderr.puts buffer[0]
+          exit(1)
+        end # end if /DEFINITION/
+      end # end of def parse
+
+      # yielded from MgNu::Parser::Prodigal
+      class Sequence
+        attr_accessor :name, :length, :definition, :features
+        attr_accessor :seqnum, :seqhdr, :gc_cont, :transl_table
+
+        def initialize(options = {})
+          @name = options.has_key?(:name) ? options[:name] : ""
+          @length = options.has_key?(:length) ? options[:length] : ""
+          @definition = options.has_key?(:definition) ? options[:definition] : ""
+          @seqnum = options.has_key?(:seqnum) ? options[:seqnum] : ""
+          @seqhdr = options.has_key?(:seqhdr) ? options[:seqhdr] : ""
+          @gc_cont = options.has_key?(:gc_cont) ? options[:gc_cont] : ""
+          @transl_table = options.has_key?(:transl_table) ? options[:transl_table] : ""
+          @features = Array.new
+        end
+
+
+        def parse_features(buffer)
+          buffer.shift if buffer[0] =~ /^FEATURES/
+          all_features = split_at_features(buffer.join("\n"))
+
+          all_features.each do |feature_str|
+            @features << MgNu::Genbank::Feature.parse(feature_str)
+          end
+        end # end parse_features
+
+        def split_at_features(str)
+          sep = "\001"
+          str.gsub(/\n(\s{5}\S)/, "\n#{sep}\\1").split(sep)      
+        end
+
+        def to_s
+          str = "DEFINITION  seqnum=#{@seqnum};seqlen=#{@length};seqhdr=\"#{@seqhdr}\";gc_cont=#{@gc_cont};transl_table=#{@transl_table}\n"
+          str += "FEATURES             Location/Qualifiers\n"
+          @features.each do |f|
+            str += "#{f.to_s}\n"
+          end
+          str += '//'
+          return str
+        end
+
+      end # end of MgNu::Parser::Prodigal::Sequence class
+    end # end of MgNu::Parser::Prodigal class
+  end # end of MgNu::Parser module
+end # end of MgNu module
+
+__END__
+
+DEFINITION  seqnum=1;seqlen=252779;seqhdr="cn_combo_scaffold_29  length_252779 read_count_231853";gc_cont=66.10;transl_table=11;uses_sd=1
+FEATURES             Location/Qualifiers
+     CDS             complement(<2..85)
+                     /note=";gc_cont=0.619;tscore=4.54;"
+     CDS             529..1245
+                     /note=";gc_cont=0.646;tscore=4.54;"
+     CDS             1322..1747
+                     /note=";gc_cont=0.688;tscore=4.54;"
+
+
+        def to_s
+          str = ""
+          str += ">Feature #{@name}\n"
+          @features.each do |f|
+            locstr = ""
+            if f.location.complement
+              if f.location.stop_continues
+                locstr += "<#{f.location.stop}\t"
+              else
+                locstr += "#{f.location.stop}\t"
+              end
+              if f.location.start_continues
+                locstr += ">#{f.location.start}\t"
+              else
+                locstr += "#{f.location.start}\t"
+              end
+            else
+              if f.location.start_continues
+                locstr += "<#{f.location.start}\t"
+              else
+                locstr += "#{f.location.start}\t"
+              end
+              if f.location.stop_continues
+                locstr += ">#{f.location.stop}\t"
+              else
+                locstr += "#{f.location.stop}\t"
+              end
+            end
+            str += "#{locstr}gene\n"
+            str += "\t\t\tgene\tgene#{count}\n"
+            f.qualifiers.sort.each do |qualifier,q|
+              str += "\t\t\t#{qualifier}\t#{q.value}\n"
+            end
+            str += "#{locstr}CDS\n"
+            f.qualifiers.sort.each do |qualifier,q|
+              str += "\t\t\t#{qualifier}\t#{q.value}\n"
+            end
+            str += "\t\t\tproduct\tgene_#{@number}p\n"
+            str += "\t\t\ttransl_table\t#{@transl_table}\n"
+          end # end of features.each
+          return str
+        end

data/lib/mgnu/parser/sam.rb

@@ -0,0 +1,115 @@
+require 'mgnu/parser/sam/header'
+require 'mgnu/parser/sam/alignment'
+
+module MgNu
+  module Parser
+    class Sam
+      include Enumerable
+      include MgNu::Loggable
+      include MgNu::Parser
+
+      FIELDS = [:name, :flag, :hit, :position, :quality, :cigar, :mate_ref,
+                :mate_pos, :distance, :sequence, :query_qual, :other]
+
+      attr_reader :file, :header #, :alignments
+
+      # create a new SAM file parser
+      def initialize(filename = nil)
+        @header = nil
+        if filename
+          if File.exists?(filename) and File.readable?(filename)
+            # # find all the reference sequences
+            # # skip header lines (^@) and then only save the 3rd column
+            # # from the sam file input, only return unique names, then
+            # # split on newlines
+            # `egrep -v '^@' #{filename} | cut -f3 | uniq`.split(/\n/).each do |ref|
+            #   if @references.has_key?(ref)
+            #     $stderr.puts "Already a reference by name (#{ref})"
+            #     $stderr.puts "... skipping"
+            #   else
+            #     @references[ref] = MgNu::Parser::Sam::Reference.new(:name => ref)
+            #   end
+            # end
+            
+            @file = File.open(filename)
+          end # end of exists and readable file checks
+        else
+          error "MgNu::Parser::Sam.new(): need a SAM file"
+          exit(1)
+        end # end of if/else filename
+      end
+
+      # override enumerables
+      # MgNu::Parser::Sam will emit a reference-object with every
+      # iteration.  Iteration happens with file-reading.
+      def each
+        header_buffer = Array.new
+        # short-term buffer hash
+        alignment_buffer = Hash.new
+        
+        @file.each do |line|
+          next if line =~ /^\s*$/
+          line.chomp!
+          if line =~ /^@/
+            header_buffer << line
+          else
+            if header_buffer.length > 0
+              @header = process_header(header_buffer)
+              header_buffer.clear
+            end
+            alignment_attrs = Hash[*FIELDS.zip(line.split("\t"))]
+            # TODO last field needs to be globbed into array
+            alignment = MgNu::Parser::Sam::Alignment.new(alignment_attrs)
+            next unless alignment.matched_and_paired?
+            key = alignment.first_read? ? :first : :second
+            if !alignment_buffer[alignment.basename]
+              alignment_buffer[alignment.basename] = { key => alignment }
+            else
+              read_pair = alignment_buffer[alignment.basename] 
+              read_pair[key] = alignment
+              yield MgNu::Parser::Sam::Pair.new(name, read_pair[:first], read_pair[:second])
+              alignment_buffer.delete(alignment.basename)
+            end
+          end
+        end
+      end
+
+      def process_header(buffer)
+        hdr = MgNu::Parser::Sam::Header.new
+        buffer.each do |line|
+          case line
+          when /^@HD/
+            if line =~ /VN:(.+)[\s\n]/
+              hdr.vn = $1
+            end
+            if line =~ /SO:(.+)[\s\n]/
+              hdr.so = $1
+            end
+          when /^@SQ/
+            ref = nil
+            if line =~ /SN:(.+)[\s\n]/
+              # verify this ref is in the @references hash (from
+              # initialize()
+              if @references.has_key?($1)
+                ref = @references[$1]
+              else
+                $stderr.puts "WARNING: reference from header not found in alignments"
+                # create a ref
+                ref = MgNu::Parser::Sam::Reference.new(:name => $1)
+                @references[$1] = ref
+              end
+            end
+
+            if line =~ /LN:(\d+)[\s\n]/
+              if ref
+                ref.ln = $1.to_i
+              end
+            end
+          end
+        end
+        return hdr
+      end # end process_header_line
+    end # end of MgNu::Parser::Sam class
+  end # end of MgNu::Parser module
+end # end of MgNu module
+__END__