bio 1.2.1 → 1.3.0

data/lib/bio/io/flatfile/buffer.rb

@@ -0,0 +1,237 @@
+#
+# = bio/io/flatfile/buffer.rb - Input stream buffer for FlatFile
+#
+#   Copyright (C) 2001-2006 Naohisa Goto <ng@bioruby.org>
+#
+# License:: The Ruby License
+#
+#  $Id:$
+#
+#
+# See documents for Bio::FlatFile::BufferedInputStream and Bio::FlatFile.
+#
+
+require 'bio/io/flatfile'
+
+module Bio
+
+  class FlatFile
+
+    # Wrapper for a IO (or IO-like) object.
+    # It can input with a buffer.
+    class BufferedInputStream
+      # Creates a new input stream wrapper
+      def initialize(io, path)
+        @io = io
+        @path = path
+        # initialize prefetch buffer
+        @buffer = ''
+      end
+
+      # Creates a new input stream wrapper from the given IO object.
+      def self.for_io(io)
+        begin
+          path = io.path
+        rescue NameError
+          path = nil
+        end
+        self.new(io, path)
+      end
+
+      # Creates a new input stream wrapper to open file _filename_
+      # by using File.open.
+      # *arg is passed to File.open.
+      #
+      # Like File.open, a block can be accepted.
+      def self.open_file(filename, *arg)
+        if block_given? then
+          File.open(filename, *arg) do |fobj|
+            yield self.new(fobj, filename)
+          end
+        else
+          fobj = File.open(filename, *arg)
+          self.new(fobj, filename)
+        end
+      end
+
+      # Creates a new input stream wrapper from URI specified as _uri_.
+      # by using OpenURI.open_uri or URI#open.
+      # _uri_ must be a String or URI object.
+      # *arg is passed to OpenURI.open_uri or URI#open.
+      #
+      # Like OpenURI.open_uri, it can accept a block.
+      def self.open_uri(uri, *arg)
+        if uri.kind_of?(URI)
+          if block_given?
+            uri.open(*arg) do |fobj|
+              yield self.new(fobj, uri.to_s)
+            end
+          else
+            fobj = uri.open(*arg)
+            self.new(fobj, uri.to_s)
+          end
+        else
+          if block_given?
+            OpenURI.open_uri(uri, *arg) do |fobj|
+              yield self.new(fobj, uri)
+            end
+          else
+            fobj = OpenURI.open_uri(uri, *arg)
+            self.new(fobj, uri)
+          end
+        end
+      end
+
+      # Pathname, filename or URI to open the object.
+      # Like File#path, returned value isn't normalized.
+      attr_reader :path
+
+      # Converts to IO object if possible
+      def to_io
+        @io.to_io
+      end
+
+      # Closes the IO object if possible
+      def close
+        @io.close
+      end
+
+      # Rewinds the IO object if possible
+      # Internal buffer in this wrapper is cleared.
+      def rewind
+        r = @io.rewind
+        @buffer = ''
+        r
+      end
+
+      # Returns current file position
+      def pos
+        @io.pos - @buffer.size
+      end
+
+      # Sets current file position if possible
+      # Internal buffer in this wrapper is cleared.
+      def pos=(p)
+        r = (@io.pos = p)
+        @buffer = ''
+        r
+      end
+
+      # Returns true if end-of-file. Otherwise, returns false.
+      #
+      # Note that it returns false if internal buffer is this wrapper
+      # is not empty,
+      def eof?
+        if @buffer.size > 0
+          false
+        else
+          @io.eof?
+        end
+      end
+
+      # Same as IO#gets.
+      #
+      # Compatibility note: the bahavior of paragraph mode (io_rs = '')
+      # may differ from that of IO#gets('').
+      def gets(io_rs = $/)
+        if @buffer.size > 0
+          if io_rs == nil then
+            r = @buffer + @io.gets(nil).to_s
+            @buffer = ''
+          else
+            if io_rs == '' then # io_rs.empty?
+              sp_rs = /((?:\r?\n){2,})/n
+            else
+              sp_rs = io_rs
+            end
+            a = @buffer.split(sp_rs, 2)
+            if a.size > 1 then
+              r = a.shift
+              r += (io_rs.empty? ? a.shift : io_rs)
+              @buffer = a.shift.to_s
+            else
+              @buffer << @io.gets(io_rs).to_s
+              a = @buffer.split(sp_rs, 2)
+              if a.size > 1 then
+                r = a.shift
+                r += (io_rs.empty? ? a.shift : io_rs)
+                @buffer = a.shift.to_s
+              else
+                r = @buffer
+                @buffer = ''
+              end
+            end
+          end
+          r
+        else
+          @io.gets(io_rs)
+        end
+      end
+
+      # Pushes back given str to the internal buffer.
+      # Returns nil.
+      # str must be read previously with the wrapper object.
+      #
+      # Note that in current implementation, the str can be everything,
+      # but please don't depend on it.
+      #
+      def ungets(str)
+        @buffer = str + @buffer
+        nil
+      end
+
+      # Same as IO#getc.
+      def getc
+        if @buffer.size > 0 then
+          r = @buffer[0]
+          @buffer = @buffer[1..-1]
+        else
+          r = @io.getc
+        end
+        r
+      end
+      
+      # Pushes back one character into the internal buffer.
+      # Unlike IO#getc, it can be called more than one time.
+      def ungetc(c)
+        @buffer = sprintf("%c", c) + @buffer
+        nil
+      end
+
+      # Gets current prefetch buffer
+      def prefetch_buffer
+        @buffer
+      end
+
+      # It does @io.gets,  and addes returned string
+      # to the internal buffer, and returns the string.
+      def prefetch_gets(*arg)
+        r = @io.gets(*arg)
+        @buffer << r if r
+        r
+      end
+      
+      # It does @io.readpartial, and addes returned string
+      # to the internal buffer, and returns the string.
+      def prefetch_readpartial(*arg)
+        r = @io.readpartial(*arg)
+        @buffer << r if r
+        r
+      end
+
+      # Skips space characters in the stream.
+      # returns nil.
+      def skip_spaces
+        ws = { ?\s => true, ?\n => true, ?\r => true, ?\t => true }
+        while r = self.getc
+          unless ws[r] then
+            self.ungetc(r)
+            break
+          end
+        end
+        nil
+      end
+    end #class BufferedInputStream
+
+  end #class FlatFile
+end #module Bio

data/lib/bio/io/flatfile/index.rb

@@ -888,13 +888,18 @@ module Bio
           self
         end
 
-        def self.external_sort_proc(sort_program = '/usr/bin/sort')
+        def self.external_sort_proc(sort_program = [ '/usr/bin/env', 
+                                                     'LC_ALL=C',
+                                                     '/usr/bin/sort' ])
           Proc.new do |out, in1, *files|
-            system(sort_program, '-o', out, in1, *files)
+            cmd = sort_program + [ '-o', out, in1, *files ]
+            system(*cmd)
           end
         end
 
-        def self.external_merge_sort_proc(sort_program = '/usr/bin/sort')
+        def self.external_merge_sort_proc(sort_program = [ '/usr/bin/env', 
+                                                           'LC_ALL=C',
+                                                           '/usr/bin/sort' ])
           Proc.new do |out, in1, *files|
             # (in1 may be sorted)
             tf_all = []
@@ -902,21 +907,26 @@ module Bio
             files.each do |fn|
               tf = Tempfile.open('sort')
               tf.close(false)
-              system(sort_program, '-o', tf.path, fn)
+              cmd = sort_program + [ '-o', tf.path, fn ]
+              system(*cmd)
               tf_all << tf
               tfn_all << tf.path
             end
-            system(sort_program, '-m', '-o', out, in1, *tfn_all)
+            cmd_fin = sort_program + [ '-m', '-o', out, in1, *tfn_all ]
+            system(*cmd_fin)
             tf_all.each do |tf|
               tf.close(true)
             end
           end
         end
 
-        def self.external_merge_proc(sort_program = '/usr/bin/sort')
+        def self.external_merge_proc(sort_program =  [ '/usr/bin/env', 
+                                                       'LC_ALL=C',
+                                                       '/usr/bin/sort' ])
           Proc.new do |out, in1, *files|
             # files (and in1) must be sorted
-            system(sort_program, '-m', '-o', out, in1, *files)
+            cmd = sort_program + [ '-m', '-o', out, in1, *files ]
+            system(*cmd)
           end
         end
 

data/lib/bio/io/flatfile/indexer.rb

@@ -525,6 +525,8 @@ module Bio
       def self.addindex_flat(db, mode, need_update, parser, options)
         require 'tempfile'
         prog = options['sort_program']
+        env = options['env_program']
+        env_args = options['env_program_arguments']
 
         return false if need_update.to_a.size == 0
 
@@ -555,7 +557,7 @@ module Bio
           fileid += 1
         end
 
-        sort_proc = chose_sort_proc(prog, mode)
+        sort_proc = chose_sort_proc(prog, mode, env, env_args)
         pfile.close(false)
         DEBUG.print "sorting primary (#{parser.primary.name})...\n"
         db.primary.file.import_tsv_files(true, mode, sort_proc, pfile.path)
@@ -571,30 +573,46 @@ module Bio
         true
       end #def
 
+      # default sort program
       DEFAULT_SORT = '/usr/bin/sort'
-      def self.chose_sort_proc(prog, mode = :new)
+
+      # default env program (run a program in a modified environment)
+      DEFAULT_ENV = '/usr/bin/env'
+
+      # default arguments for env program
+      DEFAULT_ENV_ARGS = [ 'LC_ALL=C' ]
+
+      def self.chose_sort_proc(prog, mode = :new,
+                               env = nil, env_args = nil)
         case prog
         when /^builtin$/i, /^hs$/i, /^lm$/i
           DEBUG.print "sort: internal sort routine\n"
-          sort_proc = mapfile.internal_sort_proc
+          sort_proc = Flat_1::FlatMappingFile::internal_sort_proc
         when nil, ''
           if FileTest.executable?(DEFAULT_SORT)
-            DEBUG.print "sort: #{DEFAULT_SORT}\n"
-            if mode == :new then
-              sort_proc = Flat_1::FlatMappingFile::external_sort_proc(DEFAULT_SORT)
-            else
-              sort_proc = Flat_1::FlatMappingFile::external_merge_sort_proc(DEFAULT_SORT)
-            end
+            return chose_sort_proc(DEFAULT_SORT, mode, env, env_args)
           else
             DEBUG.print "sort: internal sort routine\n"
             sort_proc = Flat_1::FlatMappingFile::internal_sort_proc
           end
         else
-          DEBUG.print "sort: #{prog}\n"
+          env_args ||= DEFAULT_ENV_ARGS
+          if env == '' or env == false then # inhibit to use env program
+            prefixes = [ prog ]
+          elsif env then # uses given env program
+            prefixes = [ env ] + env_args + [ prog ]
+          else # env == nil; uses default env program if possible
+            if FileTest.executable?(DEFAULT_ENV)
+              prefixes = [ DEFAULT_ENV ] + env_args + [ prog ]
+            else
+              prefixes = [ prog ]
+            end
+          end
+          DEBUG.print "sort: #{prefixes.join(' ')}\n"
           if mode == :new then
-            sort_proc = Flat_1::FlatMappingFile::external_sort_proc(prog)
+            sort_proc = Flat_1::FlatMappingFile::external_sort_proc(prefixes)
           else
-            sort_proc = Flat_1::FlatMappingFile::external_merge_sort_proc(prog)
+            sort_proc = Flat_1::FlatMappingFile::external_merge_sort_proc(prefixes)
           end
         end
         sort_proc

data/lib/bio/io/flatfile/splitter.rb

@@ -0,0 +1,297 @@
+#
+# = bio/io/flatfile/splitter.rb - input data splitter for FlatFile
+#
+#   Copyright (C) 2001-2008 Naohisa Goto <ng@bioruby.org>
+#
+# License:: The Ruby License
+#
+#  $Id:$
+#
+#
+# See documents for Bio::FlatFile::Splitter and Bio::FlatFile.
+#
+
+require 'bio/io/flatfile'
+
+module Bio
+
+  class FlatFile
+
+    # The Bio::FlatFile::Splitter is a namespace for flatfile splitters.
+    # Each splitter is a class to get entries from a buffered input stream.
+    #
+    # It is internally called in Bio::FlatFile.
+    # Normally, users do not need to use it directly.
+    module Splitter
+
+      # This is a template of splitter.
+      class Template
+        # Creates a new splitter.
+        def initialize(klass, bstream)
+          @dbclass = klass
+          @stream = bstream
+          @entry_pos_flag = nil
+        end
+
+        # skips leader of the entry.
+        def skip_leader
+          raise NotImplementedError
+        end
+
+        # rewind the stream
+        def rewind
+          @stream.rewind
+        end
+
+        # Gets entry as a string. (String)
+        def get_entry
+          raise NotImplementedError
+        end
+
+        # Gets entry as a data class's object
+        def get_parsed_entry
+          ent = get_entry
+          if ent then
+            self.parsed_entry = dbclass.new(ent)
+          else
+            self.parsed_entry = ent
+          end
+          parsed_entry
+        end
+
+        # the last entry string read from the stream (String)
+        attr_reader :entry
+
+        # The last parsed entry read from the stream (entry data class).
+        # Note that it is valid only after get_parsed_entry is called,
+        # and the get_entry may not affect the parsed_entry attribute.
+        attr_reader :parsed_entry
+
+        # a flag to write down entry start and end positions
+        attr_accessor :entry_pos_flag
+
+        # start position of the entry
+        attr_reader :entry_start_pos
+
+        # (end position of the entry) + 1
+        attr_reader :entry_ended_pos
+
+        #--
+        #private
+        #
+        ## to prevent warning message "warning: private attribute?",
+        ## private attributes are explicitly declared.
+        #++
+
+        # entry data class
+        attr_reader :dbclass
+        private     :dbclass
+
+        # input stream
+        attr_reader :stream
+        private     :stream
+
+        # the last entry string read from the stream
+        attr_writer :entry
+        private     :entry=
+
+        # the last entry as a parsed data object
+        attr_writer :parsed_entry
+        private     :parsed_entry=
+
+        # start position of the entry
+        attr_writer :entry_start_pos
+        private     :entry_start_pos=
+
+        # (end position of the entry) + 1
+        attr_writer :entry_ended_pos
+        private     :entry_ended_pos=
+
+        # Does stream.pos if entry_pos_flag is not nil.
+        # Otherwise, returns nil.
+        def stream_pos
+          entry_pos_flag ? stream.pos : nil
+        end
+        private :stream_pos
+      end #class Template
+
+      # Default splitter.
+      # It sees following constants in the given class.
+      # DELIMITER:: (String) delimiter indicates the end of a entry.
+      # FLATFILE_HEADER:: (String) start of a entry, located on head of a line.
+      # DELIMITER_OVERRUN:: (Integer) excess read size included in DELIMITER.
+      #
+      class Default < Template
+        # Creates a new splitter.
+        # klass:: database class
+        # bstream:: input stream. It must be a BufferedInputStream object.
+        def initialize(klass, bstream)
+          super(klass, bstream)
+
+          @delimiter = klass::DELIMITER rescue nil
+          @header = klass::FLATFILE_HEADER rescue nil
+          # for specific classes' benefit
+          unless header
+            if (defined?(Bio::GenBank) and klass == Bio::GenBank) or
+                (defined?(Bio::GenPept) and klass == Bio::GenPept)
+              @header = 'LOCUS '
+            end
+          end
+          @delimiter_overrun = klass::DELIMITER_OVERRUN rescue nil
+        end
+
+        # (String) delimiter indicates the end of a entry.
+        attr_accessor :delimiter
+
+        # (String) start of a entry, located on head of a line.
+        attr_accessor :header
+
+        # (Integer) excess read data size included in delimiter.
+        attr_accessor :delimiter_overrun
+        
+        # Skips leader of the entry.
+        #
+        # If @header is not nil, it reads till the contents of @header
+        # comes at the head of a line.
+        # If correct FLATFILE_HEADER is found, returns true.
+        # Otherwise, returns nil.
+        def skip_leader
+          if @header then
+            data = ''
+            while s = stream.gets(@header)
+              data << s
+              if data.split(/[\r\n]+/)[-1] == @header then
+                stream.ungets(@header)
+                return true
+              end
+            end
+            # @header was not found. For safety,
+            # pushes back data with removing white spaces in the head.
+            data.sub(/\A\s+/, '')
+            stream.ungets(data)
+            return nil
+          else
+            stream.skip_spaces
+            return nil
+          end
+        end
+
+        # gets a entry
+        def get_entry
+          p0 = stream_pos()
+          e  = stream.gets(@delimiter)
+          if e and @delimiter_overrun then
+            if e[-@delimiter.size, @delimiter.size ] == @delimiter then
+              overrun = e[-@delimiter_overrun, @delimiter_overrun]
+              e[-@delimiter_overrun, @delimiter_overrun] = ''
+              stream.ungets(overrun)
+            end
+          end
+          p1 = stream_pos()
+          self.entry_start_pos = p0
+          self.entry = e
+          self.entry_ended_pos = p1
+          return entry
+        end
+      end #class Defalult
+
+
+      # A splitter for line oriented text data.
+      #
+      # The given class's object must have following methods.
+      #   Klass#add_header_line(line)
+      #   Klass#add_line(line)
+      # where 'line' is a string. They normally returns self.
+      # If the line is not suitable to add to the current entry,
+      # nil or false should be returned.
+      # Then, the line is treated as (for add_header_line) the entry data
+      # or (for add_line) the next entry's data.
+      #
+      class LineOriented < Template
+        # Creates a new splitter.
+        # klass:: database class
+        # bstream:: input stream. It must be a BufferedInputStream object.
+        def initialize(klass, bstream)
+          super(klass, bstream)
+          self.flag_to_fetch_header = true
+        end
+
+        # do nothing
+        def skip_leader
+          nil
+        end
+
+        # get an entry and return the entry as a string
+        def get_entry
+          if e = get_parsed_entry then
+            entry
+          else
+            e
+          end
+        end
+
+        # get an entry and return the entry as a data class object
+        def get_parsed_entry
+          p0 = stream_pos()
+          ent = @dbclass.new()
+
+          lines = []
+          line_overrun = nil
+
+          if flag_to_fetch_header then
+            while line = stream.gets("\n")
+              unless ent.add_header_line(line) then
+                line_overrun = line
+                break
+              end
+              lines.push line
+            end
+            stream.ungets(line_overrun) if line_overrun
+            line_overrun = nil
+            self.flag_to_fetch_header = false
+          end
+              
+          while line = stream.gets("\n")
+            unless ent.add_line(line) then
+              line_overrun = line
+              break
+            end
+            lines.push line
+          end
+          stream.ungets(line_overrun) if line_overrun
+          p1 = stream_pos()
+
+          return nil if lines.empty?
+
+          self.entry_start_pos = p0
+          self.entry = lines.join('')
+          self.parsed_entry = ent
+          self.entry_ended_pos = p1
+
+          return ent
+        end
+
+        # rewinds the stream
+        def rewind
+          ret = super
+          self.flag_to_fetch_header = true
+          ret
+        end
+
+        #--
+        #private methods / attributes
+        #++
+
+        # flag to fetch header
+        attr_accessor :flag_to_fetch_header
+        private       :flag_to_fetch_header
+        private       :flag_to_fetch_header=
+
+      end #class LineOriented
+
+    end #module Splitter
+
+  end #class FlatFile
+end #module Bio
+
+