external 0.1.0

data/lib/external/base.rb

@@ -0,0 +1,85 @@
+require 'external/io'
+require 'external/chunkable'
+require 'external/enumerable'
+require 'tempfile'
+
+module External
+  
+  #--
+  # Base provides the basic array functionality shared by ExtArr and Index,
+  # essentially wrapping the IO functions required to access and utilized external
+  # array data with the standard array functions.  Bases can be opened with 
+  # in any of the IO modes; the capabilities of Base will be reduced accordingly
+  # (ie read-only Bases cannot write values using []=, for instance).  
+  #
+  # It is VERY IMPORTANT to realize that the underlying IO will be opened using the
+  # given mode.  The 'w' mode will overwrite all existing data; 'r+' is a safer mode
+  # for full read-write functionality.  Note that since Base actively scans over
+  # the IO, append modes essentially behaves like write, but does not overwrite existing
+  # data.
+  #
+  # To work properly, Base must be subclassed with methods:
+  # * length
+  # * io_fetch
+  #++
+  #
+  #
+  class Base
+    class << self
+      def open(fd=nil, mode="r", options={})
+        fd = File.open(fd, mode) unless fd == nil
+        ab = self.new(fd, options)
+        
+        if block_given?
+          begin
+            yield(ab)
+          ensure
+            ab.close
+          end
+        else
+          ab
+        end
+      end
+    end
+    
+    include External::Enumerable  
+    include External::Chunkable
+ 
+    attr_reader :io
+    
+    # Initializes a new Base given the file descriptor, mode and options.
+    # (see open_io for details on what io is opened for a given file descriptor)
+    #
+    # If mode contains an 's', then the Base will be initialized in strio 
+    # mode where the underlying IO will be a StringIO.  In this case the fd 
+    # will be used as the string to initialize the StringIO.
+    #
+    # Standard options for Base include:
+    # nil_value:: the value written to file for nils, and converted to nil on read
+    #             (default ' ')
+    # max_gap:: the maximum gap size used by Offset (default 10000)
+    # max_chunk_size:: the chunk size used by Offset (default 1M)
+    def initialize(io=nil)
+      self.io = (io.nil? ? Tempfile.new("array_base") : io)
+    end
+ 
+    # True if io is closed.
+    def closed?
+      io.closed?
+    end
+  
+    # Closes io.
+    def close
+      io.close unless io.closed?
+    end
+    
+    protected
+    
+    # Sets io and extends the input io with External::Position.
+    def io=(io)
+      io.extend External::IO unless io.kind_of?(External::IO)
+      @io = io
+    end
+
+  end
+end

data/lib/external/chunkable.rb

@@ -0,0 +1,105 @@
+module External
+  
+  # The Chunkable mixin provides methods for organizing a span or range
+  # into chunks no larger than a specified block size. 
+  module Chunkable
+    attr_accessor :length, :default_blksize
+    
+    # Returns the default span: [0, length]
+    def default_span
+      [0, length]
+    end
+
+    # Breaks the input range or span into chunks of blksize or less.  
+    # The offset and length of each chunk will be provided to the 
+    # block, if given.
+    #
+    #   blksize           # => 100
+    #   chunk(0..250)     # => [[0,100],[100,100],[200,50]]
+    #
+    #   results = []
+    #   chunk([10,190]) {|offset, length| results << [offset, length]}
+    #   results           # => [[10,100],[110,90]]
+    #
+    def chunk(range_or_span=default_span, blksize=default_blksize)
+      return collect_results(:chunk, range_or_span) unless block_given?
+      
+      rbegin, rend = range_begin_and_end(range_or_span)
+      
+      # chunk the final range to make sure that no chunks  
+      # greater than blksize are returned
+      while rend - rbegin > blksize 
+        yield(rbegin, blksize)
+        rbegin += blksize
+      end
+      yield(rbegin, rend - rbegin) if rend - rbegin > 0
+    end
+    
+    # Breaks the input range or span into chunks of blksize or less,
+    # beginning from the end of the interval.  The offset and length 
+    # of each chunk will be provided to the block, if given.
+    #
+    #   blksize                   # => 100
+    #   reverse_chunk(0..250)     # => [[150,100],[50,100],[0,50]]
+    #
+    #   results = []
+    #   reverse_chunk([10,190]) {|offset, length| results << [offset, length]}
+    #   results                   # => [[100,100],[10,90]]
+    #
+    def reverse_chunk(range_or_span=default_span, blksize=default_blksize)
+      return collect_results(:reverse_chunk, range_or_span) unless block_given?
+    
+      rbegin, rend = range_begin_and_end(range_or_span)
+
+      # chunk the final range to make sure that no chunks  
+      # greater than blksize are returned
+      while rend - rbegin > blksize 
+        rend -= blksize
+        yield(rend, blksize)
+      end
+      yield(rbegin, rend - rbegin) if rend - rbegin > 0
+    end
+
+    protected
+    
+    # Converts a range into an offset and length.  Negative values are
+    # counted back from self.length
+    #
+    #   length                # => 10
+    #   split_range(0..9)     # => [0,10]
+    #   split_range(0...9)    # => [0,9]
+    #
+    #   split_range(-1..9)    # => [9,1]
+    #   split_range(0..-1)    # => [0,10]
+    def split_range(range)
+      begin_range = range.begin + (range.begin < 0 ? self.length : 0)
+      end_range = range.end + (range.end < 0 ? self.length : 0)
+      length = end_range - begin_range - (range.exclude_end? ? 1 : 0)
+
+      [begin_range, length]
+    end
+    
+    def split_span(span)
+      span[0] += self.length if span[0] < 0
+      span
+    end
+    
+    def range_begin_and_end(range_or_span)
+      rbegin, rend = range_or_span.kind_of?(Range) ? split_range(range_or_span) : split_span(range_or_span)
+      raise ArgumentError.new("negative offset specified: #{PP.singleline_pp(range_or_span,'')}") if rbegin < 0
+      rend += rbegin
+      
+      [rbegin, rend]
+    end
+
+    private
+    
+    def collect_results(method, args) # :nodoc:
+      results = []
+      send(method, args) do |*result|
+        results << result
+      end
+      results
+    end
+  end
+end

data/lib/external/enumerable.rb

@@ -0,0 +1,137 @@
+require 'enumerator'
+
+module External
+  module Enumerable
+    # def all? # :yield: obj
+    #   not_implemented
+    # end
+    
+    # def any? # :yield: obj
+    #   not_implemented
+    # end
+    
+    # def collect # :yield: item
+    #   not_implemented
+    # end
+    
+    # def collect! # :yield: item
+    #   not_implemented
+    # end
+    
+    # def detect(ifnone=nil) # :yield: obj
+    #   not_implemented
+    # end
+    
+    # def each_cons(n) # :yield:
+    #   not_implemented
+    # end
+    
+    # def each_slice(n) # :yield:
+    #   not_implemented
+    # end
+    
+    def each_with_index(&block)
+      chunk do |offset, length|
+        self[offset, length].each_with_index do |item, i|
+          yield(item, i + offset)
+        end
+      end
+    end
+    
+    # def entries
+    #   to_a
+    # end
+    
+    # def enum_cons(n)
+    #   not_implemented
+    # end
+    
+    # def enum_slice(n)
+    #   not_implemented
+    # end
+    
+    # def enum_with_index
+    #   not_implemented
+    # end
+    
+    # def find(ifnone=nil, &block) # :yield: obj
+    #   detect(ifnone, &block)
+    # end
+    
+    # def find_all # :yield: obj
+    #   not_implemented
+    # end
+    
+    # def grep(pattern) # :yield: obj
+    #   not_implemented
+    # end
+    
+    # def include?(obj)
+    #   not_implemented
+    # end
+    
+    # def inject(init) # :yield: memo, obj
+    #   not_implemented
+    # end
+    
+    # def map(&block) # :yield: item
+    #   collect(&block)
+    # end
+    
+    # def map!(&block) # :yield: item
+    #   collect!(&block)
+    # end
+    
+    # def max # :yield: a,b
+    #   not_implemented
+    # end
+    
+    # def member?(obj)
+    #   include?(obj)
+    # end
+    
+    # def min # :yield: a,b
+    #   not_implemented
+    # end
+    
+    # def partition # :yield: obj
+    #   not_implemented
+    # end
+    
+    # def reject # :yield: item
+    #   not_implemented
+    # end
+    
+    # def reject! # :yield: item
+    #   not_implemented
+    # end
+    
+    # def select(&block) # :yield: obj
+    #   find_all(&block)
+    # end
+    
+    # def sort # :yield: a,b
+    #   not_implemented
+    # end
+    
+    # def sort! # :yield: a,b
+    #   not_implemented
+    # end
+    
+    # def sort_by # :yield: obj
+    #   not_implemented
+    # end
+    
+    # def to_a
+    #   not_implemented
+    # end
+    
+    # def to_set(klass=Set, *args, &block)
+    #   not_implemented
+    # end
+
+    # def zip(*arg) # :yield: arr
+    #   not_implemented
+    # end
+  end
+end