parslet 1.1.1 → 1.2.0

data/lib/parslet/slice.rb

@@ -0,0 +1,172 @@
+
+# A slice is a small part from the parse input. A slice mainly behaves like
+# any other string, except that it remembers where it came from (offset in
+# original input). 
+#
+# Some slices also know what parent slice they are a small part of. This
+# allows the slice to be concatenated to other slices from the same buffer by
+# reslicing it against that original buffer. 
+#
+# Why the complexity? Slices allow retaining offset information. This will
+# allow to assign line and column to each small bit of output from the parslet
+# parser. Also, while we keep that information, we might as well try to do
+# something useful with it. Reslicing the same buffers should in theory keep
+# buffer copies and allocations down. 
+#
+# == Extracting line and column
+#
+# Using the #line_and_column method, you can extract the line and column in
+# the original input where this slice starts. 
+#
+# Example: 
+#   slice.line_and_column # => [1, 13]
+#   slice.offset          # => 12
+#
+# == Likeness to strings
+#
+# Parslet::Slice behaves in many ways like a Ruby String. This likeness
+# however is not complete - many of the myriad of operations String supports
+# are not yet in Slice. You can always extract the internal string instance by
+# calling #to_s.
+#
+# These omissions are somewhat intentional. Rather than maintaining a full
+# delegation, we opt for a partial emulation that gets the job done. 
+#
+# Note also that there are some things that work with strings that will never
+# work when using slices. For instance, you cannot concatenate slices that
+# aren't from the same source or that don't join up: 
+#
+# Example: 
+#   big_slice = 'abcdef'
+#   a = big_slice.slice(0, 2)   # => "ab"@0
+#   b = big_slice.slice(4, 2)   # => "ef"@4
+#   
+#   a + b # raises Parslet::InvalidSliceOperation
+#
+# This avoids creating slices with impossible offsets or that are
+# discontinous. 
+#
+class Parslet::Slice
+  attr_reader :str, :offset
+  attr_reader :parent
+  attr_reader :source
+  
+  def initialize(string, offset, source=nil, parent=nil)
+    @str, @offset = string, offset
+    @source = source
+    @parent = parent
+  end
+  
+  # Compares slices to other slices or strings. 
+  #
+  def == other
+    str == other
+  end
+  
+  # Match regular expressions. 
+  # 
+  def match(regexp)
+    str.match(regexp)
+  end
+  
+  # Returns a slice that starts at offset start and that has length characters.
+  # Whenever possible, return parts of the parent buffer that this slice was
+  # cut out of.
+  #
+  def slice(start, length)
+    # NOTE: At a later stage, we might not want to create huge trees of slices. 
+    # The fact that the root of the tree creates slices that link to it makes
+    # the tree already rather flat. 
+    
+    if parent
+      parent.slice(offset - parent.offset, length)
+    else
+      self.class.new(str.slice(start, length), offset+start, source, self)
+    end
+  end
+  
+  # Returns a slice that starts at file offset start and that has length
+  # characters in it. 
+  #
+  def abs_slice(start, length)
+    slice(start-offset, length)
+  end
+  
+  # True if this slice can satisfy an original input request to the 
+  # range ofs, len.
+  #
+  def satisfies?(ofs, len)
+    ofs >= offset && (ofs-offset+len-1)<str.size
+  end
+  
+  def size
+    str.size
+  end
+  def +(other)
+    raise ArgumentError, 
+      "Cannot concat something other than a slice to a slice." \
+        unless other.respond_to?(:to_slice)
+          
+    raise Parslet::InvalidSliceOperation, 
+      "Cannot join slices that aren't adjacent."+
+      " (#{self.inspect} + #{other.inspect})" \
+        if offset+size != other.offset
+          
+    raise Parslet::InvalidSliceOperation, "Not from the same source." \
+      if source != other.source
+       
+    # If both slices stem from the same bigger buffer, we can reslice that 
+    # buffer to (probably) avoid a buffer copy, as long as the strings are
+    # not modified. 
+    if parent && parent == other.parent
+      return parent.abs_slice(offset, size+other.size)
+    end
+    
+    self.class.new(str + other.str, offset, source)
+  end
+  
+  # Returns a <line, column> tuple referring to the original input. 
+  #
+  def line_and_column
+    raise ArgumentError, "No source was given, cannot infer line and column." \
+      unless source
+        
+    source.line_and_column(self.offset)
+  end
+
+    
+  # Conversion operators -----------------------------------------------------
+  def to_str
+    str
+  end
+  alias to_s to_str
+  
+  def to_slice
+    self
+  end
+  def to_sym
+    str.to_sym
+  end
+  def to_int
+    Integer(str)
+  end
+  def to_i
+    str.to_i
+  end
+  def to_f
+    str.to_f
+  end
+  
+  # Inspection & Debugging ---------------------------------------------------
+
+  # Prints the slice as <code>"string"@offset</code>.
+  def inspect
+    str.inspect << "@#{offset}"
+  end
+end
+
+# Raised when trying to do an operation on slices that cannot succeed, like 
+# adding non-adjacent slices. See Parslet::Slice.
+#
+class Parslet::InvalidSliceOperation < StandardError
+end

data/lib/parslet/source.rb

@@ -1,120 +1,109 @@
 
 require 'stringio'
 
+require 'parslet/source/line_cache'
+
 # Wraps the input IO to parslet. The interface defined by this class is 
 # smaller than what IO offers, but enhances it with a #column and #line 
 # method for the current position. 
 #
 class Parslet::Source
-  attr_reader :line_ends
-  
   def initialize(io)
     if io.respond_to? :to_str
       io = StringIO.new(io)
     end
     
     @io = io
-    warn "Line counting will be off if the IO is not rewound." unless @io.pos==0
+    @virtual_position = @io.pos
+    @eof_position = nil
     
-    # Stores line endings as a simple position number. The first line always
-    # starts at 0; numbers beyond the biggest entry are on any line > size, 
-    # but probably make a scan to that position neccessary.
-    @line_ends = []
-    @line_ends.extend RangeSearch
+    @line_cache = LineCache.new
+    
+    # Stores an array of <offset, buffer> tuples. 
+    @slices = []
   end
   
+  # Reads n chars from the input and returns a Range instance. 
+  #
   def read(n)
-    start_pos = pos
-    @io.read(n).tap { |buf| scan_for_line_endings(start_pos, buf) }
+    slice = read_from_cache(@virtual_position, n)
+    @virtual_position += slice.size
+    
+    slice
   end
   
   def eof?
-    @io.eof?
+    @eof_position && @virtual_position >= @eof_position
   end
-  
   def pos
-    @io.pos
+    @virtual_position
   end
-
-  # NOTE: If you seek beyond the point that you last read, you will get 
-  # undefined behaviour. This is by design. 
   def pos=(new_pos)
-    @io.pos = new_pos
-  end
-  
-  def line_and_column(position=nil)
-    pos = (position || self.pos)
-    eol_idx = @line_ends.lbound(pos)
-    
-    if eol_idx
-      # eol_idx points to the offset that ends the current line.
-      # Let's try to find the offset that starts it: 
-      offset = eol_idx>0 && @line_ends[eol_idx-1] || 0
-      return [eol_idx+1, pos-offset+1]
-    else
-      # eol_idx is nil, that means that we're beyond the last line end that
-      # we know about. Pretend for now that we're just on the last line.
-      offset = @line_ends.last || 0
-      return [@line_ends.size+1, pos-offset+1]
-    end
+    @virtual_position = new_pos
   end
 
-  # Mixin for arrays that implicitly give a number of ranges, where one range
-  # begins where the other one ends.
-  # 
-  #   Example: 
-  #
-  #     [10, 20, 30]
-  #     # would describe [0, 10], (10, 20], (20, 30]
+  # Returns a <line, column> tuple for the given position. If no position is
+  # given, line/column information is returned for the current position given
+  # by #pos. 
   #
-  module RangeSearch
-    # Scans the array for the first number that is > than bound. Returns the 
-    # index of that number. 
-    #
-    def lbound(bound)
-      return nil if empty?
-      return nil unless last > bound
-
-      left = 0
-      right = size - 1 
-    
-      n = 10
-      loop do
-        mid = left + (right - left) / 2
-        
-        if self[mid] > bound
-          right = mid
-        else
-          # assert: self[mid] <= bound
-          left = mid+1
-        end
-        
-        if right <= left
-          return right
-        end
-      end
-    end
+  def line_and_column(position=nil)
+    @line_cache.line_and_column(position || self.pos)
   end
   
 private
+  # Minimal size of a single read
+  MIN_READ_SIZE = 10 * 1024
+  # Number of slices to keep 
+  BUFFER_CACHE_SIZE = 10
+  
+  # Reads and returns a piece of the input that contains length chars starting
+  # at offset. 
+  #
+  def read_from_cache(offset, length)
+    # Do we already have a buffer that contains the given range?
+    # Return that. 
+    slice = @slices.find { |slice| 
+      slice.satisfies?(offset, length) }
+    return slice.abs_slice(offset, length) if slice
+    
+    # Read a new buffer: Can the demand be satisfied by sequentially reading
+    # from the current position?
+    needed = offset-@io.pos+length
+    if @io.pos <= offset && needed<MIN_READ_SIZE
+      # read the slice
+      slice = read_slice(needed)
+      return slice.abs_slice(offset, length)
+    end
+    
+    # Otherwise seek and read enough so that we can satisfy the demand. 
+    @io.pos = offset
 
-  def scan_for_line_endings(start_pos, buf)
-    return unless buf
-    return unless buf.index("\n")
-    cur = -1
+    slice = read_slice(needed)
+    return slice.abs_slice(offset, length)
+  end
     
-    # If we have already read part or all of buf, we already know about
-    # line ends in that portion. remove it and correct cur (search index)
-    if @last_line_end && start_pos < @last_line_end
-      # Let's not search the range from start_pos to last_line_end again.
-      cur = @last_line_end - start_pos -1
+  def read_slice(needed)
+    start = @io.pos
+    request = [MIN_READ_SIZE, needed].max
+    buf = @io.read(request)
+    
+    # remember eof position
+    if !buf || buf.size<request
+      @eof_position = @io.pos
     end
+
+    # cache line ends
+    @line_cache.scan_for_line_endings(start, buf)
+    
+    slice = Parslet::Slice.new(buf || '', start, self)
+    
+    # Don't cache empty slices.
+    return slice unless buf
+    
+    # cache the buffer (and eject old entries)
+    @slices << slice
+    @slices.shift if @slices.size > BUFFER_CACHE_SIZE
     
-    # Scan the string for line endings; store the positions of all endings
-    # in @line_ends. 
-    while buf && cur = buf.index("\n", cur+1)
-      @last_line_end = (start_pos + cur+1)
-      @line_ends << @last_line_end
-    end 
+    slice
   end
 end

data/lib/parslet/source/line_cache.rb

@@ -0,0 +1,90 @@
+
+
+class Parslet::Source
+  # A cache for line start positions. 
+  #
+  class LineCache # :nodoc: 
+    def initialize
+      # Stores line endings as a simple position number. The first line always
+      # starts at 0; numbers beyond the biggest entry are on any line > size, 
+      # but probably make a scan to that position neccessary.
+      @line_ends = []
+      @line_ends.extend RangeSearch
+    end
+
+    # Returns a <line, column> tuple for the given input position. 
+    # 
+    def line_and_column(pos)
+      eol_idx = @line_ends.lbound(pos)
+
+      if eol_idx
+        # eol_idx points to the offset that ends the current line.
+        # Let's try to find the offset that starts it: 
+        offset = eol_idx>0 && @line_ends[eol_idx-1] || 0
+        return [eol_idx+1, pos-offset+1]
+      else
+        # eol_idx is nil, that means that we're beyond the last line end that
+        # we know about. Pretend for now that we're just on the last line.
+        offset = @line_ends.last || 0
+        return [@line_ends.size+1, pos-offset+1]
+      end
+    end
+
+    def scan_for_line_endings(start_pos, buf)
+      return unless buf
+      return unless buf.index("\n")
+      cur = -1
+
+      # If we have already read part or all of buf, we already know about
+      # line ends in that portion. remove it and correct cur (search index)
+      if @last_line_end && start_pos < @last_line_end
+        # Let's not search the range from start_pos to last_line_end again.
+        cur = @last_line_end - start_pos -1
+      end
+
+      # Scan the string for line endings; store the positions of all endings
+      # in @line_ends. 
+      while buf && cur = buf.index("\n", cur+1)
+        @last_line_end = (start_pos + cur+1)
+        @line_ends << @last_line_end
+      end 
+    end
+  end
+
+  # Mixin for arrays that implicitly give a number of ranges, where one range
+  # begins where the other one ends.
+  # 
+  #   Example: 
+  #
+  #     [10, 20, 30]
+  #     # would describe [0, 10], (10, 20], (20, 30]
+  #
+  module RangeSearch # :nodoc: 
+    # Scans the array for the first number that is > than bound. Returns the 
+    # index of that number. 
+    #
+    def lbound(bound)
+      return nil if empty?
+      return nil unless last > bound
+
+      left = 0
+      right = size - 1 
+
+      n = 10
+      loop do
+        mid = left + (right - left) / 2
+
+        if self[mid] > bound
+          right = mid
+        else
+          # assert: self[mid] <= bound
+          left = mid+1
+        end
+
+        if right <= left
+          return right
+        end
+      end
+    end
+  end
+end