ferret 0.9.6 → 0.10.0

data/lib/ferret/store/index_io.rb

@@ -1,245 +0,0 @@
-module Ferret::Store
-  # Ferret's IO Input methods are defined here. The methods read_byte and
-  # read_bytes need to be defined before this class is of any use. 
-  class IndexInput
-
-    # Reads and returns a single byte.
-    def read_byte()
-      raise NotImplementedError
-    end
-
-    # Reads a specified number of bytes into an array at the specified offset.
-    # buf:: the array to read bytes into
-    # offset:: the offset in the array to start storing bytes
-    # len:: the number of bytes to read
-    def read_bytes(buf, offset, len)
-      raise NotImplementedError
-    end
-     
-
-    # Reads four bytes and returns an int. read_uint should be used for
-    # unsigned integers for performance reasons.
-    def read_int
-      # This may be slow. I'm not sure if this is the best way to get
-      # integers from files but this is the only way I could find to get
-      # signed integers.
-      #i = read_byte
-      #return (((i&0x80)==0 ? 0 : -1) << 32) |
-             #(i << 24) |
-             #((read_byte) << 16) |
-             #((read_byte) << 8) |
-             #(read_byte)
-      i1 = read_byte
-      i2 = read_byte
-      i3 = read_byte
-      i4 = read_byte
-      res =  (((i1&0x80) == 0 ? 0 : -0x100000000)) +
-             ((i1 << 24) + (i2 << 16) + (i3 << 8) + (i4))
-      return res
-    end
-
-    # Reads eight bytes and returns a long.
-    def read_long
-      return (read_int << 32) + (read_int & 0xFFFFFFFF)
-    end
-
-    # Reads four bytes and returns a positive integer
-    def read_uint
-      return ((read_byte) << 24) | ((read_byte) << 16) |
-             ((read_byte) <<  8) |  (read_byte)
-    end
-    
-    # Reads eight bytes and returns a positive integer.
-    def read_ulong
-      return (read_uint << 32) | (read_uint & 0xFFFFFFFF)
-    end
-
-    # Reads an int stored in variable-length format.  Reads between one and
-    # five bytes.  Smaller values take fewer bytes.  Negative numbers are not
-    # supported.
-    def read_vint
-      b = read_byte
-      i = b & 0x7F # 0x7F = 0b01111111
-      shift = 7
-      
-      while b & 0x80 != 0 # 0x80 = 0b10000000
-        b = read_byte
-        i |= (b & 0x7F) << shift
-        shift += 7
-      end
-      
-      return i
-    end
-    alias :read_vlong :read_vint
-
-    # Reads a string. A string is stored as a single vint which describes
-    # the length of the string, followed by the actually string itself.
-    def read_string
-      length = read_vint
-      
-      chars = Array.new(length, ' ')
-      read_chars(chars, 0, length)
-      
-      chars.to_s
-    end
-
-    # Reads UTF-8 encoded characters into an array.
-    # buf:: the array to read characters into
-    # start:: the offset in the array to start storing characters
-    # length:: the number of characters to read
-    #
-    # TODO: Test on some actual UTF-8 documents.
-    def read_chars(buf, start, length)
-      if buf.length < (start + length)
-        # make room for the characters to read
-        buf << " " * (start + length - buf.length)
-      end
-      last = start + length
-      (start...last).each do |i|
-        buf[i] = read_byte.chr
-      end
-#        last = start + length
-#        
-#        (start...last).each do |i|
-#          b = read_byte
-#          if (b & 0x80) == 0
-#            buf[i] = (b & 0x7F).chr # don't need to worry about UTF-8 here
-#          else
-#            if (b & 0xE0) != 0xE0
-#              tmp_int = (((b & 0x1F) << 6) | (read_byte & 0x3F))
-#              buf[i] = [tmp_int].pack("C") # pack into a UTF-8 string
-#            else
-#              buf[i] = [
-#                         ((b & 0x0F) << 12) |
-#                         ((read_byte & 0x3F) << 6) |
-#                         (read_byte & 0x3F)
-#                       ].pack("U") # pack into a UTF-8 string
-#            end
-#          end
-#        end
-    end
-    
-    # Closes the stream to futher operations. 
-    def close
-      raise NotImplementedError
-    end
-
-    # Returns the current position in this file, where the next read will
-    # occur.
-    def pos
-      raise NotImplementedError
-    end
-
-    # Sets current position in this file, where the next read will occur.
-    def seek(pos)
-      raise NotImplementedError
-    end
-
-    # The number of bytes in the file. 
-    def length
-      raise NotImplementedError
-    end
-
-    # Returns a clone of this stream.
-    # 
-    # Clones of a stream access the same data, and are positioned at the same
-    # point as the stream they were cloned from.
-    # 
-    # Expert:: Subclasses must ensure that clones may be positioned at
-    # different points in the input from each other and from the stream they
-    # were cloned from.
-    #   def clone
-    #     raise NotImplementedError
-    #   end
-
-  end
-
-  # Ferret's IO Output methods are defined here. The methods write_byte and
-  # write_bytes need to be defined before this class is of any use. 
-  class IndexOutput 
-
-    # Writes a single byte.
-    def write_byte(b)
-      raise NotImplementedError
-    end
-
-    # Writes an array of bytes.
-    # buf:: the bytes to write
-    # len:: the number of bytes to write
-    def write_bytes(buf, len)
-      raise NotImplementedError
-    end
-
-    # Writes an int as four bytes.
-    def write_int(i)
-      write_byte((i >> 24) & 0xFF)
-      write_byte((i >> 16) & 0xFF)
-      write_byte((i >>  8) & 0xFF)
-      write_byte(i & 0xFF)
-    end
-    alias :write_uint :write_int
-
-    # Writes an int in a variable-length format.  Writes between one and
-    # five bytes.  Smaller values take fewer bytes.  Negative numbers are not
-    # supported.
-    def write_vint(i)
-      while i > 127
-        write_byte((i & 0x7f) | 0x80)
-        i >>= 7
-      end
-      write_byte(i)
-    end
-    alias :write_vlong :write_vint
-
-    # Writes a long as eight bytes.
-    def write_long(i)
-      write_int(i >> 32)
-      write_int(i)
-    end
-    alias :write_ulong :write_long
-
-    # Writes a string.
-    def write_string(s)
-      length = s.length()
-      write_vint(length)
-      write_chars(s, 0, length)
-    end
-
-    # Writes a sequence of UTF-8 encoded characters from a string.
-    # buf:: the source of the characters
-    # start:: the first character in the sequence
-    # length:: the number of characters in the sequence
-    def write_chars(buf, start, length)
-      last = start + length
-      (start ... last).each do |i|
-        write_byte(buf[i])
-      end
-    end
-
-    # Forces any buffered output to be written. 
-    def flush
-      raise NotImplementedError
-    end
-
-    # Closes this stream to further operations. 
-    def close
-      raise NotImplementedError
-    end
-
-    # Returns the current position in this file, where the next write will
-    # occur.
-    def pos
-      raise NotImplementedError
-    end
-
-    # Sets current position in this file, where the next write will occur.
-    def seek(pos)
-      raise NotImplementedError
-    end
-
-    # The number of bytes in the file. 
-    def length
-      raise NotImplementedError
-    end
-  end
-end

data/lib/ferret/store/ram_store.rb

@@ -1,286 +0,0 @@
-module Ferret::Store
-  require 'monitor'
-
-  class RAMDirectory < Directory
-    include MonitorMixin
-
-    def initialize(dir = nil, close_dir = false)
-      super()
-      @files = Hash.new
-      if dir
-        buf = BUFFER.clone
-        dir.each do |file|
-          os = create_output(file)    # make a place on ram disk
-          is = dir.open_input(file)   # read the current file
-          len = is.length             # and copy the file to ram disk
-          if len > buf.size
-            buf << " " * (len - buf.size)
-          end
-          is.read_bytes(buf, 0, len)
-          os.write_bytes(buf, len)
-          is.close()
-          os.close()
-        end
-        dir.close() if close_dir
-      end
-    end
-
-    # returns an array of strings, one for each file in the directory
-    def each()
-      @files.each do |path, file|
-        next if file =~ /#{LOCK_PREFIX}/
-        yield file
-      end
-    end
-
-    # Returns true if a file with the given name exists.
-    def exists?(name)
-      @files.has_key?(name)
-    end
-
-    # Returns the time the named file was last modified.
-    def modified(name)
-      @files[name].mtime
-    end
-
-    # Set the modified time of an existing file to now.
-    def touch(name)
-      if @files[name].nil?
-        @files[name] = RAMFile.new(name)
-      end
-      @files[name].mtime = Time.now
-    end
-
-    # Removes an existing file in the directory.
-    def delete(name)
-      @files.delete(name)
-    end
-
-    # Renames an existing file in the directory.
-    # If a file already exists with the new name, then it is replaced.
-    # This replacement should be atomic.
-    def rename(from, to)
-      @files[to] = @files[from]
-      @files.delete(from)
-    end
-
-    # Returns the length of a file in the directory.
-    def length(name)
-      @files[name].length
-    end
-
-    # Creates a new, empty file in the directory with the given name.
-    # Returns a stream writing this file.
-    def create_output(name)
-      file = RAMFile.new(name)
-      @files[name] = file
-      RAMIndexOutput.new(file)
-    end
-    
-    # Returns a stream reading an existing file.
-    def open_input(name)
-      raise IOError, "No file #{name}" if @files[name].nil?
-      RAMIndexInput.new(@files[name])
-    end
-    
-    def print_file(name)
-      input = RAMIndexInput.new(@files[name])
-      buf = " " * input.length
-      input.read_internal(buf, 0, input.length)
-      puts buf
-    end
-
-    # Construct a Lock.
-    def make_lock(name) 
-      RAMLock.new(LOCK_PREFIX + name + ".lck", self)
-    end
-
-
-    # Closes the store.
-    def close()
-    end
-
-    def to_s
-      str = "The files in this directory are: \n"
-      @files.each do |path, file|
-        str << path + " - " + file.size.to_s + "\n"
-      end
-      str
-    end
- 
-    class RAMIndexOutput < BufferedIndexOutput
-      def initialize(f)
-        @file = f
-        @pointer = 0
-        super()
-      end
-      
-      def length
-        return @file.length
-      end
-
-      def flush_buffer(src, len)
-        buffer_number = (@pointer / BUFFER_SIZE).to_i
-        buffer_offset = @pointer % BUFFER_SIZE
-        bytes_in_buffer = BUFFER_SIZE - buffer_offset
-        bytes_to_copy = [bytes_in_buffer, len].min
-        
-        extend_buffer_if_necessary(buffer_number)
-        
-        buffer = @file.buffers[buffer_number]
-        buffer[buffer_offset, bytes_to_copy] = src[0, bytes_to_copy]
-        
-        if bytes_to_copy < len
-          src_offset = bytes_to_copy
-          bytes_to_copy = len - bytes_to_copy
-          buffer_number += 1
-          extend_buffer_if_necessary(buffer_number)
-          buffer = @file.buffers[buffer_number]
-          buffer[0, bytes_to_copy] = src[src_offset, bytes_to_copy]
-        end
-        @pointer += len
-        @file.length = @pointer unless @pointer < @file.length
-        @file.mtime = Time.now
-      end
-
-      def reset
-        seek(0)
-        @file.length = 0
-      end
-
-      def seek(pos)
-        super(pos)
-        @pointer = pos
-      end
-
-      def close
-        super()
-        @file.mtime = Time.new
-      end
-
-      def write_to(output)
-        flush()
-        last_buffer_number = (@file.length / BUFFER_SIZE).to_i
-        last_buffer_offset = @file.length % BUFFER_SIZE
-
-        (0..last_buffer_number).each do |i|
-          len = (i == last_buffer_number ? last_buffer_offset : BUFFER_SIZE)
-          output.write_bytes(@file.buffers[i], len)
-        end
-      end
-      
-      private
-
-      def extend_buffer_if_necessary(buffer_number)
-        if buffer_number == @file.buffers.size
-          @file.buffers << RAMFile::BUFFER.clone
-        end
-      end
-
-    end
-    
-    class RAMIndexInput < BufferedIndexInput
-
-      def initialize(f)
-        @pointer = 0
-        @file = f
-        super()
-      end
-      
-      def length
-        return @file.length
-      end
-
-      def read_internal(b, offset, length)
-        remainder = length
-        start = @pointer
-        
-        while remainder != 0
-          buffer_number = (start / BUFFER_SIZE).to_i
-          buffer_offset = start % BUFFER_SIZE
-          bytes_in_buffer = BUFFER_SIZE - buffer_offset
-          
-          if bytes_in_buffer >= remainder
-            bytes_to_copy = remainder
-          else
-            bytes_to_copy = bytes_in_buffer
-          end
-          buffer = @file.buffers[buffer_number]
-          bo2 = buffer_offset
-          do2 = offset
-          b[do2, bytes_to_copy] = buffer[bo2, bytes_to_copy]
-          offset += bytes_to_copy
-          start += bytes_to_copy
-          remainder -= bytes_to_copy
-        end
-        
-        @pointer += length
-      end
-
-      def seek_internal(pos)
-        @pointer = pos
-      end
-
-      def close
-      end
-    end
-    
-    # This class contains an array of byte arrays which act as buffers to
-    # store the data in.
-    class RAMFile
-      BUFFER = " " * BUFFER_SIZE
-
-      attr_reader :buffers
-      attr_accessor :mtime
-      #attr_accessor :name
-      attr_accessor :length
-      alias :size :length
-
-
-      def initialize(name)
-        @buffers = Array.new
-        @mtime = Time.now
-        @length = 0
-      end
-    end
-
-    # A Lock is used to lock a data source (in this case a file) so that
-    # not more than one output stream can access a data source at one time.
-    class RAMLock < Lock
-      # pass the name of the file that we are going to lock
-      def initialize(lock_file, dir)
-        @lock_file = lock_file
-        @dir = dir
-      end
-
-      # obtain the lock on the data source 
-      def obtain(lock_timeout = 1) 
-        MAX_ATTEMPTS.times do
-          #@dir.synchronize do
-            # create a file if none exists. If one already exists
-            # then someone beat us to the lock so return false
-            if (! locked?) then
-              @dir.create_output(@lock_file)
-              return true
-            end
-          #end
-          # lock was not obtained so sleep for timeout then try again.
-          sleep(lock_timeout)
-        end
-        # lock could not be obtained so raise an exception
-        raise "could not obtain lock: " + @lock_file.to_s
-      end 
-
-      # Release the lock on the data source. Returns true if successful.
-      def release 
-        @dir.delete(@lock_file)
-        return true
-      end
-
-      # returns true if there is a lock on the data source
-      def locked?
-        @dir.exists?(@lock_file)
-      end
-    end
-  end
-end