ferret 0.1.0

data/lib/ferret/store.rb

@@ -0,0 +1,5 @@
+require 'ferret/store/directory'
+require 'ferret/store/index_io'
+require 'ferret/store/buffered_index_io'
+require 'ferret/store/fs_store'
+require 'ferret/store/ram_store'

data/lib/ferret/store/buffered_index_io.rb

@@ -0,0 +1,191 @@
+module Ferret::Store
+  BUFFER_SIZE = 1024
+  BUFFER = " " * BUFFER_SIZE
+
+  # Base implementation class for a buffered IndexOutput. 
+  class BufferedIndexOutput < IndexOutput 
+
+    def initialize
+      @buffer = BUFFER.clone
+      @buffer_start = 0          # position in file of buffer
+      @buffer_position = 0       # position in buffer
+    end
+
+    # Writes a single byte.
+    def write_byte(b)
+    
+      # The following code offers a 5% speed improvement over the line
+      # below. It relies on the fact that ruby will throw an error if we try
+      # and modify a character that is out of range for the string.
+      #begin
+      #  @buffer[@buffer_position] = b
+      #  @buffer_position += 1
+      #rescue IndexError
+      #  flush
+      #  @buffer[@buffer_position] = b
+      #  @buffer_position += 1
+      #end
+
+      flush if @buffer_position >= BUFFER_SIZE
+      @buffer[@buffer_position] = b
+      @buffer_position += 1
+    end
+
+    # Writes an array of bytes.
+    # buf:: the bytes to write
+    # length:: the number of bytes to write
+    def write_bytes(buf, length)
+      length.times do |i|
+        write_byte(buf[i])
+      end
+    end
+
+    # Forces any buffered output to be written. 
+    def flush()
+      flush_buffer(@buffer, @buffer_position)
+      @buffer_start += @buffer_position
+      @buffer_position = 0
+    end
+
+    # Closes this stream to further operations. 
+    def close()
+      flush()
+    end
+
+    # Get the current position in the file, where the next write will occur.
+    def pos() 
+      return @buffer_start + @buffer_position
+    end
+
+    # Set the current position in the file, where the next write will occur.
+    def seek(pos)
+      flush()
+      @buffer_start = pos
+    end
+
+    # The number of bytes in the file. 
+    def length
+      raise NotImplementedError
+    end
+    
+    private
+
+      # Expert: implements buffer write.  Writes the first len bytes from the
+      # buffer to the output.
+      # 
+      # buf:: the bytes to write
+      # len:: the number of bytes to write
+      def flush_buffer(buf, len)
+        raise NotImplementedError
+      end
+  end
+
+  # Base implementation class for buffered IndexInput
+  class BufferedIndexInput < IndexInput 
+    def initialize
+      @buffer = nil
+      @buffer_start = 0
+      @buffer_length = 0
+      @buffer_position = 0
+    end
+
+    # Read and return a single byte from the file
+    def read_byte
+      refill if (@buffer_position >= @buffer_length)
+      byte = @buffer[@buffer_position]
+      @buffer_position += 1
+      return byte
+    end
+
+    # Read +len+ bytes into +buffer+ starting at position +offset+ in +buffer+
+    #
+    # buffer:: The string buffer to read the characters into.
+    # offset:: The position in +buffer+ to start writing to.
+    # len:: the number of characters to read
+    # returns:: the buffer
+    def read_bytes(buffer, offset, len)
+      if (len < BUFFER_SIZE) 
+        offset.upto(offset+len-1) do |i| # read byte-by-byte
+          buffer[i] = read_byte
+        end
+      else                               # read all-at-once
+        start = pos()
+        seek_internal(start)
+        read_internal(buffer, offset, len)
+
+        @buffer_start = start + len      # adjust stream variables
+        @buffer_position = 0
+        @buffer_length = 0               # trigger refill on read
+      end
+      return buffer
+    end
+
+    # Get the current position in the file, where the next read will occur.
+    def pos()
+      return @buffer_start + @buffer_position
+    end
+
+    # Set the current position in the file, where the next read will occur.
+    def seek(pos)
+      if (pos >= @buffer_start and pos < (@buffer_start + @buffer_length))
+        @buffer_position = pos - @buffer_start  # seek within buffer
+      else 
+        @buffer_start = pos
+        @buffer_position = 0
+        @buffer_length = 0               # trigger refill() on read()
+        seek_internal(pos)
+      end
+    end
+
+    # Creates a clone of the BufferedIndexReader. Reading from a
+    # BufferedIndexInput should not change the state (read position) in the
+    # clone and vice-versa.
+    def clone() 
+      bii = super
+      bii.buffer = @buffer.clone if @buffer
+      return bii
+    end
+
+    attr_writer :buffer
+    protected :buffer=
+
+    private
+
+      # Expert: implements buffer refill.  Reads bytes from the current position
+      # in the input.
+      # buf:: the array to read bytes into
+      # offset:: the offset in the array to start storing bytes
+      # len:: the number of bytes to read
+      def read_internal(buf, offset, len)
+        raise NotImplementedError
+      end
+          
+      # Expert: implements seek.  Sets current position in this file, where the
+      # next read_internal will occur.
+      # pos:: the position to set to
+      def seek_internal(pos)
+        raise NotImplementedError
+      end
+
+      # Refill the buffer from the file.
+      def refill
+        start = @buffer_start + @buffer_position
+        last = start + BUFFER_SIZE
+        if (last > length())               # don't read past EOF
+          last = length()
+        end
+        @buffer_length = last - start
+        if (@buffer_length <= 0)
+          raise EOFError
+        end
+
+        if (@buffer == nil)
+          @buffer = BUFFER.clone # allocate buffer lazily
+        end
+        read_internal(@buffer, 0, @buffer_length)
+
+        @buffer_start = start
+        @buffer_position = 0
+      end
+  end
+end

data/lib/ferret/store/directory.rb

@@ -0,0 +1,139 @@
+module Ferret::Store
+  # A Directory is an object which is used to access the index storage.
+  # Ruby's IO API is not used so that we can use different storage
+  # mechanisms to store the index. Some examples are;
+  # 
+  # * File system based storage
+  # * RAM based storage
+  # * Database based storage
+  #
+  # NOTE: Once a file has been written and closed, it can no longer be
+  # modified. To make any changes to the file it must be deleted and
+  # rewritten. For this reason, the method to open a file for writing is
+  # called _create_output_, while the method to open a file for reading is
+  # called _open_input_ If there is a risk of simultaneous modifications of
+  # the files then locks should be used. See Lock to find out how.
+  class Directory
+    # returns an array of strings, one for each file in the directory
+    def each # :yeilds: file_name
+      raise NotImplementedError
+    end
+
+    # returns the number of files in the directory
+    def file_count() 
+      i = 0
+      each {|f| i += 1}
+      return i
+    end
+
+    # Returns true if a file with the given name exists.
+    def exists?(file)
+      raise NotImplementedError
+    end
+
+    # Returns the time the named file was last modified.
+    def modified(file)
+      raise NotImplementedError
+    end
+
+    # Set the modified time of an existing file to now.
+    def touch(file)
+      raise NotImplementedError
+    end
+
+    # Removes an existing file in the directory.
+    def delete(file)
+      raise NotImplementedError
+    end
+
+    # Renames an existing file in the directory.
+    # If a file already exists with the new name, then it is replaced.
+    # This replacement should be atomic.
+    def rename(from, to)
+      raise NotImplementedError
+    end
+
+    # Returns the length of a file in the directory.
+    def length(file)
+      raise NotImplementedError
+    end
+
+    # Creates a new, empty file in the directory with the given name.
+    # Returns a stream writing this file.
+    def create_output(file_name)
+      raise NotImplementedError
+    end
+
+    # Returns a stream reading an existing file.
+    def open_input(file_name)
+      raise NotImplementedError
+    end
+
+    # Construct a Lock.
+    def make_lock(lock_name)
+      raise NotImplementedError
+    end
+
+    # Closes the store.
+    def close
+      raise NotImplementedError
+    end
+
+  end
+
+  # A Lock is used to lock a data source so that not more than one 
+  # output stream can access a data source at one time. It is possible
+  # that locks could be disabled. For example a read only index stored
+  # on a CDROM would have no need for a lock.
+  #
+  # You can use a lock in two ways. Firstly:
+  #
+  #   write_lock = @directory.make_lock(LOCK_NAME)
+  #   write_lock.obtain(WRITE_LOCK_TIME_OUT)
+  #     ... # Do your file modifications # ...
+  #   write_lock.release()
+  #
+  # Alternatively you could use the while locked method. This ensures that
+  # the lock will be released once processing has finished.
+  #
+  #   write_lock = @directory.make_lock(LOCK_NAME)
+  #   write_lock.while_locked(WRITE_LOCK_TIME_OUT) do
+  #     ... # Do your file modifications # ...
+  #   end
+  class Lock
+    # Attempts made to obtain the lock before the application gives up. If
+    # you want the process to wait longer to get the lock then just increase
+    # the lock timeout
+    MAX_ATTEMPTS = 5
+
+    # Obtain the lock on the data source. If you expect to have to wait for
+    # a while on a lock then you should set the lock_timeout to a large
+    # number. This may be necessary if you are doing multiple large batch
+    # updates on an index but the default 1 second should be fine in most
+    # cases.
+    def obtain(lock_timeout = 1)
+      raise NotImplementedError
+    end
+
+    # Release the lock on the data source
+    def release
+      raise NotImplementedError
+    end
+
+    # Returns true if there is a lock on the data source
+    def locked?
+      raise NotImplementedError
+    end
+
+    # Obtains the lock, processes the block and ensures that the lock is
+    # released when the block terminates. The lock timeout is in seconds.
+    def while_locked(lock_timeout=1)
+      obtain(lock_timeout)
+      begin
+        yield
+      ensure
+        release()
+      end
+    end
+  end
+end

data/lib/ferret/store/fs_store.rb

@@ -0,0 +1,338 @@
+module Ferret::Store
+
+  require 'monitor'
+  require 'fileutils'
+  require 'digest/md5'
+
+  # This is a filesystem implementation of Directory and will be the one
+  # usually used for storing the index. This implementation stores each
+  # separate file as a separate file on the operating system. This works fine
+  # and is the most efficient solution for small to medium size indexes. For
+  # very large indexes, there may be a problem with the operating system not
+  # wanting to open to many files. One fix for this is to change the maximum
+  # open files setting in your operating system.  Alternatively you could use
+  # a compound file instead.
+  # 
+  # TODO:
+  # * need a better way of setting properties. Currently you have to
+  #   change the constants to disable locking.
+  class FSDirectory < Directory
+    include MonitorMixin
+
+    # This cache of directories ensures that there is a unique Directory
+    # instance per path, so that synchronization on the Directory can be used to
+    # synchronize access between readers and writers.
+    @@Directories = Hash.new.extend(MonitorMixin)
+
+
+    # Locks should be disabled it there is no need for them
+    LOCKS_DISABLED = false
+
+    # The lock dir is the directory where the file locks will be stored
+    LOCK_DIR = nil
+
+    # Returns the directory instance for the named location.
+    #
+    # Directories are cached, so that, for a given canonical path, the same
+    # FSDirectory instance will always be returned.  This permits
+    # synchronization on directories.
+    #
+    # path:: the path to the directory.
+    # create:: if true, create, or erase any existing contents.
+    def FSDirectory.get_directory(path, create=false)
+      dir = nil
+      @@Directories.synchronize do
+        dir = @@Directories[path]
+        if not dir then
+          dir = FSDirectory.new(path, create)
+          @@Directories[path] = dir
+        end
+        dir.refresh if create
+      end
+      dir.synchronize do
+        dir.reference()
+      end
+      return dir
+    end
+
+    # Returns true if locks have been disabled
+    def FSDirectory.locks_disabled?
+      LOCKS_DISABLED
+    end
+
+    # Set the directory where all of the locks will be stored.
+    # path:: the path to the directory where the locks will be stored.
+    #        An exception will be raised if the directory does not exist
+    def lock_dir=(path)
+      # close the old lock dir if it exists
+      @lock_dir.close() if @lock_dir
+      @lock_dir = Dir.new(path)
+    end
+
+    # Returns a Dir object of the directory where the lock is stored
+    attr_reader :lock_dir
+
+    # Remove all files and locks from this directory so we have a clean instance
+    def refresh
+      synchronize do
+        # delete all the files
+        each do |fname|
+          File.delete(dir_path(fname))
+        end
+        # clear all the locks
+        refresh_lock_dir
+        @lock_dir.each do |lock_fname|
+          next if lock_fname == '.' or lock_fname == '..'
+          File.delete(@lock_dir.path + '/' + lock_fname)
+        end
+      end
+    end
+
+    #--
+    # Directory implementation
+    #++
+
+    # Iterates through the file listing, skipping lock files if they exist
+    def each()
+      refresh_dir
+      @dir.each do |file_name|
+        # return all files except for the current and parent directories
+        # and any lock files that exist in this directory
+        next if ['.', '..'].include?(file_name)
+        next if file_name =~ Regexp.new('^' + lock_prefix)
+        yield file_name
+      end
+    end
+
+    # Returns true if a file with the given name exists.
+    def exists?(name)
+      File.exists?(dir_path(name))
+    end
+
+    # Returns the time the named file was last modified.
+    def modified(name) 
+      File.mtime(dir_path(name))
+    end
+
+    # Set the modified time of an existing file to now.
+    def touch(name) 
+      # just open the file and close it. No need to do anything with it.
+      FileUtils.touch(dir_path(name))
+    end
+
+    # Removes an existing file in the directory.
+    def delete(name)
+      begin
+        File.delete(dir_path(name))
+      rescue SystemCallError => e
+        raise IOError, e.to_s
+      end
+    end
+
+    # Renames an existing file in the directory.
+    # If a file already exists with the new name, then it is replaced.
+    # This replacement should be atomic.
+    def rename(from, to)
+      File.rename(dir_path(from), dir_path(to))
+    end
+
+
+    # Returns the length of a file in the directory.
+    def length(name)
+      File.size(dir_path(name))
+    end
+
+    # Creates a new, empty file in the directory with the given name.
+    # Returns a stream writing this file.
+    def create_output(name) 
+      FSIndexOutput.new(dir_path(name))
+    end
+
+    # Returns a stream reading an existing file.
+    def open_input(name)
+      FSIndexInput.new(dir_path(name))
+    end
+
+    # Construct a Lock.
+    def make_lock(name) 
+      FSLock.new(@lock_dir.path + "/" + lock_prefix() + name)
+    end
+
+    # Closes the store.
+    def close()
+      @ref_count -= 1
+      if (@ref_count <=0) then
+        @@Directories.synchronize do
+          @@Directories.delete(@dir.path)
+        end
+      end
+    end
+
+    def reference()
+      @ref_count += 1
+    end
+
+    # See Lock for hints as to how to use locks.
+    class FSLock < Lock
+      # pass the name of the file that we are going to lock
+      def initialize(lock_file)
+        @lock_file = lock_file
+      end
+
+      # obtain the lock on the data source 
+      def obtain(lock_timeout = 1) 
+        return true if FSDirectory.locks_disabled?
+        MAX_ATTEMPTS.times do
+          begin
+            # create a file if none exists. If one already exists
+            # then someone beat us to the lock so return false
+            File.open(@lock_file, File::WRONLY|File::EXCL|File::CREAT) {|f|}
+            return true
+          rescue SystemCallError
+            # lock was not obtained so sleep for timeout then try again.
+            sleep(lock_timeout)
+          end
+        end
+        # lock could not be obtained so raise an exception
+        raise "could not obtain lock: " + @lock_file.to_s
+      end 
+
+      # Release the lock on the data source. Returns true if successful.
+      def release 
+        return if FSDirectory.locks_disabled?
+        begin
+          File.delete(@lock_file)
+        rescue SystemCallError
+          # maybe we tried to release a lock that wasn't locked. This
+          # isn't critical so just return false
+          return false
+        end
+        return true
+      end
+
+      # returns true if there is a lock on the data source
+      def locked?
+        return false if FSDirectory.locks_disabled?
+        File.exists?(@lock_file)
+      end
+    end
+
+    # A file system output stream extending OutputStream to read from the file system
+    class FSIndexOutput < BufferedIndexOutput
+      def initialize(path)
+        super()
+        @file = File.open(path, "wb")
+      end
+
+      def close
+        super()
+        @file.close
+      end
+
+      def seek(pos)
+        super(pos)
+        @file.seek(pos)
+      end
+
+      private
+        def flush_buffer(b, size)
+          @file.syswrite(b[0...size])
+        end
+    end
+
+    # A file system input stream extending InputStream to read from the file system
+    class FSIndexInput < BufferedIndexInput
+      attr_writer :is_clone
+      attr_reader :length
+      attr_reader :file
+
+      def initialize(path)
+        @file = File.open(path, "rb")
+        @file.extend(MonitorMixin)
+        @length = File.size(path)
+        @is_clone = false
+        super()
+      end
+
+      def close
+        @file.close if not @is_clone
+      end
+
+      # We need to record if this is a clone so we know when to close the file.
+      # The file should only be closed when the original FSIndexInput is closed.
+      def clone()
+        fsii = super
+        fsii.is_clone = true
+        fsii.file.seek(@file.pos)
+        return fsii
+      end
+
+      private
+
+        def read_internal(b, offset, length)
+          @file.synchronize do
+            position = pos()
+            if position != @file.pos
+              @file.seek(position)
+            end
+            bytes = @file.read(length)
+            if bytes.nil?
+              raise EOFError, "Read past EOF in #{@file.path}"
+            end
+            b[offset, bytes.length] = bytes
+          end
+        end
+
+        def seek_internal(pos)
+          @file.seek(pos)
+        end
+
+    end
+
+    private
+      # Create a new directory from the path.
+      # path:: the path to the directory.
+      # create:: if true, create, or erase any existing contents.
+      def initialize(path, create)
+        super()
+        if create then FileUtils.mkdir_p(path) end
+        if not File.directory?(path) then
+          raise "There is no directory: #{path}. Use create = true to create one"
+        end
+        @dir = Dir.new(path)
+        # put the lock_dir here as well if no default exists.
+        if LOCK_DIR then
+          @lock_dir = Dir.new(LOCK_DIR) 
+        else
+          @lock_dir = Dir.new(path) 
+        end
+        @ref_count = 0
+      end
+
+      # Add the directory path to the file name for opening
+      def dir_path(name)
+        File.join(@dir.path, name)
+      end
+
+      # returns the lock prefix for this directory
+      def lock_prefix
+        'ferret-' + Digest::MD5.hexdigest(@dir.path)
+      end
+
+      # Unfortunately, on Windows, Dir does not refresh when rewind is called
+      # so any new files will be hidden. So we open the directory again.
+      def refresh_dir()
+        tmp = Dir.new(@dir.path)
+        @dir.close
+        @dir = tmp
+      end
+
+      def refresh_lock_dir()
+        tmp = Dir.new(@lock_dir.path)
+        @lock_dir.close
+        @lock_dir = tmp
+      end
+
+    #end private
+  end
+end