ferret 0.9.6 → 0.10.0

data/lib/ferret/store.rb

@@ -1,5 +0,0 @@
-require 'ferret/store/directory'
-require 'ferret/store/index_io'
-require 'ferret/store/buffered_index_io'
-require 'ferret/store/fs_store'
-require 'ferret/store/ram_store'

data/lib/ferret/store/buffered_index_io.rb

@@ -1,190 +0,0 @@
-module Ferret::Store
-  BUFFER_SIZE = 1024
-  BUFFER = " " * BUFFER_SIZE
-
-  # Base implementation class for a buffered IndexOutput. 
-  class BufferedIndexOutput < IndexOutput 
-
-    def initialize
-      @buffer = BUFFER.clone
-      @buffer_start = 0          # position in file of buffer
-      @buffer_position = 0       # position in buffer
-    end
-
-    # Writes a single byte.
-    def write_byte(b)
-    
-      # The following code offers a 5% speed improvement over the line
-      # below. It relies on the fact that ruby will throw an error if we try
-      # and modify a character that is out of range for the string.
-      #begin
-      #  @buffer[@buffer_position] = b
-      #  @buffer_position += 1
-      #rescue IndexError
-      #  flush
-      #  @buffer[@buffer_position] = b
-      #  @buffer_position += 1
-      #end
-
-      flush if @buffer_position >= BUFFER_SIZE
-      @buffer[@buffer_position] = b
-      @buffer_position += 1
-    end
-
-    # Writes an array of bytes.
-    # buf:: the bytes to write
-    # length:: the number of bytes to write
-    def write_bytes(buf, length)
-      length.times do |i|
-        write_byte(buf[i])
-      end
-    end
-
-    # Forces any buffered output to be written. 
-    def flush()
-      flush_buffer(@buffer, @buffer_position)
-      @buffer_start += @buffer_position
-      @buffer_position = 0
-    end
-
-    # Closes this stream to further operations. 
-    def close()
-      flush()
-    end
-
-    # Get the current position in the file, where the next write will occur.
-    def pos() 
-      return @buffer_start + @buffer_position
-    end
-
-    # Set the current position in the file, where the next write will occur.
-    def seek(pos)
-      flush()
-      @buffer_start = pos
-    end
-
-    # The number of bytes in the file. 
-    def length
-      raise NotImplementedError
-    end
-    
-    private
-
-      # Expert: implements buffer write.  Writes the first len bytes from the
-      # buffer to the output.
-      # 
-      # buf:: the bytes to write
-      # len:: the number of bytes to write
-      def flush_buffer(buf, len)
-        raise NotImplementedError
-      end
-  end
-
-  # Base implementation class for buffered IndexInput
-  class BufferedIndexInput < IndexInput 
-    def initialize
-      @buffer = nil
-      @buffer_start = 0
-      @buffer_length = 0
-      @buffer_position = 0
-    end
-
-    # Read and return a single byte from the file
-    def read_byte
-      refill if (@buffer_position >= @buffer_length)
-      byte = @buffer[@buffer_position]
-      @buffer_position += 1
-      return byte
-    end
-
-    # Read +len+ bytes into +buffer+ starting at position +offset+ in +buffer+
-    #
-    # buffer:: The string buffer to read the characters into.
-    # offset:: The position in +buffer+ to start writing to.
-    # len:: the number of characters to read
-    # returns:: the buffer
-    def read_bytes(buffer, offset, len)
-      if (len < BUFFER_SIZE) 
-        offset.upto(offset+len-1) do |i| # read byte-by-byte
-          buffer[i] = read_byte
-        end
-      else                               # read all-at-once
-        start = pos()
-        seek_internal(start)
-        read_internal(buffer, offset, len)
-
-        @buffer_start = start + len      # adjust stream variables
-        @buffer_position = 0
-        @buffer_length = 0               # trigger refill on read
-      end
-      return buffer
-    end
-
-    # Get the current position in the file, where the next read will occur.
-    def pos()
-      return @buffer_start + @buffer_position
-    end
-
-    # Set the current position in the file, where the next read will occur.
-    def seek(pos)
-      if (pos >= @buffer_start and pos < (@buffer_start + @buffer_length))
-        @buffer_position = pos - @buffer_start  # seek within buffer
-      else 
-        @buffer_start = pos
-        @buffer_position = 0
-        @buffer_length = 0               # trigger refill() on read()
-        seek_internal(pos)
-      end
-    end
-
-    # Creates a clone of the BufferedIndexReader. Reading from a
-    # BufferedIndexInput should not change the state (read position) in the
-    # clone and vice-versa.
-    def initialize_copy(o)
-      super
-      @buffer = o.buffer.clone if o.buffer
-    end
-
-    attr_reader :buffer
-    protected :buffer
-
-    private
-
-      # Expert: implements buffer refill.  Reads bytes from the current position
-      # in the input.
-      # buf:: the array to read bytes into
-      # offset:: the offset in the array to start storing bytes
-      # len:: the number of bytes to read
-      def read_internal(buf, offset, len)
-        raise NotImplementedError
-      end
-          
-      # Expert: implements seek.  Sets current position in this file, where the
-      # next read_internal will occur.
-      # pos:: the position to set to
-      def seek_internal(pos)
-        raise NotImplementedError
-      end
-
-      # Refill the buffer from the file.
-      def refill
-        start = @buffer_start + @buffer_position
-        last = start + BUFFER_SIZE
-        if (last > length())               # don't read past EOF
-          last = length()
-        end
-        @buffer_length = last - start
-        if (@buffer_length <= 0)
-          raise EOFError
-        end
-
-        if (@buffer == nil)
-          @buffer = BUFFER.clone # allocate buffer lazily
-        end
-        read_internal(@buffer, 0, @buffer_length)
-
-        @buffer_start = start
-        @buffer_position = 0
-      end
-  end
-end

data/lib/ferret/store/directory.rb

@@ -1,141 +0,0 @@
-module Ferret::Store
-  # A Directory is an object which is used to access the index storage.
-  # Ruby's IO API is not used so that we can use different storage
-  # mechanisms to store the index. Some examples are;
-  # 
-  # * File system based storage
-  # * RAM based storage
-  # * Database based storage
-  #
-  # NOTE: Once a file has been written and closed, it can no longer be
-  # modified. To make any changes to the file it must be deleted and
-  # rewritten. For this reason, the method to open a file for writing is
-  # called _create_output_, while the method to open a file for reading is
-  # called _open_input_ If there is a risk of simultaneous modifications of
-  # the files then locks should be used. See Lock to find out how.
-  class Directory
-    LOCK_PREFIX = "ferret-"
-
-    # returns an array of strings, one for each file in the directory
-    def each # :yeilds: file_name
-      raise NotImplementedError
-    end
-
-    # returns the number of files in the directory
-    def file_count() 
-      i = 0
-      each {|f| i += 1}
-      return i
-    end
-
-    # Returns true if a file with the given name exists.
-    def exists?(file)
-      raise NotImplementedError
-    end
-
-    # Returns the time the named file was last modified.
-    def modified(file)
-      raise NotImplementedError
-    end
-
-    # Set the modified time of an existing file to now.
-    def touch(file)
-      raise NotImplementedError
-    end
-
-    # Removes an existing file in the directory.
-    def delete(file)
-      raise NotImplementedError
-    end
-
-    # Renames an existing file in the directory.
-    # If a file already exists with the new name, then it is replaced.
-    # This replacement should be atomic.
-    def rename(from, to)
-      raise NotImplementedError
-    end
-
-    # Returns the length of a file in the directory.
-    def length(file)
-      raise NotImplementedError
-    end
-
-    # Creates a new, empty file in the directory with the given name.
-    # Returns a stream writing this file.
-    def create_output(file_name)
-      raise NotImplementedError
-    end
-
-    # Returns a stream reading an existing file.
-    def open_input(file_name)
-      raise NotImplementedError
-    end
-
-    # Construct a Lock.
-    def make_lock(lock_name)
-      raise NotImplementedError
-    end
-
-    # Closes the store.
-    def close
-      raise NotImplementedError
-    end
-
-  end
-
-  # A Lock is used to lock a data source so that not more than one 
-  # output stream can access a data source at one time. It is possible
-  # that locks could be disabled. For example a read only index stored
-  # on a CDROM would have no need for a lock.
-  #
-  # You can use a lock in two ways. Firstly:
-  #
-  #   write_lock = @directory.make_lock(LOCK_NAME)
-  #   write_lock.obtain(WRITE_LOCK_TIME_OUT)
-  #     ... # Do your file modifications # ...
-  #   write_lock.release()
-  #
-  # Alternatively you could use the while locked method. This ensures that
-  # the lock will be released once processing has finished.
-  #
-  #   write_lock = @directory.make_lock(LOCK_NAME)
-  #   write_lock.while_locked(WRITE_LOCK_TIME_OUT) do
-  #     ... # Do your file modifications # ...
-  #   end
-  class Lock
-    # Attempts made to obtain the lock before the application gives up. If
-    # you want the process to wait longer to get the lock then just increase
-    # the lock timeout
-    MAX_ATTEMPTS = 5
-
-    # Obtain the lock on the data source. If you expect to have to wait for
-    # a while on a lock then you should set the lock_timeout to a large
-    # number. This may be necessary if you are doing multiple large batch
-    # updates on an index but the default 1 second should be fine in most
-    # cases.
-    def obtain(lock_timeout = 1)
-      raise NotImplementedError
-    end
-
-    # Release the lock on the data source
-    def release
-      raise NotImplementedError
-    end
-
-    # Returns true if there is a lock on the data source
-    def locked?
-      raise NotImplementedError
-    end
-
-    # Obtains the lock, processes the block and ensures that the lock is
-    # released when the block terminates. The lock timeout is in seconds.
-    def while_locked(lock_timeout=1)
-      obtain(lock_timeout)
-      begin
-        yield
-      ensure
-        release()
-      end
-    end
-  end
-end

data/lib/ferret/store/fs_store.rb

@@ -1,381 +0,0 @@
-module Ferret::Store
-
-  require 'monitor'
-  require 'fileutils'
-  require 'digest/md5'
-
-  # This is a filesystem implementation of Directory and will be the one
-  # usually used for storing the index. This implementation stores each
-  # separate file as a separate file on the operating system. This works fine
-  # and is the most efficient solution for small to medium size indexes. For
-  # very large indexes, there may be a problem with the operating system not
-  # wanting to open to many files. One fix for this is to change the maximum
-  # open files setting in your operating system.  Alternatively you could use
-  # a compound file instead.
-  # 
-  # TODO:
-  # * need a better way of setting properties. Currently you have to
-  #   change the constants to disable locking.
-  class FSDirectory < Directory
-    include MonitorMixin
-
-    # This cache of directories ensures that there is a unique Directory
-    # instance per path, so that synchronization on the Directory can be used to
-    # synchronize access between readers and writers.
-    @@Directories = Hash.new.extend(MonitorMixin)
-
-
-    # Locks should be disabled it there is no need for them
-    LOCKS_DISABLED = false
-
-    # The lock dir is the directory where the file locks will be stored
-    LOCK_DIR = nil
-
-    # Create a new directory from the path.
-    # path:: the path to the directory.
-    # create:: if true, create, or erase any existing contents.
-    def initialize(path, create)
-      super()
-      if create then FileUtils.mkdir_p(path) end
-      if not File.directory?(path) then
-        raise IOError, "There is no directory: #{path}. Use create = true to create one"
-      end
-      @dir = Dir.new(path)
-      # put the lock_dir here as well if no default exists.
-      if LOCK_DIR then
-        @lock_dir = Dir.new(LOCK_DIR) 
-      else
-        @lock_dir = Dir.new(path) 
-      end
-      @ref_count = 0
-    end
-
-    class <<FSDirectory
-      alias :allocate :new
-      protected :allocate
-    end
-
-    # Returns the directory instance for the named location.
-    #
-    # Directories are cached, so that, for a given canonical path, the same
-    # FSDirectory instance will always be returned.  This permits
-    # synchronization on directories.
-    #
-    # path:: the path to the directory.
-    # create:: if true, create, or erase any existing contents.
-    def FSDirectory.new(path, create = false)
-      dir = nil
-      @@Directories.synchronize do
-        dir = @@Directories[path]
-        if not dir then
-          dir = FSDirectory.allocate(path, create)
-          @@Directories[path] = dir
-        end
-        dir.refresh if create
-      end
-      dir.synchronize do
-        dir.reference()
-      end
-      return dir
-    end
-
-    # Returns true if locks have been disabled
-    def FSDirectory.locks_disabled?
-      LOCKS_DISABLED
-    end
-
-    # Set the directory where all of the locks will be stored.
-    # path:: the path to the directory where the locks will be stored.
-    #        An exception will be raised if the directory does not exist
-    def lock_dir=(path)
-      # close the old lock dir if it exists
-      @lock_dir.close() if @lock_dir
-      @lock_dir = Dir.new(path)
-    end
-
-    # Returns a Dir object of the directory where the lock is stored
-    attr_reader :lock_dir
-
-    # Remove all files and locks from this directory so we have a clean instance
-    def refresh
-      synchronize do
-        # delete all the files
-        refresh_dir
-        each do |fname|
-          FileUtils.rm_rf(dir_path(fname))
-        end
-        # clear all the locks
-        refresh_lock_dir
-        @lock_dir.each do |lock_fname|
-          next if lock_fname == '.' or lock_fname == '..'
-          FileUtils.rm_rf(@lock_dir.path + '/' + lock_fname)
-        end
-      end
-    end
-
-    #--
-    # Directory implementation
-    #++
-
-    # Iterates through the file listing, skipping lock files if they exist
-    def each()
-      refresh_dir
-      @dir.each do |file_name|
-        # return all files except for the current and parent directories
-        # and any lock files that exist in this directory
-        next if ['.', '..'].include?(file_name)
-        next if file_name =~ Regexp.new('^' + lock_prefix)
-        yield file_name
-      end
-    end
-
-    # Returns true if a file with the given name exists.
-    def exists?(name)
-      File.exists?(dir_path(name))
-    end
-
-    # Returns the time the named file was last modified.
-    def modified(name) 
-      File.mtime(dir_path(name))
-    end
-
-    # Set the modified time of an existing file to now.
-    def touch(name) 
-      # just open the file and close it. No need to do anything with it.
-      FileUtils.touch(dir_path(name))
-    end
-
-    # Removes an existing file in the directory.
-    def delete(name)
-      begin
-        File.delete(dir_path(name))
-      rescue SystemCallError => e
-        raise IOError, e.to_s
-      end
-    end
-
-    # Renames an existing file in the directory.
-    # If a file already exists with the new name, then it is replaced.
-    # This replacement should be atomic.
-    def rename(from, to)
-      synchronize do
-        begin
-          File.rename(dir_path(from), dir_path(to))
-        rescue
-          # try again, this time forcing the delete
-          FileUtils.rm_rf(dir_path(to))
-          begin
-            FileUtils.mv(dir_path(from), dir_path(to))
-          rescue
-            FileUtils.cp(dir_path(from), dir_path(to))
-            FileUtils.rm_rf(dir_path(to))
-          end
-        end
-      end
-    end
-
-
-    # Returns the length of a file in the directory.
-    def length(name)
-      File.size(dir_path(name))
-    end
-
-    # Creates a new, empty file in the directory with the given name.
-    # Returns a stream writing this file.
-    def create_output(name) 
-      FSIndexOutput.new(dir_path(name))
-    end
-
-    # Returns a stream reading an existing file.
-    def open_input(name)
-      FSIndexInput.new(dir_path(name))
-    end
-
-    # Construct a Lock.
-    def make_lock(name) 
-      FSLock.new(@lock_dir.path + "/" + lock_prefix() + name + ".lck")
-    end
-
-    # Closes the store.
-    def close()
-      synchronize do
-        @ref_count -= 1
-        if (@ref_count <= 0) then
-          @@Directories.synchronize do
-            @@Directories.delete(@dir.path)
-            close_internal
-          end
-        end
-      end
-    end
-
-    def reference()
-      @ref_count += 1
-    end
-
-    # See Lock for hints as to how to use locks.
-    class FSLock < Lock
-      # pass the name of the file that we are going to lock
-      def initialize(lock_file)
-        @lock_file = lock_file
-        #@clean = FSLock.make_finalizer(lock_file)
-        @clean = lambda { FileUtils.rm_rf(lock_file)}
-      end
-
-      def FSLock.make_finalizer(lock_file)
-        lambda { FileUtils.rm_rf(lock_file)}
-      end
-
-      # obtain the lock on the data source 
-      def obtain(lock_timeout = 1) 
-        return true if FSDirectory.locks_disabled?
-        MAX_ATTEMPTS.times do
-          begin
-            # create a file if none exists. If one already exists
-            # then someone beat us to the lock so return false
-            File.open(@lock_file, File::WRONLY|File::EXCL|File::CREAT) {|f|}
-            ObjectSpace.define_finalizer(self, @clean)
-            return true
-          rescue SystemCallError
-            # lock was not obtained so sleep for timeout then try again.
-            sleep(lock_timeout)
-          end
-        end
-        # lock could not be obtained so raise an exception
-        raise "could not obtain lock: #{@lock_file}"
-      end 
-
-      # Release the lock on the data source. Returns true if successful.
-      def release 
-        return if FSDirectory.locks_disabled?
-        begin
-          FileUtils.rm_rf(@lock_file)
-          ObjectSpace.undefine_finalizer(self)
-        rescue SystemCallError
-          # maybe we tried to release a lock that wasn't locked. This
-          # isn't critical so just return false
-          return false
-        end
-        return true
-      end
-
-      # returns true if there is a lock on the data source
-      def locked?
-        return false if FSDirectory.locks_disabled?
-        File.exists?(@lock_file)
-      end
-    end
-
-    # A file system output stream extending OutputStream to read from the file
-    # system
-    class FSIndexOutput < BufferedIndexOutput
-      def initialize(path)
-        super()
-        @file = File.open(path, "wb")
-      end
-
-      def close
-        super()
-        @file.close
-      end
-
-      def seek(pos)
-        super(pos)
-        @file.seek(pos)
-      end
-
-      private
-        def flush_buffer(b, size)
-          @file.syswrite(b[0...size])
-        end
-    end
-
-    # A file system input stream extending InputStream to read from the file system
-    class FSIndexInput < BufferedIndexInput
-      attr_accessor :is_clone
-      attr_reader   :length, :file
-
-      def initialize(path)
-        begin
-          @file = File.open(path, "rb")
-        rescue Errno::ENOENT => e
-          raise StandardError.new(e.message)
-        end
-        @file.extend(MonitorMixin)
-        #class <<@file
-        #  attr_accessor :ref_count
-        #end
-        #@file.ref_count = 1
-        @length = File.size(path)
-        @is_clone = false
-        super()
-      end
-
-      def close
-        #@file.ref_count -= 1
-        #@file.close if @file.ref_count == 0
-        @file.close if not @is_clone
-      end
-
-      # We need to record if this is a clone so we know when to close the file.
-      # The file should only be closed when the original FSIndexInput is closed.
-      def initialize_copy(o)
-        super
-        @is_clone = true
-      end
-
-      private
-
-        def read_internal(b, offset, length)
-          #@file.synchronize do
-            position = pos()
-            if position != @file.pos
-              @file.seek(position)
-            end
-            bytes = @file.read(length)
-            if bytes.nil?
-              raise EOFError, "Read past EOF in #{@file.path}"
-            end
-            b[offset, bytes.length] = bytes
-          #end
-        end
-
-        def seek_internal(pos)
-          @file.seek(pos)
-        end
-
-    end
-
-    private
-
-      # Add the directory path to the file name for opening
-      def dir_path(name)
-        File.join(@dir.path, name)
-      end
-
-      # returns the lock prefix for this directory
-      def lock_prefix
-        LOCK_PREFIX
-      end
-
-      # Unfortunately, on Windows, Dir does not refresh when rewind is called
-      # so any new files will be hidden. So we open the directory again.
-      def refresh_dir()
-        tmp = Dir.new(@dir.path)
-        @dir.close
-        @dir = tmp
-      end
-
-      def refresh_lock_dir()
-        tmp = Dir.new(@lock_dir.path)
-        @lock_dir.close
-        @lock_dir = tmp
-      end
-
-      # This method is only used by the c extension to free the directory
-      def close_internal
-      end
-
-    #end private
-  end
-end