ferret 0.1.0

data/lib/ferret/document/field.rb

@@ -0,0 +1,304 @@
+module Ferret::Document
+  # A field is a section of a Document.  Each field has two parts, a name
+  # and a value.  Values may be free text, provided as a String or as a
+  # Reader, or they may be atomic keywords, which are not further processed.
+  # Such keywords may be used to represent dates, urls, etc.  Fields are
+  # optionally stored in the index, so that they may be returned with hits
+  # on the document.
+  class Field
+
+    # This value will be
+    # multiplied into the score of all hits on this field of this
+    # document.
+    #
+    # The boost is multiplied by Document#boost of the document
+    # containing this field.  If a document has multiple fields with the same
+    # name, all such values are multiplied together.  This product is then
+    # multipled by the value Similarity#length_norm(String,int), and
+    # rounded by Similarity#encode_norm(float) before it is stored in the
+    # index.  One should attempt to ensure that this product does not overflow
+    # the range of that encoding.
+    #
+    # See Document#set_boost(float)
+    # See Similarity#length_norm(String, int)
+    # See Similarity#encode_norm(float)
+    #
+    # Note: this value is not stored directly with the document in the index.
+    # Documents returned from IndexReader#document(int) and
+    # Hits#doc(int) may thus not have the same value present as when this field
+    # was indexed.
+    attr_accessor :boost, :data
+
+    attr_reader :name
+
+    # True iff the value of the field is to be stored in the index for
+    # return with search hits.  It is an error for this to be true if a
+    # field is Reader-valued.
+    def stored?() return @stored end
+
+    # True iff the value of the field is to be indexed, so that it may be
+    # searched on.
+    def indexed?() return @indexed end
+
+    # True iff the value of the field should be tokenized as text prior to
+    # indexing.  Un-tokenized fields are indexed as a single word and may
+    # not be Reader-valued.
+    def tokenized?() return @tokenized end
+
+    # True if the field is to be stored as a binary value. This can be used
+    # to store images or other binary data in the index if you wish
+    def binary?() return @binary end
+
+    # True if you want to compress the data that you store. This is a good
+    # idea for really large text fields. The ruby Zlib library is used to do
+    # the compression
+    def compressed?() return @compressed end
+
+    # True iff the term or terms used to index this field are stored as a
+    # term vector, available from IndexReader#term_freq_vector(). These
+    # methods do not provide access to the original content of the field,
+    # only to terms used to index it. If the original content must be
+    # preserved, use the _stored_ attribute instead.
+    #
+    # See IndexReader#term_freq_vector()
+    def store_term_vector?() return @store_term_vector end
+
+    # True if the positions of the indexed terms in this field are stored.
+    def store_positions?() return @store_position end
+
+    # True if the offsets of this field are stored. The offsets are the
+    # positions of the start and end characters of the token in the whole
+    # field string
+    def store_offsets?() return @store_offset end
+
+    class Store < Ferret::Utils::Parameter
+      # Store the original field value in the index in a compressed form.
+      # This is useful for long documents and for binary valued fields.
+      COMPRESS = Store.new("COMPRESS")
+
+      # Store the original field value in the index. This is useful for
+      # short texts like a document's title which should be displayed with
+      # the results. The value is stored in its original form, i.e. no
+      # analyzer is used before it is stored. 
+      YES = Store.new("YES")
+
+      # Do not store the field value in the index.
+      NO = Store.new("NO")
+    end
+
+    class Index < Ferret::Utils::Parameter
+      # Do not index the field value. This field can thus not be searched,
+      # but one can still access its contents provided it is Field.Store
+      # stored
+      NO = Index.new("NO")
+
+      # Index the field's value so it can be searched. An Analyzer will be
+      # used to tokenize and possibly further normalize the text before its
+      # terms will be stored in the index. This is useful for common text.
+      TOKENIZED = Index.new("TOKENIZED")
+
+      # Index the field's value without using an Analyzer, so it can be
+      # searched.  As no analyzer is used the value will be stored as a
+      # single term. This is useful for unique Ids like product numbers.
+      UNTOKENIZED = Index.new("UNTOKENIZED")
+    end
+
+    class TermVector < Ferret::Utils::Parameter
+      # Do not store term vectors. 
+      NO = TermVector.new("NO")
+
+      # Store the term vectors of each document. A term vector is a list of
+      # the document's terms and their number of occurences in that
+      # document.
+      YES = TermVector.new("YES")
+
+      # Store the term vector + token position information
+      # 
+      # See #YES
+      WITH_POSITIONS = TermVector.new("WITH_POSITIONS")
+
+      # Store the term vector + Token offset information
+      # 
+      # See #YES
+      WITH_OFFSETS = TermVector.new("WITH_OFFSETS")
+
+      # Store the term vector + Token position and offset information
+      # 
+      # See #YES See #WITH_POSITIONS See #WITH_OFFSETS
+      WITH_POSITIONS_OFFSETS = TermVector.new("WITH_POSITIONS_OFFSETS")
+    end
+
+    # Create a field by specifying its name, value and how it will
+    # be saved in the index.
+    # 
+    # name:: The name of the field
+    # value:: The string to process
+    # store:: Whether _value_ should be stored in the index
+    # index:: Whether the field should be indexed, and if so, if it should
+    #         be tokenized before indexing 
+    #
+    # store_term_vector:: Whether term vector should be stored
+    #  * the field is neither stored nor indexed
+    #  * the field is not indexed but term_vector is _TermVector::YES_
+    #
+    # binary:: Whether you want to store binary data in this field. Default is
+    #    false
+    # boost:: the boost for this field. Default is 1.0. A larger number makes
+    # this field more important.
+    def initialize(name,
+                   value,
+                   stored = Store::YES,
+                   index = Index::UNTOKENIZED,
+                   store_term_vector = TermVector::NO,
+                   binary = false,
+                   boost = 1.0)
+      if (index == Index::NO and stored == Store::NO)
+        raise ArgumentError, "it doesn't make sense to have a field that " +
+          "is neither indexed nor stored"
+      end
+      if (index == Index::NO && store_term_vector != TermVector::NO)
+        raise ArgumentError, "cannot store term vector information for a " +
+          "field that is not indexed"
+      end
+
+      # The name of the field (e.g., "date", "subject", "title", or "body")
+      @name = name
+
+      # the one and only data object for all different kind of field values
+      @data = value
+      self.stored = stored
+      self.index = index
+      self.store_term_vector = store_term_vector
+      @binary = binary
+      @boost = boost
+    end
+
+    def stored=(stored)
+      if (stored == Store::YES)
+        @stored = true
+        @compressed = false
+      elsif (stored == Store::COMPRESS)
+        @stored = true
+        @compressed = true
+      elsif (stored == Store::NO)
+        @stored = false
+        @compressed = false
+      else
+        raise "unknown stored parameter " + stored.to_s
+      end
+    end
+
+    def index=(index)
+      if (index == Index::NO)
+        @indexed = false
+        @tokenized = false
+      elsif (index == Index::TOKENIZED)
+        @indexed = true
+        @tokenized = true
+      elsif (index == Index::UNTOKENIZED)
+        @indexed = true
+        @tokenized = false
+      else
+        raise "unknown stored parameter " + index.to_s
+      end
+    end
+
+    def store_term_vector=(store_term_vector)
+      if (store_term_vector == TermVector::NO)
+        @store_term_vector = false
+        @store_position = false
+        @store_offset = false
+      elsif (store_term_vector == TermVector::YES)
+        @store_term_vector = true
+        @store_position = false
+        @store_offset = false
+      elsif (store_term_vector == TermVector::WITH_POSITIONS)
+        @store_term_vector = true
+        @store_position = true
+        @store_offset = false
+      elsif (store_term_vector == TermVector::WITH_OFFSETS)
+        @store_term_vector = true
+        @store_position = false
+        @store_offset = true
+      elsif (store_term_vector == TermVector::WITH_POSITIONS_OFFSETS)
+        @store_term_vector = true
+        @store_position = true
+        @store_offset = true
+      else
+        raise "unknown term_vector parameter " + store_term_vector.to_s
+      end
+    end
+
+    # Returns the string value of the data that is stored in this field
+    def string_value
+      if @data.instance_of? String
+        return @data
+      elsif @data.respond_to? :read
+        return @data.read()
+      else
+        # if it is binary object try to return a string representation
+        return @data.to_s
+      end
+    end
+
+    # if the data is stored as a binary, just return it.
+    def binary_value
+      return @data
+    end
+
+    # Returns the string value of the data that is stored in this field
+    def reader_value
+      if @data.respond_to? :read
+        return @data
+      elsif @data.instance_of? String
+        return Ferret::Utils::StringHelper::StringReader.new(@data)
+      else
+        # if it is binary object try to return a string representation
+        return Ferret::Utils::StringHelper::StringReader.new(@data.to_s)
+      end
+    end
+
+    # Create a stored field with binary value. Optionally the value
+    # may be compressed. But it obviously won't be tokenized or
+    # term vectored or anything like that.
+    # 
+    # name:: The name of the field
+    # value:: The binary value
+    # store:: How _value_ should be stored (compressed or not.)
+    def Field.new_binary_field(name, value, stored)
+      if (stored == Store::NO)
+        raise ArgumentError, "binary values can't be unstored"
+      end
+      Field.new(name, value, stored, Index::NO, TermVector::NO, true)
+    end
+
+    # Prints a Field for human consumption.
+    def to_s()
+      str = ""
+      if (@stored)
+        str << "stored"
+        @str << @compressed ? "/compressed," : "/uncompressed,"
+      end
+      if (@indexed) then str << "indexed," end
+      if (@tokenized) then str << "tokenized," end
+      if (@store_term_vector) then str << "store_term_vector," end
+      if (@store_offset)
+        str << "term_vector_offsets,"
+      end
+      if (@store_position)
+        str << "term_vector_position,"
+      end 
+      if (@binary) then str << "binary," end
+
+      str << '<'
+      str << @name
+      str << ':'
+
+      if (@data != null)
+        str << @data.to_s
+      end
+
+      str << '>'
+    end  
+  end
+end

data/lib/ferret/index.rb

@@ -0,0 +1,26 @@
+require 'ferret/index/index_file_names'
+require 'ferret/index/term'
+require 'ferret/index/term_buffer'
+require 'ferret/index/term_doc_enum'
+require 'ferret/index/multiple_term_doc_pos_enum'
+require 'ferret/index/term_enum'
+require 'ferret/index/term_info'
+require 'ferret/index/term_infos_io'
+require 'ferret/index/term_vector_offset_info'
+require 'ferret/index/term_vectors_io'
+require 'ferret/index/field_infos'
+require 'ferret/index/fields_io'
+require 'ferret/index/compound_file_io'
+require 'ferret/index/term_buffer'
+require 'ferret/index/segment_term_enum'
+require 'ferret/index/segment_term_vector'
+require 'ferret/index/segment_merge_info'
+require 'ferret/index/segment_merge_queue'
+require 'ferret/index/segment_infos'
+require 'ferret/index/document_writer'
+require 'ferret/index/index_reader'
+require 'ferret/index/index_writer'
+require 'ferret/index/multi_reader'
+require 'ferret/index/segment_merger'
+require 'ferret/index/segment_reader'
+require 'ferret/index/index'

data/lib/ferret/index/compound_file_io.rb

@@ -0,0 +1,343 @@
+require 'monitor'
+
+module Ferret::Index
+
+  # Class for accessing a compound stream.
+  # This class implements a directory, but is limited to only read operations.
+  # Directory methods that would normally modify data raise.
+  class CompoundFileReader < Ferret::Store::Directory
+
+    include MonitorMixin
+
+    attr_reader :directory, :file_name
+
+    # Creates a Compound File Reader which contains a single file and has
+    # pointers to the individual files within. When it is initialized, the 
+    # compound file is set and the header is read so that it is ready to read
+    # the individual files within.
+    def initialize(dir, name)
+
+      super()
+
+      @directory = dir
+      @file_name = name
+      @entries = {}
+
+      success = false
+
+      begin
+        @stream = dir.open_input(name)
+
+        # read the directory and init files
+        count = @stream.read_vint()
+        entry = nil
+        count.times() do 
+          offset = @stream.read_long()
+          id = @stream.read_string()
+
+          if (entry != nil)
+            # set length of the previous entry
+            entry.length = offset - entry.offset
+          end
+
+          entry = FileEntry.new(offset)
+          @entries[id] = entry
+        end
+
+        # set the length of the final entry
+        if (entry != nil)
+          entry.length = @stream.length() - entry.offset
+        end
+
+        success = true
+
+      ensure
+
+        if not success and (@stream != nil)
+          begin
+            @stream.close()
+          rescue IOError
+          end
+        end
+      end
+    end
+
+    def close()
+      synchronize do
+        if (@stream == nil): raise(IOError, "Already closed") end
+
+        @entries.clear()
+        @stream.close()
+        @stream = nil
+      end
+    end
+
+    def open_input(id)
+      synchronize do
+        if (@stream == nil)
+          raise(IOError, "Stream closed")
+        end
+
+        entry = @entries[id]
+        if (entry == nil)
+          raise(IOError, "No sub-file with id " + id + " found")
+        end
+        return CSIndexInput.new(@stream, entry.offset, entry.length)
+      end
+    end
+
+    # Returns an array of strings, one for each file in the directory.
+    def list()
+      return @entries.keys()
+    end
+
+    # Returns true iff a file with the given name exists.
+    def file_exists(name)
+      return @entries.key?(name)
+    end
+
+    # Returns the time the named file was last modified.
+    def modified(name)
+      return @directory.modified(@file_name)
+    end
+
+    # Set the modified time of an existing file to now.
+    def touch(name)
+      @directory.touch(@file_name)
+    end
+
+    # Not implemented
+    def delete(name) raise(UnsupportedOperationError) end
+
+    # Not implemented
+    def rename(from, to) raise(UnsupportedOperationError) end
+
+    # Returns the length of a file in the directory.
+    def file_length(name)
+      e = @entries[name]
+      if (e == nil): raise(IOError, "File " + name + " does not exist") end
+      return e.length
+    end
+
+    # Not implemented
+    def create_output(name) raise(UnsupportedOperationError) end
+
+    # Not implemented
+    def make_lock(name) raise(UnsupportedOperationError) end
+
+    # Implementation of an IndexInput that reads from a portion of the
+    # compound file.
+    class CSIndexInput < Ferret::Store::BufferedIndexInput
+      attr_reader :length
+
+      def initialize(base, file_offset, length)
+        super()
+        @base = base
+        @base.extend(MonitorMixin)
+        @file_offset = file_offset
+        @length = length
+      end
+
+      # Closes the stream to further operations.
+      def close() end
+
+      private
+        # Expert: implements buffer refill.  Reads bytes from the current
+        # position in the input.
+        #
+        # b:: the array to read bytes into
+        # offset:: the offset in the array to start storing bytes
+        # len:: the number of bytes to read
+        def read_internal(b, offset, len)
+          @base.synchronize() do
+            start = pos()
+            if(start + len > @length): raise(EOFError, "read past EOF") end
+            @base.seek(@file_offset + start)
+            @base.read_bytes(b, offset, len)
+          end
+        end
+
+        # Expert: implements seek.  Sets current position in @file, where
+        # the next {@link #read_internal(byte[],int,int)} will occur.
+        def seek_internal(pos) end
+    end
+
+    private
+      # Base info
+      class FileEntry
+        attr_accessor :offset, :length
+        def initialize(offset)
+          @offset = offset
+        end
+      end
+
+  end
+
+  # Combines multiple files into a single compound file.
+  # The file format:
+  #
+  #   * VInt fileCount
+  #   * {Directory} fileCount entries with the following structure:
+  #     + long data_offset
+  #     + UTFString extension
+  #   * {File Data} fileCount entries with the raw data of the corresponding file
+  #
+  # The fileCount integer indicates how many files are contained in this compound
+  # file. The {directory} that follows has that many entries. Each directory entry
+  # contains an encoding identifier, a long pointer to the start of this file's
+  # data section, and a UTF String with that file's extension.
+  class CompoundFileWriter
+
+    attr_reader :directory, :file_name
+
+    # Create the compound stream in the specified file. The file name is the
+    # entire name (no extensions are added).
+    def initialize(dir, name)
+      @directory = dir
+      @file_name = name
+      @ids = Set.new
+      @file_entries = []
+      @merged = false
+    end
+
+    # Add a source stream. _file_name_ is the string by which the 
+    # sub-stream will be known in the compound stream.
+    # 
+    # Throws:: IllegalStateError if this writer is closed
+    # Throws:: IllegalArgumentError if a file with the same name
+    #          has been added already
+    def add_file(file_name)
+      if @merged
+        raise(IllegalStateError, "Can't add extensions after merge has been called")
+      end
+
+      if not @ids.add?(file_name)
+        raise(IllegalArgumentError, "File " + file + " already added")
+      end
+
+      entry = FileEntry.new(file_name)
+      @file_entries << entry
+    end
+
+    # Merge files with the extensions added up to now.
+    # All files with these extensions are combined sequentially into the
+    # compound stream. After successful merge, the source files
+    # are deleted.
+    #
+    # Throws:: IllegalStateException if close() had been called before or
+    #          if no file has been added to this object
+    def close()
+
+      if @merged
+        raise(IllegalStateException, "Merge already performed")
+      end
+
+      if @file_entries.empty?
+        raise(IllegalStateException, "No entries to merge have been defined")
+      end
+
+      @merged = true
+
+      # open the compound stream
+      os = nil
+      begin
+        os = @directory.create_output(@file_name)
+
+        # Write the number of entries
+        os.write_vint(@file_entries.size)
+
+        # Write the directory with all offsets at 0.
+        # Remember the positions of directory entries so that we can
+        # adjust the offsets later
+        @file_entries.each do |fe|
+          fe.directory_offset = os.pos()
+          os.write_long(0)  # for now
+          os.write_string(fe.file_name)
+        end
+
+        # Open the files and copy their data into the stream.
+        # Remember the locations of each file's data section.
+        @file_entries.each do |fe|
+          fe.data_offset = os.pos()
+          copy_file(fe, os)
+        end
+
+        # Write the data offsets into the directory of the compound stream
+        @file_entries.each do |fe|
+          os.seek(fe.directory_offset)
+          os.write_long(fe.data_offset)
+        end
+
+        # Close the output stream. Set the os to nil before trying to
+        # close so that if an exception occurs during the close, the
+        # finally clause below will not attempt to close the stream
+        # the second time.
+        tmp = os
+        os = nil
+        tmp.close()
+
+      ensure
+        if (os != nil)
+          begin
+            os.close()
+          rescue
+          end
+        end
+      end
+    end
+
+    private
+
+      # Internal class for holding a file
+      class FileEntry
+
+        attr_accessor :file_name, :directory_offset, :data_offset
+
+        def initialize(file_name)
+          @file_name = file_name
+        end
+
+      end
+
+      # Copy the contents of the file with specified extension into the
+      # provided output stream. Use a buffer for moving data
+      # to reduce memory allocation.
+      def copy_file(source, os)
+        is = nil
+        begin
+          start_ptr = os.pos()
+
+          is = @directory.open_input(source.file_name)
+          remainder = length = is.length
+
+          buffer = Ferret::Store::BUFFER.clone
+          while (remainder > 0)
+            len = [remainder, Ferret::Store::BUFFER_SIZE].min
+            is.read_bytes(buffer, 0, len)
+            os.write_bytes(buffer, len)
+            remainder -= len
+          end
+
+          # Verify that remainder is 0
+          if (remainder != 0)
+            raise(IOError,
+              "Non-zero remainder length after copying: " + remainder.to_s +
+                " (id: " + source.file_name + ", length: " + length.to_s +
+                ", buffer size: " + Ferret::Store::BUFFER_SIZE.to_s + ")")
+          end
+
+          # Verify that the output length diff is equal to original file
+          end_ptr = os.pos()
+          diff = end_ptr - start_ptr
+          if (diff != length)
+            raise(IOError,
+              "Difference in the output file offsets " + diff.to_s +
+                " does not match the original file length " + length.to_s)
+          end
+
+        ensure
+          if (is != nil): is.close() end
+        end
+      end
+  end
+end