ferret 0.9.6 → 0.10.0

data/lib/ferret/index/compound_file_io.rb

@@ -1,338 +0,0 @@
-require 'monitor'
-
-module Ferret::Index
-
-  # Class for accessing a compound stream.
-  # This class implements a directory, but is limited to only read operations.
-  # Directory methods that would normally modify data raise.
-  class CompoundFileReader < Ferret::Store::Directory
-
-    include MonitorMixin
-
-    attr_reader :directory, :file_name
-
-    # Creates a Compound File Reader which contains a single file and has
-    # pointers to the individual files within. When it is initialized, the 
-    # compound file is set and the header is read so that it is ready to read
-    # the individual files within.
-    def initialize(dir, name)
-
-      super()
-
-      @directory = dir
-      @file_name = name
-      @entries = {}
-
-      success = false
-
-      begin
-        @stream = dir.open_input(name)
-
-        # read the directory and init files
-        count = @stream.read_vint()
-        entry = nil
-        count.times() do 
-          offset = @stream.read_long()
-          id = @stream.read_string()
-
-          if (entry != nil)
-            # set length of the previous entry
-            entry.length = offset - entry.offset
-          end
-
-          entry = FileEntry.new(offset)
-          @entries[id] = entry
-        end
-
-        # set the length of the final entry
-        if (entry != nil)
-          entry.length = @stream.length() - entry.offset
-        end
-
-        success = true
-
-      ensure
-
-        if not success and (@stream != nil)
-          begin
-            @stream.close()
-          rescue IOError
-          end
-        end
-      end
-    end
-
-    def close()
-      synchronize do
-        if (@stream == nil): raise(IOError, "Already closed") end
-
-        @entries.clear()
-        @stream.close()
-        @stream = nil
-      end
-    end
-
-    def open_input(id)
-      synchronize do
-        if (@stream == nil)
-          raise(IOError, "Stream closed")
-        end
-
-        entry = @entries[id]
-        if (entry == nil)
-          raise(IOError, "No sub-file with id " + id + " found")
-        end
-        return CSIndexInput.new(@stream, entry.offset, entry.length)
-      end
-    end
-
-    # Returns an array of strings, one for each file in the directory.
-    def list()
-      return @entries.keys()
-    end
-
-    # Returns true iff a file with the given name exists.
-    def exists?(name)
-      return @entries.key?(name)
-    end
-
-    # Returns the time the named file was last modified.
-    def modified(name)
-      return @directory.modified(@file_name)
-    end
-
-    # Set the modified time of an existing file to now.
-    def touch(name)
-      @directory.touch(@file_name)
-    end
-
-    # Not implemented
-    def remove(name) raise(NotImplementedError) end
-
-    # Not implemented
-    def rename(from, to) raise(NotImplementedError) end
-
-    # Returns the length of a file in the directory.
-    def length(name)
-      e = @entries[name]
-      if (e == nil): raise(IOError, "File " + name + " does not exist") end
-      return e.length
-    end
-
-    # Not implemented
-    def create_output(name) raise(NotImplementedError) end
-
-    # Not implemented
-    def make_lock(name) raise(NotImplementedError) end
-
-    # Implementation of an IndexInput that reads from a portion of the
-    # compound file.
-    class CSIndexInput < Ferret::Store::BufferedIndexInput
-      attr_reader :length
-
-      def initialize(base, file_offset, length)
-        super()
-        @base = base
-        @base.extend(MonitorMixin)
-        @file_offset = file_offset
-        @length = length
-      end
-
-      # Closes the stream to further operations.
-      def close() end
-
-      private
-        # Expert: implements buffer refill.  Reads bytes from the current
-        # position in the input.
-        #
-        # b:: the array to read bytes into
-        # offset:: the offset in the array to start storing bytes
-        # len:: the number of bytes to read
-        def read_internal(b, offset, len)
-          @base.synchronize() do
-            start = pos()
-            if(start + len > @length): raise(EOFError, "read past EOF") end
-            @base.seek(@file_offset + start)
-            @base.read_bytes(b, offset, len)
-          end
-        end
-
-        # Expert: implements seek.  Sets current position in @file, where
-        # the next {@link #read_internal(byte[],int,int)} will occur.
-        def seek_internal(pos) end
-    end
-
-    private
-      # Base info
-      class FileEntry
-        attr_accessor :offset, :length
-        def initialize(offset)
-          @offset = offset
-        end
-      end
-
-  end
-
-  # Combines multiple files into a single compound file.
-  # The file format:
-  #
-  #   * VInt fileCount
-  #   * {Directory} fileCount entries with the following structure:
-  #     + long data_offset
-  #     + UTFString extension
-  #   * {File Data} fileCount entries with the raw data of the corresponding file
-  #
-  # The fileCount integer indicates how many files are contained in this compound
-  # file. The {directory} that follows has that many entries. Each directory entry
-  # contains an encoding identifier, a long pointer to the start of this file's
-  # data section, and a UTF String with that file's extension.
-  class CompoundFileWriter
-
-    class StateError < Exception
-    end
-
-    attr_reader :directory, :file_name
-
-    # Create the compound stream in the specified file. The file name is the
-    # entire name (no extensions are added).
-    def initialize(dir, name)
-      @directory = dir
-      @file_name = name
-      @ids = Set.new
-      @file_entries = []
-      @merged = false
-    end
-
-    # Add a source stream. _file_name_ is the string by which the 
-    # sub-stream will be known in the compound stream.
-    # 
-    # Raises:: StateError if this writer is closed
-    # Raises:: ArgumentError if a file with the same name
-    #          has been added already
-    def add_file(file_name)
-      if @merged
-        raise(StateError, "Can't add extensions after merge has been called")
-      end
-
-      if not @ids.add?(file_name)
-        raise(ArgumentError, "File #{file_name} already added")
-      end
-
-      entry = FileEntry.new(file_name)
-      @file_entries << entry
-    end
-
-    # Merge files with the extensions added up to now.
-    # All files with these extensions are combined sequentially into the
-    # compound stream. After successful merge, the source files
-    # are deleted.
-    #
-    # Throws:: StateException if close() had been called before or
-    #          if no file has been added to this object
-    def close()
-
-      if @merged
-        raise(StateException, "Merge already performed")
-      end
-
-      if @file_entries.empty?
-        raise(StateException, "No entries to merge have been defined")
-      end
-
-      @merged = true
-
-      # open the compound stream
-      os = nil
-      begin
-        os = @directory.create_output(@file_name)
-
-        # Write the number of entries
-        os.write_vint(@file_entries.size)
-
-        # Write the directory with all offsets at 0.
-        # Remember the positions of directory entries so that we can
-        # adjust the offsets later
-        @file_entries.each do |fe|
-          fe.dir_offset = os.pos()
-          os.write_long(0)  # for now
-          os.write_string(fe.file_name)
-        end
-
-        # Open the files and copy their data into the stream.
-        # Remember the locations of each file's data section.
-        @file_entries.each do |fe|
-          fe.data_offset = os.pos()
-          copy_file(fe, os)
-        end
-
-        # Write the data offsets into the directory of the compound stream
-        @file_entries.each do |fe|
-          os.seek(fe.dir_offset)
-          os.write_long(fe.data_offset)
-        end
-
-        # Close the output stream. Set the os to nil before trying to
-        # close so that if an exception occurs during the close, the
-        # finally clause below will not attempt to close the stream
-        # the second time.
-        tmp = os
-        os = nil
-        tmp.close()
-
-      ensure
-        if (os != nil)
-          begin
-            os.close()
-          rescue
-          end
-        end
-      end
-    end
-
-    private
-
-      # Internal class for holding a file
-      FileEntry = Struct.new(:file_name, :dir_offset, :data_offset)
-
-      # Copy the contents of the file with specified extension into the
-      # provided output stream. Use a buffer for moving data
-      # to reduce memory allocation.
-      def copy_file(source, os)
-        is = nil
-        begin
-          start_ptr = os.pos()
-
-          is = @directory.open_input(source.file_name)
-          remainder = length = is.length
-
-          buffer = Ferret::Store::BUFFER.clone
-          while (remainder > 0)
-            len = [remainder, Ferret::Store::BUFFER_SIZE].min
-            is.read_bytes(buffer, 0, len)
-            os.write_bytes(buffer, len)
-            remainder -= len
-          end
-
-          # Verify that remainder is 0
-          if (remainder != 0)
-            raise(IOError,
-              "Non-zero remainder length after copying: #{remainder} " +
-              "(id: #{source.file_name}, length: #{length}, buffer size: " +
-              " #{Ferret::Store::BUFFER_SIZE})")
-          end
-
-          # Verify that the output length diff is equal to original file
-          end_ptr = os.pos()
-          diff = end_ptr - start_ptr
-          if (diff != length)
-            raise(IOError,
-              "Difference in the output file offsets #{diff}" +
-              " does not match the original file length #{length}")
-          end
-
-        ensure
-          if (is != nil): is.close() end
-        end
-      end
-  end
-end

data/lib/ferret/index/document_writer.rb

@@ -1,289 +0,0 @@
-require 'ferret/search/similarity'
-
-module Ferret::Index
-
-  class DocumentWriter 
-    # If non-nil, a message will be printed to this if max_field_length is
-    # reached.
-    attr_writer :info_stream
-
-    # directory::           The directory to write the document information to
-    # analyzer::            The analyzer to use for the document
-    # similarity::          The Similarity function writer.similarity
-    # max_field_length::    The maximum number of tokens a field may have
-    #                       writer.max_field_length
-    # term_index_interval:: The interval of terms in the index 
-    #                       writer.max_field_length
-    def initialize(directory,
-                   analyzer,
-                   similarity,
-                   max_field_length,
-                   term_index_interval = IndexWriter::DEFAULT_TERM_INDEX_INTERVAL) 
-      @directory = directory
-      @analyzer = analyzer
-      @similarity = similarity
-      @max_field_length = max_field_length
-      @term_index_interval = term_index_interval
-
-      # Keys are Terms, values are Postings.
-      # Used to buffer a document before it is written to the index.
-      @posting_table = {}
-
-      @term_buffer = Term.new("", "")
-    end
-
-    def add_document(segment, doc)
-           
-      # write field names
-      @field_infos = FieldInfos.new()
-      @field_infos << doc
-      @field_infos.write_to_dir(@directory, segment + ".fnm")
-
-      # write field values
-      fields_writer = FieldsWriter.new(@directory, segment, @field_infos)
-      begin 
-        fields_writer.add_document(doc)
-      ensure 
-        fields_writer.close()
-      end
-
-      # invert doc into posting_table
-      @posting_table.clear();                    # clear posting_table
-      arr_size = @field_infos.size
-      @field_lengths = Array.new(arr_size, 0)    # init field_lengths
-      @field_positions = Array.new(arr_size, 0)  # init field_positions
-      @field_offsets = Array.new(arr_size, 0)    # init field_offsets
-      @field_boosts = Array.new(arr_size, doc.boost) # init field_boosts
-
-      invert_document(doc)
-
-      # sort posting_table into an array
-      postings = sort_posting_table()
-
-  #    for (int i = 0; i < postings.length; i += 1) 
-  #      Posting posting = postings[i]
-  #      print(posting.term)
-  #      print(" freq=" + posting.freq)
-  #      print(" pos=")
-  #      print(posting.positions[0])
-  #      for (int j = 1; j < posting.freq; j += 1)
-  #        print("," + posting.positions[j])
-  #      puts("")
-  #    end
-
-      # write postings
-      write_postings(postings, segment)
-
-      # write norms of indexed fields
-      write_norms(segment)
-
-    end
-
-    private
-
-      # Tokenizes the fields of a document into Postings.
-      def invert_document(doc)
-             
-        fields = doc.all_fields
-        fields.each do |field|
-          field_name = field.name
-          field_info = @field_infos[field_name]
-          field_number = field_info.number
-
-          length = @field_lengths[field_number]     # length of field
-          position = @field_positions[field_number] # position in field
-          position += @analyzer.pos_inc_gap(field_name) if length > 0
-          offset = @field_offsets[field_number]     # offset field
-
-          if field_info.indexed?
-            if not field.tokenized? # un-tokenized field
-              string_value = field.string_value
-              if field_info.store_offsets?
-                add_position(field_name,
-                             string_value,
-                             position,
-                             TermVectorOffsetInfo.new(offset,
-                                      offset + string_value.length))
-                position += 1
-              else
-                add_position(field_name, string_value, position, nil)
-                position += 1
-              end
-              offset += string_value.length()
-              length += 1
-            else 
-            
-              reader = field.reader_value()
-
-              # Tokenize field and add to posting_table
-              stream = @analyzer.token_stream(field_name, reader)
-              begin 
-                last_token = nil
-                while token = stream.next
-                  position += (token.pos_inc - 1)
-                  
-                  if(field_info.store_offsets?())
-                    add_position(field_name,
-                                 token.text(),
-                                 position,
-                                 TermVectorOffsetInfo.new(
-                                   offset + token.start_offset(),
-                                   offset + token.end_offset()))
-                    position += 1
-                  else
-                    add_position(field_name, token.text(), position, nil)
-                    position += 1
-                  end
-                  
-                  last_token = token
-                  length += 1
-                  if (length > @max_field_length) 
-                    if @info_stream
-                      @info_stream.puts("max_field_length " + @max_field_length.to_s + " reached, ignoring following tokens")
-                    end
-                    break
-                  end
-                end
-                
-                if(last_token != nil)
-                  offset += last_token.end_offset() + 1
-                end
-                
-              ensure 
-                stream.close()
-              end
-            end
-
-            @field_lengths[field_number] = length    # save field length
-            @field_positions[field_number] = position    # save field position
-            @field_boosts[field_number] *= field.boost
-            @field_offsets[field_number] = offset
-          end
-        end
-      end
-
-
-      def add_position(field, text, position, tv_offset_info) 
-        @term_buffer.set!(field, text)
-        #puts("Offset: " + tv_offset_info)
-        posting = @posting_table[@term_buffer]
-        if (posting != nil) # word seen before
-          freq = posting.freq
-          posting.positions[freq] = position     # add new position
-          posting.offsets[freq] = tv_offset_info # add new position
-
-          if (tv_offset_info != nil) 
-            posting.offsets[freq] = tv_offset_info
-          end
-          posting.freq = freq + 1        # update frequency
-        else # word not seen before
-          term = Term.new(field, text)
-          @posting_table[term] = Posting.new(term, position, tv_offset_info)
-        end
-      end
-
-      def sort_posting_table() 
-        # copy @posting_table into an array
-        return @posting_table.values.sort { |x,y| x.term <=> y.term }
-      end
-
-      def write_postings(postings, segment)
-             
-        freq = nil
-        prox = nil
-        tis_writer = nil
-        tv_writer = nil
-        begin 
-          #open files for inverse index storage
-          freq = @directory.create_output(segment + ".frq")
-          prox = @directory.create_output(segment + ".prx")
-          tis_writer = TermInfosWriter.new(@directory, segment, @field_infos,
-                                    @term_index_interval)
-          ti = TermInfo.new()
-          current_field = nil
-
-          postings.each do |posting|
-            # add an entry to the dictionary with pointers to prox and freq files
-            ti.set_values!(1, freq.pos(), prox.pos(), -1)
-            tis_writer.add(posting.term, ti)
-
-            # add an entry to the freq file
-            posting_freq = posting.freq
-            if (posting_freq == 1)          # optimize freq=1
-              freq.write_vint(1)            # set low bit of doc num.
-            else 
-              freq.write_vint(0)            # the document number
-              freq.write_vint(posting_freq) # frequency in doc
-            end
-
-            last_position = 0               # write positions
-            posting.positions.each do |position|
-              prox.write_vint(position - last_position)
-              last_position = position
-            end
-            # check to see if we switched to a new field
-            term_field = posting.term.field
-            if (current_field != term_field) 
-              # changing field - see if there is something to save
-              current_field = term_field
-              fi = @field_infos[current_field]
-              if (fi.store_term_vector?) 
-                if tv_writer.nil? 
-                  tv_writer = TermVectorsWriter.new(@directory, segment, @field_infos)
-                  tv_writer.open_document()
-                end
-                tv_writer.open_field(current_field)
-
-              elsif not tv_writer.nil?
-                tv_writer.close_field()
-              end
-            end
-            if not tv_writer.nil? and tv_writer.field_open?
-                tv_writer.add_term(posting.term.text, posting_freq, posting.positions, posting.offsets)
-            end
-          end
-          if not tv_writer.nil?
-            tv_writer.close_document()
-          end
-        ensure 
-          # make an effort to close all streams we can but remember and re-raise
-          # the last exception encountered in this process
-          keep = nil
-          [freq, prox, tis_writer, tv_writer].compact.each do |obj|
-            begin
-              obj.close
-            rescue IOError => e
-              keep = e
-            end
-          end
-          raise keep if not keep.nil?
-        end
-      end
-
-      def write_norms(segment)
-        @field_infos.each_with_index do |fi, i|
-          if fi.indexed? and not fi.omit_norms?
-            norm = @field_boosts[i] * @similarity.length_norm(fi.name, @field_lengths[i])
-            norms = @directory.create_output(segment + ".f" + i.to_s)
-            begin 
-              norms.write_byte(Ferret::Search::Similarity.encode_norm(norm))
-            ensure 
-              norms.close()
-            end
-          end
-        end
-      end
-      
-  end
-
-  class Posting # info about a Term in a doc
-    attr_accessor :term, :freq, :positions, :offsets
-
-    def initialize(t, position, offset) 
-      @term = t
-      @freq = 1
-      @positions = [position]
-      @offsets = [offset]
-    end
-  end
-end