ferret 0.9.6 → 0.10.0

data/lib/ferret/index/segment_infos.rb

@@ -1,130 +0,0 @@
-module Ferret
-  module Index
-    # Holds the info for one segment.
-    #
-    # ToDo: Does the dir really need to be stored here?
-    class SegmentInfo
-      attr_accessor :name, :doc_count, :directory
-
-      def initialize(name, doc_count, dir)
-        @name = name
-        @doc_count = doc_count
-        @directory = dir
-      end
-
-      def ==(o)
-        (o.name == @name and o.doc_count == @doc_count)
-      end
-    end
-
-    class SegmentInfos < Array
-      # for compatability with Java Ferret files
-      FORMAT = -1
-      SEGMENT_FILENAME = "segments"
-      TEMPORARY_SEGMENT_FILENAME = "segments.new"
-
-      attr_reader :version    # counts how often the index has been modified
-                              # by adding or deleting docs
-      attr_accessor :counter  # used to name new segments??
-
-      # Current version number from segments file.
-      def SegmentInfos.read_current_version(directory)
-        return 0 if not directory.exists?(SEGMENT_FILENAME)
-        input = directory.open_input(SEGMENT_FILENAME)
-        @format = 0
-        @version = 0
-        begin
-          @format = input.read_int()
-          if(@format < 0)
-            if (@format < FORMAT) then raise "Unknown format version: " + @format end
-            @version = input.read_long() # read version
-          end
-        ensure
-          input.close()
-        end
-         
-        if(@format < 0)
-          return @version
-        end
-
-        # We cannot be sure about the format of the file.
-        # Therefore we have to read the whole file and cannot simply
-        # seek to the version entry.
-
-        sis = SegmentInfos.new()
-        sis.read(directory)
-        return sis.version()
-      end
-      
-      def initialize()
-        @version = Time.now.to_i * 1000
-        @counter = 0
-      end
-
-      def initialize_copy(o)
-        super
-        o.each_index {|i| self[i] = o[i].clone}
-      end
-
-      def read(directory)
-        input = directory.open_input(SEGMENT_FILENAME)
-        begin
-          @format = input.read_int()
-          if(@format < 0) # file contains explicit format info
-            # check that it is a format we can understand
-            if (@format < FORMAT) then raise "Unknown format version: " + @format end
-            @version = input.read_long()
-            @counter = input.read_int()
-          else  # file is in old format without explicit format info
-            @counter = @format
-          end
-          
-          seg_count = input.read_int()
-          seg_count.times do
-            self << SegmentInfo.new(input.read_string(),
-                                    input.read_int(),
-                                    directory)
-          end
-         
-          if(@format >= 0)   
-            # in old format the version number may be at the end of the file
-            if (input.pos() >= input.length())
-              @version = 0 # old file format without version number
-            else
-              @version = input.read_long() # read version
-            end
-          end
-        ensure
-          input.close()
-        end
-      end
-
-      def write(directory)
-        output = directory.create_output(TEMPORARY_SEGMENT_FILENAME)
-        begin
-          output.write_int(FORMAT) # write FORMAT
-          output.write_long(@version += 1) # every write changes the index
-          output.write_int(@counter) # write counter
-          output.write_int(size()) # write infos
-          each() do |si|
-            output.write_string(si.name)
-            output.write_int(si.doc_count)
-          end         
-        
-        ensure
-          output.close()
-        end
-
-        # install new segment info
-        directory.rename(TEMPORARY_SEGMENT_FILENAME, SEGMENT_FILENAME)
-      end
-
-      def to_s()
-        str = "\nSegmentInfos: <"
-        each() { |si| str << "#{si.name}:#{si.doc_count}," }
-        str[-1] = ">"
-        str
-      end
-    end
-  end
-end

data/lib/ferret/index/segment_merge_info.rb

@@ -1,49 +0,0 @@
-module Ferret
-  module Index
-    class SegmentMergeInfo 
-      attr_reader :term_enum, :reader, :base, :term_buffer
-
-      def initialize(base, term_enum, reader)
-        @base = base
-        @reader = reader
-        @term_enum = term_enum
-        @term_buffer = term_enum.term_buffer
-      end
-
-      def positions
-        @postings ||= @reader.term_positions()
-      end
-
-      def doc_map
-        if @doc_map.nil?
-          # build array which maps document numbers around deletions 
-          if (@reader.has_deletions?()) 
-            max_doc = @reader.max_doc()
-            @doc_map = Array.new(max_doc)
-            j = 0
-            max_doc.times do |i|
-              if (@reader.deleted?(i))
-                @doc_map[i] = -1
-              else
-                @doc_map[i] = j
-                j += 1
-              end
-            end
-          end
-        end
-        return @doc_map
-      end
-
-      def next?
-        @term_enum.next?
-      end
-
-      def close()
-        @term_enum.close()
-        @postings.close() if @postings
-        @reader = nil
-      end
-    end
-  end
-end
-

data/lib/ferret/index/segment_merge_queue.rb

@@ -1,16 +0,0 @@
-module Ferret::Index
-  class SegmentMergeQueue < Ferret::Utils::PriorityQueue 
-    def less_than(sti_a, sti_b) 
-      if sti_a.term_buffer == sti_b.term_buffer
-        return sti_a.base < sti_b.base
-      else
-        return sti_a.term_buffer < sti_b.term_buffer
-      end
-    end
-
-    def close()
-      @heap.each {|sti| sti.close if sti}
-      clear
-    end
-  end
-end

data/lib/ferret/index/segment_merger.rb

@@ -1,358 +0,0 @@
-module Ferret::Index
-
-  # The SegmentMerger class combines two or more Segments, represented by
-  # an IndexReader#add, into a single Segment.  After adding the
-  # appropriate readers, call the merge method to combine the segments.
-  # 
-  # If the compoundFile flag is set, then the segments will be merged
-  # into a compound file.
-  class SegmentMerger 
-
-    # dir:: The Directory to merge the other segments into
-    # name:: The name of the new segment
-    def initialize(dir, name,
-                  term_index_interval = IndexWriter::DEFAULT_TERM_INDEX_INTERVAL) 
-      @directory = dir
-      @segment = name
-      @term_index_interval = term_index_interval
-      @readers = []
-      @field_infos = nil
-      @freq_output = nil
-      @prox_output = nil
-      @term_infos_writer = nil
-      @queue = nil
-      @term_info = TermInfo.new()
-      @skip_buffer = Ferret::Store::RAMDirectory::RAMIndexOutput.new(
-                          Ferret::Store::RAMDirectory::RAMFile.new(""))
-    end
-
-    # Add an IndexReader to the collection of readers that are to be merged
-    # reader::
-    def add(reader) 
-      @readers << reader
-    end
-    alias :<< :add
-
-    # 
-    # i:: The index of the reader to return
-    # returns:: The ith reader to be merged
-    def segment_reader(i) 
-      return @readers[i]
-    end
-
-    # Merges the readers specified by the #add method into the directory
-    # passed to the constructor
-    # 
-    # returns:: The number of documents that were merged
-    # raises:: IOError
-    def merge()
-      value = merge_fields()
-      merge_terms()
-      merge_norms()
-      merge_vectors() if @field_infos.has_vectors?
-      return value
-    end
-    
-    # close all IndexReaders that have been added.  Should not be called
-    # before merge().
-    #
-    # raises:: IOError
-    def close_readers()
-      @readers.each { |reader| reader.close }
-    end
-
-    def create_compound_file(file_name)
-           
-      cfs_writer = CompoundFileWriter.new(@directory, file_name)
-
-      files = []
-      
-      # Basic files
-      IndexFileNames::COMPOUND_EXTENSIONS.each do |ext|
-        files << "#{@segment}.#{ext}"
-      end
-
-      # Field norm files
-      @field_infos.each_with_index do |fi, i|
-        if (fi.indexed? and not fi.omit_norms?) 
-          files << "#{@segment}.f#{i}"
-        end
-      end
-
-      # Vector files
-      if @field_infos.has_vectors?
-        IndexFileNames::VECTOR_EXTENSIONS.each do |ext|
-          files << "#{@segment}.#{ext}"
-        end
-      end
-
-      # Now merge all added files
-      files.each do |file|
-        cfs_writer.add_file(file)
-      end
-      
-      # Perform the merge
-      cfs_writer.close
-     
-      return files
-    end
-
-    def add_indexed(reader, field_infos, field_names,
-                    store_term_vectors,
-                    store_position_with_term_vector,
-                    store_offset_with_term_vector)
-      field_names.each do |field|
-        field_infos.add(field, true,
-                        store_term_vectors,
-                        store_position_with_term_vector,
-                        store_offset_with_term_vector,
-                        !reader.has_norms?(field))
-      end
-    end
-    private :add_indexed
-  
-
-    # 
-    # returns:: The number of documents in all of the readers
-    # raises:: IOError
-    def merge_fields()
-      @field_infos = FieldInfos.new()      # merge field names
-      doc_count = 0
-      @readers.each do |reader|
-        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR_WITH_POSITION_OFFSET), true, true, true)
-        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR_WITH_POSITION), true, true, false)
-        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR_WITH_OFFSET), true, false, true)
-        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::TERM_VECTOR), true, false, false)
-        add_indexed(reader, @field_infos, reader.get_field_names(IndexReader::FieldOption::INDEXED), false, false, false)
-        @field_infos.add_fields(reader.get_field_names(IndexReader::FieldOption::UNINDEXED), false)
-      end
-      @field_infos.write_to_dir(@directory, @segment + ".fnm")
-
-      # merge field values
-      fields_writer = FieldsWriter.new(@directory, @segment, @field_infos)
-
-      begin 
-        @readers.each do |reader|
-          max_doc = reader.max_doc()
-          max_doc.times do |j|
-            if not reader.deleted?(j) # skip deleted docs
-              fields_writer.add_document(reader.get_document(j))
-              doc_count += 1
-            end
-          end
-        end
-      ensure 
-        fields_writer.close()
-      end
-      return doc_count
-    end
-
-    # Merge the TermVectors from each of the segments into the new one.
-    # raises:: IOError
-    def merge_vectors()
-      term_vectors_writer = TermVectorsWriter.new(@directory, @segment, @field_infos)
-
-      begin 
-        @readers.each do |reader|
-          max_doc = reader.max_doc()
-          max_doc.times do |doc_num|
-            # skip deleted docs
-            next if (reader.deleted?(doc_num)) 
-            term_vectors_writer.add_all_doc_vectors(reader.get_term_vectors(doc_num))
-          end
-        end
-      ensure 
-        term_vectors_writer.close()
-      end
-    end
-
-    def merge_terms()
-      begin 
-        @freq_output = @directory.create_output(@segment + ".frq")
-        @prox_output = @directory.create_output(@segment + ".prx")
-        @term_infos_writer =
-                TermInfosWriter.new(@directory, @segment, @field_infos,
-                                    @term_index_interval)
-        @skip_interval = @term_infos_writer.skip_interval
-        @queue = SegmentMergeQueue.new(@readers.size())
-
-        merge_term_infos()
-
-      ensure 
-        [@freq_output, @prox_output, @term_infos_writer, @queue].each do |obj|
-          obj.close()
-        end
-      end
-    end
-
-    def merge_term_infos()
-      base = 0
-      @readers.each do |reader|
-        term_enum = reader.terms()
-        smi = SegmentMergeInfo.new(base, term_enum, reader)
-        base += reader.num_docs()
-        if (smi.next?)
-          @queue.push(smi) # initialize @queue
-        else
-          smi.close()
-        end
-      end
-
-      match = Array.new(@readers.size)
-
-      while (@queue.size > 0) 
-        match_size = 0     # pop matching terms
-        match[match_size] = @queue.pop
-        match_size += 1
-        #term = match[0].term
-        term_buffer = match[0].term_buffer
-        top = @queue.top
-
-        #while top and term == top.term
-        while top and term_buffer == top.term_buffer
-          match[match_size] = @queue.pop
-          match_size += 1
-          top = @queue.top
-        end
-
-        merge_term_info(match, match_size)      # add new TermInfo
-
-        while (match_size > 0) 
-          match_size -= 1
-          smi = match[match_size]
-          if (smi.next?)
-            @queue.push(smi) # restore queue
-          else
-            smi.close()      # done with a segment
-          end
-        end
-      end
-    end
-
-    # Merge one term found in one or more segments. The array <code>smis</code>
-    # contains segments that are positioned at the same term. <code>N</code>
-    # is the number of cells in the array actually occupied.
-    # 
-    # smis:: array of segments
-    # n:: number of cells in the array actually occupied
-    def merge_term_info(smis, n)
-           
-      freq_pointer = @freq_output.pos
-      prox_pointer = @prox_output.pos
-
-      df = append_postings(smis, n)      # append posting data
-
-      skip_pointer = write_skip()
-
-      if (df > 0) 
-        # add an entry to the dictionary with pointers to prox and freq files
-        @term_info.set_values!(df, freq_pointer, prox_pointer, (skip_pointer - freq_pointer))
-        @term_infos_writer.add(smis[0].term_buffer.term, @term_info)
-      end
-    end
-
-    # Process postings from multiple segments all positioned on the
-    # same term. Writes out merged entries into @freq_utput and
-    # the @prox_output streams.
-    # 
-    # smis:: array of segments
-    # n:: number of cells in the array actually occupied
-    # returns:: number of documents across all segments where this term was found
-    def append_postings(smis, n)
-      last_doc = 0
-      df = 0            # number of docs w/ term
-      reset_skip()
-      n.times do |i|
-        smi = smis[i]
-        postings = smi.positions
-        base = smi.base
-        doc_map = smi.doc_map
-  
-        postings.seek(smi.term_enum)
-        while (postings.next?) 
-          doc = postings.doc()
-          doc = doc_map[doc] if (doc_map != nil) # work around deletions
-          doc += base                            # convert to merged space
-
-          if (doc < last_doc)
-            raise "docs out of order curent doc = " + doc.to_s +
-              " and previous doc = " + last_doc.to_s
-          end
-
-          df += 1
-
-          if ((df % @skip_interval) == 0) 
-            buffer_skip(last_doc)
-          end
-
-          doc_code = (doc - last_doc) << 1    # use low bit to flag freq=1
-          last_doc = doc
-
-          freq = postings.freq
-          if (freq == 1) 
-            @freq_output.write_vint(doc_code | 1) # write doc & freq=1
-          else 
-            @freq_output.write_vint(doc_code)     # write doc
-            @freq_output.write_vint(freq)         # write frequency in doc
-          end
-
-          last_position = 0        # write position deltas
-          freq.times do |j|
-            position = postings.next_position()
-            @prox_output.write_vint(position - last_position)
-            last_position = position
-          end
-        end
-      end
-      return df
-    end
-
-    def reset_skip() 
-      @skip_buffer.reset()
-      @last_skip_doc = 0
-      @last_skip_freq_pointer = @freq_output.pos
-      @last_skip_prox_pointer = @prox_output.pos
-    end
-
-    def buffer_skip(doc)
-      freq_pointer = @freq_output.pos
-      prox_pointer = @prox_output.pos
-
-      @skip_buffer.write_vint(doc - @last_skip_doc)
-      @skip_buffer.write_vint(freq_pointer - @last_skip_freq_pointer)
-      @skip_buffer.write_vint(prox_pointer - @last_skip_prox_pointer)
-
-      @last_skip_doc = doc
-      @last_skip_freq_pointer = freq_pointer
-      @last_skip_prox_pointer = prox_pointer
-    end
-
-    def write_skip()
-      skip_pointer = @freq_output.pos
-      @skip_buffer.write_to(@freq_output)
-      return skip_pointer
-    end
-
-    def merge_norms()
-      @field_infos.each_with_index do |fi, i|
-        if (fi.indexed? and not fi.omit_norms?) 
-          output = @directory.create_output(@segment + ".f" + i.to_s)
-          begin 
-            @readers.each do |reader|
-              max_doc = reader.max_doc()
-              input = "0" * max_doc
-              reader.get_norms_into(fi.name, input, 0)
-              max_doc.times do |k|
-                if not reader.deleted?(k) 
-                  output.write_byte(input[k])
-                end
-              end
-            end
-          ensure 
-            output.close()
-          end
-        end
-      end
-    end
-  end
-end