ferret 0.9.6 → 0.10.0

data/lib/ferret/index/term_enum.rb

@@ -1,52 +0,0 @@
-module Ferret
-  module Index
-    # Abstract class for enumerating terms.
-    #
-    # Term enumerations are always ordered by Term.<=>.  Each term in
-    # the enumeration is greater than all that precede it.
-    class TermEnum
-      # Increments the enumeration to the next element.  True if one exists.
-      def next?
-        raise NotImplementedError
-      end
-
-      # Returns the current Term in the enumeration.
-      def term
-        raise NotImplementedError
-      end
-
-      # Returns the doc_freq of the current Term in the enumeration.
-      def doc_freq
-        raise NotImplementedError
-      end
-
-      # Closes the enumeration to further activity, freeing resources.
-      def close
-        raise NotImplementedError
-      end
-      
-      # Term Vector support
-      # Skips terms to the first beyond the current whose value is
-      # greater or equal to _target_.
-      #
-      # Returns true iff there is such a term.
-      #
-      # Behaves as if written:
-      #
-      #   def skip_to(target)
-      #     while (target > term)
-      #       if (!next()) return false
-      #     end
-      #     return true
-      #   end
-      #
-      # Some implementations are considerably more efficient than that.
-      def skip_to(target)
-        while (target > term)
-          return false if not next?
-        end
-        return true
-      end
-    end
-  end
-end

data/lib/ferret/index/term_info.rb

@@ -1,37 +0,0 @@
-module Ferret::Index
-  # A TermInfo is the record of information stored for a term.
-  class TermInfo
-    attr_accessor :doc_freq, :freq_pointer, :prox_pointer, :skip_offset
-
-    def initialize(df=0, fp=0, pp=0, so=0)
-      set_values!(df, fp, pp, so)
-    end
-
-    def set!(ti)
-      @doc_freq = ti.doc_freq
-      @freq_pointer = ti.freq_pointer
-      @prox_pointer = ti.prox_pointer
-      @skip_offset = ti.skip_offset
-    end
-    
-    def set_values!(df=0, fp=0, pp=0, so=0)
-      @doc_freq = df
-      @freq_pointer = fp
-      @prox_pointer = pp
-      @skip_offset = so
-    end
-
-    def ==(o)
-      return false if !o.instance_of?(TermInfo)
-      @doc_freq == o.doc_freq &&
-        @freq_pointer == o.freq_pointer &&
-        @prox_pointer == o.prox_pointer &&
-        @skip_offset == o.skip_offset
-    end
-    alias eql? ==
-
-    def to_s()
-      "TermInfo:df=#{doc_freq}:fp=#{freq_pointer}:pp=#{prox_pointer}:so=#{skip_offset}"
-    end
-  end
-end

data/lib/ferret/index/term_infos_io.rb

@@ -1,321 +0,0 @@
-require 'monitor'
-module Ferret::Index
-
-  # This stores a monotonically increasing set of <Term, TermInfo> pairs in a
-  # Directory.  A TermInfos can be written once, in order.  
-  class TermInfosWriter 
-    attr_reader :index_interval, :skip_interval, :out
-    attr_writer :other
-    # The file format version, a negative number. 
-    FORMAT = -2
-
-
-    # TODO: the default values for these two parameters should be settable
-    # from IndexWriter.  However, once that's done, folks will start setting
-    # them to ridiculous values and complaining that things don't work well,
-    # as with mergeFactor.  So, let's wait until a number of folks find that
-    # alternate values work better.  Note that both of these values are
-    # stored in the segment, so that it's safe to change these w/o
-    # rebuilding all indexes.
-
-    # Expert: The fraction of terms in the "dictionary" which should be
-    # stored in RAM.  Smaller values use more memory, but make searching
-    # slightly faster, while larger values use less memory and make
-    # searching slightly slower.  Searching is typically not dominated by
-    # dictionary lookup, so tweaking this is rarely useful.
-    #
-    # Expert: The fraction of TermDocEnum entries stored in skip
-    # tables, used to accellerate TermDocEnum#skipTo(int).  Larger
-    # values result in smaller indexes, greater acceleration, but fewer
-    # accelerable cases, while smaller values result in bigger indexes, less
-    # acceleration and more accelerable cases. More detailed experiments
-    # would be useful here. 
-    def initialize(dir, segment, fis, interval, is_index = false)
-      @index_interval = interval
-      @skip_interval = 16
-      @last_index_pointer = 0
-      @last_term = Term.new("", "")
-      @last_term_info = TermInfo.new()
-      @size = 0
-      @is_index = is_index
-      @field_infos = fis
-      @out = dir.create_output(segment + (@is_index ? ".tii" : ".tis"))
-      @out.write_int(FORMAT)                      # write format
-      @out.write_long(0)                          # leave space for size
-      @out.write_int(@index_interval)             # write @index_interval
-      @out.write_int(@skip_interval)              # write @skip_interval
-      unless is_index
-        @other = TermInfosWriter.new(dir, segment, fis, interval, true)
-        @other.other = self
-      end
-    end
-
-    # Adds a new <Term, TermInfo> pair to the set.
-    # Term must be lexicographically greater than all previous Terms added.
-    # TermInfo pointers must be positive and greater than all previous.
-    def add(term, term_info)
-      if (not @is_index and @last_term > term)
-        raise IOError, "term out of order #{term.text} < #{@last_term.text}"
-      end
-      if (term_info.freq_pointer < @last_term_info.freq_pointer)
-        raise IOError, "freq pointer out of order"
-      end
-      if (term_info.prox_pointer < @last_term_info.prox_pointer)
-        raise IOError, "prox pointer out of order"
-      end
-
-      if (not @is_index and @size % @index_interval == 0)
-        @other.add(@last_term, @last_term_info) # add an index term
-      end
-
-      write_term(term)                                 # write term
-      @out.write_vint(term_info.doc_freq)              # write doc freq
-      @out.write_vlong(term_info.freq_pointer - @last_term_info.freq_pointer)
-      @out.write_vlong(term_info.prox_pointer - @last_term_info.prox_pointer)
-      @out.write_vint(term_info.skip_offset) if (term_info.doc_freq >= @skip_interval) 
-
-      if (@is_index) 
-        @out.write_vlong(@other.out.pos() - @last_index_pointer)
-        @last_index_pointer = @other.out.pos() # write pointer
-      end
-
-      @last_term_info.set!(term_info)
-      @size += 1
-    end
-
-    # Called to complete TermInfos creation. 
-    def close()
-      @out.seek(4)          # write @size after format
-      @out.write_long(@size)
-      @out.close()
-
-      @other.close() unless @is_index
-    end
-
-    private
-      def write_term(term)
-        start = Ferret::Utils::StringHelper.string_difference(@last_term.text, term.text)
-        length = term.text.length() - start
-
-        @out.write_vint(start)                   # write shared prefix length
-        @out.write_vint(length)                  # write delta length
-        @out.write_chars(term.text, start, length)  # write delta chars
-        @out.write_vint(@field_infos.field_number(term.field)) # write field num
-        @last_term = term
-      end
-  end
-
-  
-  # This stores a monotonically increasing set of <Term, TermInfo> pairs in a
-  # Directory.  Pairs are accessed either by Term or by ordinal position the
-  # set.  
-  class TermInfosReader 
-    include MonitorMixin
-
-    def initialize(dir, seg, fis)
-      super()
-
-      @directory = dir
-      @segment = seg
-      @field_infos = fis
-
-      @orig_enum = SegmentTermEnum.new(@directory.open_input(@segment + ".tis"),
-                                       @field_infos, false)
-      @size = @orig_enum.size
-      @skip_interval = @orig_enum.skip_interval
-      @index_enum = SegmentTermEnum.new(@directory.open_input(@segment + ".tii"),
-                                       @field_infos, true)
-      @index_terms = nil
-      @index_infos = nil
-      @index_pointers = nil
-    end
-
-    def close()
-      # clear this threads cache 
-      @orig_enum.close() if (@orig_enum != nil)
-      @index_enum.close() if (@index_enum != nil)
-    end
-
-    # Returns the number of term/value pairs in the set. 
-    attr_reader :size
-    # The skip interval for the original enumerator
-    attr_reader :skip_interval
-
-
-    # Returns the TermInfo for a Term in the set, or nil. 
-    def get_term_info(term)
-      return nil if (@size == 0)
-
-      ensure_index_is_read()
-
-      # optimize sequential access: first try scanning cached enum w/o seeking
-      e = enum()
-      if e.term and term >= e.term
-        enum_offset = (e.position / e.index_interval).to_i + 1
-        if (@index_terms.length == enum_offset or
-            term < @index_terms[enum_offset]) # but before end of block
-          return scan_for_term_info(term)        # no need to seek
-        end
-      end
-
-      # random-access: must seek
-      seek_enum(get_index_offset(term))
-      return scan_for_term_info(term)
-    end
-    alias :[] :get_term_info
-
-    # Returns the nth term in the set. 
-    def get_term(position)
-      return nil if (@size == 0)
-
-      e = enum()
-      if (e != nil and
-          e.term != nil and
-          position >= e.position and
-          position < (e.position + e.index_interval))
-        return scan_for_term(position)      # can avoid seek
-      end
-
-      seek_enum((position / e.index_interval).to_i) # must seek
-      return scan_for_term(position)
-    end
-
-    def get_terms_position(term)
-      return nil if (@size == 0)
-      ensure_index_is_read
-      seek_enum(get_index_offset(term))
-
-      e = enum()
-
-      while term > e.term and e.next?
-      end
-
-      return term == e.term ? e.position : -1
-    end
-
-    # Returns an enumeration of all the Terms and TermInfos in the set. 
-    def terms() 
-      return @orig_enum.clone()
-    end
-
-    # Returns an enumeration of terms starting at or after the named term. 
-    def terms_from(term)
-      get_term_info(term)
-      return enum().clone()
-    end
-
-    private
-
-      def enum() 
-        #te_cache = Thread.current["term_enum"]
-        #if (te_cache == nil) 
-        #  te_cache = Thread.current["term_enum"] = Ferret::Utils::WeakKeyHash.new
-        #end
-        #te_cache.synchronize do
-        #  term_enum = te_cache[self]
-        #  if term_enum == nil
-        #    term_enum = terms()
-        #    te_cache[self] = term_enum
-        #  end
-        #  return term_enum
-        #end
-        term_enum = Thread.current.get_local(self)
-        if term_enum.nil?
-          Thread.current.set_local(self, term_enum = terms())
-        end
-        return term_enum
-      end
-
-      def ensure_index_is_read()
-        synchronize() do
-          return if @index_terms
-          begin
-            index_size = @index_enum.size
-
-            @index_terms = Array.new(index_size)
-            @index_infos = Array.new(index_size)
-            @index_pointers = Array.new(index_size)
-
-            i = 0
-            while @index_enum.next?
-              @index_terms[i] = @index_enum.term
-              @index_infos[i] = @index_enum.term_info
-              @index_pointers[i] = @index_enum.index_pointer
-              i += 1
-            end
-          ensure 
-            @index_enum.close()
-            @index_enum = nil
-          end
-        end
-      end
-
-      # Returns the offset of the greatest index entry which is less than or
-      # equal to term.
-      #
-      # This method is rewritten in the C extension.
-      def get_index_offset(term) 
-        lo = 0            # binary search @index_terms[]
-        hi = @index_terms.length - 1
-
-        while (hi >= lo) 
-          mid = (lo + hi) >> 1
-          delta = term <=> @index_terms[mid]
-          if (delta < 0)
-            hi = mid - 1
-          elsif (delta > 0)
-            lo = mid + 1
-          else
-            return mid
-          end
-        end
-        return hi
-      end
-
-      def seek_enum(ind_offset)
-        enum().seek(@index_pointers[ind_offset],
-            (ind_offset * enum().index_interval) - 1,
-            @index_terms[ind_offset],
-            @index_infos[ind_offset])
-      end
-
-      # Scans within block for matching term. 
-      def scan_for_term_info(term)
-        e = enum()
-        e.scan_to(term)
-        if e.term != nil and term == e.term
-          return e.term_info()
-        else
-          return nil
-        end
-      end
-
-      def scan_for_term(position)
-        e = enum()
-        while (e.position < position)
-          return nil if not e.next?
-        end
-
-        return e.term
-      end
-
-      # Returns the position of a Term in the set or -1. 
-      def get_position(term)
-        return -1 if (@size == 0)
-
-        ind_offset = get_index_offset(term)
-        seek_enum(ind_offset)
-
-        e = enum()
-        while (term > e.term and e.next?)
-        end
-
-        if (term == e.term())
-          return e.position
-        else
-          return -1
-        end
-      end
-
-  end
-end

data/lib/ferret/index/term_vector_offset_info.rb

@@ -1,20 +0,0 @@
-module Ferret::Index
-  class TermVectorOffsetInfo 
-    attr_accessor :start, :end
-
-    def initialize(start, endd) 
-      @end = endd
-      @start = start
-    end
-
-    def eql?(o) 
-      return false if !o.instance_of?(TermVectorOffsetInfo)
-      @end == o.end and @start == o.start
-    end
-    alias :== :eql?
-
-    def hash() 
-      29 * @start + @end
-    end
-  end
-end

data/lib/ferret/index/term_vectors_io.rb

@@ -1,553 +0,0 @@
-module Ferret::Index
-  # Writer works by opening a document and then opening the fields within
-  # the document and then writing out the vectors for each field.
-  # 
-  # Rough usage:
-  # 
-  #    for each document
-  #    
-  #      writer.open_document()
-  #      for each field on the document
-  #      
-  #        writer.open_field(field)
-  #        for all of the @terms
-  #          
-  #          writer.add_term(...)
-  #        end
-  #        writer.close_field
-  #      end
-  #      writer.close_document()    
-  #    end
-  # 
-  # 
-  class TermVectorsWriter 
-    STORE_POSITIONS_WITH_TERMVECTOR = 0x1
-    STORE_OFFSET_WITH_TERMVECTOR = 0x2
-    
-    FORMAT_VERSION = 2
-
-    # The size in bytes that the FORMAT_VERSION will take up at the beginning
-    # of each file 
-    FORMAT_SIZE = 4
-    
-    TVX_EXTENSION = ".tvx"
-    TVD_EXTENSION = ".tvd"
-    TVF_EXTENSION = ".tvf"
-
-    def initialize(directory, segment, field_infos)
-      @current_field = nil
-      @current_doc_pointer = -1
-     
-      # Open files for TermVector storage
-      @tvx = directory.create_output(segment + TVX_EXTENSION)
-      @tvx.write_int(FORMAT_VERSION)
-      @tvd = directory.create_output(segment + TVD_EXTENSION)
-      @tvd.write_int(FORMAT_VERSION)
-      @tvf = directory.create_output(segment + TVF_EXTENSION)
-      @tvf.write_int(FORMAT_VERSION)
-
-      @field_infos = field_infos
-      @fields = []
-      @terms = []
-    end
-
-
-    def open_document()
-      close_document()
-      @current_doc_pointer = @tvd.pos()
-    end
-
-
-    def close_document()
-           
-      if (document_open?()) 
-        close_field()
-        write_doc()
-        @fields.clear()
-        @current_doc_pointer = -1
-      end
-    end
-
-
-    def document_open?() 
-      return @current_doc_pointer != -1
-    end
-
-
-    # Start processing a field. This can be followed by a number of calls to
-    # add_term, and a final call to close_field to indicate the end of
-    # processing of this field. If a field was previously open, it is closed
-    # automatically.
-    def open_field(field)
-      field_info = @field_infos[field]
-      create_field(field_info.number,
-                   field_info.store_positions?,
-                   field_info.store_offsets?)
-    end
-    
-    # Finished processing current field. This should be followed by a call
-    # to open_field before future calls to add_term.
-    def close_field()
-      if field_open?
-        #puts("close_field()")
-
-        # save field and @terms
-        write_field()
-        @fields << @current_field
-        @terms.clear()
-        @current_field = nil
-      end
-    end
-
-    # Return true if a field is currently open. 
-    def field_open?() 
-      return @current_field != nil
-    end
-
-    # Add term to the field's term vector. Field must already be open.
-    #
-    # Terms should be added in increasing order of @terms, one call per
-    # unique termNum. ProxPointer is a pointer into the TermPosition file
-    # (prx). Freq is the number of times this term appears in this field, in
-    # this document.  raises:: IllegalStateException if document or field is
-    # not open
-    def add_term(term_text, freq, positions = nil, offsets = nil)
-      if not document_open?
-        raise IllegalStateError, "Cannot add terms when document is not open"
-      end
-      if not field_open?
-        raise IllegalStateError, "Cannot add terms when field is not open"
-      end
-      
-      add_term_internal(term_text, freq, positions, offsets)
-    end
-
-    def add_term_internal(term_text, freq, positions, offsets) 
-      @terms << TVTerm.new(term_text, freq, positions, offsets)
-    end
-
-    # Add a complete document specified by all its term vectors. If document has no
-    # term vectors, add value for @tvx.
-    # 
-    # vectors:: The documents to have their term vectors added
-    # raises:: IOException
-    def add_all_doc_vectors(vectors)
-       
-      open_document()
-
-      if vectors != nil 
-        vectors.each do |vector|
-          store_positions = (vector.size > 0 and vector.positions != nil)
-          store_offsets = (vector.size > 0 and vector.offsets != nil)
-
-          create_field(@field_infos.field_number(vector.field),
-                       store_positions, store_offsets)
-
-          vector.size.times do |j|
-            add_term_internal(vector.terms[j],
-                              vector.freqs[j],
-                              store_positions ? vector.positions[j] : nil,
-                              store_offsets ? vector.offsets[j] : nil)
-          end
-          close_field()
-        end
-      end
-      close_document()
-    end
-    
-    # Close all streams. 
-    def close()
-      begin 
-        close_document()
-      ensure 
-        # make an effort to close all streams we can but remember and re-raise
-        # the last exception encountered in this process
-        keep = nil
-        [@tvx, @tvd, @tvf].compact.each do |os|
-          begin 
-            os.close()
-          rescue IOError => e
-            keep = e
-          end
-        end
-        raise keep if (keep != nil) 
-      end
-    end
-
-    class TVField 
-      attr_accessor :number, :tvf_pointer, :store_positions, :store_offsets
-      def initialize(number, store_pos, store_off) 
-        @tvf_pointer = 0
-        @number = number
-        @store_positions = store_pos
-        @store_offsets = store_off
-      end
-    end
-
-    class TVTerm 
-      attr_accessor :term_text, :freq, :positions, :offsets
-      
-      def initialize(term_text=nil, freq=nil, positions=nil, offsets=nil)
-        @term_text = term_text
-        @freq = freq
-        @positions = positions
-        @offsets = offsets
-      end
-    end
-
-    private
-
-      def write_field()
-        # remember where this field is written
-        @current_field.tvf_pointer = @tvf.pos
-        
-        size = @terms.size
-        @tvf.write_vint(size)
-        
-        store_positions = @current_field.store_positions
-        store_offsets = @current_field.store_offsets
-        bits = 0x0
-        if (store_positions) 
-          bits |= STORE_POSITIONS_WITH_TERMVECTOR
-        end
-        if (store_offsets) 
-          bits |= STORE_OFFSET_WITH_TERMVECTOR
-        end
-        @tvf.write_byte(bits)
-        
-        last_term_text = ""
-        @terms.each do |term|
-          start = Ferret::Utils::StringHelper.string_difference(last_term_text,
-                                                                term.term_text)
-          length = term.term_text.length() - start
-          @tvf.write_vint(start)       # write shared prefix length
-          @tvf.write_vint(length)      # write delta length
-          @tvf.write_chars(term.term_text, start, length)  # write delta chars
-          @tvf.write_vint(term.freq)
-          last_term_text = term.term_text
-          
-          if (store_positions)
-            if (term.positions == nil)
-              raise IllegalStateError, "Trying to write positions that are nil!"
-            end
-            
-            # use delta encoding for positions
-            position = 0
-            term.freq.times do |j|
-              @tvf.write_vint(term.positions[j] - position)
-              position = term.positions[j]
-            end
-          end
-          
-          if (store_offsets)
-            if(term.offsets == nil)
-              raise IllegalStateError, "Trying to write offsets that are nil!"
-            end
-            
-            # use delta encoding for offsets
-            position = 0
-            term.freq.times do |j|
-              @tvf.write_vint(term.offsets[j].start - position)
-              #Save the diff between the two.
-              @tvf.write_vint(term.offsets[j].end -
-                              term.offsets[j].start)
-              position = term.offsets[j].end()
-            end
-          end
-        end
-      end
-
-      def write_doc()
-        if field_open?
-          raise IllegalStateError, "Field is still open while writing document"
-        end
-        #puts("Writing doc pointer: " + @current_doc_pointer)
-        # write document index record
-        @tvx.write_long(@current_doc_pointer)
-
-        # write document data record
-        size = @fields.size
-
-        # write the number of @fields
-        @tvd.write_vint(size)
-
-        # write field numbers
-        @fields.each { |field| @tvd.write_vint(field.number) }
-
-        # write field pointers
-        last_field_pointer = 0
-        @fields.each do |field|
-          @tvd.write_vlong(field.tvf_pointer - last_field_pointer)
-          last_field_pointer = field.tvf_pointer
-        end
-        #puts("After writing doc pointer: " + @tvx.pos())
-      end
-
-      def create_field(field_number, store_position, store_offset)
-        if not document_open? 
-          raise IllegalStateError, "Cannot open field when no document is open."
-        end
-        close_field()
-        @current_field = TVField.new(field_number, store_position, store_offset)
-      end
-  end
-
-  class TermVectorsReader
-    attr_reader :size
-
-    # accessors for clone method
-    attr_accessor :tvx, :tvd, :tvf
-    protected :tvx, :tvx=, :tvd, :tvd=, :tvf, :tvf=
-    
-
-    def initialize(d, segment, field_infos)
-     
-      if (d.exists?(segment + TermVectorsWriter::TVX_EXTENSION)) 
-        @tvx = d.open_input(segment + TermVectorsWriter::TVX_EXTENSION)
-        check_valid_format(@tvx)
-        @tvd = d.open_input(segment + TermVectorsWriter::TVD_EXTENSION)
-        @tvd_format = check_valid_format(@tvd)
-        @tvf = d.open_input(segment + TermVectorsWriter::TVF_EXTENSION)
-        @tvf_format = check_valid_format(@tvf)
-        @size = @tvx.length / 8
-      else
-        @tvx = nil
-        @tvd = nil
-        @tvf = nil
-      end
-
-      @field_infos = field_infos
-    end
-    
-    def close()
-      # make an effort to close all streams we can but remember and re-raise
-      # the last exception encountered in this process
-      keep = nil
-      [@tvx, @tvd, @tvf].compact.each do |os|
-        begin 
-          os.close()
-        rescue IOError => e
-          keep = e
-        end
-      end
-      raise keep if (keep != nil) 
-    end
-
-    # Retrieve the term vector for the given document and field
-    # doc_num:: The document number to retrieve the vector for
-    # field:: The field within the document to retrieve
-    # returns:: The TermFreqVector for the document and field or nil if there
-    #   is no termVector for this field.
-    # raises:: IOException if there is an error reading the term vector files
-    def get_field_tv(doc_num, field)
-      # Check if no term vectors are available for this segment at all
-      field = field.to_s
-      field_number = @field_infos.field_number(field)
-      result = nil
-      if (@tvx != nil) 
-        #We need to account for the FORMAT_SIZE at when seeking in the @tvx
-        #We don't need to do this in other seeks because we already have the
-        # file pointer
-        #that was written in another file
-        @tvx.seek((doc_num * 8) + TermVectorsWriter::FORMAT_SIZE)
-        #puts("TVX Pointer: " + @tvx.pos())
-        position = @tvx.read_long()
-
-        @tvd.seek(position)
-        field_count = @tvd.read_vint()
-        #puts("Num Fields: " + field_count)
-        # There are only a few fields per document. We opt for a full scan
-        # rather then requiring that they be ordered. We need to read through
-        # all of the fields anyway to get to the tvf pointers.
-        number = 0
-        found = -1
-        field_count.times do |i|
-          if @tvd_format == TermVectorsWriter::FORMAT_VERSION
-            number = @tvd.read_vint()
-          else
-            number += @tvd.read_vint()
-          end
-          if (number == field_number)
-            found = i
-          end
-        end
-
-        # This field, although valid in the segment, was not found in this
-        # document
-        if (found != -1) 
-          # Compute position in the @tvf file
-          position = 0
-          (found + 1).times do 
-            position += @tvd.read_vlong()
-          end
-
-          result = read_term_vector(field, position)
-        end
-      end
-      return result
-    end
-
-    # Return all term vectors stored for this document or nil if it could
-    # not be read in.
-    # 
-    # doc_num:: The document number to retrieve the vector for
-    # returns:: All term frequency vectors
-    # raises:: IOException if there is an error reading the term vector files 
-    def get_tv(doc_num)
-      result = nil
-      # Check if no term vectors are available for this segment at all
-      if (@tvx != nil) 
-        #We need to offset by
-        @tvx.seek((doc_num * 8) + TermVectorsWriter::FORMAT_SIZE)
-        position = @tvx.read_long()
-
-        @tvd.seek(position)
-        field_count = @tvd.read_vint()
-
-        # No fields are vectorized for this document
-        if (field_count != 0) 
-          number = 0
-          fields = Array.new(field_count)
-          
-          field_count.times do |i|
-            if @tvd_format == TermVectorsWriter::FORMAT_VERSION
-              number = @tvd.read_vint()
-            else
-              number += @tvd.read_vint()
-            end
-
-            fields[i] = @field_infos[number].name
-          end
-
-          # Compute position in the @tvf file
-          position = 0
-          tvf_pointers = Array.new(field_count)
-          field_count.times do |i|
-            position += @tvd.read_vlong()
-            tvf_pointers[i] = position
-          end
-
-          result = read_term_vectors(fields, tvf_pointers)
-        end
-      end
-      return result
-    end
-
-    def clone() 
-      
-      if (@tvx == nil or @tvd == nil or @tvf == nil)
-        return nil
-      end
-      
-      clone = self
-      clone.tvx = @tvx.clone()
-      clone.tvd = @tvd.clone()
-      clone.tvf = @tvf.clone()
-      
-      return clone
-    end
-
-    private 
-
-      def read_term_vectors(fields, tvf_pointers)
-             
-        res = Array.new(fields.length)
-        fields.length.times do |i|
-          res[i] = read_term_vector(fields[i], tvf_pointers[i])
-        end
-        return res
-      end
-
-      # field:: The field to read in
-      # tvf_pointer:: The pointer within the @tvf file where we should start reading
-      # returns:: The TermVector located at that position
-      # raises:: IOException
-      def read_term_vector(field, tvf_pointer)
-        # Now read the data from specified position
-        # We don't need to offset by the FORMAT here since the pointer
-        # already includes the offset
-        @tvf.seek(tvf_pointer)
-
-        num_terms = @tvf.read_vint()
-        # If no terms - return a constant empty termvector. However, this should
-        # never occur!
-        if (num_terms == 0) 
-          return SegmentTermVector.new(field, nil, nil)
-        end
-        
-        
-        if(@tvf_format == TermVectorsWriter::FORMAT_VERSION)
-          bits = @tvf.read_byte()
-          store_positions = (bits & TermVectorsWriter::STORE_POSITIONS_WITH_TERMVECTOR) != 0
-          store_offsets = (bits & TermVectorsWriter::STORE_OFFSET_WITH_TERMVECTOR) != 0
-        else
-          @tvf.read_vint()
-          store_positions = false
-          store_offsets = false
-        end
-
-        terms = Array.new(num_terms)
-        term_freqs = Array.new(num_terms)
-        
-        #  we may not need these, but declare them
-        positions = nil
-        offsets = nil
-        if(store_positions)
-          positions = Array.new(num_terms)
-        end
-        if(store_offsets)
-          offsets = Array.new(num_terms)
-        end
-        
-        start = 0
-        delta_length = 0
-        total_length = 0
-        buffer = "" 
-        previous_buffer = ""
-        
-        num_terms.times do |i|
-          start = @tvf.read_vint()
-          delta_length = @tvf.read_vint()
-          total_length = start + delta_length
-          @tvf.read_chars(buffer, start, delta_length)
-          terms[i] = buffer[0, total_length].to_s
-          previous_string = terms[i]
-          freq = @tvf.read_vint()
-          term_freqs[i] = freq
-          
-          if (store_positions) #read in the positions
-            pos = Array.new(freq)
-            positions[i] = pos
-            prev_position = 0
-            freq.times do |j|
-              pos[j] = prev_position + @tvf.read_vint()
-              prev_position = pos[j]
-            end
-          end
-          
-          if (store_offsets) 
-            offs = Array.new(freq)
-            offsets[i] = offs
-            prev_offset = 0
-            freq.times do |j|
-              start_offset = prev_offset + @tvf.read_vint()
-              end_offset = start_offset + @tvf.read_vint()
-              offs[j] = TermVectorOffsetInfo.new(start_offset, end_offset)
-              prev_offset = end_offset
-            end
-          end
-        end
-        
-        SegmentTermVector.new(field, terms, term_freqs, positions, offsets)
-      end
-
-      def check_valid_format(istream)
-        format = istream.read_int()
-        if (format > TermVectorsWriter::FORMAT_VERSION)
-          raise IOError, "Incompatible format version: #{format} expected #{TermVectorsWriter::FORMAT_VERSION} or less"
-        end
-        return format
-      end
-
-  end
-end