ferret 0.1.0

data/lib/ferret/index/document_writer.rb

@@ -0,0 +1,288 @@
+require 'ferret/search/similarity'
+
+module Ferret::Index
+
+  class DocumentWriter 
+    # If non-nil, a message will be printed to this if max_field_length is
+    # reached.
+    attr_writer :info_stream
+
+    # directory::           The directory to write the document information to
+    # analyzer::            The analyzer to use for the document
+    # similarity::          The Similarity function writer.similarity
+    # max_field_length::    The maximum number of tokens a field may have
+    #                       writer.max_field_length
+    # term_index_interval:: The interval of terms in the index 
+    #                       writer.max_field_length
+    def initialize(directory,
+                   analyzer,
+                   similarity,
+                   max_field_length,
+                   term_index_interval = IndexWriter::DEFAULT_TERM_INDEX_INTERVAL) 
+      @directory = directory
+      @analyzer = analyzer
+      @similarity = similarity
+      @max_field_length = max_field_length
+      @term_index_interval = term_index_interval
+
+      # Keys are Terms, values are Postings.
+      # Used to buffer a document before it is written to the index.
+      @posting_table = {}
+
+      @term_buffer = Term.new("", "")
+    end
+
+    def add_document(segment, doc)
+           
+      # write field names
+      @field_infos = FieldInfos.new()
+      @field_infos << doc
+      @field_infos.write_to_dir(@directory, segment + ".fnm")
+
+      # write field values
+      fields_writer = FieldsWriter.new(@directory, segment, @field_infos)
+      begin 
+        fields_writer.add_document(doc)
+      ensure 
+        fields_writer.close()
+      end
+
+      # invert doc into posting_table
+      @posting_table.clear();                    # clear posting_table
+      arr_size = @field_infos.size
+      @field_lengths = Array.new(arr_size, 0)    # init field_lengths
+      @field_positions = Array.new(arr_size, 0)  # init field_positions
+      @field_offsets = Array.new(arr_size, 0)    # init field_offsets
+      @field_boosts = Array.new(arr_size, doc.boost) # init field_boosts
+
+      invert_document(doc)
+
+      # sort posting_table into an array
+      postings = sort_posting_table()
+
+  #    for (int i = 0; i < postings.length; i += 1) 
+  #      Posting posting = postings[i]
+  #      print(posting.term)
+  #      print(" freq=" + posting.freq)
+  #      print(" pos=")
+  #      print(posting.positions[0])
+  #      for (int j = 1; j < posting.freq; j += 1)
+  #        print("," + posting.positions[j])
+  #      puts("")
+  #    end
+
+      # write postings
+      write_postings(postings, segment)
+
+      # write norms of indexed fields
+      write_norms(segment)
+
+    end
+
+    private
+
+      # Tokenizes the fields of a document into Postings.
+      def invert_document(doc)
+             
+        fields = doc.all_fields
+        fields.each do |field|
+          field_name = field.name
+          field_info = @field_infos[field_name]
+          field_number = field_info.number
+
+          length = @field_lengths[field_number]     # length of field
+          position = @field_positions[field_number] # position in field
+          offset = @field_offsets[field_number]     # offset field
+
+          if field_info.indexed? 
+            if not field.tokenized? # un-tokenized field
+              string_value = field.string_value
+              if field_info.store_offsets?
+                add_position(field_name,
+                             string_value,
+                             position,
+                             TermVectorOffsetInfo.new(offset,
+                                      offset + string_value.length))
+                position += 1
+              else
+                add_position(field_name, string_value, position, nil)
+                position += 1
+              end
+              offset += string_value.length()
+              length += 1
+            else 
+            
+              reader = field.reader_value()
+
+              # Tokenize field and add to posting_table
+              stream = @analyzer.token_stream(field_name, reader)
+              begin 
+                last_token = nil
+                while token = stream.next
+                  position += (token.position_increment - 1)
+                  
+                  if(field_info.store_offsets?())
+                    add_position(field_name,
+                                 token.term_text(),
+                                 position,
+                                 TermVectorOffsetInfo.new(
+                                   offset + token.start_offset(),
+                                   offset + token.end_offset()))
+                    position += 1
+                  else
+                    add_position(field_name, token.term_text(), position, nil)
+                    position += 1
+                  end
+                  
+                  last_token = token
+                  length += 1
+                  if (length > @max_field_length) 
+                    if @info_stream
+                      @info_stream.puts("max_field_length " + @max_field_length.to_s + " reached, ignoring following tokens")
+                    end
+                    break
+                  end
+                end
+                
+                if(last_token != nil)
+                  offset += last_token.end_offset() + 1
+                end
+                
+              ensure 
+                stream.close()
+              end
+            end
+
+            @field_lengths[field_number] = length    # save field length
+            @field_positions[field_number] = position    # save field position
+            @field_boosts[field_number] *= field.boost
+            @field_offsets[field_number] = offset
+          end
+        end
+      end
+
+
+      def add_position(field, text, position, tv_offset_info) 
+        @term_buffer.set!(field, text)
+        #puts("Offset: " + tv_offset_info)
+        posting = @posting_table[@term_buffer]
+        if (posting != nil) # word seen before
+          freq = posting.freq
+          posting.positions[freq] = position     # add new position
+          posting.offsets[freq] = tv_offset_info # add new position
+
+          if (tv_offset_info != nil) 
+            posting.offsets[freq] = tv_offset_info
+          end
+          posting.freq = freq + 1        # update frequency
+        else # word not seen before
+          term = Term.new(field, text)
+          @posting_table[term] = Posting.new(term, position, tv_offset_info)
+        end
+      end
+
+      def sort_posting_table() 
+        # copy @posting_table into an array
+        return @posting_table.values.sort { |x,y| x.term <=> y.term }
+      end
+
+      def write_postings(postings, segment)
+             
+        freq = nil
+        prox = nil
+        tis_writer = nil
+        tv_writer = nil
+        begin 
+          #open files for inverse index storage
+          freq = @directory.create_output(segment + ".frq")
+          prox = @directory.create_output(segment + ".prx")
+          tis_writer = TermInfosWriter.new(@directory, segment, @field_infos,
+                                    @term_index_interval)
+          ti = TermInfo.new()
+          current_field = nil
+
+          postings.each do |posting|
+            # add an entry to the dictionary with pointers to prox and freq files
+            ti.set_values!(1, freq.pos(), prox.pos(), -1)
+            tis_writer.add(posting.term, ti)
+
+            # add an entry to the freq file
+            posting_freq = posting.freq
+            if (posting_freq == 1)          # optimize freq=1
+              freq.write_vint(1)            # set low bit of doc num.
+            else 
+              freq.write_vint(0)            # the document number
+              freq.write_vint(posting_freq) # frequency in doc
+            end
+
+            last_position = 0               # write positions
+            posting.positions.each do |position|
+              prox.write_vint(position - last_position)
+              last_position = position
+            end
+            # check to see if we switched to a new field
+            term_field = posting.term.field
+            if (current_field != term_field) 
+              # changing field - see if there is something to save
+              current_field = term_field
+              fi = @field_infos[current_field]
+              if (fi.store_term_vector?) 
+                if tv_writer.nil? 
+                  tv_writer = TermVectorsWriter.new(@directory, segment, @field_infos)
+                  tv_writer.open_document()
+                end
+                tv_writer.open_field(current_field)
+
+              elsif not tv_writer.nil?
+                tv_writer.close_field()
+              end
+            end
+            if not tv_writer.nil? and tv_writer.field_open?
+                tv_writer.add_term(posting.term.text, posting_freq, posting.positions, posting.offsets)
+            end
+          end
+          if not tv_writer.nil?
+            tv_writer.close_document()
+          end
+        ensure 
+          # make an effort to close all streams we can but remember and re-raise
+          # the last exception encountered in this process
+          keep = nil
+          [freq, prox, tis_writer, tv_writer].compact.each do |obj|
+            begin
+              obj.close
+            rescue IOError => e
+              keep = e
+            end
+          end
+          raise keep if not keep.nil?
+        end
+      end
+
+      def write_norms(segment)
+        @field_infos.each_with_index do |fi, i|
+          if fi.indexed?
+            norm = @field_boosts[i] * @similarity.length_norm(fi.name, @field_lengths[i])
+            norms = @directory.create_output(segment + ".f" + i.to_s)
+            begin 
+              norms.write_byte(Ferret::Search::Similarity.encode_norm(norm))
+            ensure 
+              norms.close()
+            end
+          end
+        end
+      end
+      
+  end
+
+  class Posting # info about a Term in a doc
+    attr_accessor :term, :freq, :positions, :offsets
+
+    def initialize(t, position, offset) 
+      @term = t
+      @freq = 1
+      @positions = [position]
+      @offsets = [offset]
+    end
+  end
+end

data/lib/ferret/index/field_infos.rb

@@ -0,0 +1,259 @@
+module Ferret
+  module Index
+    # Access to the Field Info file that describes document fields and whether or
+    # not they are indexed. Each segment has a separate Field Info file. Objects
+    # of this class are thread-safe for multiple readers, but only one thread can
+    # be adding documents at a time, with no other reader or writer threads
+    # accessing this object.
+    class FieldInfos
+
+      NOT_A_FIELD = 0xffffffff # -1 in java int
+
+      # Construct a FieldInfos object using the directory and the name of the file
+      # InputStream
+      #
+      # dir:: The directory to open the InputStream from
+      # name:: The name of the file to open the InputStream from in the Directory
+      def initialize(dir = nil, name = nil)
+        @fi_array = []
+        @fi_hash = {}
+        if dir and dir.exists?(name)
+          input = dir.open_input(name)
+          begin
+            read(input)
+          ensure
+            input.close()
+          end
+        end
+      end
+
+      # Returns the number of fields that have been added to this field infos
+      # object.
+      def size
+        return @fi_array.size
+      end
+
+      # Automatically adds all of the fields from the document if they haven't
+      # been added already. Or it will update the values.
+      def add_doc_fields(doc)
+        doc.all_fields.each do |field|
+          add(field.name,
+              field.indexed?,
+              field.store_term_vector?,
+              field.store_positions?,
+              field.store_offsets?)
+        end
+      end
+      alias :<< :add_doc_fields
+      
+      # Calls the 5 param add method to add all the names in the collection
+      def add_fields(names, 
+                    indexed = true,
+                    store_term_vector = false,
+                    store_position = false,
+                    store_offset = false)
+        names.each do |name|
+          add(name, indexed, store_term_vector, store_position, store_offset)
+        end
+      end
+
+      # If the field is not yet known, adds it. If it is known, checks to make
+      # sure that the indexed flag is the same as was given previously for this
+      # field. If not - marks it as being indexed.  Same goes for the TermVector
+      # parameters.
+      # 
+      # name:: The name of the field
+      # indexed:: true if the field is indexed
+      # store_term_vector:: true if the term vector should be stored
+      # store_position:: true if the positions should be stored
+      # store_offset:: true if the offsets should be stored
+      def add(name,
+              indexed = true,
+              store_term_vector = false,
+              store_position = false,
+              store_offset = false)
+        fi = @fi_hash[name]
+        if (fi == nil)
+          fi = add_internal(name, indexed, store_term_vector, store_position, store_offset)
+        else
+          if (fi.indexed? != indexed)
+            fi.indexed = true             # once indexed, always index
+          end
+          if (fi.store_term_vector? != store_term_vector)
+            fi.store_term_vector = true   # once vector, always vector
+          end
+          if (fi.store_positions? != store_position)
+            fi.store_position = true # once vector, always vector
+          end
+          if (fi.store_offsets? != store_offset)
+            fi.store_offset = true   # once vector, always vector
+          end
+        end
+        return fi
+      end
+
+      # Returns the number of the field that goes by the field name that is
+      # passed. If there is no field of this name then -1 is returned
+      def field_number(name)
+        fi = @fi_hash[name]
+        return fi ? fi.number : NOT_A_FIELD
+      end
+
+      # Retrieve the field_info object by either field number or field name.
+      def [](index)
+        if index.is_a? Integer
+          if index == NOT_A_FIELD || index < 0 # < 0 is for C extensions
+            return FieldInfo.new("", false, NOT_A_FIELD, false)
+          end
+          return @fi_array[index]
+        else
+          return @fi_hash[index]
+        end
+      end
+
+      def name(index)
+        if index == NOT_A_FIELD || index < 0 # < 0 is for C extensions
+          return ""
+        end
+        return self[index].name
+      end
+
+      # Iterate through the field_info objects
+      def each()
+        @fi_array.each() {|fi| yield(fi) }
+      end
+
+      # Iterate through the field_info objects including the index
+      def each_with_index()
+        @fi_array.each_with_index() {|fi, i| yield(fi, i) }
+      end
+
+      # Get the number of field_infos in this object.
+      #
+      # NOTE: There is a default empty field always added at the start. This
+      # may later be used to set the default values for a field.
+      def size()
+        return @fi_array.size()
+      end
+
+      # Return true if any of the fields have store_term_vector? set to true
+      def has_vectors?()
+        @fi_array.each() { |fi| return true if fi.store_term_vector? }
+        return false
+      end
+
+      # Write the field_infos to a file specified by name in dir.
+      #
+      # dir:: the directory to write the fieldinfos to
+      # name:: the name of the file to write to.
+      def write_to_dir(dir, name)
+        output = dir.create_output(name)
+        begin
+          write(output)
+        ensure
+          output.close()
+        end
+      end
+
+      protected
+
+        # Write the field_infos to the output file
+        #
+        # output:: the file to write to
+        def write(output)
+          output.write_vint(size())
+          @fi_array.each() do |fi|
+            output.write_string(fi.name)
+            output.write_byte(get_field_info_byte(fi))
+          end
+        end
+
+        # Read the field_infos object from the input file
+        #
+        # input:: the input file to read from
+        def read(input)
+          size = input.read_vint()#read in the size
+          size.times do |i|
+            name = input.read_string()
+            bits = input.read_byte()
+            indexed = (bits & IS_INDEXED) != 0
+            store_term_vector = (bits & STORE_TERM_VECTOR) != 0
+            store_position = (bits & STORE_POSITION) != 0
+            store_offset = (bits & STORE_OFFSET) != 0
+            add_internal(name, indexed, store_term_vector, store_position, store_offset)
+          end
+        end
+
+      private
+        IS_INDEXED = 0x1;
+        STORE_TERM_VECTOR = 0x2;
+        STORE_POSITION = 0x4;
+        STORE_OFFSET = 0x8;
+
+        def add_internal(name, indexed, store_term_vector,
+                         store_position = false,
+                         store_offset = false)
+          fi = FieldInfo.new(name, indexed,
+                             @fi_array.size(),
+                             store_term_vector,
+                             store_position,
+                             store_offset)
+          @fi_array << fi
+          @fi_hash[name] = fi
+          return fi
+        end
+
+        def get_field_info_byte(fi)
+          bits = 0x0
+          if (fi.indexed?)
+            bits |= IS_INDEXED
+          end
+          if (fi.store_term_vector?)
+            bits |= STORE_TERM_VECTOR
+          end
+          if (fi.store_positions?)
+            bits |= STORE_POSITION
+          end
+          if (fi.store_offsets?)
+            bits |= STORE_OFFSET
+          end
+          return bits
+        end
+    end
+
+    class FieldInfo
+      attr_accessor :name, :number
+      attr_writer :indexed, :store_term_vector, :store_offset, :store_position
+
+      def indexed?()
+        return @indexed
+      end
+
+      def store_term_vector?()
+        return @store_term_vector
+      end
+
+      def store_offsets?()
+        return @store_offset
+      end
+      def store_positions?()
+        return @store_position
+      end
+
+      def set!(indexed, store_term_vector, store_position, store_offset)
+        @indexed = indexed
+        @store_term_vector = store_term_vector
+        @store_position = store_position
+        @store_offset = store_offset
+      end
+
+      def initialize(name, indexed, number, store_term_vector,
+                     store_position = false,
+                     store_offset = false)
+        @name = name
+        @number = number
+        set!(indexed, store_term_vector, store_position, store_offset)
+      end
+    end
+  end
+end