ferret 0.1.0

data/lib/ferret/index/fields_io.rb

@@ -0,0 +1,175 @@
+require 'zlib'
+
+
+module Ferret::Index
+
+
+  # Class responsible for access to stored document fields.
+  # 
+  # It uses <segment>.fdt and <segment>.fdx; files.
+  class FieldsReader 
+    include Ferret::Document
+    attr_reader :size
+    alias :length :size
+    
+    def initialize(d, segment, fi)
+      @field_infos = fi
+
+      @fields_stream = d.open_input(segment + ".fdt")
+      @index_stream = d.open_input(segment + ".fdx")
+
+      @size = (@index_stream.length() / 8).to_i
+    end
+
+    def close()
+      @fields_stream.close()
+      @index_stream.close()
+    end
+
+
+    def doc(n)
+      @index_stream.seek(n * 8)
+      position = @index_stream.read_long()
+      @fields_stream.seek(position)
+
+      doc = Document.new
+      @fields_stream.read_vint().times do
+        field_number = @fields_stream.read_vint()
+        fi = @field_infos[field_number]
+
+        bits = @fields_stream.read_byte()
+        
+        compressed = (bits & FieldsWriter::FIELD_IS_COMPRESSED) != 0
+        tokenize = (bits & FieldsWriter::FIELD_IS_TOKENIZED) != 0
+        binary = (bits & FieldsWriter::FIELD_IS_BINARY) != 0
+        
+        if binary
+          b = " " * @fields_stream.read_vint()
+          @fields_stream.read_bytes(b, 0, b.length)
+          if compressed
+            doc << Field.new_binary_field(fi.name,
+                                          uncompress(b),
+                                          Field::Store::COMPRESS)
+          else # No compression
+            doc << Field.new_binary_field(fi.name, b, Field::Store::YES)
+          end
+        else 
+          store = Field::Store::YES
+          if fi.indexed? and tokenize
+            index = Field::Index::TOKENIZED
+          elsif fi.indexed? and not tokenize
+            index = Field::Index::UNTOKENIZED
+          else
+            index = Field::Index::NO
+          end
+          data = nil
+          if (compressed) 
+            store = Field::Store::COMPRESS
+            b = " " * @fields_stream.read_vint()
+            @fields_stream.read_bytes(b, 0, b.length)
+            data = uncompress(b)
+          else
+            data = @fields_stream.read_string()
+          end
+          stv =  Field::TermVector::NO
+          if fi.store_term_vector? 
+            if fi.store_positions? and fi.store_offsets?
+              stv =  Field::TermVector::WITH_POSITIONS_OFFSETS 
+            elsif fi.store_positions?
+              stv =  Field::TermVector::WITH_POSITIONS
+            elsif fi.store_offsets?
+              stv =  Field::TermVector::WITH_OFFSETS 
+            else
+              stv =  Field::TermVector::YES 
+            end
+          end
+          doc << Field.new(fi.name, data, store, index, stv)
+        end
+      end
+
+      return doc
+    end
+    
+    def uncompress(input)
+      zstream = Zlib::Inflate.new
+      buf = zstream.inflate(input)
+      zstream.finish
+      zstream.close
+      buf
+    end
+  end
+
+
+  class FieldsWriter
+
+    FIELD_IS_TOKENIZED = 0X1
+    FIELD_IS_BINARY = 0X2
+    FIELD_IS_COMPRESSED = 0X4 
+
+    def initialize(dir, segment, fi)
+      @field_infos = fi
+      @fields_stream = dir.create_output(segment + ".fdt")
+      @index_stream = dir.create_output(segment + ".fdx")
+    end
+
+    def close()
+      @fields_stream.close()
+      @index_stream.close()
+    end
+
+    def add_document(doc)
+      @index_stream.write_long(@fields_stream.pos)
+      stored_count = 0
+      doc.all_fields.each() { |field| stored_count += 1 if field.stored?() }
+      @fields_stream.write_vint(stored_count)
+      
+      doc.all_fields.each() do |field|
+        if (field.stored?())
+          @fields_stream.write_vint(@field_infos.field_number(field.name))
+
+          bits = 0
+          bits |= FIELD_IS_TOKENIZED if field.tokenized?
+          bits |= FIELD_IS_BINARY if field.binary?
+          bits |= FIELD_IS_COMPRESSED if field.compressed?
+          @fields_stream.write_byte(bits)
+          
+          data = nil
+          if field.compressed?
+            if field.binary? 
+              data = compress(field.binary_value)
+            else
+              data = compress(field.string_value)
+            end
+            save_data(data)
+          else
+            if field.binary?
+              save_data(field.binary_value)
+            else
+              @fields_stream.write_string(field.string_value)
+            end
+          end
+        end
+      end
+    end
+    alias :<< :add_document
+
+    private
+    
+      def compress(input)
+        zstream = Zlib::Deflate.new(Zlib::BEST_COMPRESSION)
+        buf = zstream.deflate(input, Zlib::FINISH)
+        zstream.close
+        return buf
+      end
+
+      def save_data(data)
+        len = data.length
+        if data.is_a? Array
+          data = data.pack("C*")
+        end
+
+        @fields_stream.write_vint(len)
+        @fields_stream.write_bytes(data, len)
+      end
+  end
+end

data/lib/ferret/index/index.rb

@@ -0,0 +1,228 @@
+module Ferret::Index
+  class Index
+    include Ferret::Store
+    include Ferret::Search
+    include Ferret::Document
+
+    def initialize(options = {})
+      if options[:path]
+        @dir = FSDirectory.new(options[:path], true)
+        options[:close_dir] = true
+      elsif options[:dir]
+        @dir = options[:dir]
+      else
+        options[:create] = true # this should always be true for a new RAMDir
+        @dir = RAMDirectory.new
+      end
+
+      @options = options
+      @writer = IndexWriter.new(@dir, options)
+      options[:analyzer] = @analyzer = @writer.analyzer
+      @has_writes = false
+      @reader = nil
+      @options.delete(:create) # only want to create the first time if at all
+      @close_dir = @options.delete(:close_dir) || false # we'll hold this here
+      @default_field = @options[:default_field] || ""
+      @open = true
+    end
+
+    def close
+      if not @open
+        raise "tried to close an already closed directory"
+      end
+      @reader.close() if @reader
+      @writer.close() if @writer
+      @dir.close()
+
+      @open = false
+    end
+
+    # Get the reader for this index.
+    # NOTE:: This will close the writer from this index.
+    def reader
+      ensure_reader_open()
+      return @reader
+    end
+
+    # Get the searcher for this index.
+    # NOTE:: This will close the writer from this index.
+    def searcher
+      ensure_searcher_open()
+      return @searcher
+    end
+
+    # Get the writer for this index.
+    # NOTE:: This will close the reader from this index.
+    def writer
+      ensure_writer_open()
+      return @writer
+    end
+
+    # Adds a document to this index, using the provided analyzer instead of
+    # the local analyzer if provided.  If the document contains more than
+    # IndexWriter::MAX_FIELD_LENGTH terms for a given field, the remainder are
+    # discarded.
+    def add_document(doc, analyzer = nil)
+      ensure_writer_open()
+      fdoc = nil
+      if doc.is_a?(String)
+        fdoc = Document.new
+        fdoc << Field.new(@default_field, doc,
+                          Field::Store::YES, Field::Index::TOKENIZED)
+      elsif doc.is_a?(Array)
+        fdoc = Document.new
+        doc.each() do |field|
+          fdoc << Field.new(@default_field, field,
+                            Field::Store::YES, Field::Index::TOKENIZED)
+        end
+      elsif doc.is_a?(Hash)
+        fdoc = Document.new
+        doc.each_pair() do |field, text|
+          fdoc << Field.new(field.to_s, text.to_s,
+                            Field::Store::YES, Field::Index::TOKENIZED)
+        end
+      elsif doc.is_a?(Document)
+        fdoc = doc
+      else
+        raise ArgumentError, "Unknown document type #{doc.class}"
+      end
+      @has_writes = true
+
+      @writer.add_document(fdoc, analyzer || @writer.analyzer)
+    end
+    alias :<< :add_document
+
+    # The main search method for the index. You need to create a query to
+    # pass to this method. You can also pass a hash with one or more of the
+    # following; {filter, num_docs, first_doc, sort}
+    #
+    # query::    the query to run on the index
+    # filter::   filters docs from the search result
+    # first_doc:: The index in the results of the first doc retrieved.
+    #   Default is 0
+    # num_docs:: The number of results returned. Default is 10
+    # sort::     an array of SortFields describing how to sort the results.
+    def search(query, options = {})
+      if query.is_a?(String)
+        if @qp.nil?
+          @qp = Ferret::QueryParser.new(@default_field, options)
+        end
+        query = @qp.parse(query)
+      end
+
+      ensure_searcher_open()
+      return @searcher.search(query, options)
+    end
+
+    # See Index#search
+    #
+    # This method yields the doc and score for each hit.
+    # eg.
+    #   index.search_each() do |doc, score|
+    #     puts "hit document number #{doc} with a score of #{score}"
+    #   end
+    #
+    def search_each(query, options = {}) # :yield: doc, score
+      search(query, options).score_docs.each do |score_doc|
+        yield score_doc.doc, score_doc.score
+      end
+    end
+
+    # Retrieve the document referenced by the document number +id+, if id is
+    # an integer or the first document with term +id+ if +id+ is a term.
+    #
+    # id:: The number of the document to retrieve, or the term used as the id
+    #      for the document we wish to retrieve
+    def doc(id)
+      ensure_reader_open()
+      if id.is_a?(String)
+        t = Term.new("id", id.to_s)
+        return @reader.get_document_with_term(t)
+      elsif id.is_a?(Term)
+        return @reader.get_document_with_term(id)
+      else
+        return @reader.get_document(id)
+      end
+    end
+    alias :[] :doc
+    
+    # Delete the document referenced by the document number +id+ if +id+ is an
+    # integer or all of the documents which have the term +id+ if +id+ is a
+    # term..
+    #
+    # id:: The number of the document to delete
+    def delete(id)
+      ensure_reader_open()
+      if id.is_a?(String)
+        t = Term.new("id", id.to_s)
+        return @reader.delete_docs_with_term(t)
+      elsif id.is_a?(Term)
+        return @reader.delete_docs_with_term(id)
+      else
+        return @reader.delete(id)
+      end
+    end
+
+    # Returns true if document +n+ has been deleted 
+    def deleted?(n)
+      ensure_reader_open()
+      return @reader.deleted?(n) 
+    end
+
+    # Returns true if any documents have been deleted since the index was last
+    # flushed.
+    def has_deletions?()
+      ensure_reader_open()
+      return @reader.has_deletions?
+    end
+    
+    # Returns true if any documents have been added to the index since the
+    # last flush.
+    def has_writes?()
+      return @has_writes
+    end
+
+    # optimizes the index. This should only be called when the index will no
+    # longer be updated very often, but will be read a lot.
+    def optimize()
+      ensure_writer_open()
+      @writer.optimize()
+      @modified = true
+    end
+
+    # returns the number of documents in the index
+    def size()
+      ensure_reader_open()
+      return @reader.num_docs()
+    end
+
+    protected
+      def ensure_writer_open()
+        raise "tried to use a closed index" if not @open
+        return if @writer
+        if @reader
+          @reader.close
+          @reader = nil
+          @searcher = nil
+        end
+        @writer = IndexWriter.new(@dir, @options)
+      end
+
+      def ensure_reader_open()
+        raise "tried to use a closed index" if not @open
+        return if @reader
+        if @writer
+          @writer.close
+          @writer = nil
+        end
+        @reader = IndexReader.open(@dir, false)
+      end
+
+      def ensure_searcher_open()
+        raise "tried to use a closed index" if not @open
+        return if @searcher
+        ensure_reader_open()
+        @searcher = IndexSearcher.new(@reader)
+      end
+  end
+end

data/lib/ferret/index/index_file_names.rb

@@ -0,0 +1,33 @@
+module Ferret
+  module Index
+    # Useful constants representing filenames and extensions used by lucene
+    class IndexFileNames 
+
+      # Name of the index segment file 
+      SEGMENTS = "segments"
+      
+      # Name of the index deletable file 
+      DELETABLE = "deletable"
+      
+      # This array contains all filename extensions used by Lucene's index files, with
+      # one exception, namely the extension made up from +.f+ + a number.
+      # Also note that two of Lucene's files (+deletable+ and
+      # +segments+) don't have any filename extension.
+      INDEX_EXTENSIONS = [
+          "cfs", "fnm", "fdx", "fdt", "tii", "tis", "frq", "prx", "del",
+          "tvx", "tvd", "tvf", "tvp"
+      ]
+      
+      # File extensions of old-style index files 
+      COMPOUND_EXTENSIONS = [
+        "fnm", "frq", "prx", "fdx", "fdt", "tii", "tis"
+      ]
+      
+      # File extensions for term vector support 
+      VECTOR_EXTENSIONS = [
+        "tvx", "tvd", "tvf"
+      ]
+      
+    end
+  end
+end

data/lib/ferret/index/index_reader.rb

@@ -0,0 +1,462 @@
+require 'monitor'
+
+module Ferret::Index
+  # IndexReader is an abstract class, providing an interface for accessing an
+  # index.  Search of an index is done entirely through this abstract interface,
+  # class which implements it is searchable.
+  #
+  # Concrete subclasses of IndexReader are usually constructed with a call to
+  # one of the static <tt>open()</tt> methods, e.g. <tt>#open</tt>.
+  #
+  # For efficiency, in this API documents are often referred to via
+  # _document numbers_, non-negative integers which each name a unique
+  # document in the index.  These document numbers are ephemeral, ie they may change
+  # as documents are added to and deleted from an index.  Clients should thus not
+  # rely on a given document having the same number between sessions.
+  # 
+  # An IndexReader can be opened on a directory for which an IndexWriter is
+  # opened already, but it cannot be used to delete documents from the index then.
+  class IndexReader 
+    include MonitorMixin
+    
+    # This array contains all filename extensions used by Lucene's index files, with
+    # one exception, namely the extension made up from +.f+ + a number.
+    # Also note that two of Lucene's files (+deletable+ and
+    # +segments+) don't have any filename extension.
+    FILENAME_EXTENSIONS = ["cfs",
+                           "fnm",
+                           "fdx",
+                           "fdt",
+                           "tii",
+                           "tis",
+                           "frq",
+                           "prx",
+                           "del",
+                           "tvx",
+                           "tvd",
+                           "tvf",
+                           "tvp"]
+
+    attr_reader :directory
+    
+    class FieldOption < Ferret::Utils::Parameter
+      # all fields
+      ALL = FieldOption.new("ALL")
+      # all indexed fields
+      INDEXED = FieldOption.new("INDEXED")
+      # all fields which are not indexed
+      UNINDEXED = FieldOption.new("UNINDEXED")
+      # all fields which are indexed with termvectors enables
+      INDEXED_WITH_TERM_VECTOR = FieldOption.new("INDEXED_WITH_TERM_VECTOR")
+      # all fields which are indexed but don't have termvectors enabled
+      INDEXED_NO_TERM_VECTOR = FieldOption.new("INDEXED_NO_TERM_VECTOR")
+      # all fields where termvectors are enabled. Please note that only standard
+      # termvector fields are returned
+      TERM_VECTOR = FieldOption.new("TERM_VECTOR")
+      # all field with termvectors wiht positions enabled
+      TERM_VECTOR_WITH_POSITION = FieldOption.new("TERM_VECTOR_WITH_POSITION")
+      # all fields where termvectors with offset position are set
+      TERM_VECTOR_WITH_OFFSET = FieldOption.new("TERM_VECTOR_WITH_OFFSET")
+      # all fields where termvectors with offset and position values set
+      TERM_VECTOR_WITH_POSITION_OFFSET =
+        FieldOption.new("TERM_VECTOR_WITH_POSITION_OFFSET")
+    end
+    
+    # directory:: Directory where IndexReader files reside.
+    # segment_infos:: Used for write-l
+    # close_directory:: close the directory when the index reader is closed
+    def initialize(directory, segment_infos = nil,
+                   close_directory = false, directory_owner = false)
+      super()
+      @directory = directory
+      @close_directory = close_directory
+      @segment_infos = segment_infos
+      @directory_owner = directory_owner
+
+      @has_changes = false
+      @stale = false
+      @write_lock = nil
+
+      #ObjectSpace.define_finalizer(self, lambda { |id| @write_lock.release() if @write_lock})
+    end
+
+    # Returns an index reader to read the index in the directory
+    def IndexReader.open(directory, close_directory = true, infos = nil)
+      directory.synchronize do # in- & inter-process sync
+        commit_lock = directory.make_lock(IndexWriter::COMMIT_LOCK_NAME)
+        commit_lock.while_locked() do
+          if infos.nil?
+            infos = SegmentInfos.new()
+            infos.read(directory)
+          end
+          if (infos.size() == 1) # index is optimized
+            return SegmentReader.get(infos[0], infos, close_directory)
+          end
+          readers = Array.new(infos.size)
+          infos.size.times do |i|
+            readers[i] = SegmentReader.get(infos[i])
+          end
+          return MultiReader.new(readers, directory, infos, close_directory)
+        end
+      end
+    end
+
+    # Reads version number from segments files. The version number counts the
+    # number of changes of the index.
+    # 
+    # directory:: where the index resides.
+    # returns:: version number.
+    # raises:: IOError if segments file cannot be read.
+    def IndexReader.get_current_version(directory)
+      return SegmentInfos.read_current_version(directory)
+    end
+
+    # Return an array of term vectors for the specified document.  The array
+    # contains a vector for each vectorized field in the document.  Each vector
+    # contains terms and frequencies for all terms in a given vectorized field.
+    # If no such fields existed, the method returns nil. The term vectors that
+    # are returned my either be of type TermFreqVector or of type
+    # TermDocPosEnumVector if positions or offsets have been stored.
+    # 
+    # doc_number:: document for which term vectors are returned
+    # returns:: array of term vectors. May be nil if no term vectors have been
+    #           stored for the specified document.
+    # raises:: IOError if index cannot be accessed
+    #
+    # See Field.TermVector
+    def get_term_vectors(doc_number)
+      raise NotImplementedError
+    end
+           
+
+    
+    # Return a term vector for the specified document and field. The returned
+    # vector contains terms and frequencies for the terms in the specified
+    # field of this document, if the field had the storeTermVector flag set. If
+    # termvectors had been stored with positions or offsets, a
+    # TermDocPosEnumVector is returned.
+    # 
+    # doc_number:: document for which the term vector is returned
+    # field:: field for which the term vector is returned.
+    # returns:: term vector May be nil if field does not exist in the specified
+    #           document or term vector was not stored.
+    # raises:: IOError if index cannot be accessed
+    # See Field.TermVector
+    def get_term_vector(doc_number, field)
+      raise NotImplementedError
+    end
+           
+   
+    # Returns +true+ if an index exists at the specified directory.  If the
+    # directory does not exist or if there is no index in it.
+    #
+    # directory:: the directory to check for an index
+    # returns:: +true+ if an index exists; +false+ otherwise
+    # raises:: IOError if there is a problem with accessing the index
+    def IndexReader.index_exists?(directory)
+      return directory.exists?("segments")
+    end
+
+    # Returns the number of documents in this index. 
+    def num_docs()
+      raise NotImplementedError
+    end
+
+    # Returns one greater than the largest possible document number.
+    #
+    # This may be used to, e.g., determine how big to allocate an array which
+    # will have an element for every document number in an index.
+    def max_doc()
+      raise NotImplementedError
+    end
+
+    # Returns the stored fields of the +n+<sup>th</sup>
+    # +Document+ in this index. 
+    def get_document(n)
+      raise NotImplementedError
+    end
+
+    # Returns the first document with the term +term+. This is useful, for
+    # example, if we are indexing rows from a database. We can store the id of
+    # each row in a field in the index and use this method get the document by
+    # the id. Hence, only one document is returned.
+    #
+    # term: The term we are searching for.
+    def get_document_with_term(term)
+      docs = term_docs_for(term)
+      if (docs == nil) then return nil end
+      document = nil
+      begin 
+        document = get_document(docs.doc) if docs.next?
+      ensure 
+        docs.close()
+      end
+      return document
+    end
+
+    # Returns true if document _n_ has been deleted 
+    def deleted?(n)
+      raise NotImplementedError
+    end
+
+    # Returns true if any documents have been deleted 
+    def has_deletions?()
+      raise NotImplementedError
+    end
+    
+    # Returns the byte-encoded normalization factor for the named field of
+    # every document.  This is used by the search code to score documents.
+    # 
+    # See Field#boost
+    def get_norms(field, bytes=nil, offset=nil)
+      raise NotImplementedError
+    end
+
+    # Expert: Resets the normalization factor for the named field of the named
+    # document.  The norm represents the product of the field's Field#boost and
+    # its Similarity#length_norm length normalization.  Thus, to preserve the
+    # length normalization values when resetting this, one should base the new
+    # value upon the old.
+    # 
+    # See #get_norms
+    # See Similarity#decode_norm
+    def set_norm(doc, field, value)
+      synchronize do
+        value = Similarity.encode_norm(value) if value.is_a? Float
+        if(@directory_owner)
+          acquire_write_lock()
+        end
+        do_set_norm(doc, field, value)
+        @has_changes = true
+      end
+    end
+            
+    # Implements set_norm in subclass.
+    def do_set_norm(doc, field, value) 
+      raise NotImplementedError
+    end
+           
+    # Returns an enumeration of all the terms in the index.
+    # Each term is greater than all that precede it in the enumeration.
+    def terms()
+      raise NotImplementedError
+    end
+
+    # Returns an enumeration of all terms after a given term.
+    #
+    # Each term is greater than all that precede it in the enumeration.
+    def terms_from(t)
+      raise NotImplementedError
+    end
+
+    # Returns the number of documents containing the term +t+. 
+    def doc_freq(t)
+      raise NotImplementedError
+    end
+
+    # Returns an enumeration of all the documents which contain +term+. For each
+    # document, the document number, the frequency of the term in that document
+    # is also provided, for use in search scoring.  Thus, this method implements
+    # the mapping:
+    # 
+    #   Term => <doc_num, freq><sup>*</sup>
+    # 
+    # The enumeration is ordered by document number.  Each document number is
+    # greater than all that precede it in the enumeration.
+    def term_docs_for(term)
+      term_docs = term_docs()
+      term_docs.seek(term)
+      return term_docs
+    end
+
+    # Returns an unpositioned TermDocEnum enumerator. 
+    def term_docs()
+      raise NotImplementedError
+    end
+
+    # Returns an enumeration of all the documents which contain
+    # +term+.  For each document, in addition to the document number
+    # and frequency of the term in that document, a list of all of the ordinal
+    # positions of the term in the document is available.  Thus, this method
+    # implements the mapping:
+    #
+    #   Term => <doc_num, freq, < pos<sub>1</sub>, pos<sub>2</sub>, ...
+    #   pos<sub>freq-1</sub> > > <sup>*</sup>
+    #
+    # This positional information faciliates phrase and proximity searching.
+    # The enumeration is ordered by document number.  Each document number is
+    # greater than all that precede it in the enumeration.
+    def term_positions_for(term)
+      term_positions = term_positions()
+      term_positions.seek(term)
+      return term_positions
+    end
+
+    # Returns an unpositioned @link TermDocPosEnumendenumerator. 
+    def term_positions()
+      raise NotImplementedError
+    end
+
+    # Tries to acquire the WriteLock on this directory.
+    #
+    # This method is only valid if this IndexReader is directory owner.
+    # 
+    # raises:: IOError If WriteLock cannot be acquired.
+    def acquire_write_lock()
+      if @stale
+        raise IOError, "IndexReader out of date and no longer valid for delete, undelete, or set_norm operations"
+      end
+
+      if (@write_lock == nil) 
+        @write_lock = @directory.make_lock(IndexWriter::WRITE_LOCK_NAME)
+        if not @write_lock.obtain(IndexWriter::WRITE_LOCK_TIMEOUT) # obtain write lock
+          raise IOError, "Index locked for write: " + @write_lock
+        end
+
+        # we have to check whether index has changed since this reader was opened.
+        # if so, this reader is no longer valid for deletion
+        if (SegmentInfos.read_current_version(@directory) > @segment_infos.version()) 
+          @stale = true
+          @write_lock.release()
+          @write_lock = nil
+          raise IOError, "IndexReader out of date and no longer valid for delete, undelete, or set_norm operations"
+        end
+      end
+    end
+    
+    # Deletes the document numbered +doc_num+.  Once a document is deleted it
+    # will not appear in TermDocEnum or TermPostitions enumerations.  Attempts to
+    # read its field with the @link #documentend method will result in an error.
+    # The presence of this document may still be reflected in the @link
+    # #docFreqendstatistic, though this will be corrected eventually as the
+    # index is further modified.
+    def delete(doc_num)
+      synchronize do
+        acquire_write_lock() if @directory_owner
+        do_delete(doc_num)
+        @has_changes = true
+      end
+      return 1
+    end
+
+    # Implements deletion of the document numbered +doc_num+.
+    # Applications should call @link #delete(int)endor @link #delete(Term)end.
+    def do_delete(doc_num)
+      raise NotImplementedError
+    end
+
+    # Deletes all documents containing +term+.
+    # This is useful if one uses a document field to hold a unique ID string for
+    # the document.  Then to delete such a document, one merely constructs a
+    # term with the appropriate field and the unique ID string as its text and
+    # passes it to this method.  Returns the number of documents deleted.  See
+    # #delete for information about when this deletion will become effective.
+    def delete_docs_with_term(term)
+      docs = term_docs_for(term)
+      if (docs == nil) then return 0 end
+      n = 0
+      begin 
+        while (docs.next?) 
+          delete(docs.doc)
+          n += 1
+        end
+      ensure 
+        docs.close()
+      end
+      return n
+    end
+
+    # Undeletes all documents currently marked as deleted in this index.
+    def undelete_all()
+      synchronize do
+        acquire_write_lock() if @directory_owner
+        do_undelete_all()
+        @has_changes = true
+      end
+    end
+    
+    # Commit changes resulting from delete, undelete_all, or set_norm operations
+    # 
+    # raises:: IOError
+    def commit()
+      synchronize do
+        if @has_changes
+          if @directory_owner
+            @directory.synchronize do # in- & inter-process sync
+              commit_lock = @directory.make_lock(IndexWriter::COMMIT_LOCK_NAME)
+              commit_lock.while_locked do
+                do_commit()
+                @segment_infos.write(@directory)
+              end
+            end
+            if (@write_lock != nil) 
+              @write_lock.release()  # release write lock
+              @write_lock = nil
+            end
+          else
+            do_commit()
+          end
+        end
+        @has_changes = false
+      end
+    end
+    
+    # Closes files associated with this index.
+    # Also saves any new deletions to disk.
+    # No other methods should be called after this has been called.
+    def close()
+      synchronize do
+        commit()
+        do_close()
+        @directory.close() if @close_directory
+      end
+    end
+
+    protected 
+
+    # Implements actual undelete_all() in subclass. 
+    def do_undelete_all()
+      raise NotImplementedError
+    end
+
+    # Implements commit. 
+    def do_commit()
+      raise NotImplementedError
+    end
+    
+
+    # Implements close. 
+    def do_close()
+      raise NotImplementedError
+    end
+
+    # Get a list of unique field names that exist in this index and have the
+    # specified field option information.
+    # fld_option:: specifies which field option should be available for the
+    #              returned fields
+    # returns:: Collection of Strings indicating the names of the fields.
+    # See IndexReader.FieldOption
+    def get_field_names()
+      raise NotImplementedError
+    end
+
+    # Returns +true+ iff the index in the named directory is
+    # currently locked.
+    # directory:: the directory to check for a lock
+    # raises:: IOError if there is a problem with accessing the index
+    def IndexReader.locked?(directory)
+      return (directory.make_lock(IndexWriter::WRITE_LOCK_NAME).locked? or
+        directory.make_lock(IndexWriter::COMMIT_LOCK_NAME).locked?)
+    end
+
+    # Forcibly unlocks the index in the named directory.
+    #
+    # Caution: this should only be used by failure recovery code,
+    # when it is known that no other process nor thread is in fact
+    # currently accessing this index.
+    def IndexReader.unlock(directory)
+      directory.make_lock(IndexWriter::WRITE_LOCK_NAME).release
+      directory.make_lock(IndexWriter::COMMIT_LOCK_NAME).release
+    end
+  end
+end