ferret 0.11.6 → 0.11.8.4

data/lib/ferret.rb

@@ -23,7 +23,8 @@
 # :include: ../TUTORIAL
 $: << File.expand_path(File.join(File.dirname(__FILE__), "../ext"))
 require 'ferret_ext'
-require 'ferret_version'
+require 'ferret/version'
 require 'ferret/document'
 require 'ferret/index'
 require 'ferret/field_infos'
+require 'ferret/field_symbol'

data/lib/ferret/browser.rb

@@ -77,7 +77,7 @@ module Ferret::Browser
     end
 
     def tick_or_cross(t)
-      "<img src=\"/s/i/#{t ?'tick':'cross'}.png\" alt=\"#{t ?'yes':'no'}\"/>"
+      "<img src=\"/s/i/#{t ? 'tick' : 'cross'}.png\" alt=\"#{t ? 'yes' : 'no'}\"/>"
     end
   end
 

data/lib/ferret/field_symbol.rb

@@ -0,0 +1,94 @@
+module Ferret
+  FIELD_TYPES = %w(integer float string byte).map{|t| t.to_sym}
+
+  if defined?(BasicObject)
+    # Ruby 1.9.x
+    class BlankSlate < BasicObject
+    end
+  else
+    # Ruby 1.8.x
+    # BlankSlate is a class with no instance methods except for __send__ and
+    # __id__. It is useful for creating proxy classes. It is currently used by
+    # the FieldSymbol class which is a proxy to the Symbol class 
+    class BlankSlate
+      instance_methods.each { |m| undef_method m unless m =~ /^__|object_id/ }
+    end
+  end
+
+  # The FieldSymbolMethods module contains the methods that are added to both
+  # the Symbol class and the FieldSymbol class. These methods allow you to set
+  # the type easily set the type of a field by calling a method on a symbol.
+  #
+  # Right now this is only useful for Sorting and grouping, but some day Ferret
+  # may have typed fields, in which case these this methods will come in handy.
+  #
+  # The available types are specified in Ferret::FIELD_TYPES.
+  #
+  # == Examples
+  #
+  #   index.search(query, :sort => :title.string.desc)
+  #
+  #   index.search(query, :sort => [:price.float, :count.integer.desc])
+  #   
+  #   index.search(query, :group_by => :catalogue.string)
+  #
+  # == Note
+  #
+  # If you set the field type multiple times, the last type specified will be
+  # the type used. For example;
+  #
+  #   puts :title.integer.float.byte.string.type.inspect # => :string
+  #
+  # Calling #desc twice will set desc? to false
+  #
+  #   puts :title.desc?           # => false
+  #   puts :title.desc.desc?      # => true
+  #   puts :title.desc.desc.desc? # => false
+  module FieldSymbolMethods
+    FIELD_TYPES.each do |method|
+      define_method(method) do
+        fsym = FieldSymbol.new(self, respond_to?(:desc?) ? desc? : false)
+        fsym.type = method
+        fsym
+      end
+    end
+      
+    # Set a field to be a descending field. This only makes sense in sort
+    # specifications.
+    def desc
+      fsym = FieldSymbol.new(self, respond_to?(:desc?) ? !desc? : true)
+      fsym.type = type if respond_to? :type
+      fsym
+    end
+
+    # Return whether or not this field should be a descending field
+    def desc?
+      @desc == true
+    end
+
+    # Return the type of this field
+    def type
+      @type || nil
+    end
+  end
+
+  # See FieldSymbolMethods
+  class FieldSymbol < BlankSlate
+    include FieldSymbolMethods
+    def initialize(symbol, desc = false)
+      @symbol = symbol
+      @desc = desc
+    end
+
+    def method_missing(method, *args)
+      @symbol.__send__(method, *args)
+    end
+
+    attr_writer :type, :desc
+  end
+end
+
+# See FieldSymbolMethods
+class Symbol
+  include Ferret::FieldSymbolMethods
+end

data/lib/ferret/index.rb

@@ -1,20 +1,6 @@
 require 'monitor'
 
 module Ferret::Index
-  module SynchroLockMixin
-    def synchrolock
-      trys = 5
-      begin
-        synchronize {yield}
-      rescue Ferret::Store::Lock::LockError => e
-        if (trys -= 1) <= 0
-          raise e
-        else
-          retry
-        end
-      end
-    end
-  end
   # This is a simplified interface to the index. See the TUTORIAL for more
   # information on how to use this class.
   class Index
@@ -40,7 +26,7 @@ module Ferret::Index
     # default_input_field::   Default: "id". This specifies the default field
     #                         that will be used when you add a simple string
     #                         to the index using #add_document or <<.
-    # id_field:               Default: "id". This field is as the field to
+    # id_field::              Default: "id". This field is as the field to
     #                         search when doing searches on a term. For
     #                         example, if you do a lookup by term "cat", ie
     #                         index["cat"], this will be the field that is
@@ -75,8 +61,16 @@ module Ferret::Index
     #                         Directory object to this class and you want
     #                         Index to close it when it is closed itself then
     #                         set this to true.
-    # 
-    # Some examples;
+    # use_typed_range_query:: Default: true. Use TypedRangeQuery instead of
+    #                         the standard RangeQuery when parsing
+    #                         range queries. This is useful if you have number
+    #                         fields which you want to perform range queries
+    #                         on. You won't need to pad or normalize the data
+    #                         in the field in anyway to get correct results.
+    #                         However, performance will be a lot slower for
+    #                         large indexes, hence the default.
+    #
+    # == Examples
     #
     #   index = Index::Index.new(:analyzer => WhiteSpaceAnalyzer.new())
     #
@@ -130,7 +124,7 @@ module Ferret::Index
         @dir = RAMDirectory.new
       end
 
-      @dir.extend(MonitorMixin).extend(SynchroLockMixin)
+      @dir.extend(MonitorMixin) unless @dir.kind_of? MonitorMixin
       options[:dir] = @dir
       options[:lock_retry_time]||= 2
       @options = options
@@ -138,6 +132,9 @@ module Ferret::Index
         IndexWriter.new(options).close
       end
       options[:analyzer]||= Ferret::Analysis::StandardAnalyzer.new
+      if options[:use_typed_range_query].nil?
+        options[:use_typed_range_query] = true
+      end
 
       @searcher = nil
       @writer = nil
@@ -264,7 +261,7 @@ module Ferret::Index
     # 
     # See FieldInfos for more information on how to set field properties.
     def add_document(doc, analyzer = nil)
-      @dir.synchrolock do
+      @dir.synchronize do
         ensure_writer_open()
         if doc.is_a?(String) or doc.is_a?(Array)
           doc = {@default_input_field => doc}
@@ -281,9 +278,7 @@ module Ferret::Index
           else
             id = doc[@key].to_s
             if id
-              ensure_writer_open()
               @writer.delete(@key, id)
-              @writer.commit
             end
           end
         end
@@ -397,6 +392,50 @@ module Ferret::Index
       end
     end
 
+    # Run a query through the Searcher on the index, ignoring scoring and
+    # starting at +:start_doc+ and stopping when +:limit+ matches have been
+    # found. It returns an array of the matching document numbers.
+    #
+    # There is a big performance advange when using this search method on a
+    # very large index when there are potentially thousands of matching
+    # documents and you only want say 50 of them. The other search methods need
+    # to look at every single match to decide which one has the highest score.
+    # This search method just needs to find +:limit+ number of matches before
+    # it returns.
+    # 
+    # === Options
+    #
+    # start_doc::     Default: 0. The start document to start the search from.
+    #                 NOTE very carefully that this is not the same as the
+    #                 +:offset+ parameter used in the other search methods
+    #                 which refers to the offset in the result-set. This is the
+    #                 document to start the scan from. So if you scanning
+    #                 through the index in increments of 50 documents at a time
+    #                 you need to use the last matched doc in the previous
+    #                 search to start your next search. See the example below.
+    # limit::         Default: 50. This is the number of results you want
+    #                 returned, also called the page size. Set +:limit+ to
+    #                 +:all+ to return all results.
+    # TODO: add option to return loaded documents instead
+    #
+    # === Options
+    #
+    #   start_doc = 0
+    #   begin
+    #     results = @searcher.scan(query, :start_doc => start_doc)
+    #     yield results # or do something with them
+    #     start_doc = results.last
+    #     # start_doc will be nil now if results is empty, ie no more matches
+    #   end while start_doc
+    def scan(query, options = {})
+      @dir.synchronize do
+        ensure_searcher_open()
+        query = do_process_query(query)
+
+        @searcher.scan(query, options)
+      end
+    end
+
     # Retrieves a document/documents from the index. The method for retrieval
     # depends on the type of the argument passed.
     #
@@ -408,7 +447,7 @@ module Ferret::Index
     #
     # If +arg+ is a String then search for the first document with +arg+ in
     # the +id+ field. The +id+ field is either :id or whatever you set
-    # :id_field parameter to when you create the Index object.
+    # +:id_field+ parameter to when you create the Index object.
     def doc(*arg)
       @dir.synchronize do
         id = arg[0]
@@ -424,6 +463,38 @@ module Ferret::Index
     end
     alias :[] :doc
 
+    # Retrieves the term_vector for a document. The document can be referenced
+    # by either a string id to match the id field or an integer corresponding
+    # to Ferret's document number.
+    #
+    # See Ferret::Index::IndexReader#term_vector
+    def term_vector(id, field)
+      @dir.synchronize do
+        ensure_reader_open()
+        if id.kind_of?(String) or id.kind_of?(Symbol)
+          term_doc_enum = @reader.term_docs_for(@id_field, id.to_s)
+          if term_doc_enum.next?
+            id = term_doc_enum.doc
+          else
+            return nil
+          end
+        end
+        return @reader.term_vector(id, field)
+      end
+    end
+
+    # iterate through all documents in the index. This method preloads the
+    # documents so you don't need to call #load on the document to load all the
+    # fields.
+    def each
+      @dir.synchronize do
+        ensure_reader_open
+        (0...@reader.max_doc).each do |i|
+          yield @reader[i].load unless @reader.deleted?(i)
+        end
+      end
+    end
+
     # Deletes a document/documents from the index. The method for determining
     # the document to delete depends on the type of the argument passed.
     #
@@ -431,18 +502,28 @@ module Ferret::Index
     # document number. Will raise an error if the document does not exist.
     #
     # If +arg+ is a String then search for the documents with +arg+ in the
-    # +id+ field. The +id+ field is either :id or whatever you set :id_field
+    # +id+ field. The +id+ field is either :id or whatever you set +:id_field+
     # parameter to when you create the Index object. Will fail quietly if the
     # no document exists.
+    #
+    # If +arg+ is a Hash or an Array then a batch delete will be performed.
+    # If +arg+ is an Array then it will be considered an array of +id+'s. If
+    # it is a Hash, then its keys will be used instead as the Array of
+    # document +id+'s. If the +id+ is an Integer then it is considered a
+    # Ferret document number and the corresponding document will be deleted.
+    # If the +id+ is a String or a Symbol then the +id+ will be considered a
+    # term and the documents that contain that term in the +:id_field+ will be
+    # deleted.
     def delete(arg)
-      @dir.synchrolock do
-        ensure_writer_open()
+      @dir.synchronize do
         if arg.is_a?(String) or arg.is_a?(Symbol)
           ensure_writer_open()
           @writer.delete(@id_field, arg.to_s)
         elsif arg.is_a?(Integer)
           ensure_reader_open()
           cnt = @reader.delete(arg)
+        elsif arg.is_a?(Hash) or arg.is_a?(Array)
+          batch_delete(arg)
         else
           raise ArgumentError, "Cannot delete for arg of type #{arg.class}"
         end
@@ -457,7 +538,7 @@ module Ferret::Index
     #         string (in which case it is parsed by the standard query parser)
     #         or an actual query object.
     def query_delete(query)
-      @dir.synchrolock do
+      @dir.synchronize do
         ensure_writer_open()
         ensure_searcher_open()
         query = do_process_query(query)
@@ -479,13 +560,14 @@ module Ferret::Index
     # Update the document referenced by the document number +id+ if +id+ is an
     # integer or all of the documents which have the term +id+ if +id+ is a
     # term..
+    # For batch update of set of documents, for performance reasons, see batch_update
     #
     # id::      The number of the document to update. Can also be a string
     #           representing the value in the +id+ field. Also consider using
     #           the :key attribute.
     # new_doc:: The document to replace the old document with
     def update(id, new_doc)
-      @dir.synchrolock do
+      @dir.synchronize do
         ensure_writer_open()
         delete(id)
         if id.is_a?(String) or id.is_a?(Symbol)
@@ -498,6 +580,73 @@ module Ferret::Index
       end
     end
 
+    # Batch updates the documents in an index. You can pass either a Hash or
+    # an Array.
+    #
+    # === Array (recommended)
+    #
+    # If you pass an Array then each value needs to be a Document or a Hash
+    # and each of those documents must have an +:id_field+ which will be used
+    # to delete the old document that this document is replacing.
+    #
+    # === Hash
+    #
+    # If you pass a Hash then the keys of the Hash will be considered the
+    # +id+'s and the values will be the new documents to replace the old ones
+    # with.If the +id+ is an Integer then it is considered a Ferret document
+    # number and the corresponding document will be deleted.  If the +id+ is a
+    # String or a Symbol then the +id+ will be considered a term and the
+    # documents that contain that term in the +:id_field+ will be deleted.
+    #
+    # Note: No error will be raised if the document does not currently
+    # exist. A new document will simply be created.
+    #
+    # == Examples
+    #
+    #   # will replace the documents with the +id+'s id:133 and id:254
+    #   @index.batch_update({
+    #       '133' => {:id => '133', :content => 'yada yada yada'},
+    #       '253' => {:id => '253', :content => 'bla bla bal'}
+    #     })
+    #
+    #   # will replace the documents with the Ferret Document numbers 2 and 92
+    #   @index.batch_update({
+    #       2  => {:id => '133', :content => 'yada yada yada'},
+    #       92 => {:id => '253', :content => 'bla bla bal'}
+    #     })
+    #
+    #   # will replace the documents with the +id+'s id:133 and id:254
+    #   # this is recommended as it guarantees no duplicate keys
+    #   @index.batch_update([
+    #       {:id => '133', :content => 'yada yada yada'},
+    #       {:id => '253', :content => 'bla bla bal'}
+    #     ])
+    #
+    # docs:: A Hash of id/document pairs. The set of documents to be updated
+    def batch_update(docs)
+      @dir.synchronize do
+        ids = values = nil
+        case docs
+        when Array
+          ids = docs.collect{|doc| doc[@id_field].to_s}
+          if ids.include?(nil)
+            raise ArgumentError, "all documents must have an #{@id_field} " 
+                                 "field when doing a batch update"
+          end
+        when Hash
+          ids = docs.keys
+          docs = docs.values
+        else
+          raise ArgumentError, "must pass Hash or Array, not #{docs.class}"
+        end
+        batch_delete(ids)
+        ensure_writer_open()
+        docs.each {|new_doc| @writer << new_doc }
+        flush()
+      end
+    end
+
+
     # Update all the documents returned by the query.
     #
     # query::   The query to find documents you wish to update. Can either be
@@ -523,12 +672,12 @@ module Ferret::Index
     #     #=> {:id => "28", :title => "My Oh My", :artist => "David Gray"}
     #
     def query_update(query, new_val)
-      @dir.synchrolock do
+      @dir.synchronize do
         ensure_writer_open()
         ensure_searcher_open()
         docs_to_add = []
         query = do_process_query(query)
-        @searcher.search_each(query) do |id, score|
+        @searcher.search_each(query, :limit => :all) do |id, score|
           document = @searcher[id].load
           if new_val.is_a?(Hash)
             document.merge!(new_val)
@@ -568,7 +717,8 @@ module Ferret::Index
           end
           @reader.commit
         elsif @writer
-          @writer.commit
+          @writer.close
+          @writer = nil
         end
       end
     end
@@ -577,7 +727,7 @@ module Ferret::Index
     # optimizes the index. This should only be called when the index will no
     # longer be updated very often, but will be read a lot.
     def optimize()
-      @dir.synchrolock do
+      @dir.synchronize do
         ensure_writer_open()
         @writer.optimize()
         @writer.close()
@@ -605,7 +755,7 @@ module Ferret::Index
     #
     # After this completes, the index is optimized.
     def add_indexes(indexes)
-      @dir.synchrolock do
+      @dir.synchronize do
         ensure_writer_open()
         indexes = [indexes].flatten   # make sure we have an array
         return if indexes.size == 0 # nothing to do
@@ -648,7 +798,7 @@ module Ferret::Index
         elsif directory.is_a?(Ferret::Store::Directory)
           @dir = directory
         end
-        @dir.extend(MonitorMixin).extend(SynchroLockMixin)
+        @dir.extend(MonitorMixin) unless @dir.kind_of? MonitorMixin
         @options[:dir] = @dir
         @options[:create_if_missing] = true
         add_indexes([old_dir])
@@ -690,7 +840,7 @@ module Ferret::Index
     # Returns the field_infos object so that you can add new fields to the
     # index.
     def field_infos
-      @dir.synchrolock do
+      @dir.synchronize do
         ensure_writer_open()
         return @writer.field_infos
       end
@@ -778,6 +928,43 @@ module Ferret::Index
           @writer = nil
         end
       end
+
+      # If +docs+ is a Hash or an Array then a batch delete will be performed.
+      # If +docs+ is an Array then it will be considered an array of +id+'s. If
+      # it is a Hash, then its keys will be used instead as the Array of
+      # document +id+'s. If the +id+ is an Integers then it is considered a
+      # Ferret document number and the corresponding document will be deleted.
+      # If the +id+ is a String or a Symbol then the +id+ will be considered a
+      # term and the documents that contain that term in the +:id_field+ will
+      # be deleted.
+      #
+      # docs:: An Array of docs to be deleted, or a Hash (in which case the keys
+      # are used)
+      def batch_delete(docs)
+        docs = docs.keys if docs.is_a?(Hash)
+        raise ArgumentError, "must pass Array or Hash" unless docs.is_a? Array
+        ids = []
+        terms = []
+        docs.each do |doc|
+          case doc
+          when String then  terms << doc
+          when Symbol then  terms << doc.to_s
+          when Integer then ids   << doc
+          else
+            raise ArgumentError, "Cannot delete for arg of type #{id.class}"
+          end
+        end
+        if ids.size > 0
+          ensure_reader_open
+          ids.each {|id| @reader.delete(id)}
+        end
+        if terms.size > 0
+          ensure_writer_open()
+          @writer.delete(@id_field, terms)
+        end
+        return self
+      end
+
   end
 end