mongoid_fulltext 0.3.6 → 0.4.0

data/.rspec

@@ -0,0 +1,2 @@
+--color
+

data/README.md

@@ -207,6 +207,10 @@ Additional indexing/query options can be used as parameters to `fulltext_search_
 * `index_full_words`: index full words, which improves exact matches, default is `true` 
 * `apply_prefix_scoring_to_all_words`: score n-grams at beginning of words higher, default is `true`
 * `max_ngrams_to_search`: maximum number of ngrams to query at any given time, default is `6`
+* `max_candidate_set_size`: maximum number of candidate ngrams to examine for a given query.
+  Defaults to 1000. If you're seeing poor results, you can try increasing this value to consider
+  more ngrams per query (changing this parameter does not require a re-index.) The amount of time
+  a search takes is directly proportional to this parameter's value.
 
 Array filters
 -------------
@@ -243,14 +247,19 @@ Building the index
 
 The fulltext index is built and maintained incrementally by hooking into `before_save` and 
 `before_destroy` callbacks on each model that's being indexed. If you want to build an index
-on existing models, you can call the `update_ngram_index` method on each instance:
+on existing models, you can call the `update_ngram_index` method on the class or each instance:
 
-    Artwork.all.each { |artwork| artwork.update_ngram_index }
+    Artwork.update_ngram_index
+    Artwork.find(id).update_ngram_index
 
-You can also remove instances in bulk from the index with the `remove_from_ngram_index`
+You can remove all or individual instances from the index with the `remove_from_ngram_index`
 method:
 
-    Artwork.all.each { |artwork| artwork.remove_from_ngram_index }
+    Artwork.remove_from_ngram_index
+    Artwork.find(id).remove_from_ngram_index
+
+The methods on the model level perform bulk removal operations and are therefore faster that 
+updating or removing records individually.
 
 Running the specs
 -----------------

data/VERSION

@@ -1 +1 @@
-0.3.6
+0.4.0

data/lib/mongoid_fulltext.rb

@@ -8,7 +8,7 @@ module Mongoid::FullTextSearch
   class UnspecifiedIndexError < StandardError; end
 
   module ClassMethods
-
+  
     def fulltext_search_in(*args)
       self.mongoid_fulltext_config = {} if self.mongoid_fulltext_config.nil?
       options = args.last.is_a?(Hash) ? args.pop : {}
@@ -24,7 +24,8 @@ module Mongoid::FullTextSearch
         :ngram_width => 3,
         :max_ngrams_to_search => 6,
         :apply_prefix_scoring_to_all_words => true,
-        :index_full_words => true
+        :index_full_words => true,
+        :max_candidate_set_size => 1000
       }
       
       config.update(options)
@@ -35,14 +36,41 @@ module Mongoid::FullTextSearch
       config[:word_separators] = Hash[config[:word_separators].split('').map{ |ch| [ch,ch] }]
       self.mongoid_fulltext_config[index_name] = config
 
-      coll = collection.db.collection(index_name)
-      coll.ensure_index([['ngram', Mongo::ASCENDING]])
-      coll.ensure_index([['document_id', Mongo::ASCENDING]])
+      ensure_indexes(index_name, config)
       
       before_save :update_ngram_index
       before_destroy :remove_from_ngram_index
     end
 
+    def ensure_indexes(index_name, config)
+      db = collection.db
+      coll = db.collection(index_name)
+
+      filter_indexes = (config[:filters] || []).map do |key,value|
+        ["filter_values.#{key}", Mongo::ASCENDING]
+      end
+      index_definition = [['ngram', Mongo::ASCENDING], ['score', Mongo::DESCENDING]].concat(filter_indexes)
+
+      # Since the definition of the index could have changed, we'll clean up by
+      # removing any indexes that aren't on the exact 
+      correct_keys = index_definition.map{ |field_def| field_def[0] }
+      all_filter_keys = filter_indexes.map{ |field_def| field_def[0] }
+      coll.index_information.each do |name, definition|
+        keys = definition['key'].keys
+        next if !keys.member?('ngram')
+        all_filter_keys |= keys.find_all{ |key| key.starts_with?('filter_values.') }
+        coll.drop_index(name) if keys & correct_keys != correct_keys
+      end
+
+      if all_filter_keys.length > filter_indexes.length
+        filter_indexes = all_filter_keys.map { |key| [key, Mongo::ASCENDING] }
+        index_definition = [['ngram', Mongo::ASCENDING], ['score', Mongo::DESCENDING]].concat(filter_indexes)        
+      end
+
+      coll.ensure_index(index_definition, name: 'fts_index')
+      coll.ensure_index([['document_id', Mongo::ASCENDING]]) # to make removes fast
+    end
+
     def fulltext_search(query_string, options={})
       max_results = options.has_key?(:max_results) ? options.delete(:max_results) : 10
       return_scores = options.has_key?(:return_scores) ? options.delete(:return_scores) : false
@@ -51,61 +79,84 @@ module Mongoid::FullTextSearch
         raise UnspecifiedIndexError, error_message % self.name, caller
       end
       index_name = options.has_key?(:index) ? options.delete(:index) : self.mongoid_fulltext_config.keys.first
-      
-      # options hash should only contain filters after this point      
+
+      # Options hash should only contain filters after this point
+
       ngrams = all_ngrams(query_string, self.mongoid_fulltext_config[index_name])
       return [] if ngrams.empty?
-      
-      query = {'ngram' => {'$in' => ngrams.keys}}
-      query.update(Hash[options.map { |key,value| [ 'filter_values.%s' % key, { '$all' => [ value ].flatten } ] }])
-      map = <<-EOS
-        function() {
-          emit(this['document_id'], {'class': this['class'], 'score': this['score']*ngrams[this['ngram']] })
-        }
-      EOS
-      reduce = <<-EOS
-        function(key, values) {
-          score = 0.0
-          for (i in values) {
-            score += values[i]['score']
-          }
-          return({'class': values[0]['class'], 'score': score})
-        }
-      EOS
-      mr_options = {:scope => {:ngrams => ngrams }, :query => query, :raw => true}
-      rc_options = { :return_scores => return_scores }
+
+      # For each ngram, construct the query we'll use to pull index documents and 
+      # get a count of the number of index documents containing that n-gram
+      ordering = [['score', Mongo::DESCENDING]]
+      limit = self.mongoid_fulltext_config[index_name][:max_candidate_set_size]
       coll = collection.db.collection(index_name)
-      if collection.db.connection.server_version >= '1.7.4'
-        mr_options[:out] = {:inline => 1}
-        results = coll.map_reduce(map, reduce, mr_options)['results'].sort_by{ |x| -x['value']['score'] }
-        max_results = results.count if max_results.nil?
-        instantiate_mapreduce_results(results.first(max_results), rc_options)
-      else
-        result_collection = coll.map_reduce(map, reduce, mr_options)['result']
-        results = collection.db.collection(result_collection).find.sort(['value.score',-1])
-        results = results.limit(max_results) if !max_results.nil?
-        models = instantiate_mapreduce_results(results, rc_options)
-        collection.db.collection(result_collection).drop
-        models
+      cursors = ngrams.map do |ngram| 
+        query = {'ngram' => ngram[0]}
+        query.update(Hash[options.map { |key,value| [ 'filter_values.%s' % key, { '$all' => [ value ].flatten } ] }])
+        count = coll.find(query).count
+        {:ngram => ngram, :count => count, :query => query}
+      end.sort_by!{ |record| record[:count] }
+
+      # Using the queries we just constructed and the n-gram frequency counts we
+      # just computed, pull in about *:max_candidate_set_size* candidates by 
+      # considering the n-grams in order of increasing frequency. When we've 
+      # spent all *:max_candidate_set_size* candidates, pull the top-scoring 
+      # *max_results* candidates for each remaining n-gram.
+      results_so_far = 0
+      candidates_list = cursors.map do |doc|
+        next if doc[:count] == 0
+        query_options = {}
+        if results_so_far >= limit
+          query_options = {:sort => ordering, :limit => max_results}
+        elsif doc[:count] > limit - results_so_far
+          query_options = {:sort => ordering, :limit => limit - results_so_far}
+        end
+        results_so_far += doc[:count]
+        ngram_score = ngrams[doc[:ngram][0]]
+        Hash[coll.find(doc[:query], query_options).map do |candidate|
+               [candidate['document_id'], 
+                {clazz: candidate['class'], score: candidate['score'] * ngram_score}]
+             end]
+      end.compact
+      
+      # Finally, score all candidates by matching them up with other candidates that are
+      # associated with the same document. This is similar to how you might process a
+      # boolean AND query, except that with an AND query, you'd stop after considering
+      # the first candidate list and matching its candidates up with candidates from other
+      # lists, whereas here we want the search to be a little fuzzier so we'll run through
+      # all candidate lists, removing candidates as we match them up.
+      all_scores = []
+      while !candidates_list.empty?
+        candidates = candidates_list.pop
+        scores = candidates.map do |candidate_id, data|
+          {:id => candidate_id, 
+           :clazz => data[:clazz], 
+           :score => data[:score] + candidates_list.map{ |others| (others.delete(candidate_id) || {score: 0})[:score] }.sum
+           }
+        end
+        all_scores.concat(scores)
       end
+      all_scores.sort_by!{ |document| -document[:score] }
+
+      instantiate_mapreduce_results(all_scores[0..max_results-1], { :return_scores => return_scores })
     end
     
     def instantiate_mapreduce_result(result)
-      Object::const_get(result['value']['class']).find(:first, :conditions => {:id => result['_id']})
+      result[:clazz].constantize.find(:first, :conditions => {'_id' => result[:id]})
     end
     
     def instantiate_mapreduce_results(results, options)
       if (options[:return_scores])
-        results.map { |result| [ instantiate_mapreduce_result(result), result['value']['score'] ] }.find_all { |result| ! result[0].nil? }
+        results.map { |result| [ instantiate_mapreduce_result(result), result[:score] ] }.find_all { |result| ! result[0].nil? }
       else
-        results.map { |result| instantiate_mapreduce_result(result) }.find_all { |result| ! result.nil? }
+        results.map { |result| instantiate_mapreduce_result(result) }.compact
       end
     end
 
     # returns an [ngram, score] [ngram, position] pair
     def all_ngrams(str, config, bound_number_returned = true)
       return {} if str.nil? or str.length < config[:ngram_width]
-      filtered_str = str.downcase.split('').map{ |ch| config[:alphabet][ch] }.find_all{ |ch| !ch.nil? }.join('')
+      filtered_str = str.downcase.split('').map{ |ch| config[:alphabet][ch] }.compact.join('')
       
       if bound_number_returned
         step_size = [((filtered_str.length - config[:ngram_width]).to_f / config[:max_ngrams_to_search]).ceil, 1].max
@@ -141,7 +192,20 @@ module Mongoid::FullTextSearch
       
       ngram_hash
     end
-
+    
+    def remove_from_ngram_index
+      self.mongoid_fulltext_config.each_pair do |index_name, fulltext_config|
+        coll = collection.db.collection(index_name)
+        coll.remove({'class' => self.name})
+      end
+    end
+    
+    def update_ngram_index
+      self.all.each do |model|
+        model.update_ngram_index
+      end
+    end
+    
   end
 
   def update_ngram_index
@@ -162,7 +226,7 @@ module Mongoid::FullTextSearch
           rescue 
             # Suppress any exceptions caused by filters
           end
-        end.find_all{ |x| !x.nil? }]
+        end.compact]
       end
       # insert new ngrams in external index
       ngrams.each_pair do |ngram, score|

data/mongoid_fulltext.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{mongoid_fulltext}
-  s.version = "0.3.6"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aaron Windsor"]
-  s.date = %q{2011-05-27}
+  s.date = %q{2011-07-19}
   s.description = %q{Full-text search for the Mongoid ORM, using n-grams extracted from text}
   s.email = %q{aaron.windsor@gmail.com}
   s.extra_rdoc_files = [
@@ -18,6 +18,7 @@ Gem::Specification.new do |s|
   ]
   s.files = [
     ".document",
+    ".rspec",
     "Gemfile",
     "LICENSE",
     "README.md",
@@ -32,6 +33,8 @@ Gem::Specification.new do |s|
     "spec/models/external_artwork_no_fields_supplied.rb",
     "spec/models/filtered_artist.rb",
     "spec/models/filtered_artwork.rb",
+    "spec/models/filtered_other.rb",
+    "spec/models/gallery/basic_artwork.rb",
     "spec/models/multi_external_artwork.rb",
     "spec/models/multi_field_artist.rb",
     "spec/models/multi_field_artwork.rb",
@@ -42,7 +45,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/aaw/mongoid_fulltext}
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.6.2}
   s.summary = %q{Full-text search for the Mongoid ORM}
   s.test_files = [
     "spec/models/advanced_artwork.rb",
@@ -52,6 +55,8 @@ Gem::Specification.new do |s|
     "spec/models/external_artwork_no_fields_supplied.rb",
     "spec/models/filtered_artist.rb",
     "spec/models/filtered_artwork.rb",
+    "spec/models/filtered_other.rb",
+    "spec/models/gallery/basic_artwork.rb",
     "spec/models/multi_external_artwork.rb",
     "spec/models/multi_field_artist.rb",
     "spec/models/multi_field_artwork.rb",
@@ -61,7 +66,6 @@ Gem::Specification.new do |s|
   ]
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then

data/spec/models/filtered_other.rb

@@ -0,0 +1,11 @@
+# This is some other model that lives in the same index with FilteredArtist and FilteredArtwork,
+# to make sure different filters can co-exist in the same index and are indexed properly.
+class FilteredOther
+  include Mongoid::Document
+  include Mongoid::FullTextSearch
+  field :name
+  fulltext_search_in :name, :index_name => 'mongoid_fulltext.artworks_and_artists',
+                     :filters => { :is_fuzzy => lambda { |x| true },
+                                   :is_awesome => lambda { |x| false }
+                                  }
+end

data/spec/models/gallery/basic_artwork.rb

@@ -0,0 +1,9 @@
+module Gallery
+  class BasicArtwork
+    include Mongoid::Document
+    include Mongoid::FullTextSearch
+
+    field :title
+    fulltext_search_in :title
+  end
+end

data/spec/mongoid/fulltext_spec.rb

@@ -62,6 +62,23 @@ module Mongoid
       end
             
     end
+    
+    context "with default settings" do
+      
+      let!(:flower_myth) { Gallery::BasicArtwork.create(:title => 'Flower Myth') }
+      let!(:flowers)     { Gallery::BasicArtwork.create(:title => 'Flowers') }
+      let!(:lowered)     { Gallery::BasicArtwork.create(:title => 'Lowered') }
+      let!(:cookies)     { Gallery::BasicArtwork.create(:title => 'Cookies') }
+      let!(:empty)       { Gallery::BasicArtwork.create(:title => '') }
+      
+      it "returns exact matches for model within a module" do
+        Gallery::BasicArtwork.fulltext_search('Flower Myth', :max_results => 1).first.should == flower_myth
+        Gallery::BasicArtwork.fulltext_search('Flowers', :max_results => 1).first.should == flowers
+        Gallery::BasicArtwork.fulltext_search('Cookies', :max_results => 1).first.should == cookies
+        Gallery::BasicArtwork.fulltext_search('Lowered', :max_results => 1).first.should == lowered
+      end
+      
+    end
 
     context "with default settings" do
 
@@ -310,6 +327,26 @@ module Mongoid
       end
 
     end
+
+    context "with different filters applied to multiple models" do
+      let!(:foo_artwork)    { FilteredArtwork.create(:title => 'foo') }
+      let!(:bar_artist)     { FilteredArtist.create(:full_name => 'bar') }
+      let!(:baz_other)      { FilteredOther.create(:name => 'baz') }
+
+      # These three models are all indexed by the same mongoid_fulltext index, but have different filters
+      # applied. The index created on the mongoid_fulltext collection should include the ngram and score
+      # fields as well as the union of all the filter fields to allow for efficient lookups.
+
+      it "creates a proper index for searching efficiently" do
+        index_collection = FilteredArtwork.collection.db.collection('mongoid_fulltext.artworks_and_artists')
+        ngram_indexes = index_collection.index_information.find_all{ |name, definition| definition['key'].has_key?('ngram') }
+        ngram_indexes.length.should == 1
+        keys = ngram_indexes.first[1]['key'].keys
+        expected_keys = ['ngram','score', 'filter_values.is_fuzzy', 'filter_values.is_awesome', 
+                         'filter_values.is_foobar', 'filter_values.is_artwork', 'filter_values.is_artist'].sort
+        keys.sort.should == expected_keys
+      end
+    end
     
     context "with partitions applied to a model" do
       
@@ -318,9 +355,11 @@ module Mongoid
       let!(:artist_0) { PartitionedArtist.create(:full_name => 'foobar', :exhibitions => [ ]) }
 
       it "allows partitioned searches" do
-        PartitionedArtist.fulltext_search('foobar').should == [ artist_2, artist_1, artist_0 ]
+        artists_by_exhibition_length = [ artist_0, artist_1, artist_2 ].sort_by{ |x| x.exhibitions.length }
+        PartitionedArtist.fulltext_search('foobar').sort_by{ |x| x.exhibitions.length }.should == artists_by_exhibition_length
         PartitionedArtist.fulltext_search('foobar', :exhibitions => [ "Armory NY" ]).should == [ artist_2 ]
-        PartitionedArtist.fulltext_search('foobar', :exhibitions => [ "Art Basel 2011" ]).should == [ artist_2, artist_1 ]
+        art_basel_only = PartitionedArtist.fulltext_search('foobar', :exhibitions => [ "Art Basel 2011" ]).sort_by{ |x| x.exhibitions.length }
+        art_basel_only.should == [ artist_1, artist_2 ].sort_by{ |x| x.exhibitions.length }
         PartitionedArtist.fulltext_search('foobar', :exhibitions => [ "Art Basel 2011", "Armory NY" ]).should == [ artist_2 ]
       end
 
@@ -343,6 +382,56 @@ module Mongoid
         first_result[1].is_a?(Float).should be_true
       end
     end
+
+    context "remove_from_ngram_index" do
+      let!(:flowers1)     { BasicArtwork.create(:title => 'Flowers 1') }
+      let!(:flowers2)     { BasicArtwork.create(:title => 'Flowers 1') }
+
+      it "removes all records from the index" do
+        BasicArtwork.remove_from_ngram_index
+        BasicArtwork.fulltext_search('flower').length.should == 0
+      end
+      
+      it "removes a single record from the index" do
+        flowers1.remove_from_ngram_index
+        BasicArtwork.fulltext_search('flower').length.should == 1        
+      end      
+    end
+    
+    context "update_ngram_index" do
+      let!(:flowers1)     { BasicArtwork.create(:title => 'Flowers 1') }
+      let!(:flowers2)     { BasicArtwork.create(:title => 'Flowers 2') }
+
+      context "from scratch" do
+
+        before(:each) do
+          Mongoid.master["mongoid_fulltext.index_basicartwork_0"].remove
+        end
+
+        it "updates index on a single record" do
+          flowers1.update_ngram_index
+          BasicArtwork.fulltext_search('flower').length.should == 1
+        end
+        
+        it "updates index on all records" do
+          BasicArtwork.update_ngram_index
+          BasicArtwork.fulltext_search('flower').length.should == 2
+        end
+
+      end
+      
+      context "incremental" do
+      
+        it "removes an existing record" do
+          coll = Mongoid.master["mongoid_fulltext.index_basicartwork_0"]
+          Mongoid.master.stub(:collection).with("mongoid_fulltext.index_basicartwork_0").and_return { coll }
+          coll.should_receive(:remove).once.with({'document_id' => flowers1._id})
+          flowers1.update_ngram_index
+        end
+        
+      end
+
+    end
     
   end
 end

data/spec/spec_helper.rb

@@ -11,10 +11,11 @@ Mongoid.configure do |config|
 end
 
 require File.expand_path("../../lib/mongoid_fulltext", __FILE__)
-Dir["#{File.dirname(__FILE__)}/models/*.rb"].each { |f| require f }
+Dir["#{File.dirname(__FILE__)}/models/**/*.rb"].each { |f| require f }
 
 Rspec.configure do |c|
   c.before(:all)  { DatabaseCleaner.strategy = :truncation }
   c.before(:each) { DatabaseCleaner.clean }
+  c.after(:all) { Mongoid.master.command({'repairDatabase' => 1}) }
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid_fulltext
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-05-27 00:00:00.000000000 -04:00
+date: 2011-07-19 00:00:00.000000000 -04:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongoid
-  requirement: &82366030 !ruby/object:Gem::Requirement
+  requirement: &70446910 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +22,10 @@ dependencies:
         version: 2.0.0
   type: :development
   prerelease: false
-  version_requirements: *82366030
+  version_requirements: *70446910
 - !ruby/object:Gem::Dependency
   name: database_cleaner
-  requirement: &82365790 !ruby/object:Gem::Requirement
+  requirement: &70444760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -33,10 +33,10 @@ dependencies:
         version: 0.6.0
   type: :development
   prerelease: false
-  version_requirements: *82365790
+  version_requirements: *70444760
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &82365550 !ruby/object:Gem::Requirement
+  requirement: &70397170 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -44,10 +44,10 @@ dependencies:
         version: 2.5.0
   type: :development
   prerelease: false
-  version_requirements: *82365550
+  version_requirements: *70397170
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &82365310 !ruby/object:Gem::Requirement
+  requirement: &70386800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -55,7 +55,7 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *82365310
+  version_requirements: *70386800
 description: Full-text search for the Mongoid ORM, using n-grams extracted from text
 email: aaron.windsor@gmail.com
 executables: []
@@ -65,6 +65,7 @@ extra_rdoc_files:
 - README.md
 files:
 - .document
+- .rspec
 - Gemfile
 - LICENSE
 - README.md
@@ -79,6 +80,8 @@ files:
 - spec/models/external_artwork_no_fields_supplied.rb
 - spec/models/filtered_artist.rb
 - spec/models/filtered_artwork.rb
+- spec/models/filtered_other.rb
+- spec/models/gallery/basic_artwork.rb
 - spec/models/multi_external_artwork.rb
 - spec/models/multi_field_artist.rb
 - spec/models/multi_field_artwork.rb
@@ -99,6 +102,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 858859311
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -119,6 +125,8 @@ test_files:
 - spec/models/external_artwork_no_fields_supplied.rb
 - spec/models/filtered_artist.rb
 - spec/models/filtered_artwork.rb
+- spec/models/filtered_other.rb
+- spec/models/gallery/basic_artwork.rb
 - spec/models/multi_external_artwork.rb
 - spec/models/multi_field_artist.rb
 - spec/models/multi_field_artwork.rb