mongoid_fulltext 0.4.5 → 0.5.0

data/README.md

@@ -50,6 +50,16 @@ To return a pair of `[ result, score ]` instead of an array of results, pass the
 
     Artist.fulltext_search("vince vangogh", { :return_scores => true })
 
+The larger a score is, the better mongoid_fulltext thinks the match is. The scores have the following rough 
+interpretation that you can use to make decisions about whether the match is good enough:
+
+* If a prefix of your query matches something indexed, or if your query matches a prefix of something
+  indexed (for example, searching for "foo" finds "myfoo" or searching for "myfoo" finds "foo"), you
+  can expect a score of at least 1 for the match.
+* If an entire word in your query matches an entire word that's indexed and you have the `index_full_words`
+  option turned on (it's turned on by default), you can expect a score of at least 2 for the match.
+* If neither of the above criteria are met, you can expect a score less than one.
+
 If you don't specify a field to index, the default is the result of `to_s` called on the object.
 The following definition will index the first and last name of an artist:
 
@@ -219,10 +229,10 @@ Additional indexing/query options can be used as parameters to `fulltext_search_
   and then finally stripped, as before.
 * `update_if`: controls whether or not the index will be updated. This can be set to a symbol,
   string, or proc. If the result of evaluating the value is true, the index will be updated.
-** When set to a symbol, the symbol is sent to the document.
-** When set to a string, the string is evaluated within the document's instance.
-** When set to a proc, the proc is called, and the document is given to the proc as the first arg.
-** When set to any other type of object, the document's index will not be updated.
+    * When set to a symbol, the symbol is sent to the document.
+    * When set to a string, the string is evaluated within the document's instance.
+    * When set to a proc, the proc is called, and the document is given to the proc as the first arg.
+    * When set to any other type of object, the document's index will not be updated.
 
 Array filters
 -------------

data/VERSION

@@ -1 +1 @@
-0.4.5
+0.5.0

data/lib/mongoid_fulltext.rb

@@ -153,7 +153,6 @@ module Mongoid::FullTextSearch
         all_scores.concat(scores)
       end
       all_scores.sort!{ |document1, document2| -document1[:score] <=> -document2[:score] }
-
       instantiate_mapreduce_results(all_scores[0..max_results-1], { :return_scores => return_scores })
     end
     
@@ -189,7 +188,7 @@ module Mongoid::FullTextSearch
         step_size = 1
       end
       
-      # array of ngrams
+      # Create an array of records of the form {:ngram => x, :score => y} for all ngrams that occur in the input string
       ngram_ary = (0..filtered_str.length - config[:ngram_width]).step(step_size).map do |i|
         if i == 0 or (config[:apply_prefix_scoring_to_all_words] and \
                       config[:word_separators].has_key?(filtered_str[i-1].chr))
@@ -197,25 +196,22 @@ module Mongoid::FullTextSearch
         else
           score = Math.sqrt(2.0/filtered_str.length)
         end
-        [filtered_str[i..i+config[:ngram_width]-1], score]
+        {:ngram => filtered_str[i..i+config[:ngram_width]-1], :score => score}
       end
+
+      # If an ngram appears multiple times in the query string, keep the max score
+      ngram_ary = ngram_ary.group_by{ |h| h[:ngram] }.map{ |key, values| {:ngram => key, :score => values.map{ |v| v[:score] }.max} }
       
       if (config[:index_full_words])
         filtered_str.split(Regexp.compile(config[:word_separators].keys.join)).each do |word|
           if word.length >= config[:ngram_width]
-            ngram_ary << [ word, 1 ]
+            ngram_ary << {:ngram => word, :score => 1}
           end
         end
       end
-      
-      ngram_hash = {}
-      
-      # deduplicate, and keep the highest score
-      ngram_ary.each do |ngram, score, position|        
-        ngram_hash[ngram] = [ngram_hash[ngram] || 0, score].max
-      end
-      
-      ngram_hash
+
+      # If an ngram appears as a full word and an ngram, keep the sum of the two scores
+      Hash[ngram_ary.group_by{ |h| h[:ngram] }.map{ |key, values| [key, values.map{ |v| v[:score] }.sum] }]
     end
     
     def remove_from_ngram_index

data/mongoid_fulltext.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{mongoid_fulltext}
-  s.version = "0.4.5"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aaron Windsor"]
-  s.date = %q{2011-10-05}
+  s.date = %q{2011-10-11}
   s.description = %q{Full-text search for the Mongoid ORM, using n-grams extracted from text}
   s.email = %q{aaron.windsor@gmail.com}
   s.extra_rdoc_files = [

data/spec/models/external_artist.rb

@@ -3,4 +3,9 @@ class ExternalArtist
   include Mongoid::FullTextSearch
   field :full_name
   fulltext_search_in :full_name, :index_name => 'mongoid_fulltext.artworks_and_artists'
+
+  def to_s
+    full_name
+  end
+
 end

data/spec/models/external_artwork.rb

@@ -3,4 +3,9 @@ class ExternalArtwork
   include Mongoid::FullTextSearch
   field :title
   fulltext_search_in :title, :index_name => 'mongoid_fulltext.artworks_and_artists'
+
+  def to_s
+    title
+  end
+
 end

data/spec/mongoid/fulltext_spec.rb

@@ -391,7 +391,7 @@ module Mongoid
       end
 
     end
-    
+
     context "using search options" do
       let!(:patterns)    { BasicArtwork.create(:title => 'Flower Patterns') }
       let!(:flowers)     { BasicArtwork.create(:title => 'Flowers') }
@@ -410,6 +410,46 @@ module Mongoid
       end
     end
 
+    context "returning scores" do
+      # Since we return scores, let's make some weak guarantees about what they actually mean
+      
+      let!(:mao_yan)      { ExternalArtist.create(:full_name => "Mao Yan") }
+      let!(:mao)          { ExternalArtwork.create(:title => "Mao by Andy Warhol") }
+      let!(:maox)         { ExternalArtwork.create(:title => "Maox by Randy Morehall") }
+      let!(:somao)        { ExternalArtwork.create(:title => "Somao by Randy Morehall") }
+
+      it "returns basic matches that don't match a whole word and aren't prefixes with score < 1" do
+        ['paox', 'porehall'].each do |query|
+          results = ExternalArtist.fulltext_search(query, { :return_scores => true })
+          results.length.should > 0
+          results.map{ |result| result[-1] }.inject(true){ |accum, item| accum &= (item < 1) }.should be_true
+        end
+      end
+      
+      it "returns prefix matches with a score >= 1 but < 2" do
+        ['warho', 'rand'].each do |query|
+          results = ExternalArtist.fulltext_search(query, { :return_scores => true })
+          results.length.should > 0
+          results.map{ |result| result[-1] if result[0].to_s.starts_with?(query)}
+                 .compact
+                 .inject(true){ |accum, item| accum &= (item >= 1 and item < 2) }
+                 .should be_true
+        end
+      end
+      
+      it "returns full-word matches with a score >= 2" do
+        ['andy', 'warhol', 'mao'].each do |query|
+          results = ExternalArtist.fulltext_search(query, { :return_scores => true })
+          results.length.should > 0
+          results.map{ |result| result[-1] if result[0].to_s.split(' ').member?(query) }
+                 .compact
+                 .inject(true){ |accum, item| accum &= (item >= 2) }
+                 .should be_true
+        end
+      end
+      
+    end
+
     context "remove_from_ngram_index" do
       let!(:flowers1)     { BasicArtwork.create(:title => 'Flowers 1') }
       let!(:flowers2)     { BasicArtwork.create(:title => 'Flowers 1') }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid_fulltext
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-05 00:00:00.000000000 -04:00
+date: 2011-10-11 00:00:00.000000000 -04:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode_utils
-  requirement: &83773400 !ruby/object:Gem::Requirement
+  requirement: &79126690 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +22,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *83773400
+  version_requirements: *79126690
 - !ruby/object:Gem::Dependency
   name: mongoid
-  requirement: &83773150 !ruby/object:Gem::Requirement
+  requirement: &79116990 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -33,10 +33,10 @@ dependencies:
         version: 2.0.0
   type: :development
   prerelease: false
-  version_requirements: *83773150
+  version_requirements: *79116990
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &83772860 !ruby/object:Gem::Requirement
+  requirement: &79116510 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -44,10 +44,10 @@ dependencies:
         version: 1.3.0
   type: :development
   prerelease: false
-  version_requirements: *83772860
+  version_requirements: *79116510
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &83772590 !ruby/object:Gem::Requirement
+  requirement: &79116050 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -55,10 +55,10 @@ dependencies:
         version: 2.5.0
   type: :development
   prerelease: false
-  version_requirements: *83772590
+  version_requirements: *79116050
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &83772310 !ruby/object:Gem::Requirement
+  requirement: &79115620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -66,7 +66,7 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *83772310
+  version_requirements: *79115620
 description: Full-text search for the Mongoid ORM, using n-grams extracted from text
 email: aaron.windsor@gmail.com
 executables: []
@@ -118,7 +118,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 700135439
+      hash: -203766451
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: