mongoid_fulltext 0.5.2 → 0.5.3

data/README.md

@@ -214,7 +214,9 @@ Additional indexing/query options can be used as parameters to `fulltext_search_
 * `alphabet`: letters to index, default is `abcdefghijklmnopqrstuvwxyz0123456789 `
 * `word_separators`: word separators, default is the space character.
 * `ngram_width`: ngram width, default is `3`
-* `index_full_words`: index full words, which improves exact matches, default is `true` 
+* `index_full_words`: index full words, which improves exact matches, default is `true`.
+* `index_short_prefixes`: index a prefix of each full word of length `(ngram_width-1)`. Useful if
+  you use a larger ngram_width than the default of 3. Default for this option is `false`.
 * `stop_words`: a hash of words to avoid indexing as full words. Used only if `index_full_words`
   is set to `true`. Defaults to a hash containing a list of common English stop words.
 * `apply_prefix_scoring_to_all_words`: score n-grams at beginning of words higher, default is `true`

data/VERSION

@@ -1 +1 @@
-0.5.2
+0.5.3

data/lib/mongoid_fulltext.rb

@@ -29,6 +29,7 @@ module Mongoid::FullTextSearch
         :max_ngrams_to_search => 6,
         :apply_prefix_scoring_to_all_words => true,
         :index_full_words => true,
+        :index_short_prefixes => false,
         :max_candidate_set_size => 1000,
         :remove_accents => true,
         :stop_words => Hash[['i', 'a', 's', 't', 'me', 'my', 'we', 'he', 'it', 'am', 'is', 'be', 'do', 'an', 'if', 
@@ -218,19 +219,32 @@ module Mongoid::FullTextSearch
 
       # If an ngram appears multiple times in the query string, keep the max score
       ngram_array = ngram_array.group_by{ |h| h[:ngram] }.map{ |key, values| {:ngram => key, :score => values.map{ |v| v[:score] }.max} }
-      
+
+      # Add 'short prefix' records to the array: prefixes of the string that are length (ngram_width - 1)
+      if config[:index_short_prefixes]
+        prefixes_seen = {}
+        filtered_str.split(Regexp.compile(config[:word_separators].keys.join)).each do |word|
+          next if word.length < config[:ngram_width]-1
+          prefix = word[0...config[:ngram_width]-1]
+          if prefixes_seen[prefix].nil? and (config[:stop_words][word].nil? or word == filtered_str)
+            ngram_array << {:ngram => prefix, :score => 1}
+            prefixes_seen[prefix] = true
+          end
+        end
+      end
+
       # Add records to the array of ngrams for each full word in the string that isn't a stop word
-      if (config[:index_full_words])
+      if config[:index_full_words]
         full_words_seen = {}
         filtered_str.split(Regexp.compile(config[:word_separators].keys.join)).each do |word|
-          if word.length > 1 and full_words_seen[word].nil? and config[:stop_words][word].nil?
+          if word.length > 1 and full_words_seen[word].nil? and (config[:stop_words][word].nil? or word == filtered_str)
             ngram_array << {:ngram => word, :score => 1}
             full_words_seen[word] = true
           end
         end
       end
 
-      # If an ngram appears as a full word and an ngram, keep the sum of the two scores
+      # If an ngram appears as any combination of full word, short prefix, and ngram, keep the sum of the two scores
       Hash[ngram_array.group_by{ |h| h[:ngram] }.map{ |key, values| [key, values.map{ |v| v[:score] }.sum] }]
     end
     

data/mongoid_fulltext.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{mongoid_fulltext}
-  s.version = "0.5.2"
+  s.version = "0.5.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aaron Windsor"]
-  s.date = %q{2011-11-05}
+  s.date = %q{2011-11-08}
   s.description = %q{Full-text search for the Mongoid ORM, using n-grams extracted from text}
   s.email = %q{aaron.windsor@gmail.com}
   s.extra_rdoc_files = [
@@ -42,6 +42,7 @@ Gem::Specification.new do |s|
     "spec/models/multi_field_artist.rb",
     "spec/models/multi_field_artwork.rb",
     "spec/models/partitioned_artist.rb",
+    "spec/models/short_prefixes_artwork.rb",
     "spec/models/stopwords_artwork.rb",
     "spec/mongoid/fulltext_spec.rb",
     "spec/spec_helper.rb"
@@ -67,6 +68,7 @@ Gem::Specification.new do |s|
     "spec/models/multi_field_artist.rb",
     "spec/models/multi_field_artwork.rb",
     "spec/models/partitioned_artist.rb",
+    "spec/models/short_prefixes_artwork.rb",
     "spec/models/stopwords_artwork.rb",
     "spec/mongoid/fulltext_spec.rb",
     "spec/spec_helper.rb"

data/spec/models/short_prefixes_artwork.rb

@@ -0,0 +1,11 @@
+class ShortPrefixesArtwork
+  include Mongoid::Document
+  include Mongoid::FullTextSearch
+
+  field :title
+  fulltext_search_in :title,
+  :ngram_width => 4,
+  :index_short_prefixes => true,
+  :index_full_words => false
+
+end

data/spec/mongoid/fulltext_spec.rb

@@ -467,6 +467,33 @@ module Mongoid
 
     end
 
+    context "indexing short prefixes" do
+      let!(:dimethyl_mercury)   { ShortPrefixesArtwork.create(:title => "Dimethyl Mercury by Damien Hirst") }
+      let!(:volume)             { ShortPrefixesArtwork.create(:title => "Volume by Dadamaino") }
+      let!(:damaged)            { ShortPrefixesArtwork.create(:title => "Damaged: Photographs from the Chicago Daily News 1902-1933 (Governor) by Lisa Oppenheim") }
+      let!(:frozen)             { ShortPrefixesArtwork.create(:title => "Frozen Fountain XXX by Evelyn Rosenberg") }
+      let!(:skull)              { ShortPrefixesArtwork.create(:title => "Skull by Andy Warhol") }
+
+      it "finds the most relevant items with prefix indexing" do
+        ShortPrefixesArtwork.fulltext_search("damien").first.should == dimethyl_mercury
+        ShortPrefixesArtwork.fulltext_search("dami").first.should == dimethyl_mercury
+        ShortPrefixesArtwork.fulltext_search("dama").first.should == damaged
+        ShortPrefixesArtwork.fulltext_search("dam").first.should_not == volume
+        ShortPrefixesArtwork.fulltext_search("dadamaino").first.should == volume
+        ShortPrefixesArtwork.fulltext_search("kull").first.should == skull
+      end
+
+      it "doesn't index prefixes of stopwords" do
+        # damaged has the word "from" in it, which shouldn't get indexed.
+        ShortPrefixesArtwork.fulltext_search("fro").should == [frozen]
+      end
+
+      it "does index prefixes that would be stopwords taken alone" do
+        # skull has the word "andy" in it, which should get indexed as "and" even though "and" is a stopword
+        ShortPrefixesArtwork.fulltext_search("and").should == [skull]
+      end
+    end
+
     context "remove_from_ngram_index" do
       let!(:flowers1)     { BasicArtwork.create(:title => 'Flowers 1') }
       let!(:flowers2)     { BasicArtwork.create(:title => 'Flowers 1') }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid_fulltext
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-05 00:00:00.000000000 -04:00
+date: 2011-11-08 00:00:00.000000000 -05:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode_utils
-  requirement: &86209940 !ruby/object:Gem::Requirement
+  requirement: &87410860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +22,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *86209940
+  version_requirements: *87410860
 - !ruby/object:Gem::Dependency
   name: mongoid
-  requirement: &86209400 !ruby/object:Gem::Requirement
+  requirement: &87410620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -33,10 +33,10 @@ dependencies:
         version: 2.0.0
   type: :development
   prerelease: false
-  version_requirements: *86209400
+  version_requirements: *87410620
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &86209080 !ruby/object:Gem::Requirement
+  requirement: &87410380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -44,10 +44,10 @@ dependencies:
         version: 1.3.0
   type: :development
   prerelease: false
-  version_requirements: *86209080
+  version_requirements: *87410380
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &86208570 !ruby/object:Gem::Requirement
+  requirement: &87410140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -55,10 +55,10 @@ dependencies:
         version: 2.5.0
   type: :development
   prerelease: false
-  version_requirements: *86208570
+  version_requirements: *87410140
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &86208060 !ruby/object:Gem::Requirement
+  requirement: &87409900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -66,7 +66,7 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *86208060
+  version_requirements: *87409900
 description: Full-text search for the Mongoid ORM, using n-grams extracted from text
 email: aaron.windsor@gmail.com
 executables: []
@@ -100,6 +100,7 @@ files:
 - spec/models/multi_field_artist.rb
 - spec/models/multi_field_artwork.rb
 - spec/models/partitioned_artist.rb
+- spec/models/short_prefixes_artwork.rb
 - spec/models/stopwords_artwork.rb
 - spec/mongoid/fulltext_spec.rb
 - spec/spec_helper.rb
@@ -119,7 +120,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 444903309
+      hash: 591721469
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -148,6 +149,7 @@ test_files:
 - spec/models/multi_field_artist.rb
 - spec/models/multi_field_artwork.rb
 - spec/models/partitioned_artist.rb
+- spec/models/short_prefixes_artwork.rb
 - spec/models/stopwords_artwork.rb
 - spec/mongoid/fulltext_spec.rb
 - spec/spec_helper.rb