mongoid_fulltext 0.5.0 → 0.5.1

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.5.1

data/lib/mongoid_fulltext.rb

@@ -29,7 +29,19 @@ module Mongoid::FullTextSearch
         :apply_prefix_scoring_to_all_words => true,
         :index_full_words => true,
         :max_candidate_set_size => 1000,
-        :remove_accents => true
+        :remove_accents => true,
+        :stop_words => Hash[['i', 'a', 's', 't', 'me', 'my', 'we', 'he', 'it', 'am', 'is', 'be', 'do', 'an', 'if', 
+                             'or', 'as', 'of', 'at', 'by', 'to', 'up', 'in', 'on', 'no', 'so', 'our', 'you', 'him', 
+                             'his', 'she', 'her', 'its', 'who', 'are', 'was', 'has', 'had', 'did', 'the', 'and', 
+                             'but', 'for', 'out', 'off', 'why', 'how', 'all', 'any', 'few', 'nor', 'not', 'own', 
+                             'too', 'can', 'don', 'now', 'ours', 'your', 'hers', 'they', 'them', 'what', 'whom', 
+                             'this', 'that', 'were', 'been', 'have', 'does', 'with', 'into', 'from', 'down', 'over', 
+                             'then', 'once', 'here', 'when', 'both', 'each', 'more', 'most', 'some', 'such', 'only', 
+                             'same', 'than', 'very', 'will', 'just', 'yours', 'their', 'which', 'these', 'those', 
+                             'being', 'doing', 'until', 'while', 'about', 'after', 'above', 'below', 'under', 
+                             'again', 'there', 'where', 'other', 'myself', 'itself', 'theirs', 'having', 'during', 
+                             'before', 'should', 'himself', 'herself', 'because', 'against', 'between', 'through', 
+                             'further', 'yourself', 'ourselves', 'yourselves', 'themselves'].map{ |x| [x,true] }]
       }
       
       config.update(options)
@@ -180,7 +192,7 @@ module Mongoid::FullTextSearch
         filtered_str = UnicodeUtils.nfkd(filtered_str).gsub(/[^\x00-\x7F]/,'')
       end
 
-      filtered_str = filtered_str.downcase.split('').map{ |ch| config[:alphabet][ch] }.compact.join('')
+      filtered_str = filtered_str.mb_chars.downcase.to_s.split('').map{ |ch| config[:alphabet][ch] }.compact.join('')
       
       if bound_number_returned
         step_size = [((filtered_str.length - config[:ngram_width]).to_f / config[:max_ngrams_to_search]).ceil, 1].max
@@ -203,9 +215,11 @@ module Mongoid::FullTextSearch
       ngram_ary = ngram_ary.group_by{ |h| h[:ngram] }.map{ |key, values| {:ngram => key, :score => values.map{ |v| v[:score] }.max} }
       
       if (config[:index_full_words])
+        full_words_seen = {}
         filtered_str.split(Regexp.compile(config[:word_separators].keys.join)).each do |word|
-          if word.length >= config[:ngram_width]
+          if word.length >= config[:ngram_width] and full_words_seen[word].nil? and config[:stop_words][word].nil?
             ngram_ary << {:ngram => word, :score => 1}
+            full_words_seen[word] = true
           end
         end
       end

data/mongoid_fulltext.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{mongoid_fulltext}
-  s.version = "0.5.0"
+  s.version = "0.5.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aaron Windsor"]
-  s.date = %q{2011-10-11}
+  s.date = %q{2011-11-02}
   s.description = %q{Full-text search for the Mongoid ORM, using n-grams extracted from text}
   s.email = %q{aaron.windsor@gmail.com}
   s.extra_rdoc_files = [
@@ -42,6 +42,7 @@ Gem::Specification.new do |s|
     "spec/models/multi_field_artist.rb",
     "spec/models/multi_field_artwork.rb",
     "spec/models/partitioned_artist.rb",
+    "spec/models/stopwords_artwork.rb",
     "spec/mongoid/fulltext_spec.rb",
     "spec/spec_helper.rb"
   ]
@@ -66,6 +67,7 @@ Gem::Specification.new do |s|
     "spec/models/multi_field_artist.rb",
     "spec/models/multi_field_artwork.rb",
     "spec/models/partitioned_artist.rb",
+    "spec/models/stopwords_artwork.rb",
     "spec/mongoid/fulltext_spec.rb",
     "spec/spec_helper.rb"
   ]

data/spec/models/stopwords_artwork.rb

@@ -0,0 +1,10 @@
+class StopwordsArtwork
+  include Mongoid::Document
+  include Mongoid::FullTextSearch
+
+  field :title
+  fulltext_search_in :title, 
+  :index_full_words => true,
+  :stop_words => { 'and' => true }
+
+end

data/spec/mongoid/fulltext_spec.rb

@@ -450,6 +450,16 @@ module Mongoid
       
     end
 
+    context "with stop words defined" do
+      let!(:flowers)      { StopwordsArtwork.create(:title => "Flowers by Andy Warhol") }
+      let!(:many_ands)    { StopwordsArtwork.create(:title => "Foo and bar and baz and foobar") }
+
+      it "doesn't give a full-word score boost to stopwords" do
+        StopwordsArtwork.fulltext_search("andy").map{ |a| a.title }.should == [flowers.title, many_ands.title]
+        StopwordsArtwork.fulltext_search("warhol and other stuff").map{ |a| a.title }.should == [flowers.title, many_ands.title]
+      end
+    end
+
     context "remove_from_ngram_index" do
       let!(:flowers1)     { BasicArtwork.create(:title => 'Flowers 1') }
       let!(:flowers2)     { BasicArtwork.create(:title => 'Flowers 1') }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid_fulltext
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-11 00:00:00.000000000 -04:00
+date: 2011-11-02 00:00:00.000000000 -04:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode_utils
-  requirement: &79126690 !ruby/object:Gem::Requirement
+  requirement: &87323870 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +22,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *79126690
+  version_requirements: *87323870
 - !ruby/object:Gem::Dependency
   name: mongoid
-  requirement: &79116990 !ruby/object:Gem::Requirement
+  requirement: &87323630 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -33,10 +33,10 @@ dependencies:
         version: 2.0.0
   type: :development
   prerelease: false
-  version_requirements: *79116990
+  version_requirements: *87323630
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &79116510 !ruby/object:Gem::Requirement
+  requirement: &87323390 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -44,10 +44,10 @@ dependencies:
         version: 1.3.0
   type: :development
   prerelease: false
-  version_requirements: *79116510
+  version_requirements: *87323390
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &79116050 !ruby/object:Gem::Requirement
+  requirement: &87323150 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -55,10 +55,10 @@ dependencies:
         version: 2.5.0
   type: :development
   prerelease: false
-  version_requirements: *79116050
+  version_requirements: *87323150
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &79115620 !ruby/object:Gem::Requirement
+  requirement: &87322910 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -66,7 +66,7 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *79115620
+  version_requirements: *87322910
 description: Full-text search for the Mongoid ORM, using n-grams extracted from text
 email: aaron.windsor@gmail.com
 executables: []
@@ -100,6 +100,7 @@ files:
 - spec/models/multi_field_artist.rb
 - spec/models/multi_field_artwork.rb
 - spec/models/partitioned_artist.rb
+- spec/models/stopwords_artwork.rb
 - spec/mongoid/fulltext_spec.rb
 - spec/spec_helper.rb
 has_rdoc: true
@@ -118,7 +119,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -203766451
+      hash: 136864689
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -147,5 +148,6 @@ test_files:
 - spec/models/multi_field_artist.rb
 - spec/models/multi_field_artwork.rb
 - spec/models/partitioned_artist.rb
+- spec/models/stopwords_artwork.rb
 - spec/mongoid/fulltext_spec.rb
 - spec/spec_helper.rb