mongoid_fulltext 0.4.3 → 0.4.4

data/Gemfile

@@ -1,12 +1,12 @@
 source "http://rubygems.org"
-# Add dependencies required to use your gem here.
-# Example:
-#   gem "activesupport", ">= 2.3.5"
+
+gem "unicode_utils", "~> 1.0.0"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "mongoid", "~> 2.0.0"
-  gem 'rspec', '~> 2.5.0' 
+  gem "bson_ext", "~> 1.3.0"
+  gem "rspec", "~> 2.5.0" 
   gem "jeweler", "~> 1.5.2"
 end

data/README.md

@@ -211,6 +211,12 @@ Additional indexing/query options can be used as parameters to `fulltext_search_
   Defaults to 1000. If you're seeing poor results, you can try increasing this value to consider
   more ngrams per query (changing this parameter does not require a re-index.) The amount of time
   a search takes is directly proportional to this parameter's value.
+* `remove_accents`: remove accents on accented characters or not. Defaults to true.  If a string 
+  is encoded in UTF-8, we strip the accents using NFKD normalization (via an external library, 
+  `unicode_utils`. If a string is encoded in ASCII-8BIT, we assume it has been passed via a
+  URL, for instance we might have "%C3%A9" which is how an "e-accute" ("é") gets passed
+  through a web-browser. These are then changed to their UTF-8 equivalents (via the `CGI` gem) 
+  and then finally stripped, as before.
 
 Array filters
 -------------

data/VERSION

@@ -1 +1 @@
-0.4.3
+0.4.4

data/lib/mongoid_fulltext.rb

@@ -1,4 +1,5 @@
 require 'mongoid_indexes'
+require 'unicode_utils'
 
 module Mongoid::FullTextSearch
   extend ActiveSupport::Concern
@@ -27,7 +28,8 @@ module Mongoid::FullTextSearch
         :max_ngrams_to_search => 6,
         :apply_prefix_scoring_to_all_words => true,
         :index_full_words => true,
-        :max_candidate_set_size => 1000
+        :max_candidate_set_size => 1000,
+        :remove_accents => true
       }
       
       config.update(options)
@@ -43,6 +45,7 @@ module Mongoid::FullTextSearch
     end
     
     def create_fulltext_indexes
+      return unless self.mongoid_fulltext_config
       self.mongoid_fulltext_config.each_pair do |index_name, fulltext_config|
         fulltext_search_ensure_indexes(index_name, fulltext_config)
       end
@@ -169,7 +172,16 @@ module Mongoid::FullTextSearch
     # returns an [ngram, score] [ngram, position] pair
     def all_ngrams(str, config, bound_number_returned = true)
       return {} if str.nil? or str.length < config[:ngram_width]
-      filtered_str = str.downcase.split('').map{ |ch| config[:alphabet][ch] }.compact.join('')
+
+      filtered_str = String.new(str)
+      if config[:remove_accents]
+        if str.encoding.name == "ASCII-8BIT"
+          filtered_str = CGI.unescape(filtered_str)
+        end
+        filtered_str = UnicodeUtils.nfkd(filtered_str).gsub(/[^\x00-\x7F]/,'')
+      end
+
+      filtered_str = filtered_str.downcase.split('').map{ |ch| config[:alphabet][ch] }.compact.join('')
       
       if bound_number_returned
         step_size = [((filtered_str.length - config[:ngram_width]).to_f / config[:max_ngrams_to_search]).ceil, 1].max

data/mongoid_fulltext.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{mongoid_fulltext}
-  s.version = "0.4.3"
+  s.version = "0.4.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aaron Windsor"]
-  s.date = %q{2011-08-03}
+  s.date = %q{2011-08-31}
   s.description = %q{Full-text search for the Mongoid ORM, using n-grams extracted from text}
   s.email = %q{aaron.windsor@gmail.com}
   s.extra_rdoc_files = [
@@ -27,6 +27,7 @@ Gem::Specification.new do |s|
     "lib/mongoid_fulltext.rb",
     "lib/mongoid_indexes.rb",
     "mongoid_fulltext.gemspec",
+    "spec/models/accentless_artwork.rb",
     "spec/models/advanced_artwork.rb",
     "spec/models/basic_artwork.rb",
     "spec/models/external_artist.rb",
@@ -36,6 +37,7 @@ Gem::Specification.new do |s|
     "spec/models/filtered_artwork.rb",
     "spec/models/filtered_other.rb",
     "spec/models/gallery/basic_artwork.rb",
+    "spec/models/hidden_dragon.rb",
     "spec/models/multi_external_artwork.rb",
     "spec/models/multi_field_artist.rb",
     "spec/models/multi_field_artwork.rb",
@@ -49,6 +51,7 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.6.2}
   s.summary = %q{Full-text search for the Mongoid ORM}
   s.test_files = [
+    "spec/models/accentless_artwork.rb",
     "spec/models/advanced_artwork.rb",
     "spec/models/basic_artwork.rb",
     "spec/models/external_artist.rb",
@@ -58,6 +61,7 @@ Gem::Specification.new do |s|
     "spec/models/filtered_artwork.rb",
     "spec/models/filtered_other.rb",
     "spec/models/gallery/basic_artwork.rb",
+    "spec/models/hidden_dragon.rb",
     "spec/models/multi_external_artwork.rb",
     "spec/models/multi_field_artist.rb",
     "spec/models/multi_field_artwork.rb",
@@ -70,16 +74,22 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<unicode_utils>, ["~> 1.0.0"])
       s.add_development_dependency(%q<mongoid>, ["~> 2.0.0"])
+      s.add_development_dependency(%q<bson_ext>, ["~> 1.3.0"])
       s.add_development_dependency(%q<rspec>, ["~> 2.5.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
     else
+      s.add_dependency(%q<unicode_utils>, ["~> 1.0.0"])
       s.add_dependency(%q<mongoid>, ["~> 2.0.0"])
+      s.add_dependency(%q<bson_ext>, ["~> 1.3.0"])
       s.add_dependency(%q<rspec>, ["~> 2.5.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
     end
   else
+    s.add_dependency(%q<unicode_utils>, ["~> 1.0.0"])
     s.add_dependency(%q<mongoid>, ["~> 2.0.0"])
+    s.add_dependency(%q<bson_ext>, ["~> 1.3.0"])
     s.add_dependency(%q<rspec>, ["~> 2.5.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
   end

data/spec/models/accentless_artwork.rb

@@ -0,0 +1,7 @@
+class AccentlessArtwork
+  include Mongoid::Document
+  include Mongoid::FullTextSearch
+
+  field :title
+  fulltext_search_in :title, :remove_accents => false
+end

data/spec/models/hidden_dragon.rb

@@ -0,0 +1,6 @@
+# a Mongoid model that doesn't have a fulltext search index, but includes the Mongoid::FullTextSearch module
+class HiddenDragon
+  include Mongoid::Document
+  include Mongoid::FullTextSearch
+  
+end

data/spec/mongoid/fulltext_spec.rb

@@ -6,6 +6,8 @@ module Mongoid
     context "with several config options defined" do
 
       let!(:abcdef) { AdvancedArtwork.create(:title => 'abcdefg hijklmn') }
+      let!(:cesar) { AccentlessArtwork.create(:title => "C\u00e9sar Galicia") }
+      let!(:julio) { AccentlessArtwork.create(:title => "Julio Cesar Morales") } 
 
       it "should recognize all options" do
         # AdvancedArtwork is defined with an ngram_width of 4 and a different alphabet (abcdefg)
@@ -13,6 +15,9 @@ module Mongoid
         AdvancedArtwork.fulltext_search('abcd').first.should == abcdef
         AdvancedArtwork.fulltext_search('defg').first.should == abcdef
         AdvancedArtwork.fulltext_search('hijklmn').should == []
+        # AccentlessArtwork is just like BasicArtwork, except that we set :remove_accents to false,
+        # so this behaves like the ``old'' version of fulltext_search
+        AccentlessArtwork.fulltext_search("cesar").first.should == julio
       end
 
     end
@@ -24,6 +29,17 @@ module Mongoid
       let!(:lowered)     { BasicArtwork.create(:title => 'Lowered') }
       let!(:cookies)     { BasicArtwork.create(:title => 'Cookies') }
       let!(:empty)       { BasicArtwork.create(:title => '') }
+      let!(:cesar) { BasicArtwork.create(:title => "C\u00e9sar Galicia") }
+      let!(:julio) { BasicArtwork.create(:title => "Julio Cesar Morales") }
+      let!(:csar) { BasicArtwork.create(:title => "Csar") }
+
+      it "forgets accents" do
+        BasicArtwork.fulltext_search('cesar', :max_results => 1).first.should == cesar
+        BasicArtwork.fulltext_search('cesar g', :max_results => 1).first.should == cesar
+        BasicArtwork.fulltext_search("C\u00e9sar", :max_results => 1).first.should == cesar
+        BasicArtwork.fulltext_search("C\303\251sar", :max_results => 1).first.should == cesar
+        BasicArtwork.fulltext_search("c%C3%A9sar".encode("ASCII-8BIT"), :max_results => 1).first.should == cesar
+      end
 
       it "returns exact matches" do
         BasicArtwork.fulltext_search('Flower Myth', :max_results => 1).first.should == flower_myth
@@ -443,6 +459,7 @@ module Mongoid
       end
 
       context "mongoid indexes" do
+      
         it "can re-create dropped indexes" do
           # there're no indexes by default as Mongoid.autocreate_indexes is set to false
           # but mongo will automatically attempt to index _id in the background
@@ -471,6 +488,10 @@ module Mongoid
           }
         end
         
+        it "doesn't fail on models that don't have a fulltext index" do
+          lambda { HiddenDragon.create_indexes }.should_not raise_error
+        end
+        
       end
 
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid_fulltext
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,23 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-03 00:00:00.000000000 -04:00
+date: 2011-08-31 00:00:00.000000000 -04:00
 default_executable: 
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: unicode_utils
+  requirement: &82521100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *82521100
 - !ruby/object:Gem::Dependency
   name: mongoid
-  requirement: &82382470 !ruby/object:Gem::Requirement
+  requirement: &82510440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +33,21 @@ dependencies:
         version: 2.0.0
   type: :development
   prerelease: false
-  version_requirements: *82382470
+  version_requirements: *82510440
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: &82510030 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *82510030
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &82381540 !ruby/object:Gem::Requirement
+  requirement: &82509570 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -33,10 +55,10 @@ dependencies:
         version: 2.5.0
   type: :development
   prerelease: false
-  version_requirements: *82381540
+  version_requirements: *82509570
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &82380580 !ruby/object:Gem::Requirement
+  requirement: &82509170 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -44,7 +66,7 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *82380580
+  version_requirements: *82509170
 description: Full-text search for the Mongoid ORM, using n-grams extracted from text
 email: aaron.windsor@gmail.com
 executables: []
@@ -63,6 +85,7 @@ files:
 - lib/mongoid_fulltext.rb
 - lib/mongoid_indexes.rb
 - mongoid_fulltext.gemspec
+- spec/models/accentless_artwork.rb
 - spec/models/advanced_artwork.rb
 - spec/models/basic_artwork.rb
 - spec/models/external_artist.rb
@@ -72,6 +95,7 @@ files:
 - spec/models/filtered_artwork.rb
 - spec/models/filtered_other.rb
 - spec/models/gallery/basic_artwork.rb
+- spec/models/hidden_dragon.rb
 - spec/models/multi_external_artwork.rb
 - spec/models/multi_field_artist.rb
 - spec/models/multi_field_artwork.rb
@@ -94,7 +118,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 867015793
+      hash: 518227181
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -108,6 +132,7 @@ signing_key:
 specification_version: 3
 summary: Full-text search for the Mongoid ORM
 test_files:
+- spec/models/accentless_artwork.rb
 - spec/models/advanced_artwork.rb
 - spec/models/basic_artwork.rb
 - spec/models/external_artist.rb
@@ -117,6 +142,7 @@ test_files:
 - spec/models/filtered_artwork.rb
 - spec/models/filtered_other.rb
 - spec/models/gallery/basic_artwork.rb
+- spec/models/hidden_dragon.rb
 - spec/models/multi_external_artwork.rb
 - spec/models/multi_field_artist.rb
 - spec/models/multi_field_artwork.rb