mongoid_fulltext 0.5.7 → 0.5.8

data/Gemfile

@@ -6,7 +6,7 @@ gem "unicode_utils", "~> 1.0.0"
 # Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "mongoid", "~> 2.0.0"
-  gem "bson_ext", "~> 1.3.0"
+  gem "bson_ext", "~> 1.5.2"
   gem "rspec", "~> 2.5.0" 
   gem "jeweler", "~> 1.5.2"
 end

data/README.md

@@ -10,6 +10,11 @@ lets you do a fuzzy string search across relatively short strings, which makes i
 autocomplete boxes based on the display names of your Rails models but not appropriate for, say, 
 indexing hundreds of thousands of HTML documents.
 
+Install
+--------------
+
+    gem 'mongoid_fulltext'
+
 Some examples:
 --------------
     
@@ -217,26 +222,23 @@ Indexing Options
 
 Additional indexing/query options can be used as parameters to `fulltext_search_in`.
 
-* `alphabet`: letters to index, default is `abcdefghijklmnopqrstuvwxyz0123456789 `
+* `alphabet`: letters to index, default is `abcdefghijklmnopqrstuvwxyz0123456789 `.
 * `word_separators`: word separators, default is the space character.
-* `ngram_width`: ngram width, default is `3`
+* `ngram_width`: ngram width, default is `3`.
 * `index_full_words`: index full words, which improves exact matches, default is `true`.
 * `index_short_prefixes`: index a prefix of each full word of length `(ngram_width-1)`. Useful if
   you use a larger ngram_width than the default of 3. Default for this option is `false`.
 * `stop_words`: a hash of words to avoid indexing as full words. Used only if `index_full_words`
   is set to `true`. Defaults to a hash containing a list of common English stop words.
-* `apply_prefix_scoring_to_all_words`: score n-grams at beginning of words higher, default is `true`
-* `max_ngrams_to_search`: maximum number of ngrams to query at any given time, default is `6`
+* `apply_prefix_scoring_to_all_words`: score n-grams at beginning of words higher, default is `true`.
+* `max_ngrams_to_search`: maximum number of ngrams to query at any given time, default is `6`.
 * `max_candidate_set_size`: maximum number of candidate ngrams to examine for a given query.
   Defaults to 1000. If you're seeing poor results, you can try increasing this value to consider
   more ngrams per query (changing this parameter does not require a re-index.) The amount of time
   a search takes is directly proportional to this parameter's value.
-* `remove_accents`: remove accents on accented characters or not. Defaults to true.  If a string 
-  is encoded in UTF-8, we strip the accents using NFKD normalization (via an external library, 
-  `unicode_utils`. If a string is encoded in ASCII-8BIT, we assume it has been passed via a
-  URL, for instance we might have "%C3%A9" which is how an "e-accute" ("é") gets passed
-  through a web-browser. These are then changed to their UTF-8 equivalents (via the `CGI` gem) 
-  and then finally stripped, as before.
+* `remove_accents`: remove accents on accented characters, default is `true`. 
+  We strip the accents using [NFKD normalization](http://unicode-utils.rubyforge.org/UnicodeUtils.html#method-c-compatibility_decomposition)
+  using an external library, `unicode_utils`.
 * `update_if`: controls whether or not the index will be updated. This can be set to a symbol,
   string, or proc. If the result of evaluating the value is true, the index will be updated.
     * When set to a symbol, the symbol is sent to the document.
@@ -275,6 +277,20 @@ at both the Art Basel and the New York Armory exhibition.
     # Artists at both the Art Basel and the New York Armory exhibition
     Artist.fulltext_search('foobar', :exhibitions => [ "Art Basel", "New York Armory" ])
 
+    # Note that the following explicit syntax may be used to achieve the
+    # same result as above
+    Artist.fulltext_search('foobar', :exhibitions => {:all => [ "Art Basel", "New York Armory" ]})
+
+If you want to find all artists that are at either the Art Basel or the
+New York Armory exhibition, then you may specify the `:any` operator in
+the filter.
+
+    # Artists at either the Art Basel or the New York Armory exhibition
+    Artist.fulltext_search('foobar', :exhibitions => {:any => [ "Art Basel", "New York Armory" ]})
+
+Note that `:all` and `:any` are currently the only supported operators
+for the array filters.
+
 Building the index
 ------------------
 

data/VERSION

@@ -1 +1 @@
-0.5.7
+0.5.8

data/lib/mongoid_fulltext.rb

@@ -10,6 +10,7 @@ module Mongoid::FullTextSearch
   end
 
   class UnspecifiedIndexError < StandardError; end
+  class UnknownFilterQueryOperator < StandardError; end
 
   module ClassMethods
   
@@ -86,7 +87,7 @@ module Mongoid::FullTextSearch
         next if !keys.member?('ngram')
         all_filter_keys |= keys.find_all{ |key| key.starts_with?('filter_values.') }
         if keys & correct_keys != correct_keys
-          Mongoid.logger.info "Droping #{name} [#{keys & correct_keys} <=> #{correct_keys}]"
+          Mongoid.logger.info "Dropping #{name} [#{keys & correct_keys} <=> #{correct_keys}]" if Mongoid.logger
           coll.drop_index(name)
         end
       end
@@ -96,9 +97,9 @@ module Mongoid::FullTextSearch
         index_definition = [['ngram', Mongo::ASCENDING], ['score', Mongo::DESCENDING]].concat(filter_indexes)        
       end
       
-      Mongoid.logger.info "Ensuring fts_index on #{coll.name}: #{index_definition}"
+      Mongoid.logger.info "Ensuring fts_index on #{coll.name}: #{index_definition}" if Mongoid.logger
       coll.ensure_index(index_definition, { :name => 'fts_index' })
-      Mongoid.logger.info "Ensuring document_id index on #{coll.name}"
+      Mongoid.logger.info "Ensuring document_id index on #{coll.name}" if Mongoid.logger
       coll.ensure_index([['document_id', Mongo::ASCENDING]]) # to make removes fast
     end
     
@@ -123,7 +124,7 @@ module Mongoid::FullTextSearch
       coll = collection.db.collection(index_name)
       cursors = ngrams.map do |ngram| 
         query = {'ngram' => ngram[0]}
-        query.update(Hash[options.map { |key,value| [ 'filter_values.%s' % key, { '$all' => [ value ].flatten } ] }])
+        query.update(map_query_filters options)
         count = coll.find(query).count
         {:ngram => ngram, :count => count, :query => query}
       end.sort!{ |record1, record2| record1[:count] <=> record2[:count] }
@@ -187,7 +188,7 @@ module Mongoid::FullTextSearch
       return {} if str.nil?
 
       if config[:remove_accents]
-        str = UnicodeUtils.nfkd(CGI.unescape(str)).gsub(/[^\x00-\x7F]/,'')
+        str = UnicodeUtils.nfkd(str).gsub(/[^\x00-\x7F]/,'')
       end
 
       # Remove any characters that aren't in the alphabet and aren't word separators
@@ -268,6 +269,23 @@ module Mongoid::FullTextSearch
       end
     end
     
+    private
+    # Take a list of filters to be mapped so they can update the query
+    # used upon the fulltext search of the ngrams
+    def map_query_filters filters
+      Hash[filters.map {|key,value|
+        case value
+          when Hash then
+            if value.has_key? :any then format_query_filter('$in',key,value[:any])
+            elsif value.has_key? :all then format_query_filter('$all',key,value[:all])
+            else raise UnknownFilterQueryOperator, value.keys.join(","), caller end
+          else format_query_filter('$all',key,value)
+        end
+      }]
+    end
+    def format_query_filter operator, key, value
+      ['filter_values.%s' % key, {operator => [value].flatten}]
+    end
   end
 
   def update_ngram_index

data/mongoid_fulltext.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "mongoid_fulltext"
-  s.version = "0.5.7"
+  s.version = "0.5.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aaron Windsor"]
-  s.date = "2012-01-12"
+  s.date = "2012-03-08"
   s.description = "Full-text search for the Mongoid ORM, using n-grams extracted from text"
   s.email = "aaron.windsor@gmail.com"
   s.extra_rdoc_files = [
@@ -82,20 +82,20 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<unicode_utils>, ["~> 1.0.0"])
       s.add_development_dependency(%q<mongoid>, ["~> 2.0.0"])
-      s.add_development_dependency(%q<bson_ext>, ["~> 1.3.0"])
+      s.add_development_dependency(%q<bson_ext>, ["~> 1.5.2"])
       s.add_development_dependency(%q<rspec>, ["~> 2.5.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
     else
       s.add_dependency(%q<unicode_utils>, ["~> 1.0.0"])
       s.add_dependency(%q<mongoid>, ["~> 2.0.0"])
-      s.add_dependency(%q<bson_ext>, ["~> 1.3.0"])
+      s.add_dependency(%q<bson_ext>, ["~> 1.5.2"])
       s.add_dependency(%q<rspec>, ["~> 2.5.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
     end
   else
     s.add_dependency(%q<unicode_utils>, ["~> 1.0.0"])
     s.add_dependency(%q<mongoid>, ["~> 2.0.0"])
-    s.add_dependency(%q<bson_ext>, ["~> 1.3.0"])
+    s.add_dependency(%q<bson_ext>, ["~> 1.5.2"])
     s.add_dependency(%q<rspec>, ["~> 2.5.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
   end

data/spec/models/filtered_artwork.rb

@@ -1,10 +1,12 @@
 class FilteredArtwork
   include Mongoid::Document
   include Mongoid::FullTextSearch
-  field :title
+  field :title, type: String
+  field :colors, type: Array, default: []
   fulltext_search_in :title, :index_name => 'mongoid_fulltext.artworks_and_artists',
                      :filters => { :is_foobar => lambda { |x| x.title == 'foobar' },
                                    :is_artwork => lambda { |x| true },
-                                   :is_artist => lambda { |x| false }
+                                   :is_artist => lambda { |x| false },
+                                   :colors? => lambda { |x| x.colors }
                                   }
 end

data/spec/mongoid/fulltext_spec.rb

@@ -29,16 +29,28 @@ module Mongoid
       let!(:lowered)     { BasicArtwork.create(:title => 'Lowered') }
       let!(:cookies)     { BasicArtwork.create(:title => 'Cookies') }
       let!(:empty)       { BasicArtwork.create(:title => '') }
-      let!(:cesar) { BasicArtwork.create(:title => "C\u00e9sar Galicia") }
-      let!(:julio) { BasicArtwork.create(:title => "Julio Cesar Morales") }
-      let!(:csar) { BasicArtwork.create(:title => "Csar") }
-
+      let!(:cesar)       { BasicArtwork.create(:title => "C\u00e9sar Galicia") }
+      let!(:julio)       { BasicArtwork.create(:title => "Julio Cesar Morales") }
+      let!(:csar)        { BasicArtwork.create(:title => "Csar") }
+      let!(:percent)     { BasicArtwork.create(:title => "Untitled (cal%desert)") }
+      
+      it "returns empty for empties" do
+        BasicArtwork.fulltext_search(nil, :max_results => 1).should == []
+        BasicArtwork.fulltext_search("", :max_results => 1).should == []
+      end
+      
+      it "finds percents" do
+        BasicArtwork.fulltext_search("cal%desert".force_encoding("ASCII-8BIT"), :max_results => 1).first.should == percent
+        BasicArtwork.fulltext_search("cal%desert".force_encoding("UTF-8"), :max_results => 1).first.should == percent
+      end
+      
       it "forgets accents" do
         BasicArtwork.fulltext_search('cesar', :max_results => 1).first.should == cesar
         BasicArtwork.fulltext_search('cesar g', :max_results => 1).first.should == cesar
         BasicArtwork.fulltext_search("C\u00e9sar", :max_results => 1).first.should == cesar
-        BasicArtwork.fulltext_search("C\303\251sar", :max_results => 1).first.should == cesar
-        BasicArtwork.fulltext_search("c%C3%A9sar".encode("ASCII-8BIT"), :max_results => 1).first.should == cesar
+        BasicArtwork.fulltext_search("C\303\251sar".force_encoding("UTF-8"), :max_results => 1).first.should == cesar
+        BasicArtwork.fulltext_search(CGI.unescape("c%C3%A9sar"), :max_results => 1).first.should == cesar
+        BasicArtwork.fulltext_search(CGI.unescape("c%C3%A9sar".encode("ASCII-8BIT")), :max_results => 1).first.should == cesar
       end
 
       it "returns exact matches" do
@@ -370,9 +382,10 @@ module Mongoid
         ngram_indexes.length.should == 1
         keys = ngram_indexes.first[1]['key'].keys
         expected_keys = ['ngram','score', 'filter_values.is_fuzzy', 'filter_values.is_awesome', 
-                         'filter_values.is_foobar', 'filter_values.is_artwork', 'filter_values.is_artist'].sort
+                         'filter_values.is_foobar', 'filter_values.is_artwork', 'filter_values.is_artist', 'filter_values.colors?'].sort
         keys.sort.should == expected_keys
       end
+
     end
     
     context "with partitions applied to a model" do
@@ -639,32 +652,18 @@ module Mongoid
           # but mongo will automatically attempt to index _id in the background
           Mongoid.master["mongoid_fulltext.index_basicartwork_0"].index_information.size.should <= 1
           BasicArtwork.create_indexes
-          Mongoid.master["mongoid_fulltext.index_basicartwork_0"].index_information.should == 
-          { 
-            "_id_" => { 
-              "name" => "_id_", 
-              "ns" => "mongoid_fulltext_test.mongoid_fulltext.index_basicartwork_0", 
-              "key" => { "_id" => 1 }, 
-              "v" => 0
-            }, 
-            "fts_index" => { 
-              "name" => "fts_index", 
-              "ns" => "mongoid_fulltext_test.mongoid_fulltext.index_basicartwork_0", 
-              "key" => { "ngram" => 1, "score" => -1 }, 
-              "v" => 0
-            }, 
-            "document_id_1" => {
-              "name" => "document_id_1", 
-              "ns" => "mongoid_fulltext_test.mongoid_fulltext.index_basicartwork_0", 
-              "key" => { "document_id" => 1 }, 
-              "v"=>0 
-            }
-          }
+          expected_indexes = ['_id_', 'fts_index', 'document_id_1'].sort
+          Mongoid.master["mongoid_fulltext.index_basicartwork_0"].index_information.keys.sort.should == expected_indexes
         end
         
         it "doesn't fail on models that don't have a fulltext index" do
           lambda { HiddenDragon.create_indexes }.should_not raise_error
         end
+
+        it "doesn't blow up when the Mongoid.logger is set to false" do
+          Mongoid.logger = false
+          BasicArtwork.create_indexes
+        end
         
       end
 
@@ -680,5 +679,100 @@ module Mongoid
       end
     end
 
+    # For =~ operator documentation
+    # https://github.com/dchelimsky/rspec/blob/master/lib/spec/matchers/match_array.rb#L53
+    
+    context "with artwork that returns an array of colors as a filter" do
+      let!(:title) {"title"}
+      let!(:nomatch) {"nomatch"}
+      let!(:red) {"red"}
+      let!(:green) {"green"}
+      let!(:blue) {"blue"}
+      let!(:yellow) {"yellow"}
+      let!(:brown) {"brown"}
+
+      let!(:rgb_artwork) {FilteredArtwork.create(:title => "#{title} rgb", :colors => [red,green,blue])}
+      let!(:holiday_artwork) {FilteredArtwork.create(:title => "#{title} holiday", :colors => [red,green])}
+      let!(:aqua_artwork) {FilteredArtwork.create(:title => "#{title} aqua", :colors => [green,blue])}
+
+      context "with a fulltext search passing red, green, and blue to the colors filter" do
+        it "should return the rgb artwork" do
+          FilteredArtwork.fulltext_search(title, :colors? => [red,green,blue]).should == [rgb_artwork]
+        end
+      end
+
+      context "with a fulltext search passing blue and red to the colors filter" do
+        it "should return the rgb artwork" do
+          FilteredArtwork.fulltext_search(title, :colors? => [blue,red]).should == [rgb_artwork]
+        end
+      end
+
+      context "with a fulltext search passing green to the colors filter" do
+        it "should return all artwork" do
+          FilteredArtwork.fulltext_search(title, :colors? => [green]).should =~ [rgb_artwork,holiday_artwork,aqua_artwork]
+        end
+      end
+
+      context "with a fulltext search passing no colors to the filter" do
+        it "should return all artwork" do
+          FilteredArtwork.fulltext_search(title).should =~ [rgb_artwork,holiday_artwork,aqua_artwork]
+        end
+      end
+
+      context "with a fulltext search passing green and yellow to the colors filter" do
+        it "should return no artwork" do
+          FilteredArtwork.fulltext_search(title, :colors? => [green,yellow]).should == []
+        end
+      end
+
+      context "with the query operator overridden to use $in instead of the default $all" do
+        context "with a fulltext search passing green and yellow to the colors filter" do
+          it "should return all of the artwork" do
+            FilteredArtwork.fulltext_search(title, :colors? => {:any => [green,yellow]}).should =~ [rgb_artwork,holiday_artwork,aqua_artwork]
+          end
+        end
+
+        context "with a fulltext search passing brown and yellow to the colors filter" do
+          it "should return none of the artwork" do
+            FilteredArtwork.fulltext_search(title, :colors? => {:any => [brown,yellow]}).should == []
+          end
+        end
+
+        context "with a fulltext search passing blue to the colors filter" do
+          it "should return the rgb and aqua artwork" do
+            FilteredArtwork.fulltext_search(title, :colors? => {:any => [blue]}).should == [rgb_artwork,aqua_artwork]
+          end
+        end
+
+        context "with a fulltext search term that won't match" do
+          it "should return none of the artwork" do
+            FilteredArtwork.fulltext_search(nomatch, :colors? => {:any => [green,yellow]}).should == []
+          end
+        end
+      end
+
+      context "with the query operator overridden to use $all" do
+        context "with a fulltext search passing red, green, and blue to the colors filter" do
+          it "should return the rgb artwork" do
+            FilteredArtwork.fulltext_search(title, :colors? => {:all => [red,green,blue]}).should == [rgb_artwork]
+          end
+        end
+
+        context "with a fulltext search passing green to the colors filter" do
+          it "should return all artwork" do
+            FilteredArtwork.fulltext_search(title, :colors? => {:all => [green]}).should =~ [rgb_artwork,holiday_artwork,aqua_artwork]
+          end
+        end
+      end
+
+      context "with an unknown query operator used to override the default $all" do
+        context "with a fulltext search passing red, green, and blue to the colors filter" do
+          it "should raise an error" do
+            -> {FilteredArtwork.fulltext_search(title, :colors? => {:unknown => [red,green,blue]})}.should raise_error(Mongoid::FullTextSearch::UnknownFilterQueryOperator)
+          end
+        end
+      end
+    end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid_fulltext
 version: !ruby/object:Gem::Version
-  version: 0.5.7
+  version: 0.5.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-12 00:00:00.000000000Z
+date: 2012-03-08 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode_utils
-  requirement: &2154109500 !ruby/object:Gem::Requirement
+  requirement: &2165513880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *2154109500
+  version_requirements: *2165513880
 - !ruby/object:Gem::Dependency
   name: mongoid
-  requirement: &2154107440 !ruby/object:Gem::Requirement
+  requirement: &2165513400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,21 +32,21 @@ dependencies:
         version: 2.0.0
   type: :development
   prerelease: false
-  version_requirements: *2154107440
+  version_requirements: *2165513400
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &2154105600 !ruby/object:Gem::Requirement
+  requirement: &2165512920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *2154105600
+  version_requirements: *2165512920
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2154103900 !ruby/object:Gem::Requirement
+  requirement: &2165512440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 2.5.0
   type: :development
   prerelease: false
-  version_requirements: *2154103900
+  version_requirements: *2165512440
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2154099020 !ruby/object:Gem::Requirement
+  requirement: &2165511960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *2154099020
+  version_requirements: *2165511960
 description: Full-text search for the Mongoid ORM, using n-grams extracted from text
 email: aaron.windsor@gmail.com
 executables: []
@@ -119,7 +119,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2225884234054153231
+      hash: -3796604883696425002
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: