mongoid-haystack 1.0.0 → 1.1.0

data/README.md

@@ -0,0 +1,249 @@
+NAME
+----
+
+  mongoid-haystack.rb
+
+DESCRIPTION
+-----------
+
+  mongoid-haystack provides a zero-config, POLS, pure mongo, fulltext search
+  solution for your mongoid models.
+
+SYNOPSIS
+--------
+
+````ruby
+
+  # simple usage is simple
+  #
+    class Article
+      include Mongoid::Document
+      include Mongoid::Haystack
+
+      field(:content, :type => String)
+    end
+
+    Article.create!(:content => 'teh cats')
+
+    results = Article.search('cat')
+
+    article = results.first.model
+
+
+  # haystack stems the search terms and does score based sorting all using a
+  # fast b-tree 
+  #
+    a = Article.create!(:content => 'cats are awesome')
+    b = Article.create!(:content => 'dogs eat cats')
+    c = Article.create!(:content => 'dogs dogs dogs')
+
+    results = Article.search('dogs cats').models
+    results == [b, a, c] #=> true
+
+    results = Article.search('awesome').models
+    results == [a] #=> true
+
+
+  # cross models searching is supported out of the box, and models can
+  # customise how they are indexed:
+  #
+  # - a global score lets some models appear hight in the global results
+  # - keywords count more than fulltext 
+  #
+    class Article
+      include Mongoid::Document
+      include Mongoid::Haystack
+
+      field(:title, :type => String)
+      field(:content, :type => String)
+
+      def to_haystack
+        { :score => 11, :keywords => title, :fulltext => content }
+      end
+    end
+
+    class Comment
+      include Mongoid::Document
+      include Mongoid::Haystack
+
+      field(:content, :type => String)
+
+      def to_haystack
+        { :score => -11, :fulltext => content }
+      end
+    end
+
+    a1 = Article.create!(:title => 'hot pants', :content => 'teh b 52s rock')
+    a2 = Article.create!(:title => 'boring title', :content => 'but hot content that rocks')
+
+    c = Comment.create!(:content => 'those guys rock')
+
+    results = Mongoid::Haystack.search('rock')
+    results.count #=> 3
+
+    models = results.models
+    models == [a1, a2, c]  #=> true. articles first beause we generally score them higher
+
+    results = Mongoid::Haystack.search('hot')
+    models = results.models
+    models == [a1, a2]  #=> true. because keywords score highter than general fulltext
+
+
+  # by default searching returns Mongoid::Haystack::Index objects. you'll want
+  # to expand these results to the models they reference in your views, but
+  # avoid doing an N+1 query.  to do this simply call #models on the result set
+  # and the models will be eager loaded using only as many queries as their are
+  # model types in your result set
+  #
+
+    @results = Mongoid::Haystack.search('needle').page(params[:page]).per(10)
+    @models = @results.models
+
+
+  # you can decorate your search items with arbirtrary meta data and filter
+  # searches by it later.  this too uses a b-tree index.
+  #
+    class Article
+      include Mongoid::Document
+      include Mongoid::Haystack
+
+      belongs_to :author, :class_name => '::User'
+
+      field(:title, :type => String)
+      field(:content, :type => String)
+
+      def to_haystack
+        { 
+          :score    => author.popularity,
+          :keywords => title,
+          :fulltext => content,
+          :facets   => {:author_id => author.id}
+        }
+      end
+    end
+
+    a = 
+      author.articles.create!(
+        :title => 'iggy and keith',
+        :content => 'seen the needles and the damage done...'
+      )
+
+    author_articles = Article.search('needle', :facets => {:author_id => author.id})
+
+
+````
+
+DESCRIPTION
+-----------
+
+there two main pathways to understand in the code.  shit going into the
+index, and shit coming out.
+
+shit going in entails:
+
+- stem and stopword the search terms.
+- create or update a new token for each
+- create an index item reference all the tokens with precomputed scores
+
+for example the terms 'dog dogs cat' might result in these tokens
+
+````javascript
+
+  [
+    {
+      '_id'   : '0x1',
+      'value' : 'dog',
+      'count' : 2
+    },
+
+
+    {
+      '_id'   : '0x2',
+      'value' : 'cat',
+      'count' : 1
+    }
+  ]
+
+````
+
+  and this index item
+
+
+````javascript
+
+    {
+      '_id'        : '50c11759a04745961e000001'
+
+      'model_type' : 'Article',
+      'model_id'   : '50c11775a04745461f000001'
+
+      'tokens'     : ['0x1', '0x2'],
+
+      'score'      : 10,
+
+      'keyword_scores' : {
+        '0x1' : 2,
+        '0x2' : 1
+      },
+
+      'fulltext_scores' : {
+      }
+    }
+
+
+````
+
+being built
+
+in addition, some other information is tracked such and the total number of
+search tokens every discovered in the corpus
+  
+
+
+a few things to notice:
+  
+- the tokens are counted and auto-id'd using hex notation and a sequence
+  generator.  the reason for this is so that their ids are legit hash keys
+  in the keyword and fulltext score hashes.
+
+- the data structure above allows both filtering for index items that have
+  certain tokens, but also ordering them based on global, keyword, and
+  fulltext score without resorting to map-reduce: a b-tree index can be
+  used.
+
+- all tokens have their text/stem stored exactly once.  aka: we do not store
+  'hugewords' all over the place but store it once and count occurances of
+  it to keep the total index much smaller
+
+
+
+
+pulling objects back out in a search involved these logical steps:
+
+- filter the search terms through the same tokenizer as when indexed
+
+- lookup tokens for each of the tokens in the search string
+
+- using the count for each token, plus the global token count that has been
+  tracked we can decide to order the results by relatively rare words first
+  and, all else being equal (same rarity bin: 0.10, 0.20, 0.30, etc.), the
+  order in which the user typed the words
+
+- this approach is applies and is valid whether we are doing a union (or) or
+  intersection (all) search and regardless of whether facets are included in
+  the search.  facets, however, never affect the order unless done so by the
+  user manually.  eg
+
+````ruby
+
+  results =
+    Mongoid::Haystack.
+      search('foo bar', :facets => {:hotness.gte => 11}).
+        order_by('facets.hotness' => :desc)
+
+````
+  
+
+SEE ALSO
+--------
+  tests: <a href='https://github.com/ahoward/mongoid-haystack/blob/master/test/mongoid-haystack_test.rb'>./test/mongoid-haystack_test.rb<a/>

data/lib/mongoid-haystack/index.rb

@@ -73,12 +73,13 @@ module Mongoid
 
             keyword_scores = Hash.new{|h,k| h[k] = 0}
             fulltext_scores = Hash.new{|h,k| h[k] = 0}
+            token_ids = []
 
             Token.values_for(keywords).each do |value|
               token = Token.add(value)
               id = token.id
 
-              index.tokens.push(id)
+              token_ids.push(id)
               keyword_scores[id] += 1
             end
 
@@ -86,7 +87,7 @@ module Mongoid
               token = Token.add(value)
               id = token.id
 
-              index.tokens.push(id)
+              token_ids.push(id)
               fulltext_scores[id] += 1
             end
 
@@ -96,7 +97,7 @@ module Mongoid
             index.score = score if score
             index.facets = facets if facets
 
-            index.tokens = index.tokens.uniq
+            index.token_ids = token_ids
 
             index.save!
           end
@@ -105,16 +106,12 @@ module Mongoid
         def remove(*args)
           models_for(*args) do |model|
             index = where(:model_type => model.class.name, :model_id => model.id).first
-
-            if index
-              subtract(index)
-              index.destroy
-            end
+            index.destroy if index
           end
         end
 
         def subtract(index)
-          tokens = Token.where(:id.in => index.tokens)
+          tokens = index.tokens
 
           n = 0
 
@@ -145,9 +142,11 @@ module Mongoid
         end
       end
 
+      before_destroy{|index| Index.subtract(index)}
+
       belongs_to(:model, :polymorphic => true)
 
-      field(:tokens, :type => Array, :default => [])
+      has_and_belongs_to_many(:tokens, :class_name => '::Mongoid::Haystack::Token', :inverse_of => nil)
       field(:score, :type => Integer, :default => 0)
       field(:keyword_scores, :type => Hash, :default => proc{ Hash.new{|h,k| h[k] = 0} })
       field(:fulltext_scores, :type => Hash, :default => proc{ Hash.new{|h,k| h[k] = 0} })
@@ -156,7 +155,7 @@ module Mongoid
       index({:model_type => 1})
       index({:model_id => 1})
 
-      index({:tokens => 1})
+      index({:token_ids => 1})
       index({:score => 1})
       index({:keyword_scores => 1})
       index({:fulltext_scores => 1})

data/lib/mongoid-haystack/search.rb

@@ -5,15 +5,34 @@ module Mongoid
       options = Map.options_for!(args)
       search = args.join(' ')
 
+      conditions = {}
+      order = []
+
+      op = :token_ids.in
+
+    #
+      case
+        when options[:all]
+          op = :token_ids.all
+          search += Coerce.string(options[:all]) 
+
+        when options[:any]
+          op = :token_ids.in
+          search += Coerce.string(options[:any]) 
+
+        when options[:in]
+          op = :token_ids.in
+          search += Coerce.string(options[:in]) 
+      end
+
     #
       tokens = search_tokens_for(search)
+      token_ids = tokens.map{|token| token.id}
 
     #
-      conditions = {}
-      conditions[:tokens.in] = tokens.map{|token| token.id}
+      conditions[op] = token_ids
 
     #
-      order = []
       order.push(["score", :desc])
 
       tokens.each do |token|
@@ -26,7 +45,7 @@ module Mongoid
 
     #
       if options[:facets]
-        conditions[:facets] = options[:facets]
+        conditions[:facets] = {'$elemMatch' => options[:facets]}
       end
 
     #
@@ -36,7 +55,9 @@ module Mongoid
       end
 
     #
-      Index.where(conditions).order_by(order)
+      Index.where(conditions).order_by(order).tap do |results|
+        results.extend(Denormalize)
+      end
     end
 
     def search_tokens_for(search)
@@ -62,7 +83,7 @@ module Mongoid
           options[:types] = Array(options[:types]).flatten.compact
           options[:types].push(self)
           args.push(options)
-          Haystack.search(*args, &block)
+          results = Haystack.search(*args, &block)
         end
 
         after_save do |doc|
@@ -80,6 +101,8 @@ module Mongoid
             nil
           end
         end
+
+        has_one(:haystack_index, :as => :model, :class_name => '::Mongoid::Haystack::Index')
       end
 
       InstanceMethods = proc do
@@ -92,5 +115,73 @@ module Mongoid
         other.class_eval(&InstanceMethods)
       end
     end
+
+    module Denormalize
+      def denormalize
+        ::Mongoid::Haystack.denormalize(self)
+        self
+      end
+
+      def models
+        denormalize
+        map(&:model)
+      end
+    end
+
+    def Haystack.denormalize(results)
+      queries = Hash.new{|h,k| h[k] = []}
+
+      results = results.to_a.flatten.compact
+
+      results.each do |result|
+        model_type = result[:model_type]
+        model_id = result[:model_id]
+        model_class = model_type.constantize
+        queries[model_class].push(model_id)
+      end
+
+      index = Hash.new{|h,k| h[k] = {}}
+
+      queries.each do |model_class, model_ids|
+        models = 
+          begin
+            model_class.find(model_ids)
+          rescue Mongoid::Errors::DocumentNotFound
+            model_ids.map do |model_id|
+              begin
+                model_class.find(model_id)
+              rescue Mongoid::Errors::DocumentNotFound
+                nil
+              end
+            end
+          end
+
+        models.each do |model|
+          index[model.class.name] ||= Hash.new
+          next unless model
+          index[model.class.name][model.id.to_s] = model
+        end
+      end
+
+      to_ignore = []
+
+      results.each_with_index do |result, i|
+        model = index[result['model_type']][result['model_id'].to_s]
+
+        if model.nil?
+          to_ignore.push(i)
+          next
+        else
+          result.model = model
+        end
+
+        result.model.freeze
+        result.freeze
+      end
+
+      to_ignore.reverse.each{|i| results.delete_at(i)}
+
+      results.to_a
+    end
   end
 end

data/lib/mongoid-haystack/stemming.rb

@@ -1,77 +1,80 @@
 # encoding: utf-8
 
-module Stemming
-  def stem(*args)
-    string = args.join(' ')
-    words = string.scan(/[\w._-]+/)
-    stems = []
-    words.each do |word|
-      word = word.downcase
-      stem = word.stem.downcase
-      next if Stopwords.stopword?(word)
-      next if Stopwords.stopword?(stem)
-      stems.push(stem)
-    end
-    stems
-  end
+module Mongoid
+  module Haystack
+    module Stemming
+      def stem(*args)
+        string = args.join(' ')
+        words = Util.words_for(*args)
+        stems = []
+        words.each do |word|
+          stem = word.stem.downcase
+          next if Stopwords.stopword?(word)
+          next if Stopwords.stopword?(stem)
+          stems.push(stem)
+        end
+        stems
+      end
 
-  alias_method('for', 'stem')
+      alias_method('for', 'stem')
 
-  module Stopwords
-    dirname = __FILE__.sub(/\.rb\Z/, '')
-    glob = File.join(dirname, 'stopwords', '*.txt')
+      module Stopwords
+        dirname = __FILE__.sub(/\.rb\Z/, '')
+        glob = File.join(dirname, 'stopwords', '*.txt')
 
-    List = {}
+        List = {}
 
-    Dir.glob(glob).each do |wordlist|
-      basename = File.basename(wordlist)
-      name = basename.split(/\./).first
+        Dir.glob(glob).each do |wordlist|
+          basename = File.basename(wordlist)
+          name = basename.split(/\./).first
 
-      open(wordlist) do |fd|
-        lines = fd.readlines
-        words = lines.map{|line| line.strip}
-        words.delete_if{|word| word.empty?}
-        words.push('')
-        List[name] = words
-      end
-    end
+          open(wordlist) do |fd|
+            lines = fd.readlines
+            words = lines.map{|line| line.strip}
+            words.delete_if{|word| word.empty?}
+            words.push('')
+            List[name] = words
+          end
+        end
 
-    unless defined?(All)
-      All = []
-      All.concat(List['english'])
-      All.concat(List['full_english'])
-      All.concat(List['extended_english'])
-      #All.concat(List['full_french'])
-      #All.concat(List['full_spanish'])
-      #All.concat(List['full_portuguese'])
-      #All.concat(List['full_italian'])
-      #All.concat(List['full_german'])
-      #All.concat(List['full_dutch'])
-      #All.concat(List['full_norwegian'])
-      #All.concat(List['full_danish'])
-      #All.concat(List['full_russian'])
-      #All.concat(List['full_russian_koi8_r'])
-      #All.concat(List['full_finnish'])
-      All.sort!
-      All.uniq!
-    end
+        unless defined?(All)
+          All = []
+          All.concat(List['english'])
+          All.concat(List['full_english'])
+          All.concat(List['extended_english'])
+          #All.concat(List['full_french'])
+          #All.concat(List['full_spanish'])
+          #All.concat(List['full_portuguese'])
+          #All.concat(List['full_italian'])
+          #All.concat(List['full_german'])
+          #All.concat(List['full_dutch'])
+          #All.concat(List['full_norwegian'])
+          #All.concat(List['full_danish'])
+          #All.concat(List['full_russian'])
+          #All.concat(List['full_russian_koi8_r'])
+          #All.concat(List['full_finnish'])
+          All.sort!
+          All.uniq!
+        end
 
-    unless defined?(Index)
-      Index = {}
+        unless defined?(Index)
+          Index = {}
 
-      All.each do |word|
-        Index[word] = word
+          All.each do |word|
+            Index[word] = word
+          end
+        end
+
+        def stopword?(word)
+          !!Index[word]
+        end
+
+        extend(Stopwords)
       end
-    end
 
-    def stopword?(word)
-      !!Index[word]
+      extend(Stemming)
     end
-
-    extend(Stopwords)
   end
-
-  extend(Stemming)
 end
 
 if $0 == __FILE__

data/lib/mongoid-haystack/token.rb

@@ -4,33 +4,20 @@ module Mongoid
       include Mongoid::Document
 
       class << Token
-        def values_for(*args, &block)
-          string = args.join(' ')
-          values = string.scan(/[^\s]+/)
-          Stemming.stem(*values)
+        def values_for(*args)
+          Haystack.stems_for(*args)
         end
 
         def add(value)
-          token = nil
-          created = nil
-
-          Haystack.find_or_create(
-            proc do
-              token = where(:value => value).first
-              created = false if token
-              token
-            end,
-
-            proc do
-              token = create!(:value => value)
-              created = true if token
-              token
-            end
-          )
+          token =
+            Haystack.find_or_create(
+              ->{ where(:value => value).first },
+              ->{ create!(:value => value) }
+            )
 
           token.inc(:count, 1)
 
-          Count[:tokens].inc(1) #if created
+          Count[:tokens].inc(1)
 
           token
         end

data/lib/mongoid-haystack/util.rb

@@ -33,9 +33,6 @@ module Mongoid
         models.map{|model| model.destroy_all}
       end
 
-      def stem(*args, &block)
-        Stemming.stem(*args, &block)
-      end
 
       def find_or_create(finder, creator)
         doc = finder.call()
@@ -59,6 +56,23 @@ module Mongoid
         end
       end
 
+      def words_for(*args)
+        string = args.flatten.compact.join(' ').scan(/\w+/).join(' ')
+        words = []
+        UnicodeUtils.each_word(string) do |word|
+          word = UnicodeUtils.nfkd(word.strip)
+          word.gsub!(/\A(?:[^\w]|_|\s)+/, '')  # leading punctuation/spaces
+          word.gsub!(/(?:[^\w]|_|\s+)+\Z/, '') # trailing punctuation/spaces
+          next if word.empty?
+          words.push(word)
+        end
+        words
+      end
+
+      def stems_for(*args, &block)
+        Stemming.stem(*args, &block)
+      end
+
       extend Util
     end
 

data/lib/mongoid-haystack.rb

@@ -2,7 +2,7 @@
 #
   module Mongoid
     module Haystack
-      const_set :Version, '1.0.0'
+      const_set :Version, '1.1.0'
 
       class << Haystack
         def version
@@ -11,9 +11,11 @@
 
         def dependencies
           {
-            'mongoid'     => [ 'mongoid'     , '~> 3.0' ] ,
-            'map'         => [ 'map'         , '~> 6.2' ] ,
-            'fattr'       => [ 'fattr'       , '~> 2.2' ] ,
+            'mongoid'       => [ 'mongoid'       , '~> 3.0'   ] , 
+            'map'           => [ 'map'           , '~> 6.2'   ] , 
+            'fattr'         => [ 'fattr'         , '~> 2.2'   ] , 
+            'coerce'        => [ 'coerce'        , '~> 0.0.3' ] , 
+            'unicode_utils' => [ 'unicode_utils' , '~> 1.4.0' ] , 
           }
         end
 
@@ -66,6 +68,9 @@
         end
       end
 
+      require 'unicode_utils/u'
+      require 'unicode_utils/each_word'
+
       load Haystack.libdir('stemming.rb')
       load Haystack.libdir('util.rb')
       load Haystack.libdir('count.rb')
@@ -74,6 +79,10 @@
       load Haystack.libdir('index.rb')
       load Haystack.libdir('search.rb')
 
+      def Haystack.included(other)
+        other.send(:include, Search)
+      end
+
       extend Haystack
     end
   end

data/mongoid-haystack.gemspec

@@ -3,13 +3,14 @@
 
 Gem::Specification::new do |spec|
   spec.name = "mongoid-haystack"
-  spec.version = "1.0.0"
+  spec.version = "1.1.0"
   spec.platform = Gem::Platform::RUBY
   spec.summary = "mongoid-haystack"
   spec.description = "a mongoid 3 zero-config, zero-integration, POLS pure mongo fulltext solution"
 
   spec.files =
-["Rakefile",
+["README.md",
+ "Rakefile",
  "lib",
  "lib/app",
  "lib/app/models",
@@ -63,6 +64,10 @@ Gem::Specification::new do |spec|
   
     spec.add_dependency(*["fattr", "~> 2.2"])
   
+    spec.add_dependency(*["coerce", "~> 0.0.3"])
+  
+    spec.add_dependency(*["unicode_utils", "~> 1.4.0"])
+  
 
   spec.extensions.push(*[])
 

data/test/helper.rb

@@ -7,22 +7,35 @@ require_relative 'testing'
 require_relative '../lib/mongoid-haystack.rb'
 
 Mongoid::Haystack.connect!
+Mongoid::Haystack.reset!
 
 class A
   include Mongoid::Document
   field(:content, :type => String)
   def to_s; content; end
+
+  field(:a)
+  field(:b)
+  field(:c)
 end
 
 class B
   include Mongoid::Document
   field(:content, :type => String)
   def to_s; content; end
+
+  field(:a)
+  field(:b)
+  field(:c)
 end
 
 class C
   include Mongoid::Document
   field(:content, :type => String)
   def to_s; content; end
+
+  field(:a)
+  field(:b)
+  field(:c)
 end
 

data/test/mongoid-haystack_test.rb

@@ -1,15 +1,6 @@
 require_relative 'helper'
 
 Testing Mongoid::Haystack do
-##
-#
-  Mongoid::Haystack.reset!
-
-  setup do
-    [A, B, C].map{|m| m.destroy_all}
-    Mongoid::Haystack.destroy_all
-  end
-
 ##
 #
   testing 'that models can, at minimum, be indexed and searched' do
@@ -49,7 +40,7 @@ Testing Mongoid::Haystack do
 ##
 #
   testing 'that basic stemming can be performed' do
-    assert{ Mongoid::Haystack.stem('dogs cats') == %w[ dog cat ] }
+    assert{ Mongoid::Haystack.stems_for('dogs cats fishes') == %w[ dog cat fish ] }
   end
 
   testing 'that words are stemmed when they are indexed' do
@@ -80,14 +71,12 @@ Testing Mongoid::Haystack do
   end
 
   testing 'that removing a model from the index decrements counts appropriately' do
-  #
     a = A.create!(:content => 'dog')
     b = A.create!(:content => 'cat')
     c = A.create!(:content => 'cats dogs')
 
     assert{ Mongoid::Haystack.index(A) }
 
-  #
     assert{ Mongoid::Haystack.search('cat').first }
 
     assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 2 }
@@ -116,4 +105,159 @@ Testing Mongoid::Haystack do
     assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 0 }
     assert{ Mongoid::Haystack::Token.where(:value => 'dog').first.count == 0 }
   end
+
+##
+#
+  testing 'that search uses a b-tree index' do
+    a = A.create!(:content => 'dog')
+
+    assert{ Mongoid::Haystack.index(A) }
+    assert{ Mongoid::Haystack.search('dog').explain['cursor'] =~ /BtreeCursor/i }
+  end
+
+##
+#
+  testing 'that classes can export a custom [score|keywords|fulltext] for the search index' do
+    k = new_klass do 
+      def to_haystack
+        colors.push(color = colors.shift)
+
+        {
+          :score => score,
+
+          :keywords => "cats #{ color }",
+
+          :fulltext => 'now is the time for all good men...'
+        }
+      end
+
+      def self.score
+        @score ||= 0
+      ensure
+        @score += 1
+      end
+
+      def score
+        self.class.score
+      end
+
+      def self.colors
+        @colors ||= %w( black white )
+      end
+
+      def colors
+        self.class.colors
+      end
+    end
+
+    a = k.create!(:content => 'dog')
+    b = k.create!(:content => 'dogs too')
+
+    assert{ a.haystack_index.score == 0 }
+    assert{ b.haystack_index.score == 1 }
+
+    assert do
+      a.haystack_index.tokens.map(&:value).sort ==
+        ["black", "cat", "good", "men", "time"]
+    end
+    assert do
+      b.haystack_index.tokens.map(&:value).sort ==
+        ["cat", "good", "men", "time", "white"]
+    end
+
+    assert{ Mongoid::Haystack.search('cat').count == 2 }
+    assert{ Mongoid::Haystack.search('black').count == 1 }
+    assert{ Mongoid::Haystack.search('white').count == 1 }
+    assert{ Mongoid::Haystack.search('good men').count == 2 }
+  end
+
+##
+#
+  testing 'that set intersection and union are supported via search' do
+    a = A.create!(:content => 'dog')
+    b = A.create!(:content => 'dog cat')
+    c = A.create!(:content => 'dog cat fish')
+
+    assert{ Mongoid::Haystack.index(A) }
+
+    assert{ Mongoid::Haystack.search(:any => 'dog').count == 3 }
+    assert{ Mongoid::Haystack.search(:any => 'dog cat').count == 3 }
+    assert{ Mongoid::Haystack.search(:any => 'dog cat fish').count == 3 }
+
+    assert{ Mongoid::Haystack.search(:all => 'dog').count == 3 }
+    assert{ Mongoid::Haystack.search(:all => 'dog cat').count == 2 }
+    assert{ Mongoid::Haystack.search(:all => 'dog cat fish').count == 1 }
+  end
+
+##
+#
+  testing 'that classes can export custom facets and then search them, again using a b-tree index' do
+    k = new_klass do
+      field(:to_haystack, :type => Hash, :default => proc{ Hash.new })
+    end
+
+    a = k.create!(:content => 'hello kitty', :to_haystack => { :keywords => 'cat', :facets => {:x => 42.0}})
+    b = k.create!(:content => 'hello kitty', :to_haystack => { :keywords => 'cat', :facets => {:x => 4.20}})
+
+    assert{ Mongoid::Haystack.search('cat').where(:facets => {'x' => 42.0}).first.model == a }
+    assert{ Mongoid::Haystack.search('cat').where(:facets => {'x' => 4.20}).first.model == b }
+
+    assert{ Mongoid::Haystack.search('cat').where('facets.x' => 42.0).first.model == a }
+    assert{ Mongoid::Haystack.search('cat').where('facets.x' => 4.20).first.model == b }
+
+    assert{ Mongoid::Haystack.search('cat').where('facets' => {'x' => 42.0}).explain['cursor'] =~ /BtreeCursor/ }
+    assert{ Mongoid::Haystack.search('cat').where('facets' => {'x' => 4.20}).explain['cursor'] =~ /BtreeCursor/ }
+
+    assert{ Mongoid::Haystack.search('cat').where('facets.x' => 42.0).explain['cursor'] =~ /BtreeCursor/ }
+    assert{ Mongoid::Haystack.search('cat').where('facets.x' => 4.20).explain['cursor'] =~ /BtreeCursor/ }
+  end
+
+##
+#
+  testing 'that keywords are considered more highly than fulltext' do
+    k = new_klass do
+      field(:title)
+      field(:body)
+
+      def to_haystack
+        { :keywords => title, :fulltext => body }
+      end
+    end
+
+    a = k.create!(:title => 'the cats', :body => 'like to meow')
+    b = k.create!(:title => 'the dogs', :body => 'do not like to meow, they bark at cats')
+
+    assert{ Mongoid::Haystack.search('cat').count == 2 }
+    assert{ Mongoid::Haystack.search('cat').first.model == a }
+
+    assert{ Mongoid::Haystack.search('meow').count == 2 }
+    assert{ Mongoid::Haystack.search('bark').count == 1 }
+    assert{ Mongoid::Haystack.search('dog').first.model == b }
+  end
+
+protected
+
+  def new_klass(&block)
+    Object.send(:remove_const, :K) if Object.send(:const_defined?, :K)
+
+    k = Class.new(A) do
+      self.default_collection_name = :ks
+      def self.name() 'K' end
+      include ::Mongoid::Haystack::Search
+      class_eval(&block) if block
+    end
+
+    Object.const_set(:K, k)
+
+    k
+  end
+
+  H = Mongoid::Haystack
+  T = Mongoid::Haystack::Token
+  I = Mongoid::Haystack::Index
+
+  setup do
+    [A, B, C].map{|m| m.destroy_all}
+    Mongoid::Haystack.destroy_all
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid-haystack
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -59,12 +59,45 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: coerce
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+- !ruby/object:Gem::Dependency
+  name: unicode_utils
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.0
 description: a mongoid 3 zero-config, zero-integration, POLS pure mongo fulltext solution
 email: ara.t.howard@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- README.md
 - Rakefile
 - lib/app/models/mongoid/haystack/count.rb
 - lib/app/models/mongoid/haystack/index.rb