mongoid-haystack 1.2.1 → 1.3.0

data/article

@@ -0,0 +1,19 @@
+
+good abstractions are the backbone of reusable code, and although most people
+these days are abstracting via objects, class, and functional paradigms little
+attention is paid to the lowly function signature.
+
+a good one allows modular methods to be composed into compound ones without
+fugly code contortions - keeping code nice and readable.
+
+although many people spend time considering carefully their method names and
+argument lists, few people stop to consider the role of the block on their
+methods and how it affects reusability.
+
+that's unfortunate since fully groking the block is what unleashes the
+expressive power ruby's internal iterators provide.
+
+
+
+let's say we have a simple function that parses text into paragraphs.  a naive
+implementation might look like:

data/lib/mongoid-haystack.rb

@@ -2,7 +2,7 @@
 #
   module Mongoid
     module Haystack
-      const_set :Version, '1.2.1'
+      const_set(:Version, '1.3.0') unless const_defined?(:Version)
 
       class << Haystack
         def version
@@ -11,13 +11,13 @@
 
         def dependencies
           {
-            'mongoid'       => [ 'mongoid'       , '~> 3.0.14' ] , 
-            'moped'         => [ 'moped'         , '~> 1.3.1'  ] , 
-            'origin'        => [ 'origin'        , '~> 1.0.11' ] , 
-            'map'           => [ 'map'           , '~> 6.2'    ] , 
-            'fattr'         => [ 'fattr'         , '~> 2.2'    ] , 
-            'coerce'        => [ 'coerce'        , '~> 0.0.3'  ] , 
-            'unicode_utils' => [ 'unicode_utils' , '~> 1.4.0'  ] , 
+            'mongoid'       => [ 'mongoid'       , '~> 3.0' ] , 
+            'moped'         => [ 'moped'         , '~> 1.3' ] , 
+            'origin'        => [ 'origin'        , '~> 1.0' ] , 
+            'map'           => [ 'map'           , '~> 6.2' ] , 
+            'fattr'         => [ 'fattr'         , '~> 2.2' ] , 
+            'coerce'        => [ 'coerce'        , '~> 0.0' ] , 
+            'unicode_utils' => [ 'unicode_utils' , '~> 1.4' ] , 
           }
         end
 

data/lib/mongoid-haystack/index.rb

@@ -174,23 +174,36 @@ module Mongoid
         end
       end
 
-      before_destroy{|index| Index.subtract(index)}
+      before_destroy do |index|
+        Index.subtract(index)
+      end
+
+      before_validation do |index|
+        index.size = tokens.count
+      end
 
       belongs_to(:model, :polymorphic => true)
 
       has_and_belongs_to_many(:tokens, :class_name => '::Mongoid::Haystack::Token', :inverse_of => nil)
 
+      field(:size, :type => Integer, :default => nil)
       field(:score, :type => Integer, :default => 0)
       field(:keyword_scores, :type => Hash, :default => proc{ Hash.new{|h,k| h[k] = 0} })
       field(:fulltext_scores, :type => Hash, :default => proc{ Hash.new{|h,k| h[k] = 0} })
       field(:facets, :type => Hash, :default => {})
 
+      %w( size score ).each do |f|
+        validates_presence_of(f)
+      end
+
       index({:model_type => 1, :model_id => 1}, :unique => true)
 
       index({:token_ids => 1})
       index({:score => 1})
+      index({:size => 1})
       index({:keyword_scores => 1})
       index({:fulltext_scores => 1})
+      index({:facets => 1})
     end
   end
 end

data/lib/mongoid-haystack/search.rb

@@ -2,6 +2,10 @@ module Mongoid
   module Haystack
     module Search
       ClassMethods = proc do
+        def mongoid_haystack_searchable?
+          true
+        end
+
         def search(*args, &block)
           options = Map.options_for!(args)
           options[:types] = Array(options[:types]).flatten.compact
@@ -52,8 +56,10 @@ module Mongoid
       def Search.included(other)
         super
       ensure
-        other.instance_eval(&ClassMethods)
-        other.class_eval(&InstanceMethods)
+        unless other.respond_to?(:mongoid_haystack_searchable?)
+          other.instance_eval(&ClassMethods)
+          other.class_eval(&InstanceMethods)
+        end
       end
     end
 
@@ -100,6 +106,8 @@ module Mongoid
         order.push(["fulltext_scores.#{ token.id }", :desc])
       end
 
+      order.push(["size", :asc])
+
     #
       if options[:facets]
         conditions[:facets] = {'$elemMatch' => options[:facets]}
@@ -201,16 +209,22 @@ module Mongoid
 
     def search_tokens_for(search)
       values = Token.values_for(search.to_s)
-      tokens = Token.where(:value.in => values).to_a
+      tokens = []
 
-      positions = {}
-      tokens.each_with_index{|token, index| positions[token] = index + 1}
+      Token.where(:value.in => values).each do |token|
+        index = values.index(token.value)
+        tokens[index] = token
+      end
 
       total = Token.total.to_f
 
-      tokens.sort! do |a,b|
-        [b.rarity_bin(total), positions[b]] <=> [a.rarity_bin(total), positions[a]]
-      end
+      rarity = {}
+      tokens.map{|token| rarity[token] = token.rarity_bin(total)}
+
+      position = {}
+      tokens.each_with_index{|token, i| position[token] = i + 1}
+
+      tokens.sort!{ |a, b| [rarity[b], position[a]] <=> [rarity[a], position[b]] }
 
       tokens
     end

data/lib/mongoid-haystack/stemming.rb

@@ -3,17 +3,19 @@
 module Mongoid
   module Haystack
     module Stemming
-      def stem(*args)
+      def stem(*args, &block)
         string = args.join(' ')
         words = Util.words_for(*args)
-        stems = []
+        list = []
+
         words.each do |word|
           stem = word.stem.downcase
           next if Stopwords.stopword?(word)
           next if Stopwords.stopword?(stem)
-          stems.push(stem)
+          block ? block.call(stem) : list.push(stem)
         end
-        stems
+
+        block ? nil : list
       end
 
       alias_method('for', 'stem')
@@ -37,6 +39,8 @@ module Mongoid
           end
         end
 
+      # TODO - this needs to be configurable...
+      #
         unless defined?(All)
           All = []
           All.concat(List['english'])

data/lib/mongoid-haystack/token.rb

@@ -5,7 +5,7 @@ module Mongoid
 
       class << Token
         def values_for(*args)
-          Haystack.stems_for(*args)
+          Haystack.tokens_for(*args)
         end
 
         def add(value)

data/lib/mongoid-haystack/util.rb

@@ -3,8 +3,8 @@ module Mongoid
     module Util 
       def models
         [
-          Mongoid::Haystack::Token,
           Mongoid::Haystack::Index,
+          Mongoid::Haystack::Token,
           Mongoid::Haystack::Sequence
         ]
       end
@@ -59,28 +59,119 @@ module Mongoid
         end
       end
 
-      def words_for(*args)
+      def token_tree_for(*args, &block)
+        tree = []
+
+        phrases_for(*args) do |phrase|
+          #next if stopword?(phrase)
+
+          if block
+            block.call(:phrase, phrase)
+          else
+            tree.push([phrase, words = []])
+          end
+
+          words_for(phrase) do |word|
+            #next if phrase == word
+            #next if stopword?(word)
+
+            if block
+              block.call(:word, word)
+            else
+              words.push([word, stems = []])
+            end
+
+            stems_for(word) do |stem|
+              #next if word == stem
+
+              if block
+                block.call(:stem, stem)
+              else
+                stems.push(stem)
+              end
+            end
+          end
+        end
+
+        block ? nil : tree
+      end
+
+      def tokens_for(*args, &block)
+        list = []
+
+        token_tree_for(*args).each do |phrase, words|
+          next if stopword?(phrase)
+          block ? block.call(phrase) : list.push(phrase) 
+
+          words.each do |word, stems|
+            next if stopword?(word)
+
+            unless word == phrase
+              block ? block.call(word) : list.push(word) 
+            end
+
+            stems.each do |stem|
+              next if stopword?(stem)
+
+              unless stem == phrase or stem == word
+                block ? block.call(stem) : list.push(stem)
+              end
+            end
+          end
+        end
+
+        block ? nil : list
+      end
+
+      def phrases_for(*args, &block)
+        string = args.join(' ')
+        string.strip!
+
+        phrases = string.split(/\s+/)
+
+        list = []
+
+        phrases.each do |phrase|
+          strip!(phrase)
+          next if phrase.empty?
+          block ? block.call(phrase) : list.push(phrase)
+        end
+
+        block ? nil : list
+      end
+
+      def words_for(*args, &block)
         string = args.join(' ')
         string.gsub!(/_+/, '-')
         string.gsub!(/[^\w]/, ' ')
 
-        words = []
+        list = []
 
         UnicodeUtils.each_word(string) do |word|
-          word = UnicodeUtils.nfkd(word.strip)
-          word.gsub!(/\A(?:[^\w]|_|\s)+/, '')  # leading punctuation/spaces
-          word.gsub!(/(?:[^\w]|_|\s+)+\Z/, '') # trailing punctuation/spaces
+          strip!(word)
           next if word.empty?
-          words.push(word)
+          block ? block.call(word) : list.push(word)
         end
 
-        words
+        block ? nil : list
       end
 
       def stems_for(*args, &block)
         Stemming.stem(*args, &block)
       end
 
+      def stopword?(word)
+        word = UnicodeUtils.nfkd(word.to_s.strip.downcase)
+        word.empty? or Stemming::Stopwords.stopword?(word)
+      end
+
+      def strip!(word)
+        word.replace(UnicodeUtils.nfkd(word.to_s.strip))
+        word.gsub!(/\A(?:[^\w]|_|\s)+/, '')  # leading punctuation/spaces
+        word.gsub!(/(?:[^\w]|_|\s+)+\Z/, '') # trailing punctuation/spaces
+        word
+      end
+
       extend Util
     end
 

data/mongoid-haystack.gemspec

@@ -3,7 +3,7 @@
 
 Gem::Specification::new do |spec|
   spec.name = "mongoid-haystack"
-  spec.version = "1.2.1"
+  spec.version = "1.3.0"
   spec.platform = Gem::Platform::RUBY
   spec.summary = "mongoid-haystack"
   spec.description = "a mongoid 3 zero-config, zero-integration, POLS pure mongo fulltext solution"
@@ -12,6 +12,7 @@ Gem::Specification::new do |spec|
 ["HISTORY",
  "README.md",
  "Rakefile",
+ "article",
  "lib",
  "lib/app",
  "lib/app/models",
@@ -57,19 +58,19 @@ Gem::Specification::new do |spec|
   spec.test_files = nil
 
   
-    spec.add_dependency(*["mongoid", "~> 3.0.14"])
+    spec.add_dependency(*["mongoid", "~> 3.0"])
   
-    spec.add_dependency(*["moped", "~> 1.3.1"])
+    spec.add_dependency(*["moped", "~> 1.3"])
   
-    spec.add_dependency(*["origin", "~> 1.0.11"])
+    spec.add_dependency(*["origin", "~> 1.0"])
   
     spec.add_dependency(*["map", "~> 6.2"])
   
     spec.add_dependency(*["fattr", "~> 2.2"])
   
-    spec.add_dependency(*["coerce", "~> 0.0.3"])
+    spec.add_dependency(*["coerce", "~> 0.0"])
   
-    spec.add_dependency(*["unicode_utils", "~> 1.4.0"])
+    spec.add_dependency(*["unicode_utils", "~> 1.4"])
   
 
   spec.extensions.push(*[])

data/test/helper.rb

@@ -11,6 +11,7 @@ Mongoid::Haystack.reset!
 
 class A
   include Mongoid::Document
+  include Mongoid::Haystack
   field(:content, :type => String)
   def to_s; content; end
 
@@ -21,6 +22,7 @@ end
 
 class B
   include Mongoid::Document
+  include Mongoid::Haystack
   field(:content, :type => String)
   def to_s; content; end
 
@@ -31,6 +33,7 @@ end
 
 class C
   include Mongoid::Document
+  include Mongoid::Haystack
   field(:content, :type => String)
   def to_s; content; end
 

data/test/mongoid-haystack_test.rb

@@ -48,6 +48,25 @@ Testing Mongoid::Haystack do
     assert{ Mongoid::Haystack.search('cat dog').map(&:model) == [d, c, b, a] }
   end
 
+##
+#
+  testing 'that word specificity affects the sort' do
+    a = A.create!(:content => 'cat@dog.com')
+    b = A.create!(:content => 'dogs')
+    c = A.create!(:content => 'dog')
+    d = A.create!(:content => 'cats')
+    e = A.create!(:content => 'cat')
+
+    assert{ Mongoid::Haystack.index(A) }
+
+    assert{ Mongoid::Haystack.search('cat@dog.com').map(&:model) == [a, e, d, c, b] }
+    assert{ Mongoid::Haystack.search('cat').map(&:model) == [e, d, a] }
+    assert{ Mongoid::Haystack.search('cats').map(&:model) == [d, e, a] }
+    assert{ Mongoid::Haystack.search('dog').map(&:model) == [c, b, a] }
+    assert{ Mongoid::Haystack.search('dogs').map(&:model) == [b, c, a] }
+    #assert{ Mongoid::Haystack.search('dog').map(&:model) == [c, b, a] }
+  end
+
 ##
 #
   testing 'that basic stemming can be performed' do
@@ -76,45 +95,59 @@ Testing Mongoid::Haystack do
 
     assert{ Mongoid::Haystack.index(A) }
 
-    assert{ Mongoid::Haystack::Token.count == 2 }
-    assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat dog ) }
-    assert{ Mongoid::Haystack::Token.total == 3 }
+    assert{ Mongoid::Haystack::Token.count == 3 }
+    assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat dog dogs ) }
+    assert{ Mongoid::Haystack::Token.total == 4 }
   end
 
   testing 'that removing a model from the index decrements counts appropriately' do
-    a = A.create!(:content => 'dog')
-    b = A.create!(:content => 'cat')
-    c = A.create!(:content => 'cats dogs')
+    2.times do |i|
+      A.destroy_all
+
+      a = A.create!(:content => 'dog')
+      b = A.create!(:content => 'cat')
+      c = A.create!(:content => 'cats dogs')
+
+      remove = proc do |model|
+        assert{
+          if i == 0
+            assert{ Mongoid::Haystack.unindex(model) }
+          else
+            assert{ model.destroy; true }
+          end
+        }
+      end
 
-    assert{ Mongoid::Haystack.index(A) }
+      assert{ Mongoid::Haystack.index(A) }
 
-    assert{ Mongoid::Haystack.search('cat').first }
+      %w( cat dog cats dogs ).each do |search|
+        assert{ Mongoid::Haystack.search(search).first }
+      end
 
-    assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 2 }
-    assert{ Mongoid::Haystack::Token.where(:value => 'dog').first.count == 2 }
-    assert{ Mongoid::Haystack::Token.total == 4 }
-    assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat dog ) }
-    assert{ Mongoid::Haystack.unindex(c) }
-    assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat dog ) }
-    assert{ Mongoid::Haystack::Token.total == 2 }
-    assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 1 }
-    assert{ Mongoid::Haystack::Token.where(:value => 'dog').first.count == 1 }
-
-    assert{ Mongoid::Haystack::Token.total == 2 }
-    assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat dog ) }
-    assert{ Mongoid::Haystack.unindex(b) }
-    assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat dog ) }
-    assert{ Mongoid::Haystack::Token.total == 1 }
-    assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 0 }
-    assert{ Mongoid::Haystack::Token.where(:value => 'dog').first.count == 1 }
-
-    assert{ Mongoid::Haystack::Token.total == 1 }
-    assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat dog ) }
-    assert{ Mongoid::Haystack.unindex(a) }
-    assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat dog ) }
-    assert{ Mongoid::Haystack::Token.total == 0 }
-    assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 0 }
-    assert{ Mongoid::Haystack::Token.where(:value => 'dog').first.count == 0 }
+      assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 2 }
+      assert{ Mongoid::Haystack::Token.where(:value => 'dog').first.count == 2 }
+      assert{ Mongoid::Haystack::Token.total == 6 }
+      assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat cats dog dogs ) }
+
+      remove[ c ]
+      assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat cats dog dogs ) }
+      assert{ Mongoid::Haystack::Token.total == 2 }
+      assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 1 }
+      assert{ Mongoid::Haystack::Token.where(:value => 'dog').first.count == 1 }
+
+
+      remove[ b ]
+      assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat cats dog dogs ) }
+      assert{ Mongoid::Haystack::Token.total == 1 }
+      assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 0 }
+      assert{ Mongoid::Haystack::Token.where(:value => 'dog').first.count == 1 }
+
+      remove[ a ]
+      assert{ Mongoid::Haystack::Token.all.map(&:value).sort == %w( cat cats dog dogs ) }
+      assert{ Mongoid::Haystack::Token.total == 0 }
+      assert{ Mongoid::Haystack::Token.where(:value => 'cat').first.count == 0 }
+      assert{ Mongoid::Haystack::Token.where(:value => 'dog').first.count == 0 }
+    end
   end
 
 ##
@@ -129,7 +162,7 @@ Testing Mongoid::Haystack do
 ##
 #
   testing 'that classes can export a custom [score|keywords|fulltext] for the search index' do
-    k = new_klass do 
+    k = new_klass do
       def to_haystack
         colors.push(color = colors.shift)
 
@@ -148,6 +181,10 @@ Testing Mongoid::Haystack do
         @score += 1
       end
 
+      def self.score=(score)
+        @score = score.to_i
+      end
+
       def score
         self.class.score
       end
@@ -169,11 +206,11 @@ Testing Mongoid::Haystack do
 
     assert do
       a.haystack_index.tokens.map(&:value).sort ==
-        ["black", "cat", "good", "men", "time"]
+        ["black", "cat", "cats", "good", "men", "time"]
     end
     assert do
       b.haystack_index.tokens.map(&:value).sort ==
-        ["cat", "good", "men", "time", "white"]
+        ["cat", "cats", "good", "men", "time", "white"]
     end
 
     assert{ Mongoid::Haystack.search('cat').count == 2 }
@@ -363,6 +400,14 @@ Testing Mongoid::Haystack do
     end
   end
 
+  test '.tokens_for' do
+   {
+    'cats-and-dogs Cats!and?dogs foo-bar! The end. and trees' => %w( cats-and-dogs cats cat dogs dog Cats!and?dogs Cats cat dogs dog foo-bar foo bar end trees tree )
+   }.each do |src, dst|
+     assert{ Mongoid::Haystack.tokens_for(src) == dst }
+   end
+  end
+
 protected
 
   def new_klass(&block)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongoid-haystack
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-12 00:00:00.000000000 Z
+date: 2013-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongoid
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.0.14
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.0.14
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: moped
   requirement: !ruby/object:Gem::Requirement
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: origin
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.11
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.11
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: map
   requirement: !ruby/object:Gem::Requirement
@@ -98,7 +98,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: '0.0'
 - !ruby/object:Gem::Dependency
   name: unicode_utils
   requirement: !ruby/object:Gem::Requirement
@@ -114,7 +114,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: '1.4'
 description: a mongoid 3 zero-config, zero-integration, POLS pure mongo fulltext solution
 email: ara.t.howard@gmail.com
 executables: []
@@ -132,6 +132,7 @@ files:
 - HISTORY
 - README.md
 - Rakefile
+- article
 - lib/app/models/mongoid/haystack/index.rb
 - lib/app/models/mongoid/haystack/sequence.rb
 - lib/app/models/mongoid/haystack/token.rb
@@ -180,8 +181,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: codeforpeople
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: mongoid-haystack
 test_files: []
+has_rdoc: