mongoid-fts 1.1.1 → 2.0.0

data/lib/mongoid-fts/util.rb

@@ -0,0 +1,322 @@
+module Mongoid
+  module FTS
+    module Util 
+    #
+      def fts_models
+        [
+          Mongoid::FTS::Index
+        ]
+      end
+
+      def reset!
+        Mongoid::FTS.setup!(:warn => true)
+
+        fts_models.each do |model|
+          model.destroy_all
+
+          begin
+            model.collection.indexes.drop
+          rescue Object => e
+          end
+
+          begin
+            model.collection.drop
+          rescue Object => e
+          end
+
+          begin
+            model.create_indexes
+          rescue Object => e
+          end
+        end
+      end
+
+      def create_indexes
+        fts_models.each{|model| model.create_indexes}
+      end
+
+      def destroy_all
+        fts_models.map{|model| model.destroy_all}
+      end
+
+    #
+      def find_in_batches(queries = {})
+        models =
+          queries.map do |model_class, model_ids|
+            unless model_class.is_a?(Class)
+              model_class = eval(model_class.to_s)
+            end
+
+            model_ids = Array(model_ids)
+
+            begin
+              model_class.find(model_ids)
+            rescue Mongoid::Errors::DocumentNotFound
+              model_ids.map do |model_id|
+                begin
+                  model_class.find(model_id)
+                rescue Mongoid::Errors::DocumentNotFound
+                  nil
+                end
+              end
+            end
+          end
+
+        models.flatten!
+        models.compact!
+        models
+      end
+
+      def find_or_create(finder, creator)
+        doc = finder.call()
+        return doc if doc
+
+        n, max = 0, 2
+
+        begin
+          creator.call()
+        rescue Object => e
+          n += 1
+          raise if n > max
+          sleep(rand(0.1))
+          finder.call() or retry
+        end
+      end
+
+    #
+      def terms_for(*args, &block)
+        options = Map.options_for!(args)
+
+        words = words_for(*args)
+
+        list = options[:list] || []
+
+        words.each do |word|
+          word = word.downcase
+          next if stopword?(word)
+
+          stems = stems_for(word)
+
+          stems.each do |stem|
+            [stem, unidecode(stem)].uniq.each do |stem|
+              next if stopword?(stem)
+
+              block ? block.call(stem) : list.push(stem)
+
+              substems = stem.split(/_/)
+
+              if options[:subterms] and substems.size > 1
+                substems.each do |substem|
+                  terms_for(substem.gsub(/_+/, '-'), :list => list)
+                end
+              end
+            end
+          end
+        end
+
+        list.uniq!
+
+        block ? nil : list
+      end
+
+      def words_for(*args, &block)
+        options = Map.options_for!(args)
+
+        string = args.join(' ')
+
+        list = []
+
+        UnicodeUtils.each_word(string) do |word|
+          word = strip(utf8ify(word))
+
+          next if word.empty?
+
+          block ? block.call(word) : list.push(word)
+        end
+
+        block ? nil : list
+      end
+
+      def stems_for(*args, &block)
+        options = Map.options_for!(args)
+
+        words = Coerce.list_of_strings(*args).map{|word| utf8ify(word)}
+
+        Stemming.stem(*words)
+      end
+
+      def literals_for(*args)
+        words = FTS.normalized_array(args)
+
+        return words.map{|word| "__#{ Digest::MD5.hexdigest(word) }__"}
+      end
+
+      def stopword?(word)
+        word = utf8ify(word)
+        word.empty? or Stemming::Stopwords.stopword?(word)
+      end
+
+      def strip(word)
+        word = utf8ify(word)
+        word.gsub!(/\A(?:[^\w]|_|\s)+/, '')  # leading punctuation/spaces
+        word.gsub!(/(?:[^\w]|_|\s+)+\Z/, '') # trailing punctuation/spaces
+        word
+      end
+
+      def fuzzy(*args)
+        strings = Coerce.list_of_strings(args).map{|string| utf8ify(string)}
+
+        list = []
+
+        strings.each do |string|
+          list.push(*ngrams_for(string))
+
+          decoded = unidecode(string)
+
+          unless decoded == string
+            list.push(*ngrams_for(decoded))
+          end
+        end
+
+        list.uniq
+      end
+      alias_method(:fuzzy_for, :fuzzy)
+
+      def ngrams_for(*args)
+        options = Map.options_for!(args)
+
+        strings = Coerce.list_of_strings(args).map{|string| utf8ify(string)}
+
+        list = []
+
+        sizes = options[:sizes] || [2,3]
+
+        strings.each do |string|
+          chars = Util.chars('_' + string + '_')
+
+          sizes.each do |size|
+            (chars.size - (size - 1)).times do |i|
+              ngram = chars[i, size].join
+              list.push(ngram)
+            end
+          end
+
+        end
+
+        list
+      end
+
+      def chars(string)
+        chars = []
+        UnicodeUtils.each_grapheme(string.to_s){|g| chars.push(g)}
+        chars
+      end
+
+      def unidecode(string)
+        Stringex::Unidecoder.decode(utf8ify(string.to_s))
+      end
+
+      def utf8ify(string)
+        UnicodeUtils.nfkd(
+          begin
+            string.force_encoding('UTF-8')
+          rescue
+            string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+          end
+        )
+      end
+
+      def normalized_array(*array)
+        array.flatten.map{|_| _.to_s.strip}.select{|_| !_.empty?}.uniq
+      end
+
+      def list_of_strings(*args)
+        args.flatten.compact.map{|arg| arg.to_s}.select{|arg| !arg.empty?}.uniq
+      end
+
+    #
+      def index(*args, &block)
+        if args.empty? and block.nil?
+          Index
+        else
+          args.each do |arg|
+            case arg
+              when Class
+                arg.all.each{|model| Index.add(model)}
+              else
+                Index.add(arg, &block)
+            end
+          end
+        end
+      end
+
+      def unindex(*args, &block)
+        Index.remove(*args, &block)
+      end
+
+      def index!(*args, &block)
+        Index.add!(*args, &block)
+      end
+
+      def unindex!(*args, &block)
+        Index.remove!(*args, &block)
+      end
+
+    #
+      def models
+        @models ||= []
+      end
+
+    #
+      def session
+        @session ||= Mongoid::Sessions.default
+      end
+
+      def session=(session)
+        @session = session
+      end
+
+      def enable!(*args)
+        options = Map.options_for!(args)
+
+        unless options.has_key?(:warn)
+          options[:warn] = true
+        end
+
+        begin
+          session = Mongoid::Sessions.default
+          session.with(database: :admin).command({ setParameter: 1, textSearchEnabled: true })
+        rescue Object => e
+          unless e.is_a?(Mongoid::Errors::NoSessionsConfig)
+            warn "failed to enable search with #{ e.class }(#{ e.message })"
+          end
+        end
+      end
+
+      def setup!(*args)
+        enable!(*args)
+        Index.setup!
+      end
+
+      def connect!
+        Mongoid.configure do |config|
+          config.connect_to('mongoid-fts')
+        end
+      end
+
+      def boolean_and(*strings)
+        strings = Coerce.list_of_strings(*strings)
+        strings.map{|s| '"%s"' % s.gsub('"', '')}.join(' ')
+      end
+
+      def boolean_or(*strings)
+        strings = Coerce.list_of_strings(*strings)
+        strings.join(' ')
+      end
+
+      extend Util
+    end
+
+    extend Util
+  end
+end

data/mongoid-fts.gemspec

@@ -3,7 +3,7 @@
 
 Gem::Specification::new do |spec|
   spec.name = "mongoid-fts"
-  spec.version = "1.1.1"
+  spec.version = "2.0.0"
   spec.platform = Gem::Platform::RUBY
   spec.summary = "mongoid-fts"
   spec.description = "enable mongodb's new fulltext simply and quickly on your mongoid models, including pagination."
@@ -18,9 +18,37 @@ Gem::Specification::new do |spec|
  "lib/app/mongoid",
  "lib/app/mongoid/fts",
  "lib/app/mongoid/fts/index.rb",
- "lib/mongoid",
+ "lib/mongoid-fts",
  "lib/mongoid-fts.rb",
- "mongoid-fts.gemspec"]
+ "lib/mongoid-fts/able.rb",
+ "lib/mongoid-fts/error.rb",
+ "lib/mongoid-fts/index.rb",
+ "lib/mongoid-fts/rails.rb",
+ "lib/mongoid-fts/raw.rb",
+ "lib/mongoid-fts/results.rb",
+ "lib/mongoid-fts/stemming",
+ "lib/mongoid-fts/stemming.rb",
+ "lib/mongoid-fts/stemming/stopwords",
+ "lib/mongoid-fts/stemming/stopwords/english.txt",
+ "lib/mongoid-fts/stemming/stopwords/extended_english.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_danish.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_dutch.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_english.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_finnish.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_french.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_german.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_italian.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_norwegian.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_portuguese.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_russian.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_russiankoi8_r.txt",
+ "lib/mongoid-fts/stemming/stopwords/full_spanish.txt",
+ "lib/mongoid-fts/util.rb",
+ "mongoid-fts.gemspec",
+ "test",
+ "test/helper.rb",
+ "test/mongoid-fts_test.rb",
+ "test/testing.rb"]
 
   spec.executables = []
   
@@ -35,6 +63,12 @@ Gem::Specification::new do |spec|
   
     spec.add_dependency(*["coerce", "~> 0.0"])
   
+    spec.add_dependency(*["unicode_utils", "~> 1.4"])
+  
+    spec.add_dependency(*["stringex", "~> 2.0"])
+  
+    spec.add_dependency(*["fast-stemmer", "~> 1.0"])
+  
 
   spec.extensions.push(*[])
 

data/test/helper.rb

@@ -0,0 +1,44 @@
+# -*- encoding : utf-8 -*-
+
+# this triggers mongoid to load rails...
+# module Rails; end
+
+require_relative 'testing'
+require_relative '../lib/mongoid-fts.rb'
+
+Mongoid::FTS.connect!
+Mongoid::FTS.reset!
+
+class A
+  include Mongoid::Document
+  include Mongoid::FTS
+  field(:content, :type => String)
+  def to_s; content; end
+
+  field(:a)
+  field(:b)
+  field(:c)
+end
+
+class B
+  include Mongoid::Document
+  include Mongoid::FTS
+  field(:content, :type => String)
+  def to_s; content; end
+
+  field(:a)
+  field(:b)
+  field(:c)
+end
+
+class C
+  include Mongoid::Document
+  include Mongoid::FTS
+  field(:content, :type => String)
+  def to_s; content; end
+
+  field(:a)
+  field(:b)
+  field(:c)
+end
+

data/test/mongoid-fts_test.rb

@@ -0,0 +1,177 @@
+# encoding: utf-8
+#
+require_relative 'helper'
+
+Testing Mongoid::FTS do
+#
+  testing 'wording' do
+    assert{ Mongoid::FTS.words_for('dogs cats fishes') == %w[ dogs cats fishes ] }
+    assert{ Mongoid::FTS.words_for('foo-bar baz_bub') == %w[ foo bar baz_bub ] }
+  end
+
+#
+  testing 'stemming' do
+    assert{ Mongoid::FTS.stems_for('dogs cats fishes') == %w[ dog cat fish ] }
+  end
+
+#
+  testing 'terming' do
+    assert{ Mongoid::FTS.terms_for('dogs and the cats and those fishes') == %w[ dog cat fish ] }
+    assert{ Mongoid::FTS.terms_for('the foo-bar and then baz_bub') == %w[ foo bar baz_bub ] }
+    assert{ Mongoid::FTS.terms_for('the foo-bar and then baz_bub', :subterms => true) == %w[ foo bar baz_bub baz bub ] }
+  end
+
+#
+  testing 'fuzzy' do
+    assert{ 
+      actual = Mongoid::FTS.fuzzy("über")
+      expected = ["_ü", "üb", "be", "er", "r_", "_üb", "übe", "ber", "er_", "_u", "ub", "_ub", "ube"]
+
+      actual.zip(expected).all? do |a,b|
+        a = Mongoid::FTS.utf8ify(a)
+        b = Mongoid::FTS.utf8ify(b)
+        a == b
+      end
+    }
+  end
+
+#
+  testing 'that models can, at minimum, be indexed and searched' do
+    a = A.create!(:content => 'dogs')
+    b = B.create!(:content => 'cats')
+
+    assert{ Mongoid::FTS.index(a) }
+    assert{ Mongoid::FTS.index(b) }
+
+    assert{ Mongoid::FTS.search('dog') == [a] }
+    assert{ Mongoid::FTS.search('cat') == [b] }
+    assert{ Mongoid::FTS.search('Cat') == [b] }
+  end
+
+#
+  testing 'fuzzy search' do
+    a = A.create!(:title => 'über')
+
+    assert{ Mongoid::FTS.index(a) }
+
+    assert{ Mongoid::FTS.search('uber') == [a] }
+    assert{ Mongoid::FTS.search('üb') == [a] }
+  end
+
+#
+  testing 'that rare words float to the front of the results' do
+    a = A.create!(:content => 'dog')
+    b = A.create!(:content => 'dog dog')
+    c = A.create!(:content => 'dog dog dog')
+    d = A.create!(:content => 'dog dog dog cat')
+    
+    assert{ Mongoid::FTS.index(A) }
+    assert{ Mongoid::FTS.search('cat dog') == [d] }
+  end
+
+#
+  testing 'that word specificity affects the search' do
+    a = A.create!(:content => 'cat@dog.com')
+    b = A.create!(:content => 'dogs')
+    c = A.create!(:content => 'dog')
+    d = A.create!(:content => 'cats')
+    e = A.create!(:content => 'cat')
+
+    assert{ Mongoid::FTS.index(A) }
+
+    assert{ Mongoid::FTS.search('cat@dog.com') == [a] }
+    assert{ Mongoid::FTS.search('cat') == [e, d, a] }
+    assert{ Mongoid::FTS.search('dog') == [c, b, a] }
+  end
+
+#
+  testing 'that set intersection and union are supported via search' do
+    a = A.create!(:content => 'dog')
+    b = A.create!(:content => 'dog cat')
+    c = A.create!(:content => 'dog cat fish')
+
+    assert{ Mongoid::FTS.index(A) }
+
+    assert{ Mongoid::FTS.search(:any => 'dog').count == 3 }
+    assert{ Mongoid::FTS.search(:any => 'dog cat').count == 3 }
+    assert{ Mongoid::FTS.search(:any => 'dog cat fish').count == 3 }
+
+    assert{ Mongoid::FTS.search(:all => 'dog').count == 3 }
+    assert{ Mongoid::FTS.search(:all => 'dog cat').count == 2 }
+    assert{ Mongoid::FTS.search(:all => 'dog cat fish').count == 1 }
+  end
+
+#
+  testing 'that keywords are considered more highly than fulltext' do
+    a = A.create!(:title => 'the cats', :content => 'like to meow')
+    b = A.create!(:title => 'the dogs', :content => 'do not like to meow, they bark at cats')
+
+    assert{ Mongoid::FTS.search('cat').count == 2 }
+    assert{ Mongoid::FTS.search('cat').first == a }
+
+    assert{ Mongoid::FTS.search('meow').count == 2 }
+    assert{ Mongoid::FTS.search('bark').count == 1 }
+    assert{ Mongoid::FTS.search('dog').first == b }
+  end
+
+#
+  testing 'basic pagination' do
+     11.times{|i| A.create! :content => "cats #{ i }" }
+
+     assert{ A.search('cat').paginate(:page => 1, :size => 2).to_a.size == 2 }
+     assert{ A.search('cat').paginate(:page => 2, :size => 5).to_a.size == 5 }
+
+     accum = []
+
+     n = 6
+     size = 2
+     (1..n).each do |page|
+       list = assert{ A.search('cat').paginate(:page => page, :size => size) }
+       accum.push(*list)
+       assert{ list.num_pages == n }
+       assert{ list.total_pages == n }
+       assert{ list.current_page == page }
+     end
+
+     a = accum.map{|i| i}.sort_by{|m| m.content}
+     b = A.all.sort_by{|m| m.content}
+
+     assert{ a == b }
+  end
+
+protected
+
+  def new_klass(&block)
+    if Object.send(:const_defined?, :K)
+      Object.const_get(:K).destroy_all
+      Object.send(:remove_const, :K)
+    end
+
+    k = Class.new(A) do
+      self.default_collection_name = :ks
+      def self.name() 'K' end
+    end
+
+    Object.const_set(:K, k)
+
+    k.class_eval do
+      include ::Mongoid::FTS
+      class_eval(&block) if block
+    end
+
+    k
+  end
+
+  setup do
+    [A, B, C].map{|m| m.destroy_all}
+    Mongoid::FTS.destroy_all
+  end
+
+=begin
+  H = Mongoid::FTS
+  T = Mongoid::FTS::Token
+  I = Mongoid::FTS::Index
+
+  at_exit{ K.destroy_all if defined?(K) }
+=end
+end