mongoid_fulltext 0.6.1 → 0.7.0

data/lib/mongoid/indexable.rb

@@ -0,0 +1,13 @@
+# hook onto model index creation to create related FT indexes
+module Mongoid
+  module Indexable
+    module ClassMethods
+      alias_method :create_fulltext_indexes_hook, :create_indexes
+
+      def create_indexes
+        create_fulltext_indexes if respond_to?(:create_fulltext_indexes)
+        create_fulltext_indexes_hook
+      end
+    end
+  end
+end

data/lib/mongoid/indexes.rb

@@ -0,0 +1,13 @@
+# hook onto model index creation to create related FT indexes
+module Mongoid
+  module Indexes
+    module ClassMethods
+      alias_method :create_fulltext_indexes_hook, :create_indexes
+
+      def create_indexes
+        create_fulltext_indexes if respond_to?(:create_fulltext_indexes)
+        create_fulltext_indexes_hook
+      end
+    end
+  end
+end

data/lib/mongoid_fulltext.rb

@@ -1,341 +1 @@
-require 'mongoid_indexes'
-require 'unicode_utils'
-require 'cgi'
-
-module Mongoid::FullTextSearch
-  extend ActiveSupport::Concern
-
-  included do
-    cattr_accessor :mongoid_fulltext_config
-  end
-
-  class UnspecifiedIndexError < StandardError; end
-  class UnknownFilterQueryOperator < StandardError; end
-
-  module ClassMethods
-  
-    def fulltext_search_in(*args)
-      self.mongoid_fulltext_config = {} if self.mongoid_fulltext_config.nil?
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      if options.has_key?(:index_name)
-        index_name = options[:index_name]
-      else
-        index_name = 'mongoid_fulltext.index_%s_%s' % [self.name.downcase, self.mongoid_fulltext_config.count]
-      end
-
-      config = {
-        :alphabet => 'abcdefghijklmnopqrstuvwxyz0123456789 ',
-        :word_separators => "-_ \n\t",
-        :ngram_width => 3,
-        :max_ngrams_to_search => 6,
-        :apply_prefix_scoring_to_all_words => true,
-        :index_full_words => true,
-        :index_short_prefixes => false,
-        :max_candidate_set_size => 1000,
-        :remove_accents => true,
-        :reindex_immediately => true,
-        :stop_words => Hash[['i', 'a', 's', 't', 'me', 'my', 'we', 'he', 'it', 'am', 'is', 'be', 'do', 'an', 'if',
-                             'or', 'as', 'of', 'at', 'by', 'to', 'up', 'in', 'on', 'no', 'so', 'our', 'you', 'him',
-                             'his', 'she', 'her', 'its', 'who', 'are', 'was', 'has', 'had', 'did', 'the', 'and',
-                             'but', 'for', 'out', 'off', 'why', 'how', 'all', 'any', 'few', 'nor', 'not', 'own',
-                             'too', 'can', 'don', 'now', 'ours', 'your', 'hers', 'they', 'them', 'what', 'whom',
-                             'this', 'that', 'were', 'been', 'have', 'does', 'with', 'into', 'from', 'down', 'over',
-                             'then', 'once', 'here', 'when', 'both', 'each', 'more', 'most', 'some', 'such', 'only',
-                             'same', 'than', 'very', 'will', 'just', 'yours', 'their', 'which', 'these', 'those',
-                             'being', 'doing', 'until', 'while', 'about', 'after', 'above', 'below', 'under',
-                             'again', 'there', 'where', 'other', 'myself', 'itself', 'theirs', 'having', 'during',
-                             'before', 'should', 'himself', 'herself', 'because', 'against', 'between', 'through',
-                             'further', 'yourself', 'ourselves', 'yourselves', 'themselves'].map{ |x| [x,true] }]
-      }
-      
-      config.update(options)
-
-      args = [:to_s] if args.empty?
-      config[:ngram_fields] = args
-      config[:alphabet] = Hash[config[:alphabet].split('').map{ |ch| [ch,ch] }]
-      config[:word_separators] = Hash[config[:word_separators].split('').map{ |ch| [ch,ch] }]
-      self.mongoid_fulltext_config[index_name] = config
-      
-      before_save(:update_ngram_index) if config[:reindex_immediately]
-      before_destroy :remove_from_ngram_index
-    end
-    
-    def create_fulltext_indexes
-      return unless self.mongoid_fulltext_config
-      self.mongoid_fulltext_config.each_pair do |index_name, fulltext_config|
-        fulltext_search_ensure_indexes(index_name, fulltext_config)
-      end
-    end
-
-    def fulltext_search_ensure_indexes(index_name, config)
-      db = collection.database
-      coll = db[index_name]
-
-      # The order of filters matters when the same index is used from two or more collections.
-      filter_indexes = (config[:filters] || []).map do |key,value|
-        ["filter_values.#{key}", 1]
-      end.sort_by { |filter_index| filter_index[0] }
-      
-      index_definition = [['ngram', 1], ['score', -1]].concat(filter_indexes)
-
-      # Since the definition of the index could have changed, we'll clean up by
-      # removing any indexes that aren't on the exact.
-      correct_keys = index_definition.map{ |field_def| field_def[0] }
-      all_filter_keys = filter_indexes.map{ |field_def| field_def[0] }
-      coll.indexes.each do |idef|
-        keys = idef['key'].keys
-        next if !keys.member?('ngram')
-        all_filter_keys |= keys.find_all{ |key| key.starts_with?('filter_values.') }
-        if keys & correct_keys != correct_keys
-          Mongoid.logger.info "Dropping #{idef['name']} [#{keys & correct_keys} <=> #{correct_keys}]" if Mongoid.logger
-          coll.indexes.drop(idef['key'])
-        end
-      end
-
-      if all_filter_keys.length > filter_indexes.length
-        filter_indexes = all_filter_keys.map {|key| [key, 1] }.sort_by { |filter_index| filter_index[0] }
-        index_definition = [['ngram', 1], ['score', -1]].concat(filter_indexes)
-      end
-      
-      Mongoid.logger.info "Ensuring fts_index on #{coll.name}: #{index_definition}" if Mongoid.logger
-      coll.indexes.create(Hash[index_definition], { :name => 'fts_index' })
-
-      Mongoid.logger.info "Ensuring document_id index on #{coll.name}" if Mongoid.logger
-      coll.indexes.create('document_id' => 1) # to make removes fast
-    end
-    
-    def fulltext_search(query_string, options={})
-      max_results = options.has_key?(:max_results) ? options.delete(:max_results) : 10
-      return_scores = options.has_key?(:return_scores) ? options.delete(:return_scores) : false
-      if self.mongoid_fulltext_config.count > 1 and !options.has_key?(:index)
-        error_message = '%s is indexed by multiple full-text indexes. You must specify one by passing an :index_name parameter'
-        raise UnspecifiedIndexError, error_message % self.name, caller
-      end
-      index_name = options.has_key?(:index) ? options.delete(:index) : self.mongoid_fulltext_config.keys.first
-
-      # Options hash should only contain filters after this point
-
-      ngrams = all_ngrams(query_string, self.mongoid_fulltext_config[index_name])
-      return [] if ngrams.empty?
-
-      # For each ngram, construct the query we'll use to pull index documents and
-      # get a count of the number of index documents containing that n-gram
-      ordering = {'score' => -1}
-      limit = self.mongoid_fulltext_config[index_name][:max_candidate_set_size]
-      coll = collection.database[index_name]
-      cursors = ngrams.map do |ngram|
-        query = {'ngram' => ngram[0]}
-        query.update(map_query_filters options)
-        count = coll.find(query).count
-        {:ngram => ngram, :count => count, :query => query}
-      end.sort!{ |record1, record2| record1[:count] <=> record2[:count] }
-
-      # Using the queries we just constructed and the n-gram frequency counts we
-      # just computed, pull in about *:max_candidate_set_size* candidates by
-      # considering the n-grams in order of increasing frequency. When we've
-      # spent all *:max_candidate_set_size* candidates, pull the top-scoring
-      # *max_results* candidates for each remaining n-gram.
-      results_so_far = 0
-      candidates_list = cursors.map do |doc|
-        next if doc[:count] == 0
-        query_result = coll.find(doc[:query])
-        if results_so_far >= limit
-          query_result = query_result.sort(ordering).limit(max_results)
-        elsif doc[:count] > limit - results_so_far
-          query_result = query_result.sort(ordering).limit(limit - results_so_far)
-        end
-        results_so_far += doc[:count]
-        ngram_score = ngrams[doc[:ngram][0]]
-        Hash[query_result.map do |candidate|
-               [candidate['document_id'],
-                {:clazz => candidate['class'], :score => candidate['score'] * ngram_score}]
-             end]
-      end.compact
-      
-      # Finally, score all candidates by matching them up with other candidates that are
-      # associated with the same document. This is similar to how you might process a
-      # boolean AND query, except that with an AND query, you'd stop after considering
-      # the first candidate list and matching its candidates up with candidates from other
-      # lists, whereas here we want the search to be a little fuzzier so we'll run through
-      # all candidate lists, removing candidates as we match them up.
-      all_scores = []
-      while !candidates_list.empty?
-        candidates = candidates_list.pop
-        scores = candidates.map do |candidate_id, data|
-          {:id => candidate_id,
-           :clazz => data[:clazz],
-           :score => data[:score] + candidates_list.map{ |others| (others.delete(candidate_id) || {:score => 0})[:score] }.sum
-           }
-        end
-        all_scores.concat(scores)
-      end
-      all_scores.sort!{ |document1, document2| -document1[:score] <=> -document2[:score] }
-      instantiate_mapreduce_results(all_scores[0..max_results-1], { :return_scores => return_scores })
-    end
-    
-    def instantiate_mapreduce_result(result)
-      result[:clazz].constantize.find(result[:id])
-    end
-    
-    def instantiate_mapreduce_results(results, options)
-      if (options[:return_scores])
-        results.map { |result| [ instantiate_mapreduce_result(result), result[:score] ] }.find_all { |result| ! result[0].nil? }
-      else
-        results.map { |result| instantiate_mapreduce_result(result) }.compact
-      end
-    end
-
-    def all_ngrams(str, config, bound_number_returned = true)
-      return {} if str.nil?
-
-      if config[:remove_accents]
-        if defined?(UnicodeUtils)
-          str = UnicodeUtils.nfkd(str)
-        elsif defined?(DiacriticsFu)
-          str = DiacriticsFu::escape(str)
-        end
-      end
-
-      # Remove any characters that aren't in the alphabet and aren't word separators
-      filtered_str = str.mb_chars.downcase.to_s.split('').find_all{ |ch| config[:alphabet][ch] or config[:word_separators][ch] }.join('')
-      
-      # Figure out how many ngrams to extract from the string. If we can't afford to extract all ngrams,
-      # step over the string in evenly spaced strides to extract ngrams. For example, to extract 3 3-letter
-      # ngrams from 'abcdefghijk', we'd want to extract 'abc', 'efg', and 'ijk'.
-      if bound_number_returned
-        step_size = [((filtered_str.length - config[:ngram_width]).to_f / config[:max_ngrams_to_search]).ceil, 1].max
-      else
-        step_size = 1
-      end
-      
-      # Create an array of records of the form {:ngram => x, :score => y} for all ngrams that occur in the
-      # input string using the step size that we just computed. Let score(x,y) be the score of string x
-      # compared with string y - assigning scores to ngrams with the square root-based scoring function
-      # below and multiplying scores of matching ngrams together yields a score function that has the
-      # property that score(x,y) > score(x,z) for any string z containing y and score(x,y) > score(x,z)
-      # for any string z contained in y.
-      ngram_array = (0..filtered_str.length - config[:ngram_width]).step(step_size).map do |i|
-        if i == 0 or (config[:apply_prefix_scoring_to_all_words] and \
-                      config[:word_separators].has_key?(filtered_str[i-1].chr))
-          score = Math.sqrt(1 + 1.0/filtered_str.length)
-        else
-          score = Math.sqrt(2.0/filtered_str.length)
-        end
-        {:ngram => filtered_str[i..i+config[:ngram_width]-1], :score => score}
-      end
-
-      # If an ngram appears multiple times in the query string, keep the max score
-      ngram_array = ngram_array.group_by{ |h| h[:ngram] }.map{ |key, values| {:ngram => key, :score => values.map{ |v| v[:score] }.max} }
-      
-      if config[:index_short_prefixes] or config[:index_full_words]
-        split_regex_def = config[:word_separators].keys.map{ |k| Regexp.escape(k) }.join
-        split_regex = Regexp.compile("[#{split_regex_def}]")
-        all_words = filtered_str.split(split_regex)
-      end
-
-      # Add 'short prefix' records to the array: prefixes of the string that are length (ngram_width - 1)
-      if config[:index_short_prefixes]
-        prefixes_seen = {}
-        all_words.each do |word|
-          next if word.length < config[:ngram_width]-1
-          prefix = word[0...config[:ngram_width]-1]
-          if prefixes_seen[prefix].nil? and (config[:stop_words][word].nil? or word == filtered_str)
-            ngram_array << {:ngram => prefix, :score => 1 + 1.0/filtered_str.length}
-            prefixes_seen[prefix] = true
-          end
-        end
-      end
-
-      # Add records to the array of ngrams for each full word in the string that isn't a stop word
-      if config[:index_full_words]
-        full_words_seen = {}
-        all_words.each do |word|
-          if word.length > 1 and full_words_seen[word].nil? and (config[:stop_words][word].nil? or word == filtered_str)
-            ngram_array << {:ngram => word, :score => 1 + 1.0/filtered_str.length}
-            full_words_seen[word] = true
-          end
-        end
-      end
-
-      # If an ngram appears as any combination of full word, short prefix, and ngram, keep the sum of the two scores
-      Hash[ngram_array.group_by{ |h| h[:ngram] }.map{ |key, values| [key, values.map{ |v| v[:score] }.sum] }]
-    end
-    
-    def remove_from_ngram_index
-      self.mongoid_fulltext_config.each_pair do |index_name, fulltext_config|
-        coll = collection.database[index_name]
-        coll.find({'class' => self.name}).remove_all
-      end
-    end
-    
-    def update_ngram_index
-      self.all.each do |model|
-        model.update_ngram_index
-      end
-    end
-    
-    private
-    # Take a list of filters to be mapped so they can update the query
-    # used upon the fulltext search of the ngrams
-    def map_query_filters filters
-      Hash[filters.map {|key,value|
-        case value
-          when Hash then
-            if value.has_key? :any then format_query_filter('$in',key,value[:any])
-            elsif value.has_key? :all then format_query_filter('$all',key,value[:all])
-            else raise UnknownFilterQueryOperator, value.keys.join(","), caller end
-          else format_query_filter('$all',key,value)
-        end
-      }]
-    end
-    def format_query_filter operator, key, value
-      ['filter_values.%s' % key, {operator => [value].flatten}]
-    end
-  end
-
-  def update_ngram_index
-    self.mongoid_fulltext_config.each_pair do |index_name, fulltext_config|
-      if condition = fulltext_config[:update_if]
-        case condition
-        when Symbol;  next unless self.send condition
-        when String;  next unless instance_eval condition
-        when Proc;    next unless condition.call self
-        else;         next
-        end
-      end
-
-      # remove existing ngrams from external index
-      coll = collection.database[index_name.to_sym]
-      coll.find({'document_id' => self._id}).remove_all
-      # extract ngrams from fields
-      field_values = fulltext_config[:ngram_fields].map { |field| self.send(field) }
-      ngrams = field_values.inject({}) { |accum, item| accum.update(self.class.all_ngrams(item, fulltext_config, false))}
-      return if ngrams.empty?
-      # apply filters, if necessary
-      filter_values = nil
-      if fulltext_config.has_key?(:filters)
-        filter_values = Hash[fulltext_config[:filters].map do |key,value|
-          begin
-            [key, value.call(self)]
-          rescue
-            # Suppress any exceptions caused by filters
-          end
-        end.compact]
-      end
-      # insert new ngrams in external index
-      ngrams.each_pair do |ngram, score|
-        index_document = {'ngram' => ngram, 'document_id' => self._id, 'score' => score, 'class' => self.class.name}
-        index_document['filter_values'] = filter_values if fulltext_config.has_key?(:filters)
-        coll.insert(index_document)
-      end
-    end
-  end
-
-  def remove_from_ngram_index
-    self.mongoid_fulltext_config.each_pair do |index_name, fulltext_config|
-      coll = collection.database[index_name]
-      coll.find({'document_id' => self._id}).remove_all
-    end
-  end
-
-end
+require 'mongoid/full_text_search'

data/mongoid_fulltext.gemspec

@@ -1,85 +1,19 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
-# -*- encoding: utf-8 -*-
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'mongoid/full_text_search/version'
 
 Gem::Specification.new do |s|
-  s.name = "mongoid_fulltext"
-  s.version = "0.6.1"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Aaron Windsor"]
-  s.date = "2013-04-03"
-  s.description = "Full-text search for the Mongoid ORM, using n-grams extracted from text"
-  s.email = "aaron.windsor@gmail.com"
-  s.extra_rdoc_files = [
-    "LICENSE",
-    "README.md"
-  ]
-  s.files = [
-    ".document",
-    ".rspec",
-    ".travis.yml",
-    "CHANGELOG.md",
-    "Gemfile",
-    "LICENSE",
-    "README.md",
-    "Rakefile",
-    "VERSION",
-    "lib/mongoid_fulltext.rb",
-    "lib/mongoid_indexes.rb",
-    "mongoid_fulltext.gemspec",
-    "spec/config/mongoid.yml",
-    "spec/models/accentless_artwork.rb",
-    "spec/models/advanced_artwork.rb",
-    "spec/models/basic_artwork.rb",
-    "spec/models/delayed_artwork.rb",
-    "spec/models/external_artist.rb",
-    "spec/models/external_artwork.rb",
-    "spec/models/external_artwork_no_fields_supplied.rb",
-    "spec/models/filtered_artist.rb",
-    "spec/models/filtered_artwork.rb",
-    "spec/models/filtered_other.rb",
-    "spec/models/gallery/basic_artwork.rb",
-    "spec/models/hidden_dragon.rb",
-    "spec/models/multi_external_artwork.rb",
-    "spec/models/multi_field_artist.rb",
-    "spec/models/multi_field_artwork.rb",
-    "spec/models/partitioned_artist.rb",
-    "spec/models/russian_artwork.rb",
-    "spec/models/short_prefixes_artwork.rb",
-    "spec/models/stopwords_artwork.rb",
-    "spec/mongoid/fulltext_spec.rb",
-    "spec/spec_helper.rb"
-  ]
-  s.homepage = "http://github.com/aaw/mongoid_fulltext"
-  s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.25"
-  s.summary = "Full-text search for the Mongoid ORM"
-
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<mongoid>, ["~> 3.0"])
-      s.add_runtime_dependency(%q<unicode_utils>, ["~> 1.0.0"])
-      s.add_development_dependency(%q<bundler>, [">= 0"])
-      s.add_development_dependency(%q<rspec>, ["~> 2.10.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.8.3"])
-    else
-      s.add_dependency(%q<mongoid>, ["~> 3.0"])
-      s.add_dependency(%q<unicode_utils>, ["~> 1.0.0"])
-      s.add_dependency(%q<bundler>, [">= 0"])
-      s.add_dependency(%q<rspec>, ["~> 2.10.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
-    end
-  else
-    s.add_dependency(%q<mongoid>, ["~> 3.0"])
-    s.add_dependency(%q<unicode_utils>, ["~> 1.0.0"])
-    s.add_dependency(%q<bundler>, [">= 0"])
-    s.add_dependency(%q<rspec>, ["~> 2.10.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
-  end
+  s.name = 'mongoid_fulltext'
+  s.version = Mongoid::FullTextSearch::VERSION
+  s.authors = ['Aaron Windsor']
+  s.email = 'aaron.windsor@gmail.com'
+  s.platform = Gem::Platform::RUBY
+  s.required_rubygems_version = '>= 1.3.6'
+  s.files = `git ls-files`.split("\n")
+  s.require_paths = ['lib']
+  s.homepage = 'https://github.com/artsy/mongoid_fulltext'
+  s.licenses = ['MIT']
+  s.summary = 'Full-text search for the Mongoid ORM, using n-grams extracted from text.'
+  s.add_dependency 'mongoid', '>= 3.0'
+  s.add_dependency 'mongoid-compatibility'
+  s.add_dependency 'unicode_utils'
 end
-