mongoid_fulltext 0.6.1 → 0.7.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MGNlMDBlNzg5YmVkODU3ZTIyYTFiNGI5N2M2ZTRkYTdmODkwNTA5OQ==
+  data.tar.gz: !binary |-
+    OWFjYzllZThlODIyOGQzMzJkN2MwYjc4Y2U3Y2I4ODBlMDUwZDA4Yw==
+SHA512:
+  metadata.gz: !binary |-
+    ODViNTM3ZWYyMDdiYjk4NjBkNzBlZGEzNzM2YTIxNWYwZmI3ZjBmMzQ0ZTAz
+    MzUwODY1MWNiNGFmNWIwYWVkMTRkMTc1YjcxM2RjYzMwZjJiOGVlOTEyZjcy
+    OGZiYmQ0YmVkMGJhMWIwZjg5YjFkNDc1M2ZlM2NiZmU3MThkMWE=
+  data.tar.gz: !binary |-
+    MWMyNmYzYmI4MzUxYjYwOWZmM2RmMDgxYzUxOTg2Zjg2NDFhMTRhOGFlYjU2
+    NmEyNmU0ZWRhZjRiZmFiNGZhNDY3NTc1YThlODRjNjNkZmI0YThhN2RmZDdi
+    OTNmNGE5MzM4ZjQ1MTk5YTM0OGNhZjcxZjk0YTdkZmEwNDAxNDE=

data/.gitignore

@@ -0,0 +1,47 @@
+# rcov generated
+coverage
+
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+
+# jeweler generated
+pkg
+
+# RVM
+.rvmrc
+
+# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+#
+# * Create a file at ~/.gitignore
+# * Include files you want ignored
+# * Run: git config --global core.excludesfile ~/.gitignore
+#
+# After doing this, these files will be ignored in all your git projects,
+# saving you from having to 'pollute' every project you touch with them
+#
+# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
+#
+# For MacOS:
+
+.DS_Store
+
+# For TextMate
+*.tmproj
+tmtags
+
+# For emacs:
+*~
+\#*
+.\#*
+
+# For vim:
+*.swp
+
+Gemfile.lock

data/.rspec

@@ -1,2 +1,2 @@
 --color
-
+--format documentation

data/.rubocop.yml

@@ -0,0 +1,6 @@
+AllCops:
+  Exclude:
+    - vendor/**/*
+    - bin/**/*
+
+inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -0,0 +1,101 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2015-09-18 15:56:53 -0400 using RuboCop version 0.34.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Configuration parameters: AllowSafeAssignment.
+Lint/AssignmentInCondition:
+  Exclude:
+    - 'lib/mongoid/full_text_search.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AlignWith, SupportedStyles, AutoCorrect.
+Lint/EndAlignment:
+  Enabled: false
+
+# Offense count: 2
+Lint/HandleExceptions:
+  Exclude:
+    - 'lib/mongoid/full_text_search.rb'
+    - 'spec/mongoid/full_text_search_spec.rb'
+
+# Offense count: 1
+Lint/NonLocalExitFromIterator:
+  Exclude:
+    - 'lib/mongoid/full_text_search.rb'
+
+# Offense count: 4
+Lint/UselessAssignment:
+  Exclude:
+    - 'spec/mongoid/full_text_search_spec.rb'
+
+# Offense count: 5
+Metrics/AbcSize:
+  Max: 106
+
+# Offense count: 4
+Metrics/CyclomaticComplexity:
+  Max: 22
+
+# Offense count: 262
+# Configuration parameters: AllowURI, URISchemes.
+Metrics/LineLength:
+  Max: 174
+
+# Offense count: 5
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 50
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/ModuleLength:
+  Max: 224
+
+# Offense count: 4
+Metrics/PerceivedComplexity:
+  Max: 25
+
+# Offense count: 1
+Style/AsciiComments:
+  Exclude:
+    - 'spec/mongoid/full_text_search_spec.rb'
+
+# Offense count: 1
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/ClassAndModuleChildren:
+  Exclude:
+    - 'lib/mongoid/full_text_search.rb'
+
+# Offense count: 1
+Style/ConstantName:
+  Exclude:
+    - 'spec/models/russian_artwork.rb'
+
+# Offense count: 22
+Style/Documentation:
+  Enabled: false
+
+# Offense count: 3
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/FormatString:
+  Exclude:
+    - 'lib/mongoid/full_text_search.rb'
+    - 'spec/models/external_artwork_no_fields_supplied.rb'
+
+# Offense count: 2
+Style/MultilineBlockChain:
+  Exclude:
+    - 'lib/mongoid/full_text_search.rb'
+
+# Offense count: 4
+# Configuration parameters: Methods.
+Style/SingleLineBlockParams:
+  Exclude:
+    - 'lib/mongoid/full_text_search.rb'
+    - 'spec/mongoid/full_text_search_spec.rb'

data/.travis.yml

@@ -1,7 +1,15 @@
 rvm:
+  - 2.2
+  - 2.1
+  - 2.0
   - 1.9.3
-  - ruby-head
+  - rbx-2.2.10
+  - jruby-19mode
+
 env:
-  - MONGOID=3.0.0
-  - MONGOID=3.1.0
+  - MONGOID_VERSION=3.0.0
+  - MONGOID_VERSION=3.1.0
+  - MONGOID_VERSION=4.0
+  - MONGOID_VERSION=5.0
+
 services: mongodb

data/CHANGELOG.md

@@ -1,10 +1,17 @@
+0.7.0 (9/18/2015)
+-----------------
+
+* Compatible with Mongoid 4 and 5 - [@dblock](https://github.com/dblock).
+* Rewritten .gemspec, removed Jeweler - [@dblock](https://github.com/dblock).
+* Added RuboCop - [@dblock](https://github.com/dblock).
+
 0.6.1 (4/3/2013)
---------------------
+----------------
 
 * [#6](https://github.com/artsy/mongoid_fulltext/pull/6): Upgrade to Mongoid ~> 3.0 - [@simi](https://github.com/simi).
 
 0.6.0 (7/16/2012)
---------------------
+-----------------
 
 * [#2](https://github.com/artsy/mongoid_fulltext/pull/2): Upgrade to Mongoid 3.0 - [@volmer](https://github.com/volmer).
 * [#1](https://github.com/artsy/mongoid_fulltext/pull/1): Fix: downcase destroys non-latin strings - [@netoneko](https://github.com/netoneko).

data/Gemfile

@@ -1,15 +1,25 @@
-source "http://rubygems.org"
+source 'http://rubygems.org'
 
-if ENV['TRAVIS']
-  gem "mongoid", "~> #{ENV['MONGOID']}"
+case version = ENV['MONGOID_VERSION'] || '5'
+when /5/
+  gem 'mongoid', '~> 5.0'
+when /4/
+  gem 'mongoid', '~> 4.0'
+when /3.1.0/
+  gem 'mongoid', '~> 3.1.0'
+when /3.0.0/
+  gem 'mongoid', '~> 3.0.0'
 else
-  gem "mongoid", "~> 3.0"
+  gem 'mongoid', version
 end
 
-gem "unicode_utils", "~> 1.0.0"
+gemspec
 
-group :development, :test do
-  gem "bundler"
-  gem "rspec", "~> 2.10.0"
-  gem "jeweler", "~> 1.8.3"
+group :test do
+  gem 'rspec'
+end
+
+group :development do
+  gem 'rake'
+  gem 'rubocop', '0.34.1'
 end

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2011-2012 by Artsy, Inc.
+Copyright (c) 2011-2015 by Artsy, Inc. & Contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,19 +1,22 @@
-Mongoid Fulltext Search [![Build Status](https://secure.travis-ci.org/artsy/mongoid_fulltext.png)](http://travis-ci.org/artsy/mongoid_fulltext)
+Mongoid Fulltext Search
 =======================
 
+[![Build Status](https://secure.travis-ci.org/artsy/mongoid_fulltext.svg)](http://travis-ci.org/artsy/mongoid_fulltext)
+[![Gem Version](https://badge.fury.io/rb/mongoid_fulltext.svg)](http://badge.fury.io/rb/mongoid_fulltext)
+
 Full-text search using n-gram matching for the Mongoid ODM. Tested on MongoDB 1.6 and above, but
 probably works on earlier versions as well.
 
-MongoDB currently has no native full-text search capabilities, so this gem is a good fit for cases
-where you want something a little less than a full-blown indexing service like Solr. mongoid_fulltext
+MongoDB introduced full-text search capabilities in v2.4, so this gem is a good fit for cases
+where you want something a little less than a full-blown indexing service. The mongoid_fulltext gem
 lets you do a fuzzy string search across relatively short strings, which makes it good for populating
 autocomplete boxes based on the display names of your Rails models but not appropriate for, say,
 indexing hundreds of thousands of HTML documents.
 
 Install
---------------
+-------
 
-Version 0.6.0 or newer of this gem requires Ruby 1.9.3 and Mongoid 3.0. 
+Version 0.6.1 or newer of this gem requires Ruby 1.9.3 or newer and works with Mongoid 3, 4 and 5.
 Use version 0.5.x for Mongoid 2.4.x and Ruby 1.8.7, 1.9.2 or 1.9.3.
 
 For Ruby 1.8.7 and/or Mongoid 2.x use [mongoid_fulltext 0.5.x](https://github.com/artsy/mongoid_fulltext/tree/0.5-stable).
@@ -22,8 +25,8 @@ For Ruby 1.8.7 and/or Mongoid 2.x use [mongoid_fulltext 0.5.x](https://github.co
 gem 'mongoid_fulltext'
 ```
 
-Some examples:
---------------
+Examples
+--------
 
 Suppose you have an `Artist` model and want to index each artist's name:
 
@@ -395,7 +398,7 @@ Fork the project. Make your feature addition or bug fix with tests. Send a pull
 Copyright and License
 ---------------------
 
-MIT License, see [LICENSE](https://github.com/aaw/mongoid_fulltext/blob/master/LICENSE) for details.
+MIT License, see [LICENSE](LICENSE) for details.
 
-(c) 2011-2012 [Art.sy Inc.](http://artsy.github.com)
+(c) 2011-2015 [Artsy Inc.](http://artsy.github.io)
 

data/Rakefile

@@ -1,36 +1,16 @@
 require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'rake'
-require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
 
-require 'jeweler'
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
-  gem.name = "mongoid_fulltext"
-  gem.homepage = "http://github.com/aaw/mongoid_fulltext"
-  gem.license = "MIT"
-  gem.summary = %Q{Full-text search for the Mongoid ORM}
-  gem.description = %Q{Full-text search for the Mongoid ORM, using n-grams extracted from text}
-  gem.email = "aaron.windsor@gmail.com"
-  gem.authors = ["Aaron Windsor"]
-  # Include your dependencies below. Runtime dependencies are required when using your gem,
-  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
-  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
-  #  gem.add_development_dependency 'rspec', '> 1.2.3'
-end
-Jeweler::RubygemsDotOrgTasks.new
+Bundler.setup :default, :development
 
+require 'rspec/core'
+require 'rspec/core/rake_task'
 
-desc "Run all tests"
 RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = "spec/**/*_spec.rb"
+  spec.pattern = FileList['spec/**/*_spec.rb']
 end
 
-task :default => :spec
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new(:rubocop)
+
+task default: [:rubocop, :spec]

data/lib/mongoid/full_text_search/version.rb

@@ -0,0 +1,5 @@
+module Mongoid
+  module FullTextSearch
+    VERSION = '0.7.0'
+  end
+end

data/lib/mongoid/full_text_search.rb

@@ -0,0 +1,372 @@
+require 'mongoid'
+require 'mongoid/compatibility'
+if Mongoid::Compatibility::Version.mongoid3?
+  require_relative 'indexes'
+else
+  require_relative 'indexable'
+end
+require 'unicode_utils'
+require 'cgi'
+
+module Mongoid::FullTextSearch
+  extend ActiveSupport::Concern
+
+  included do
+    cattr_accessor :mongoid_fulltext_config
+  end
+
+  class UnspecifiedIndexError < StandardError; end
+  class UnknownFilterQueryOperator < StandardError; end
+
+  module ClassMethods
+    def fulltext_search_in(*args)
+      self.mongoid_fulltext_config = {} if mongoid_fulltext_config.nil?
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      if options.key?(:index_name)
+        index_name = options[:index_name]
+      else
+        index_name = 'mongoid_fulltext.index_%s_%s' % [name.downcase, mongoid_fulltext_config.count]
+      end
+
+      config = {
+        alphabet: 'abcdefghijklmnopqrstuvwxyz0123456789 ',
+        word_separators: "-_ \n\t",
+        ngram_width: 3,
+        max_ngrams_to_search: 6,
+        apply_prefix_scoring_to_all_words: true,
+        index_full_words: true,
+        index_short_prefixes: false,
+        max_candidate_set_size: 1000,
+        remove_accents: true,
+        reindex_immediately: true,
+        stop_words: Hash[%w(i a s t me my we he it am is be do an if
+                            or as of at by to up in on no so our you him
+                            his she her its who are was has had did the and
+                            but for out off why how all any few nor not own
+                            too can don now ours your hers they them what whom
+                            this that were been have does with into from down over
+                            then once here when both each more most some such only
+                            same than very will just yours their which these those
+                            being doing until while about after above below under
+                            again there where other myself itself theirs having during
+                            before should himself herself because against between through
+                            further yourself ourselves yourselves themselves).map { |x| [x, true] }]
+      }
+
+      config.update(options)
+
+      args = [:to_s] if args.empty?
+      config[:ngram_fields] = args
+      config[:alphabet] = Hash[config[:alphabet].split('').map { |ch| [ch, ch] }]
+      config[:word_separators] = Hash[config[:word_separators].split('').map { |ch| [ch, ch] }]
+      mongoid_fulltext_config[index_name] = config
+
+      before_save(:update_ngram_index) if config[:reindex_immediately]
+      before_destroy :remove_from_ngram_index
+    end
+
+    def create_fulltext_indexes
+      return unless mongoid_fulltext_config
+      mongoid_fulltext_config.each_pair do |index_name, fulltext_config|
+        fulltext_search_ensure_indexes(index_name, fulltext_config)
+      end
+    end
+
+    def fulltext_search_ensure_indexes(index_name, config)
+      db = collection.database
+      coll = db[index_name]
+
+      # The order of filters matters when the same index is used from two or more collections.
+      filter_indexes = (config[:filters] || []).map do |key, _value|
+        ["filter_values.#{key}", 1]
+      end.sort_by { |filter_index| filter_index[0] }
+
+      index_definition = [['ngram', 1], ['score', -1]].concat(filter_indexes)
+
+      # Since the definition of the index could have changed, we'll clean up by
+      # removing any indexes that aren't on the exact.
+      correct_keys = index_definition.map { |field_def| field_def[0] }
+      all_filter_keys = filter_indexes.map { |field_def| field_def[0] }
+      coll.indexes.each do |idef|
+        keys = idef['key'].keys
+        next unless keys.member?('ngram')
+        all_filter_keys |= keys.find_all { |key| key.starts_with?('filter_values.') }
+        next unless keys & correct_keys != correct_keys
+        Mongoid.logger.info "Dropping #{idef['name']} [#{keys & correct_keys} <=> #{correct_keys}]" if Mongoid.logger
+        if Mongoid::Compatibility::Version.mongoid5?
+          coll.indexes.drop_one(idef['key'])
+        else
+          coll.indexes.drop(idef['key'])
+        end
+      end
+
+      if all_filter_keys.length > filter_indexes.length
+        filter_indexes = all_filter_keys.map { |key| [key, 1] }.sort_by { |filter_index| filter_index[0] }
+        index_definition = [['ngram', 1], ['score', -1]].concat(filter_indexes)
+      end
+
+      Mongoid.logger.info "Ensuring fts_index on #{coll.name}: #{index_definition}" if Mongoid.logger
+      if Mongoid::Compatibility::Version.mongoid5?
+        coll.indexes.create_one(Hash[index_definition], name: 'fts_index')
+      else
+        coll.indexes.create(Hash[index_definition], name: 'fts_index')
+      end
+
+      Mongoid.logger.info "Ensuring document_id index on #{coll.name}" if Mongoid.logger
+      if Mongoid::Compatibility::Version.mongoid5?
+        coll.indexes.create_one('document_id' => 1) # to make removes fast
+      else
+        coll.indexes.create('document_id' => 1) # to make removes fast
+      end
+    end
+
+    def fulltext_search(query_string, options = {})
+      max_results = options.key?(:max_results) ? options.delete(:max_results) : 10
+      return_scores = options.key?(:return_scores) ? options.delete(:return_scores) : false
+      if mongoid_fulltext_config.count > 1 && !options.key?(:index)
+        error_message = '%s is indexed by multiple full-text indexes. You must specify one by passing an :index_name parameter'
+        fail UnspecifiedIndexError, error_message % name, caller
+      end
+      index_name = options.key?(:index) ? options.delete(:index) : mongoid_fulltext_config.keys.first
+
+      # Options hash should only contain filters after this point
+
+      ngrams = all_ngrams(query_string, mongoid_fulltext_config[index_name])
+      return [] if ngrams.empty?
+
+      # For each ngram, construct the query we'll use to pull index documents and
+      # get a count of the number of index documents containing that n-gram
+      ordering = { 'score' => -1 }
+      limit = mongoid_fulltext_config[index_name][:max_candidate_set_size]
+      coll = collection.database[index_name]
+      cursors = ngrams.map do |ngram|
+        query = { 'ngram' => ngram[0] }
+        query.update(map_query_filters options)
+        count = coll.find(query).count
+        { ngram: ngram, count: count, query: query }
+      end.sort! { |record1, record2| record1[:count] <=> record2[:count] }
+
+      # Using the queries we just constructed and the n-gram frequency counts we
+      # just computed, pull in about *:max_candidate_set_size* candidates by
+      # considering the n-grams in order of increasing frequency. When we've
+      # spent all *:max_candidate_set_size* candidates, pull the top-scoring
+      # *max_results* candidates for each remaining n-gram.
+      results_so_far = 0
+      candidates_list = cursors.map do |doc|
+        next if doc[:count] == 0
+        query_result = coll.find(doc[:query])
+        if results_so_far >= limit
+          query_result = query_result.sort(ordering).limit(max_results)
+        elsif doc[:count] > limit - results_so_far
+          query_result = query_result.sort(ordering).limit(limit - results_so_far)
+        end
+        results_so_far += doc[:count]
+        ngram_score = ngrams[doc[:ngram][0]]
+        Hash[query_result.map do |candidate|
+          [candidate['document_id'],
+           { clazz: candidate['class'], score: candidate['score'] * ngram_score }]
+        end]
+      end.compact
+
+      # Finally, score all candidates by matching them up with other candidates that are
+      # associated with the same document. This is similar to how you might process a
+      # boolean AND query, except that with an AND query, you'd stop after considering
+      # the first candidate list and matching its candidates up with candidates from other
+      # lists, whereas here we want the search to be a little fuzzier so we'll run through
+      # all candidate lists, removing candidates as we match them up.
+      all_scores = []
+      until candidates_list.empty?
+        candidates = candidates_list.pop
+        scores = candidates.map do |candidate_id, data|
+          { id: candidate_id,
+            clazz: data[:clazz],
+            score: data[:score] + candidates_list.map { |others| (others.delete(candidate_id) || { score: 0 })[:score] }.sum
+           }
+        end
+        all_scores.concat(scores)
+      end
+      all_scores.sort! { |document1, document2| -document1[:score] <=> -document2[:score] }
+      instantiate_mapreduce_results(all_scores[0..max_results - 1], return_scores: return_scores)
+    end
+
+    def instantiate_mapreduce_result(result)
+      result[:clazz].constantize.find(result[:id])
+    end
+
+    def instantiate_mapreduce_results(results, options)
+      if options[:return_scores]
+        results.map { |result| [instantiate_mapreduce_result(result), result[:score]] }.find_all { |result| !result[0].nil? }
+      else
+        results.map { |result| instantiate_mapreduce_result(result) }.compact
+      end
+    end
+
+    def all_ngrams(str, config, bound_number_returned = true)
+      return {} if str.nil?
+
+      if config[:remove_accents]
+        if defined?(UnicodeUtils)
+          str = UnicodeUtils.nfkd(str)
+        elsif defined?(DiacriticsFu)
+          str = DiacriticsFu.escape(str)
+        end
+      end
+
+      # Remove any characters that aren't in the alphabet and aren't word separators
+      filtered_str = str.mb_chars.downcase.to_s.split('').find_all { |ch| config[:alphabet][ch] || config[:word_separators][ch] }.join('')
+
+      # Figure out how many ngrams to extract from the string. If we can't afford to extract all ngrams,
+      # step over the string in evenly spaced strides to extract ngrams. For example, to extract 3 3-letter
+      # ngrams from 'abcdefghijk', we'd want to extract 'abc', 'efg', and 'ijk'.
+      if bound_number_returned
+        step_size = [((filtered_str.length - config[:ngram_width]).to_f / config[:max_ngrams_to_search]).ceil, 1].max
+      else
+        step_size = 1
+      end
+
+      # Create an array of records of the form {:ngram => x, :score => y} for all ngrams that occur in the
+      # input string using the step size that we just computed. Let score(x,y) be the score of string x
+      # compared with string y - assigning scores to ngrams with the square root-based scoring function
+      # below and multiplying scores of matching ngrams together yields a score function that has the
+      # property that score(x,y) > score(x,z) for any string z containing y and score(x,y) > score(x,z)
+      # for any string z contained in y.
+      ngram_array = (0..filtered_str.length - config[:ngram_width]).step(step_size).map do |i|
+        if i == 0 || (config[:apply_prefix_scoring_to_all_words] && \
+                      config[:word_separators].key?(filtered_str[i - 1].chr))
+          score = Math.sqrt(1 + 1.0 / filtered_str.length)
+        else
+          score = Math.sqrt(2.0 / filtered_str.length)
+        end
+        { ngram: filtered_str[i..i + config[:ngram_width] - 1], score: score }
+      end
+
+      # If an ngram appears multiple times in the query string, keep the max score
+      ngram_array = ngram_array.group_by { |h| h[:ngram] }.map { |key, values| { ngram: key, score: values.map { |v| v[:score] }.max } }
+
+      if config[:index_short_prefixes] || config[:index_full_words]
+        split_regex_def = config[:word_separators].keys.map { |k| Regexp.escape(k) }.join
+        split_regex = Regexp.compile("[#{split_regex_def}]")
+        all_words = filtered_str.split(split_regex)
+      end
+
+      # Add 'short prefix' records to the array: prefixes of the string that are length (ngram_width - 1)
+      if config[:index_short_prefixes]
+        prefixes_seen = {}
+        all_words.each do |word|
+          next if word.length < config[:ngram_width] - 1
+          prefix = word[0...config[:ngram_width] - 1]
+          if prefixes_seen[prefix].nil? && (config[:stop_words][word].nil? || word == filtered_str)
+            ngram_array << { ngram: prefix, score: 1 + 1.0 / filtered_str.length }
+            prefixes_seen[prefix] = true
+          end
+        end
+      end
+
+      # Add records to the array of ngrams for each full word in the string that isn't a stop word
+      if config[:index_full_words]
+        full_words_seen = {}
+        all_words.each do |word|
+          if word.length > 1 && full_words_seen[word].nil? && (config[:stop_words][word].nil? || word == filtered_str)
+            ngram_array << { ngram: word, score: 1 + 1.0 / filtered_str.length }
+            full_words_seen[word] = true
+          end
+        end
+      end
+
+      # If an ngram appears as any combination of full word, short prefix, and ngram, keep the sum of the two scores
+      Hash[ngram_array.group_by { |h| h[:ngram] }.map { |key, values| [key, values.map { |v| v[:score] }.sum] }]
+    end
+
+    def remove_from_ngram_index
+      mongoid_fulltext_config.each_pair do |index_name, _fulltext_config|
+        coll = collection.database[index_name]
+        if Mongoid::Compatibility::Version.mongoid5?
+          coll.find('class' => name).delete_many
+        else
+          coll.find('class' => name).remove_all
+        end
+      end
+    end
+
+    def update_ngram_index
+      all.each(&:update_ngram_index)
+    end
+
+    private
+
+    # Take a list of filters to be mapped so they can update the query
+    # used upon the fulltext search of the ngrams
+    def map_query_filters(filters)
+      Hash[filters.map do|key, value|
+        case value
+        when Hash then
+          if value.key? :any then format_query_filter('$in', key, value[:any])
+          elsif value.key? :all then format_query_filter('$all', key, value[:all])
+          else fail UnknownFilterQueryOperator, value.keys.join(','), caller end
+        else format_query_filter('$all', key, value)
+        end
+      end]
+    end
+
+    def format_query_filter(operator, key, value)
+      ['filter_values.%s' % key, { operator => [value].flatten }]
+    end
+  end
+
+  def update_ngram_index
+    mongoid_fulltext_config.each_pair do |index_name, fulltext_config|
+      if condition = fulltext_config[:update_if]
+        case condition
+        when Symbol then  next unless send condition
+        when String then  next unless instance_eval condition
+        when Proc then    next unless condition.call self
+        else; next
+        end
+      end
+
+      # remove existing ngrams from external index
+      coll = collection.database[index_name.to_sym]
+      if Mongoid::Compatibility::Version.mongoid5?
+        coll.find('document_id' => _id).delete_many
+      else
+        coll.find('document_id' => _id).remove_all
+      end
+      # extract ngrams from fields
+      field_values = fulltext_config[:ngram_fields].map { |field| send(field) }
+      ngrams = field_values.inject({}) { |accum, item| accum.update(self.class.all_ngrams(item, fulltext_config, false)) }
+      return if ngrams.empty?
+      # apply filters, if necessary
+      filter_values = nil
+      if fulltext_config.key?(:filters)
+        filter_values = Hash[fulltext_config[:filters].map do |key, value|
+          begin
+            [key, value.call(self)]
+          rescue
+            # Suppress any exceptions caused by filters
+          end
+        end.compact]
+      end
+      # insert new ngrams in external index
+      ngrams.each_pair do |ngram, score|
+        index_document = { 'ngram' => ngram, 'document_id' => _id, 'score' => score, 'class' => self.class.name }
+        index_document['filter_values'] = filter_values if fulltext_config.key?(:filters)
+        if Mongoid::Compatibility::Version.mongoid5?
+          coll.insert_one(index_document)
+        else
+          coll.insert(index_document)
+        end
+      end
+    end
+  end
+
+  def remove_from_ngram_index
+    mongoid_fulltext_config.each_pair do |index_name, _fulltext_config|
+      coll = collection.database[index_name]
+      if Mongoid::Compatibility::Version.mongoid5?
+        coll.find('document_id' => _id).delete_many
+      else
+        coll.find('document_id' => _id).remove_all
+      end
+    end
+  end
+end