mongoid-haystack 1.0.0

data/lib/app/models/mongoid/haystack/sequence.rb

@@ -0,0 +1 @@
+Mongoid::Haystack::Sequence

data/lib/app/models/mongoid/haystack/token.rb

@@ -0,0 +1 @@
+Mongoid::Haystack::Token

data/lib/mongoid-haystack.rb

@@ -0,0 +1,79 @@
+##
+#
+  module Mongoid
+    module Haystack
+      const_set :Version, '1.0.0'
+
+      class << Haystack
+        def version
+          const_get :Version
+        end
+
+        def dependencies
+          {
+            'mongoid'     => [ 'mongoid'     , '~> 3.0' ] ,
+            'map'         => [ 'map'         , '~> 6.2' ] ,
+            'fattr'       => [ 'fattr'       , '~> 2.2' ] ,
+          }
+        end
+
+        def libdir(*args, &block)
+          @libdir ||= File.expand_path(__FILE__).sub(/\.rb$/,'')
+          args.empty? ? @libdir : File.join(@libdir, *args)
+        ensure
+          if block
+            begin
+              $LOAD_PATH.unshift(@libdir)
+              block.call()
+            ensure
+              $LOAD_PATH.shift()
+            end
+          end
+        end
+
+        def load(*libs)
+          libs = libs.join(' ').scan(/[^\s+]+/)
+          libdir{ libs.each{|lib| Kernel.load(lib) } }
+        end
+      end
+
+      begin
+        require 'rubygems'
+      rescue LoadError
+        nil
+      end
+
+      if defined?(gem)
+        dependencies.each do |lib, dependency|
+          gem(*dependency)
+          require(lib)
+        end
+      end
+
+      begin
+        require 'pry'
+      rescue LoadError
+        nil
+      end
+
+      begin
+        require 'fast_stemmer'
+      rescue LoadError
+        begin
+          require 'stemmer'
+        rescue LoadError
+          abort("mongoid-haystack requires either the 'fast-stemmer' or 'ruby-stemmer' gems")
+        end
+      end
+
+      load Haystack.libdir('stemming.rb')
+      load Haystack.libdir('util.rb')
+      load Haystack.libdir('count.rb')
+      load Haystack.libdir('sequence.rb')
+      load Haystack.libdir('token.rb')
+      load Haystack.libdir('index.rb')
+      load Haystack.libdir('search.rb')
+
+      extend Haystack
+    end
+  end

data/lib/mongoid-haystack/count.rb

@@ -0,0 +1,28 @@
+module Mongoid
+  module Haystack
+    class Count
+      include Mongoid::Document
+
+      field(:name, :type => String)
+      field(:value, :type => Integer, :default => 0)
+
+      index({:name => 1}, {:unique => true})
+      index({:value => 1})
+
+      def Count.for(name)
+        Haystack.find_or_create(
+          ->{ where(:name => name.to_s).first },
+          ->{ create!(:name => name.to_s) }
+        )
+      end
+
+      def Count.[](name)
+        Count.for(name)
+      end
+
+      def inc(n = 1)
+        super(:value, n)
+      end
+    end
+  end
+end

data/lib/mongoid-haystack/index.rb

@@ -0,0 +1,165 @@
+module Mongoid
+  module Haystack
+    def Haystack.index(*args, &block)
+      Index.add(*args, &block)
+    end
+
+    def Haystack.unindex(*args, &block)
+      Index.remove(*args, &block)
+    end
+
+    def Haystack.reindex!(*args, &block)
+      Index.all.each do |index|
+        model =
+          begin
+            index.model
+          rescue Object => e
+            index.destroy
+            next
+          end
+
+        index(model)
+      end
+    end
+
+    class Index
+      include Mongoid::Document
+
+      class << Index
+        def add(*args)
+          models_for(*args) do |model|
+            config = nil
+
+            if model.respond_to?(:to_haystack)
+              config = Map.for(model.to_haystack)
+            else
+              keywords = []
+                %w( keywords title ).each do |attr|
+                  if model.respond_to?(attr)
+                    keywords.push(*model.send(attr))
+                    break
+                  end
+                end
+
+              fulltext = []
+                %w( fulltext text content body description to_s ).each do |attr|
+                  if model.respond_to?(attr)
+                    fulltext.push(*model.send(attr))
+                    break
+                  end
+                end
+
+              config =
+                Map.for(
+                  :keywords => keywords,
+                  :fulltext => fulltext
+                )
+            end
+
+            keywords = Array(config[:keywords]).join(' ')
+            fulltext = Array(config[:fulltext]).join(' ')
+            facets   = Map.for(config[:facets] || {})
+            score    = config[:score]
+
+            index =
+              Haystack.find_or_create(
+                ->{ where(:model => model).first },
+                ->{ new(:model => model) },
+              )
+
+            if index.persisted?
+              Index.subtract(index)
+            end
+
+            keyword_scores = Hash.new{|h,k| h[k] = 0}
+            fulltext_scores = Hash.new{|h,k| h[k] = 0}
+
+            Token.values_for(keywords).each do |value|
+              token = Token.add(value)
+              id = token.id
+
+              index.tokens.push(id)
+              keyword_scores[id] += 1
+            end
+
+            Token.values_for(fulltext).each do |value|
+              token = Token.add(value)
+              id = token.id
+
+              index.tokens.push(id)
+              fulltext_scores[id] += 1
+            end
+
+            index.keyword_scores = keyword_scores
+            index.fulltext_scores = fulltext_scores
+
+            index.score = score if score
+            index.facets = facets if facets
+
+            index.tokens = index.tokens.uniq
+
+            index.save!
+          end
+        end
+
+        def remove(*args)
+          models_for(*args) do |model|
+            index = where(:model_type => model.class.name, :model_id => model.id).first
+
+            if index
+              subtract(index)
+              index.destroy
+            end
+          end
+        end
+
+        def subtract(index)
+          tokens = Token.where(:id.in => index.tokens)
+
+          n = 0
+
+          tokens.each do |token|
+            keyword_score = index.keyword_scores[token.id].to_i
+            fulltext_score = index.fulltext_scores[token.id].to_i
+
+            i = keyword_score + fulltext_score
+            token.inc(:count, -i)
+
+            n += i
+          end
+
+          Count[:tokens].inc(-n)
+        end
+
+        def models_for(*args, &block)
+          args.flatten.compact.each do |arg|
+            if arg.respond_to?(:persisted?)
+              model = arg
+              block.call(model)
+            else
+              arg.all.each do |model|
+                block.call(model)
+              end
+            end
+          end
+        end
+      end
+
+      belongs_to(:model, :polymorphic => true)
+
+      field(:tokens, :type => Array, :default => [])
+      field(:score, :type => Integer, :default => 0)
+      field(:keyword_scores, :type => Hash, :default => proc{ Hash.new{|h,k| h[k] = 0} })
+      field(:fulltext_scores, :type => Hash, :default => proc{ Hash.new{|h,k| h[k] = 0} })
+      field(:facets, :type => Hash, :default => {})
+
+      index({:model_type => 1})
+      index({:model_id => 1})
+
+      index({:tokens => 1})
+      index({:score => 1})
+      index({:keyword_scores => 1})
+      index({:fulltext_scores => 1})
+    end
+  end
+end

data/lib/mongoid-haystack/search.rb

@@ -0,0 +1,96 @@
+module Mongoid
+  module Haystack
+    def search(*args, &block)
+    #
+      options = Map.options_for!(args)
+      search = args.join(' ')
+
+    #
+      tokens = search_tokens_for(search)
+
+    #
+      conditions = {}
+      conditions[:tokens.in] = tokens.map{|token| token.id}
+
+    #
+      order = []
+      order.push(["score", :desc])
+
+      tokens.each do |token|
+        order.push(["keyword_scores.#{ token.id }", :desc])
+      end
+
+      tokens.each do |token|
+        order.push(["fulltext_scores.#{ token.id }", :desc])
+      end
+
+    #
+      if options[:facets]
+        conditions[:facets] = options[:facets]
+      end
+
+    #
+      if options[:types]
+        model_types = Array(options[:types]).map{|type| type.name}
+        conditions[:model_type.in] = model_types
+      end
+
+    #
+      Index.where(conditions).order_by(order)
+    end
+
+    def search_tokens_for(search)
+      values = Token.values_for(search.to_s)
+      tokens = Token.where(:value.in => values).to_a
+
+      positions = {}
+      tokens.each_with_index{|token, index| positions[token] = index + 1}
+
+      t = Count[:tokens].value.to_f
+
+      tokens.sort! do |a,b|
+        [b.rarity_bin(t), positions[b]] <=> [a.rarity_bin(t), positions[a]]
+      end
+
+      tokens
+    end
+
+    module Search
+      ClassMethods = proc do
+        def search(*args, &block)
+          options = Map.options_for!(args)
+          options[:types] = Array(options[:types]).flatten.compact
+          options[:types].push(self)
+          args.push(options)
+          Haystack.search(*args, &block)
+        end
+
+        after_save do |doc|
+          begin
+            Mongoid::Haystack::Index.add(doc) if doc.persisted?
+          rescue Object
+            nil
+          end
+        end
+
+        after_destroy do |doc|
+          begin
+            Mongoid::Haystack::Index.remove(doc)
+          rescue Object
+            nil
+          end
+        end
+      end
+
+      InstanceMethods = proc do
+      end
+
+      def Search.included(other)
+        super
+      ensure
+        other.instance_eval(&ClassMethods)
+        other.class_eval(&InstanceMethods)
+      end
+    end
+  end
+end

data/lib/mongoid-haystack/sequence.rb

@@ -0,0 +1,55 @@
+module Mongoid
+  module Haystack
+    class Sequence
+      include Mongoid::Document
+
+      field(:name, :type => String)
+
+      field(:value, :default => 0, :type => Integer)
+
+      validates_presence_of(:name)
+      validates_uniqueness_of(:name)
+
+      validates_presence_of(:value)
+
+      index({:name => 1}, {:unique => true})
+
+      Cache = Hash.new
+
+      class << self
+        def for(name)
+          name = name.to_s
+
+          Cache[name] ||= (
+            Haystack.find_or_create(
+              ->{ where(:name => name).first },
+              ->{ create!(:name => name) }
+            )
+          )
+        end
+
+        alias_method('[]', 'for')
+
+        def sequence_name_for(klass, fieldname)
+          "#{ klass.name.gsub(/::/, '.').downcase }-#{ fieldname }"
+        end
+      end
+
+      after_destroy do |sequence|
+        Cache.delete(sequence.name)
+      end
+
+      def next
+        inc(:value, 1)
+      end
+
+      def current_value
+        reload.value
+      end
+
+      def reset!
+        update_attributes!(:value => 0)
+      end
+    end
+  end
+end

data/lib/mongoid-haystack/stemming.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+
+module Stemming
+  def stem(*args)
+    string = args.join(' ')
+    words = string.scan(/[\w._-]+/)
+    stems = []
+    words.each do |word|
+      word = word.downcase
+      stem = word.stem.downcase
+      next if Stopwords.stopword?(word)
+      next if Stopwords.stopword?(stem)
+      stems.push(stem)
+    end
+    stems
+  end
+
+  alias_method('for', 'stem')
+
+  module Stopwords
+    dirname = __FILE__.sub(/\.rb\Z/, '')
+    glob = File.join(dirname, 'stopwords', '*.txt')
+
+    List = {}
+
+    Dir.glob(glob).each do |wordlist|
+      basename = File.basename(wordlist)
+      name = basename.split(/\./).first
+
+      open(wordlist) do |fd|
+        lines = fd.readlines
+        words = lines.map{|line| line.strip}
+        words.delete_if{|word| word.empty?}
+        words.push('')
+        List[name] = words
+      end
+    end
+
+    unless defined?(All)
+      All = []
+      All.concat(List['english'])
+      All.concat(List['full_english'])
+      All.concat(List['extended_english'])
+      #All.concat(List['full_french'])
+      #All.concat(List['full_spanish'])
+      #All.concat(List['full_portuguese'])
+      #All.concat(List['full_italian'])
+      #All.concat(List['full_german'])
+      #All.concat(List['full_dutch'])
+      #All.concat(List['full_norwegian'])
+      #All.concat(List['full_danish'])
+      #All.concat(List['full_russian'])
+      #All.concat(List['full_russian_koi8_r'])
+      #All.concat(List['full_finnish'])
+      All.sort!
+      All.uniq!
+    end
+
+    unless defined?(Index)
+      Index = {}
+
+      All.each do |word|
+        Index[word] = word
+      end
+    end
+
+    def stopword?(word)
+      !!Index[word]
+    end
+
+    extend(Stopwords)
+  end
+
+  extend(Stemming)
+end
+
+if $0 == __FILE__
+  p Stemming.stem("the foobars foo-bars foos bars cat and mountains")
+end