wj-mongoid-elasticsearch 0.0.1

data/lib/mongoid/elasticsearch/index.rb

@@ -0,0 +1,62 @@
+module Mongoid
+  module Elasticsearch
+    class Index
+      def initialize(es)
+        @es = es
+      end
+
+      def klass
+        @es.klass
+      end
+
+      def name
+        klass.es_index_name
+      end
+
+      def type
+        klass.model_name.collection.singularize
+      end
+
+      def options
+        klass.es_index_options
+      end
+
+      def indices
+        @es.client.indices
+      end
+
+      def exists?
+        indices.exists index: name
+      end
+
+      def create
+        unless options == {} || exists?
+          force_create
+        end
+      end
+
+      def force_create
+        indices.create index: name, body: options
+      end
+
+      def delete
+        if exists?
+          force_delete
+        end
+      end
+
+      def force_delete
+        indices.delete index: name
+      end
+
+      def refresh
+        indices.refresh index: name
+      end
+
+      def reset
+        delete
+        create
+      end
+    end
+  end
+end

data/lib/mongoid/elasticsearch/indexing.rb

@@ -0,0 +1,25 @@
+module Mongoid
+  module Elasticsearch
+    module Indexing
+      extend ActiveSupport::Concern
+      included do
+        def as_indexed_json
+          serializable_hash.reject { |k, v| %w(_id c_at u_at created_at updated_at).include?(k) }
+        end
+        
+        def es_index?
+          true
+        end
+
+        def es_update
+          if destroyed? || !es_index?
+            self.class.es.remove_item(self)
+          else
+            self.class.es.index_item(self)
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/mongoid/elasticsearch/monkeypatches.rb

@@ -0,0 +1,56 @@
+# Quick fix for https://github.com/elasticsearch/elasticsearch-ruby/issues/43
+
+require 'elasticsearch/api/actions/search'
+module Elasticsearch
+  module API
+    module Actions
+      def search(arguments={})
+        arguments[:index] = '_all' if ! arguments[:index] && arguments[:type]
+
+        valid_params = [
+          :analyzer,
+          :analyze_wildcard,
+          :default_operator,
+          :df,
+          :explain,
+          :fields,
+          :from,
+          :ignore_indices,
+          :ignore_unavailable,
+          :allow_no_indices,
+          :expand_wildcards,
+          :indices_boost,
+          :lenient,
+          :lowercase_expanded_terms,
+          :preference,
+          :q,
+          :routing,
+          :scroll,
+          :search_type,
+          :size,
+          :sort,
+          :source,
+          :_source,
+          :_source_include,
+          :_source_exclude,
+          :stats,
+          :suggest_field,
+          :suggest_mode,
+          :suggest_size,
+          :suggest_text,
+          :timeout,
+          :version ]
+
+        method = 'GET'
+        path   = Utils.__pathify( Utils.__listify(arguments[:index]), Utils.__listify(arguments[:type]), '_search' )
+
+        params = Utils.__validate_and_extract_params arguments, valid_params
+        body   = arguments[:body]
+
+        params[:fields] = Utils.__listify(params[:fields]) if params[:fields]
+
+        perform_request(method, path, params, body).body
+      end
+    end
+  end
+end

data/lib/mongoid/elasticsearch/pagination.rb

@@ -0,0 +1,63 @@
+# credits: https://github.com/karmi/retire/blob/master/lib/tire/results/pagination.rb
+
+
+module Mongoid
+  module Elasticsearch
+    # Adds support for WillPaginate and Kaminari
+    module Pagination
+      def total_entries
+        total
+      end
+
+      def per_page
+        (@options[:per_page] || @options[:per] || @options[:size] || 10 ).to_i
+      end
+
+      def total_pages
+        ( total.to_f / per_page ).ceil
+      end
+
+      def current_page
+        if @options[:page]
+          @options[:page].to_i
+        else
+          (per_page + @options[:from].to_i) / per_page
+        end
+      end
+
+      def previous_page
+        current_page > 1 ? (current_page - 1) : nil
+      end
+
+      def next_page
+        current_page < total_pages ? (current_page + 1) : nil
+      end
+
+      def offset
+        per_page * (current_page - 1)
+      end
+
+      def out_of_bounds?
+        current_page > total_pages
+      end
+
+      # Kaminari support
+      #
+      alias :limit_value :per_page
+      alias :total_count :total_entries
+      alias :num_pages :total_pages
+      alias :offset_value :offset
+      alias :out_of_range? :out_of_bounds?
+
+      def first_page?
+        current_page == 1
+      end
+
+      def last_page?
+        total_pages == 0 || current_page == total_pages
+      end
+
+
+    end
+  end
+end

data/lib/mongoid/elasticsearch/railtie.rb

@@ -0,0 +1,11 @@
+module Mongoid::Elasticsearch
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      require File.expand_path('../tasks', __FILE__)
+    end
+    
+    config.after_initialize do
+      Mongoid::Elasticsearch.create_all_indexes! if Mongoid::Elasticsearch.autocreate_indexes
+    end
+  end
+end

data/lib/mongoid/elasticsearch/response.rb

@@ -0,0 +1,193 @@
+# partially based on https://github.com/karmi/retire/blob/master/lib/tire/results/collection.rb
+
+require 'mongoid/elasticsearch/pagination'
+
+module Mongoid
+  module Elasticsearch
+    class Response
+      include Enumerable
+      include Pagination
+
+      attr_reader :time, :total, :options, :facets, :max
+      attr_reader :response
+
+      def initialize(client, query, multi, model, options)
+        @client  = client
+        @query   = query
+        @multi   = multi
+        @model   = model
+        @wrapper = options[:wrapper]
+        @options = options
+      end
+
+      def perform!
+        response = @client.search(@query)
+        @options = options
+        @time = response['took'].to_i
+        @total = response['hits']['total'].to_i rescue nil
+        @facets = response['facets']
+        @max_score = response['hits']['max_score'].to_f rescue nil
+        response
+      end
+
+      def total
+        if @total.nil?
+          perform!
+          @total
+        else
+          @total
+        end
+      end
+
+      def raw_response
+        @raw_response ||= perform!
+      end
+
+      def hits
+        @hits ||= raw_response['hits']['hits']
+      end
+
+      def results
+        return [] if failure?
+        @results ||= begin
+          case @wrapper
+          when :load
+            if @multi
+              multi_with_load
+            else
+              records = @model.find(hits.map { |h| h['_id'] })
+              hits.map do |item|
+                records.detect do |record|
+                  record.id.to_s == item['_id'].to_s
+                end
+              end
+            end
+          when :mash
+            hits.map do |h|
+              s = h.delete('_source')
+              m = Hashie::Mash.new(h.merge(s))
+              if defined?(Moped::BSON)
+                m.id = Moped::BSON::ObjectId.from_string(h['_id'])
+              else
+                m.id = BSON::ObjectId.from_string(h['_id'])
+              end
+              m._id = m.id
+              m
+            end
+          when :model
+            multi_without_load
+          else
+            hits
+          end
+
+        end
+      end
+
+      def error
+        raw_response['error']
+      end
+
+      def success?
+        error.to_s.empty?
+      end
+
+      def failure?
+        !success?
+      end
+
+      def each(&block)
+        results.each(&block)
+      end
+
+      def to_ary
+        results
+      end
+
+      def inspect
+        "#<Mongoid::Elasticsearch::Response @size:#{@results.nil? ? 'not run yet' : size} @results:#{@results.inspect} @raw_response=#{@raw_response}>"
+      end
+
+      def count
+        # returns approximate counts, for now just using search_type: 'count',
+        # which is exact
+        # @total ||= @client.count(@query)['count']
+
+        @total ||= @client.search(@query.merge(search_type: 'count'))['hits']['total']
+      end
+
+      def size
+        results.size
+      end
+      alias_method :length, :size
+
+      private
+
+      def find_klass(type)
+        raise NoMethodError, "You have tried to eager load the model instances, " +
+                             "but Mongoid::Elasticsearch cannot find the model class because " +
+                             "document has no _type property." unless type
+
+        begin
+          klass = type.camelize.singularize.constantize
+        rescue NameError => e
+          raise NameError, "You have tried to eager load the model instances, but " +
+                           "Mongoid::Elasticsearch cannot find the model class '#{type.camelize}' " +
+                           "based on _type '#{type}'.", e.backtrace
+        end
+      end
+
+      def multi_with_load
+        return [] if hits.empty?
+
+        records = {}
+        hits.group_by { |item| item['_type'] }.each do |type, items|
+          klass = find_klass(type)
+          records[type] = klass.find(items.map { |h| h['_id'] })
+        end
+
+        # Reorder records to preserve the order from search results
+        hits.map do |item|
+          records[item['_type']].detect do |record|
+            record.id.to_s == item['_id'].to_s
+          end
+        end
+      end
+
+      def multi_without_load
+        hits.map do |h|
+          klass = find_klass(h['_type'])
+          h[:_highlight] = h.delete('highlight') if h.key?('highlight')
+          source = h.delete('_source')
+          if defined?(Moped::BSON)
+            source.each do |k,v|
+              if v.is_a?(Hash) && v.has_key?("$oid")
+                source[k] = Moped::BSON::ObjectId.from_string(v["$oid"])
+              end
+            end
+          else
+            source.each do |k,v|
+              if v.is_a?(Hash) && v.has_key?("$oid")
+                source[k] = BSON::ObjectId.from_string(v["$oid"])
+              end
+            end
+          end
+          begin
+            m = klass.new(h.merge(source))
+            if defined?(Moped::BSON)
+              m.id = Moped::BSON::ObjectId.from_string(h['_id'])
+            else
+              m.id = BSON::ObjectId.from_string(h['_id'])
+            end
+          rescue Mongoid::Errors::UnknownAttribute
+            klass.class_eval <<-RUBY, __FILE__, __LINE__+1
+              attr_accessor :_type, :_score, :_source, :_highlight
+            RUBY
+            m = klass.new(h.merge(source))
+          end
+          m.new_record = false
+          m
+        end
+      end
+    end
+  end
+end

data/lib/mongoid/elasticsearch/tasks.rb

@@ -0,0 +1,28 @@
+# require this file to load the tasks
+require 'rake'
+
+# Require sitemap_generator at runtime. If we don't do this the ActionView helpers are included
+# before the Rails environment can be loaded by other Rake tasks, which causes problems
+# for those tasks when rendering using ActionView.
+namespace :es do
+  # Require sitemap_generator only. When installed as a plugin the require will fail, so in
+  # that case, load the environment first.
+  task :require do
+    Rake::Task['environment'].invoke
+    require 'ruby-progressbar'
+  end
+  desc "create all indexes"
+  task :create => ['es:require'] do
+    Mongoid::Elasticsearch.create_all_indexes!
+  end
+  desc "reindex all ES models"
+  task :reindex => ['es:require'] do
+    Mongoid::Elasticsearch.registered_models.each do |model_name|
+      pb = nil
+      model_name.constantize.es.index_all do |steps, step|
+        pb = ProgressBar.create(title: model_name, total: steps, format: '%t: %p%% %a |%b>%i| %E') if pb.nil?
+        pb.increment
+      end
+    end
+  end
+end

data/lib/mongoid/elasticsearch/utils.rb

@@ -0,0 +1,15 @@
+# coding: utf-8
+# source: https://github.com/karmi/retire/blob/master/lib/tire/utils.rb
+require 'uri'
+
+module Mongoid
+  module Elasticsearch
+    module Utils
+      def clean(s)
+        s.to_s.gsub(/\P{Word}+/, ' ').gsub(/ +/, ' ').strip
+      end
+
+      extend self
+    end
+  end
+end

data/lib/mongoid/elasticsearch/version.rb

@@ -0,0 +1,5 @@
+module Mongoid
+  module Elasticsearch
+    VERSION = "0.0.1"
+  end
+end

data/lib/mongoid/elasticsearch.rb

@@ -0,0 +1,129 @@
+require 'elasticsearch'
+require 'mongoid/elasticsearch/version'
+
+require 'active_support/concern'
+
+require 'mongoid/elasticsearch/utils'
+require 'mongoid/elasticsearch/es'
+require 'mongoid/elasticsearch/callbacks'
+require 'mongoid/elasticsearch/index'
+require 'mongoid/elasticsearch/indexing'
+require 'mongoid/elasticsearch/response'
+
+require 'mongoid/elasticsearch/monkeypatches'
+
+module Mongoid
+  module Elasticsearch
+    mattr_accessor :autocreate_indexes
+    self.autocreate_indexes = true
+    
+    mattr_accessor :prefix
+    self.prefix = ''
+
+    mattr_accessor :client_options
+    self.client_options = {}
+
+    mattr_accessor :registered_indexes
+    self.registered_indexes = []
+    
+    mattr_accessor :registered_models
+    self.registered_models = []
+
+    extend ActiveSupport::Concern
+    included do
+      def self.es
+        @__es__ ||= Mongoid::Elasticsearch::Es.new(self)
+      end
+
+      # Add elasticsearch to the model
+      # @option index_name [String] name of the index for this model
+      # @option index_options [Hash] Index options to be passed to Elasticsearch
+      # when creating an index
+      # @option client_options [Hash] Options for Elasticsearch::Client.new
+      # @option wrapper [Symbol] Select what wrapper to use for results
+      # possible options:
+      # :model - creates a new model instance, set its attributes, and marks it as persisted
+      # :mash - Hashie::Mash for object-like access  (perfect for simple models, needs gem 'hashie')
+      # :none - raw hash
+      # :load - load models from Mongo by IDs
+      def self.elasticsearch!(options = {})
+        options = {
+          prefix_name: true,
+          index_name: nil,
+          client_options: {},
+          index_options: {},
+          index_mappings: nil,
+          wrapper: :model,
+          callbacks: true,
+          skip_create: false
+        }.merge(options)
+        
+        if options[:wrapper] == :model
+          attr_accessor :_type, :_score, :_source
+        end
+
+        cattr_accessor :es_client_options, :es_index_name, :es_index_options, :es_wrapper, :es_skip_create
+
+        self.es_client_options = Mongoid::Elasticsearch.client_options.dup.merge(options[:client_options])
+        self.es_index_name     = (options[:prefix_name] ? Mongoid::Elasticsearch.prefix : '') + (options[:index_name] || model_name.plural)
+        self.es_index_options  = options[:index_options]
+        self.es_wrapper        = options[:wrapper]
+        self.es_skip_create    = options[:skip_create]
+
+        Mongoid::Elasticsearch.registered_indexes.push self.es_index_name
+        Mongoid::Elasticsearch.registered_indexes.uniq!
+        
+        Mongoid::Elasticsearch.registered_models.push self.name
+        Mongoid::Elasticsearch.registered_models.uniq!
+        
+        unless options[:index_mappings].nil?
+          self.es_index_options = self.es_index_options.deep_merge({
+            :mappings => {
+              es.index.type.to_sym => {
+                :properties => options[:index_mappings]
+              }
+            }
+          })
+        end
+
+        include Indexing
+        include Callbacks if options[:callbacks]
+      end
+    end
+
+    def self.create_all_indexes!
+      # puts "creating ES indexes"
+      Mongoid::Elasticsearch.registered_models.each do |model_name|
+        model = model_name.constantize
+        model.es.index.create unless model.es_skip_create
+      end
+    end
+
+    # search multiple models
+    def self.search(query, options = {})
+      if query.is_a?(String)
+        query = {q: Utils.clean(query)}
+      end
+      # use `_all` or empty string to perform the operation on all indices
+      # regardless whether they are managed by Mongoid::Elasticsearch or not
+      unless query.key?(:index)
+        query.merge!(index: Mongoid::Elasticsearch.registered_indexes.join(','), ignore_indices: 'missing', ignore_unavailable: true)
+      end
+
+      page = options[:page]
+      per_page = options[:per_page]
+
+      query[:size] = ( per_page.to_i ) if per_page
+      query[:from] = ( page.to_i <= 1 ? 0 : (per_page.to_i * (page.to_i-1)) ) if page && per_page
+
+      options[:wrapper] ||= :model
+
+      client = ::Elasticsearch::Client.new Mongoid::Elasticsearch.client_options.dup
+      Response.new(client, query, true, nil, options)
+    end
+  end
+end
+
+if defined? Rails
+  require 'mongoid/elasticsearch/railtie'
+end

data/lib/mongoid-elasticsearch.rb

@@ -0,0 +1 @@
+require 'mongoid/elasticsearch'

data/mongoid-elasticsearch.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mongoid/elasticsearch/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "wj-mongoid-elasticsearch"
+  spec.version       = Mongoid::Elasticsearch::VERSION
+  spec.authors       = ["glebtv"]
+  spec.email         = ["glebtv@gmail.com"]
+  spec.description   = %q{Simple and easy integration of mongoid with the new elasticsearch gem}
+  spec.summary       = %q{Simple and easy integration of mongoid with the new elasticsearch gem}
+  spec.homepage      = "https://github.com/rs-pro/mongoid-elasticsearch"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "mongoid", [">= 3.0", "< 6.0"]
+  spec.add_dependency "elasticsearch", "~> 1.0.13"
+  spec.add_dependency "ruby-progressbar"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "kaminari"
+  spec.add_development_dependency "database_cleaner"
+  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "hashie"
+  spec.add_development_dependency "mongoid-slug", '~> 5.0.0'
+  spec.add_development_dependency "glebtv-httpclient"
+end

data/spec/models/article.rb

@@ -0,0 +1,32 @@
+class Article
+  include Mongoid::Document
+  include Mongoid::Timestamps::Short
+  #include ActiveModel::ForbiddenAttributesProtection
+
+  field :name
+
+  include Mongoid::Slug
+  slug :name
+
+  field :tags
+
+  include Mongoid::Elasticsearch
+  i_fields = {
+    name:     {type: 'string', analyzer: 'snowball'},
+    raw:      {type: 'string', index: :not_analyzed}
+  }
+
+  if Gem::Version.new(::Elasticsearch::Client.new.info['version']['number']) > Gem::Version.new('0.90.2')
+    i_fields[:suggest] = {type: 'completion'} 
+  end
+
+  elasticsearch! index_name: 'mongoid_es_news', prefix_name: false, index_mappings: {
+    name: {
+      type: 'multi_field',
+      fields: i_fields
+    },
+    _slugs:   {type: 'string', index: :not_analyzed},
+    tags: {type: 'string', include_in_all: false}
+  }, wrapper: :load
+end
+

data/spec/models/namespaced.rb

@@ -0,0 +1,22 @@
+module Namespaced
+  class Model
+    include Mongoid::Document
+    include Mongoid::Timestamps::Short
+    #include ActiveModel::ForbiddenAttributesProtection
+
+    field :name
+
+    include Mongoid::Elasticsearch
+    elasticsearch! index_options: {
+      'namespaced/model' => {
+        mappings: {
+          properties: {
+            name: {
+              type: 'string'
+            }
+          }
+        }
+      }
+    }
+  end
+end

data/spec/models/no_autocreate.rb

@@ -0,0 +1,9 @@
+class NoAutocreate
+  include Mongoid::Document
+
+  field :name, type: String
+
+  include Mongoid::Elasticsearch
+  elasticsearch! skip_create: true
+end
+