mongoid-elasticsearch 0.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 227f98096d3f8725368150c8a275001d4b308fe8
+  data.tar.gz: 5385ffd8dd90ca8397731a1ab73f0f63cda66a59
+SHA512:
+  metadata.gz: 86ae14cde369b32a520d6502af4a33b717f357c22cc7eaaf4c5cf2d585c93371dd38522588f447360f3ee3463b89da9264cebcea41531fe892de953793c7564a
+  data.tar.gz: 696b7ba51d63adf247a37b5cd633985b2303a5914c838c0509f003197346d94eabc5933dd7ac64560610637aa5e1ba6ef1f834d775686dcc04e53a8f3d8b3641

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1 @@
+--color

data/.ruby-gemset

@@ -0,0 +1 @@
+mongoid-elasticsearch

data/.ruby-version

@@ -0,0 +1 @@
+2.0.0

data/.travis.yml

@@ -0,0 +1,19 @@
+services: 
+  - mongodb
+  - elasticsearch
+
+notifications:
+  email: false
+
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - jruby-19mode
+  # mongoid 4 has problems
+  #- rbx-19mode
+
+gemfile:
+  - Gemfile
+  - gemfiles/mongoid-4.0.gemfile
+

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mongoid_rating.gemspec
+gem "mongoid", github: "mongoid/mongoid"
+gemspec

data/MIT-LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 glebtv
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,117 @@
+# Mongoid::Elasticsearch
+
+[![Build Status](https://travis-ci.org/rs-pro/mongoid-elasticsearch.png?branch=master)](https://travis-ci.org/rs-pro/mongoid-elasticsearch)
+[![Coverage Status](https://coveralls.io/repos/rs-pro/mongoid-elasticsearch/badge.png?branch=master)](https://coveralls.io/r/rs-pro/mongoid-elasticsearch?branch=master)
+[![Gem Version](https://badge.fury.io/rb/mongoid-elasticsearch.png)](http://badge.fury.io/rb/mongoid-elasticsearch)
+[![Dependency Status](https://gemnasium.com/rs-pro/mongoid-elasticsearch.png)](https://gemnasium.com/rs-pro/mongoid-elasticsearch)
+
+
+Use [Elasticsearch](http://www.elasticsearch.org/) with mongoid with just a few
+lines of code
+
+Allows easy usage of [the new Elasticsearch gem](https://github.com/elasticsearch/elasticsearch-ruby)
+with [Mongoid 4](https://github.com/mongoid/mongoid)
+
+## Features
+
+- Uses new elasticsearch gem
+- Has a simple high-level API
+- No weird undocumented DSL, just raw JSON for queries and index definitions
+- Allows for full power of elasticsearch when it's necessary
+- Indexes are automatically created if they don't exist on app boot
+- Works out of the box with zero configuration
+- Multi-model search with real model instances and pagination
+- Whole test suite is run against a real ES instance, no mocks
+
+This gem is very simple and does not try hide any part of the ES REST api, it
+  just adds some shortcuts for prefixing index names, automatic updating of the index
+  when models are added\changed, search with pagination, wrapping results in
+  a model instance, ES 0.90.3 new completion suggester, etc (new features coming
+  soon)
+
+## Why use it (alternatives list):
+
+- [(re)Tire](https://github.com/karmi/retire) - retired
+- [RubberBand](https://github.com/grantr/rubberband) - EOL
+- [Elasticsearch gem](https://github.com/elasticsearch/elasticsearch-ruby) - too low-level and complex
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mongoid-elasticsearch'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mongoid-elasticsearch
+
+## Usage
+
+### Basic:
+
+    class Post
+      include Mongoid::Document
+      include Mongoid::Elasticsearch
+      elasticsearch!
+    end
+
+    Post.es.search 'test text' # shortcut for Post.es.search({q: 'test text'})
+    result = Post.es.search query: {...}, facets: {...} etc
+    result.raw_response
+    result.results # by default returns an Enumerable with Post instances exactly
+                   # like they were loaded from MongoDB
+    Post.es.index.create # create index (done automatically on app boot)
+    Post.es.index.destroy # drop index
+    Post.es.index.reset # recreate index
+    Post.es.index.refresh # force index update (useful for specs)
+    Post.es.client # Elasticsearch::Client instance
+    Post.es.completion('te') # requires ES 0.90.3
+
+    # Search multiple models
+    # By default only searches in indexes managed by Mongoid::Elasticsearch
+    # to ignore other apps indexes in same ES instance
+    response = Mongoid::Elasticsearch.search 'test'
+
+
+more docs: http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions
+
+ES docs: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/index.html
+
+### Advanced:
+
+    include Mongoid::Elasticsearch
+    # define index_options (raw elasticsearch index definition
+    # 
+    elasticsearch! index_name: 'mongoid_es_news', prefix_name: false, index_options: {}, index_mappings: {
+      name: {
+        type: 'multi_field',
+        fields: {
+          name:     {type: 'string', analyzer: 'snowball'},
+          raw:      {type: 'string', index: :not_analyzed},
+          suggest:  {type: 'completion'} 
+        }
+      },
+      tags: {type: 'string', include_in_all: false}
+    }, wrapper: :load
+    # customize what gets sent to elasticsearch:
+    def as_indexed_json
+      # id field is properly added automatically
+      {
+        name: name,
+        excerpt: excerpt
+      }
+    end
+    
+See more examples in specs.
+    
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+desc 'Default: run specs.'
+task :default => :spec
+
+desc "Run specs"
+RSpec::Core::RakeTask.new

data/gemfiles/mongoid-4.0.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "mongoid", github: "mongoid/mongoid", branch: "master"
+
+gemspec path: "../"

data/lib/mongoid/elasticsearch/callbacks.rb

@@ -0,0 +1,18 @@
+module Mongoid
+  module Elasticsearch
+    module Callbacks
+      extend ActiveSupport::Concern
+
+      included do
+        after_save do
+          es_update
+        end
+
+        after_destroy do
+          es_update
+        end
+      end
+    end
+  end
+end
+

data/lib/mongoid/elasticsearch/es.rb

@@ -0,0 +1,74 @@
+module Mongoid
+  module Elasticsearch
+    class Es
+      attr_reader :klass, :version
+
+      def initialize(klass)
+        @klass = klass
+        @version = Gem::Version.new(client.info['version']['number'])
+      end
+
+      def client
+        # dup is needed because Elasticsearch::Client.new changes options hash inplace
+        @client ||= ::Elasticsearch::Client.new klass.es_client_options.dup
+      end
+
+      def index
+        @index ||= Index.new(self)
+      end
+
+      def search(query, options = {})
+        if query.is_a?(String)
+          query = {q: Utils.clean(query)}
+        end
+
+        page = options[:page]
+        options[:per_page] ||= 50
+        per_page = options[:per_page]
+
+        query[:size] = ( per_page.to_i ) if per_page
+        query[:from] = ( page.to_i <= 1 ? 0 : (per_page.to_i * (page.to_i-1)) ) if page && per_page
+
+        Response.new(client, query.merge(type_options(true)), false, klass, klass.es_wrapper, options)
+      end
+
+      def options_for(obj, escape = false)
+        {id: obj.id.to_s}.merge type_options(escape)
+      end
+
+      def type_options(escape = false)
+        {index: index.name, type: escape ? Utils.escape(index.type) : index.type}
+      end
+
+      def index_item(obj)
+        client.index({body: obj.as_indexed_json}.merge(options_for(obj, true)))
+      end
+
+      def remove_item(obj)
+        client.delete(options_for(obj, true).merge(ignore: 404))
+      end
+
+      def all(options = {})
+        search({match_all: {}}, options)
+      end
+
+      def completion_supported?
+        @version > Gem::Version.new('0.90.2')
+      end
+
+      def completion(text, field = "suggest")
+        raise "Completion not supported in ES #{@version}" unless completion_supported?
+        body = {
+          q: {
+            text: Utils.clean(text),
+            completion: {
+              field: field
+            }
+          }
+        }
+        results = client.suggest(index: index.name, body: body)
+        results['q'][0]['options']
+      end
+    end
+  end
+end

data/lib/mongoid/elasticsearch/index.rb

@@ -0,0 +1,62 @@
+module Mongoid
+  module Elasticsearch
+    class Index
+      def initialize(es)
+        @es = es
+      end
+
+      def klass
+        @es.klass
+      end
+
+      def name
+        klass.es_index_name
+      end
+
+      def type
+        klass.model_name.collection
+      end
+
+      def options
+        klass.es_index_options
+      end
+
+      def indices
+        @es.client.indices
+      end
+
+      def exists?
+        indices.exists index: name
+      end
+
+      def create
+        unless options == {} || exists?
+          force_create
+        end
+      end
+
+      def force_create
+        indices.create index: name, body: options
+      end
+
+      def delete
+        if exists?
+          force_delete
+        end
+      end
+
+      def force_delete
+        indices.delete index: name
+      end
+
+      def refresh
+        indices.refresh index: name
+      end
+
+      def reset
+        delete
+        create
+      end
+    end
+  end
+end

data/lib/mongoid/elasticsearch/indexing.rb

@@ -0,0 +1,21 @@
+module Mongoid
+  module Elasticsearch
+    module Indexing
+      extend ActiveSupport::Concern
+      included do
+        def as_indexed_json
+          serializable_hash.reject { |k, v| %w(_id c_at u_at created_at updated_at).include?(k) }
+        end
+
+        def es_update
+          if destroyed?
+            self.class.es.remove_item(self)
+          else
+            self.class.es.index_item(self)
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/mongoid/elasticsearch/pagination.rb

@@ -0,0 +1,63 @@
+# credits: https://github.com/karmi/retire/blob/master/lib/tire/results/pagination.rb
+
+
+module Mongoid
+  module Elasticsearch
+    # Adds support for WillPaginate and Kaminari
+    module Pagination
+      def total_entries
+        total
+      end
+
+      def per_page
+        (@options[:per_page] || @options[:size] || 10 ).to_i
+      end
+
+      def total_pages
+        ( total.to_f / per_page ).ceil
+      end
+
+      def current_page
+        if @options[:page]
+          @options[:page].to_i
+        else
+          (per_page + @options[:from].to_i) / per_page
+        end
+      end
+
+      def previous_page
+        current_page > 1 ? (current_page - 1) : nil
+      end
+
+      def next_page
+        current_page < total_pages ? (current_page + 1) : nil
+      end
+
+      def offset
+        per_page * (current_page - 1)
+      end
+
+      def out_of_bounds?
+        current_page > total_pages
+      end
+
+      # Kaminari support
+      #
+      alias :limit_value :per_page
+      alias :total_count :total_entries
+      alias :num_pages :total_pages
+      alias :offset_value :offset
+      alias :out_of_range? :out_of_bounds?
+
+      def first_page?
+        current_page == 1
+      end
+
+      def last_page?
+        current_page == total_pages
+      end
+
+
+    end
+  end
+end

data/lib/mongoid/elasticsearch/response.rb

@@ -0,0 +1,176 @@
+# partially based on https://github.com/karmi/retire/blob/master/lib/tire/results/collection.rb
+
+require 'mongoid/elasticsearch/pagination'
+
+module Mongoid
+  module Elasticsearch
+    class Response
+      include Enumerable
+      include Pagination
+
+      attr_reader :time, :total, :options, :facets, :max
+      attr_reader :response
+
+      def initialize(client, query, multi, model, wrapper, options)
+        @client  = client
+        @query   = query
+        @multi   = multi
+        @model   = model
+        @wrapper = wrapper
+        @options = options
+      end
+
+      def perform!
+        response = @client.search(@query)
+        @options = options
+        @time = response['took'].to_i
+        @total = response['hits']['total'].to_i rescue nil
+        @facets = response['facets']
+        @max_score = response['hits']['max_score'].to_f rescue nil
+        response
+      end
+
+      def total
+        if @total.nil?
+          perform!
+          @total
+        else
+          @total
+        end
+      end
+
+      def raw_response
+        @raw_response ||= perform!
+      end
+
+      def hits
+        @hits ||= raw_response['hits']['hits'].map { |d| d.update '_type' => Utils.unescape(d['_type']) }
+      end
+
+      def results
+        return [] if failure?
+        @results ||= begin
+          case @wrapper
+          when :load
+            if @multi
+              multi_with_load
+            else
+              @model.find(hits.map { |h| h['_id'] })
+            end
+          when :mash
+            hits.map do |h|
+              m = Hashie::Mash.new(h)
+              m.id = BSON::ObjectId.from_string(h['_id'])
+              m._id = m.id
+              m
+            end
+          when :model
+            if @multi
+              multi_without_load
+            else
+              hits.map do |h|
+                model_from_hash(h)
+              end
+            end
+          else
+            hits
+          end
+
+        end
+      end
+
+      def error
+        raw_response['error']
+      end
+
+      def success?
+        error.to_s.empty?
+      end
+
+      def failure?
+        !success?
+      end
+
+      def each(&block)
+        results.each(&block)
+      end
+
+      def to_ary
+        results
+      end
+
+      def inspect
+        "#<Mongoid::Elasticsearch::Response @size:#{size} @results:#{results.inspect} @error=#{success? ? "none" : error} @raw_response=#{raw_response}>"
+      end
+
+      def count
+        # returns approximate counts, for now just using search_type: 'count',
+        # which is exact
+        # @total ||= @client.count(@query)['count']
+
+        @total ||= @client.search(@query.merge(search_type: 'count'))['hits']['total']
+      end
+
+      def size
+        results.size
+      end
+      alias_method :length, :size
+
+      private
+      def model_from_hash(h)
+        source = h.delete('_source')
+        m = @model.new(h.merge(source))
+        m.new_record = false
+        m
+      end
+
+      def find_klass(type)
+        raise NoMethodError, "You have tried to eager load the model instances, " +
+                             "but Mongoid::Elasticsearch cannot find the model class because " +
+                             "document has no _type property." unless type
+
+        begin
+          klass = type.camelize.singularize.constantize
+        rescue NameError => e
+          raise NameError, "You have tried to eager load the model instances, but " +
+                           "Mongoid::Elasticsearch cannot find the model class '#{type.camelize}' " +
+                           "based on _type '#{type}'.", e.backtrace
+        end
+      end
+
+      def multi_with_load
+        return [] if hits.empty?
+
+        records = {}
+        hits.group_by { |item| item['_type'] }.each do |type, items|
+          klass = find_klass(type)
+          records[type] = klass.find(items.map { |h| h['_id'] })
+        end
+
+        # Reorder records to preserve the order from search results
+        hits.map do |item|
+          records[item['_type']].detect do |record|
+            record.id.to_s == item['_id'].to_s
+          end
+        end
+      end
+
+      def multi_without_load
+        hits.map do |h|
+          klass = find_klass(h['_type'])
+          source = h.delete('_source')
+          begin
+            m = klass.new(h.merge(source))
+          rescue Mongoid::Errors::UnknownAttribute
+            klass.class_eval <<-RUBY, __FILE__, __LINE__+1
+              attr_accessor :_type, :_score, :_source
+            RUBY
+            m = klass.new(h.merge(source))
+          end
+          m.new_record = false
+          m
+        end
+      end
+    end
+  end
+end

data/lib/mongoid/elasticsearch/slash_monkeypatch.rb

@@ -0,0 +1,14 @@
+# Monkeypatch for slashes to work
+
+module Elasticsearch
+  module API
+    # Generic utility methods
+    #
+    module Utils
+      alias_method :__pathify_without_slashes, :__pathify
+      def __pathify(*segments)
+        __pathify_without_slashes(*segments).gsub('%252F', '%2F')
+      end
+    end
+  end
+end

data/lib/mongoid/elasticsearch/utils.rb

@@ -0,0 +1,24 @@
+# source: https://github.com/karmi/retire/blob/master/lib/tire/utils.rb
+require 'uri'
+
+module Mongoid
+  module Elasticsearch
+    module Utils
+
+      def escape(s)
+        URI.encode_www_form_component(s.to_s)
+      end
+
+      def unescape(s)
+        s = s.to_s.respond_to?(:force_encoding) ? s.to_s.force_encoding(Encoding::UTF_8) : s.to_s
+        URI.decode_www_form_component(s)
+      end
+
+      def clean(s)
+        s.to_s.gsub(/\W+/, ' ').gsub(/ +/, '').strip
+      end
+
+      extend self
+    end
+  end
+end

data/lib/mongoid/elasticsearch/version.rb

@@ -0,0 +1,5 @@
+module Mongoid
+  module Elasticsearch
+    VERSION = "0.2.1"
+  end
+end