wj-mongoid-elasticsearch 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e3d100f567a331d026d6f11af5855250a175d000
+  data.tar.gz: 19941f55187e0dfae9fe24939174b2d321d8ace4
+SHA512:
+  metadata.gz: aa17df21a93b08424b850ff8acc18be1242d0d3b8c34f646d1cce05d3521ff30708b815d4f21dacde93505c6155647aacdf849141e4cedc26af0193c9968200f
+  data.tar.gz: 9e2de6a5c3c73e9a17594c4134e04b23d999424240e81c7fc413d42d92a665bc39a741110326b8972ee954c5ea4eb6330e4bbeb34d37a3b5098ce6e3f0781eba

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -0,0 +1,16 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1 @@
+--color

data/.ruby-gemset

@@ -0,0 +1 @@
+mongoid-elasticsearch

data/.ruby-version

@@ -0,0 +1 @@
+2.2.3

data/.travis.yml

@@ -0,0 +1,14 @@
+services: 
+  - mongodb
+  - elasticsearch
+
+notifications:
+  email: false
+
+language: ruby
+rvm:
+  - 2.2.3
+
+gemfile:
+  - Gemfile
+

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+## 0.8.1 (March 4, 2014) ##
+
+* New method "Model.without_es_update!" to temporally disable callbacks (ie, for testing) (by [@xronos-i-am](https://github.com/xronos-i-am))
+
+## 0.8.0 (March 4, 2014) ##
+
+* fix results with :load wrapper #6 (by [@xronos-i-am](https://github.com/xronos-i-am))
+* Added option to prevent automatic creating of index. #7 (thx [@intrica](https://github.com/intrica))
+* use after_initalize to create indexes later in the app boot process
+
+Set Mongoid::Elasticsearch.autocreate_indexes to false in an initalizer to prevent automatic creation for all indexes.
+
+You can always use ```rake es:create``` to create all indexes or call Mongoid::Elasticsearch.create_all_indexes!.
+
+Indexes defined with skip_create: true are not created with all other indexes and must be created manually with Model.es.index.create

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mongoid-elasticsearch.gemspec
+gem 'database_cleaner', github: 'DatabaseCleaner/database_cleaner'
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,149 @@
+GIT
+  remote: git://github.com/DatabaseCleaner/database_cleaner.git
+  revision: 49ca0f3e8ac4c096129442d0f942d39f2c0d7dde
+  specs:
+    database_cleaner (1.5.2)
+
+PATH
+  remote: .
+  specs:
+    mongoid-elasticsearch (0.9.1)
+      elasticsearch (~> 1.0.13)
+      mongoid (>= 3.0, < 6.0)
+      ruby-progressbar
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (4.2.6)
+      actionview (= 4.2.6)
+      activesupport (= 4.2.6)
+      rack (~> 1.6)
+      rack-test (~> 0.6.2)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (4.2.6)
+      activesupport (= 4.2.6)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    activemodel (4.2.6)
+      activesupport (= 4.2.6)
+      builder (~> 3.1)
+    activesupport (4.2.6)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    bson (4.1.0)
+    builder (3.2.2)
+    coveralls (0.8.13)
+      json (~> 1.8)
+      simplecov (~> 0.11.0)
+      term-ansicolor (~> 1.3)
+      thor (~> 0.19.1)
+      tins (~> 1.6.0)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    elasticsearch (1.0.17)
+      elasticsearch-api (= 1.0.17)
+      elasticsearch-transport (= 1.0.17)
+    elasticsearch-api (1.0.17)
+      multi_json
+    elasticsearch-transport (1.0.17)
+      faraday
+      multi_json
+    erubis (2.7.0)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    glebtv-httpclient (3.2.8)
+      lru_redux
+    hashie (3.4.3)
+    i18n (0.7.0)
+    json (1.8.3)
+    kaminari (0.16.3)
+      actionpack (>= 3.0.0)
+      activesupport (>= 3.0.0)
+    loofah (2.0.3)
+      nokogiri (>= 1.5.9)
+    lru_redux (1.1.0)
+    mini_portile2 (2.0.0)
+    minitest (5.8.4)
+    mongo (2.2.4)
+      bson (~> 4.0)
+    mongoid (5.1.2)
+      activemodel (~> 4.0)
+      mongo (~> 2.1)
+      origin (~> 2.2)
+      tzinfo (>= 0.3.37)
+    mongoid-compatibility (0.3.1)
+      activesupport
+      mongoid (>= 2.0)
+    mongoid-slug (5.0.0)
+      mongoid (>= 3.0)
+      mongoid-compatibility
+      stringex (~> 2.0)
+    multi_json (1.11.2)
+    multipart-post (2.0.0)
+    nokogiri (1.6.7.2)
+      mini_portile2 (~> 2.0.0.rc2)
+    origin (2.2.0)
+    rack (1.6.4)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails-deprecated_sanitizer (1.0.3)
+      activesupport (>= 4.2.0.alpha)
+    rails-dom-testing (1.0.7)
+      activesupport (>= 4.2.0.beta, < 5.0)
+      nokogiri (~> 1.6.0)
+      rails-deprecated_sanitizer (>= 1.0.1)
+    rails-html-sanitizer (1.0.3)
+      loofah (~> 2.0)
+    rake (11.1.2)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.4)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    ruby-progressbar (1.7.5)
+    simplecov (0.11.2)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    stringex (2.6.0)
+    term-ansicolor (1.3.2)
+      tins (~> 1.0)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    tins (1.6.0)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  coveralls
+  database_cleaner!
+  glebtv-httpclient
+  hashie
+  kaminari
+  mongoid-elasticsearch!
+  mongoid-slug (~> 5.0.0)
+  rake
+  rspec
+
+BUNDLED WITH
+   1.11.2

data/MIT-LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 glebtv
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,258 @@
+# Mongoid::Elasticsearch
+
+[![Build Status](https://travis-ci.org/rs-pro/mongoid-elasticsearch.png?branch=master)](https://travis-ci.org/rs-pro/mongoid-elasticsearch)
+[![Coverage Status](https://coveralls.io/repos/rs-pro/mongoid-elasticsearch/badge.png?branch=master)](https://coveralls.io/r/rs-pro/mongoid-elasticsearch?branch=master)
+[![Gem Version](https://badge.fury.io/rb/mongoid-elasticsearch.png)](http://badge.fury.io/rb/mongoid-elasticsearch)
+[![Dependency Status](https://www.versioneye.com/user/projects/53e73fe735080d1e4d00009c/badge.svg)](https://www.versioneye.com/user/projects/53e73fe735080d1e4d00009c)
+[![Issues](http://img.shields.io/github/issues/rs-pro/mongoid-elasticsearch.svg)](https://github.com/rs-pro/mongoid-elasticsearch/issues)
+[![License](http://img.shields.io/:license-mit-blue.svg)](https://github.com/rs-pro/mongoid-elasticsearch/blob/master/MIT-LICENSE.txt)
+
+
+Use [Elasticsearch](http://www.elasticsearch.org/) with mongoid with just a few
+lines of code
+
+Allows easy usage of [the new Elasticsearch gem](https://github.com/elasticsearch/elasticsearch-ruby)
+with [Mongoid 4](https://github.com/mongoid/mongoid)
+
+## Features
+
+- Uses new elasticsearch gem
+- Has a simple high-level API
+- No weird undocumented DSL, just raw JSON for queries and index definitions
+- Allows for full power of elasticsearch when it's necessary
+- Indexes are automatically created if they don't exist on app boot
+- Works out of the box with zero configuration
+- Multi-model search with real model instances and pagination
+- Whole test suite is run against a real ES instance, no mocks
+
+This gem is very simple and does not try hide any part of the ES REST api, it
+  just adds some shortcuts for prefixing index names, automatic updating of the index
+  when models are added\changed, search with pagination, wrapping results in
+  a model instance, ES 0.90.3 new completion suggester, etc (new features coming
+  soon)
+
+## Alternatives list:
+
+- [Elasticsearch gem](https://github.com/elasticsearch/elasticsearch-ruby) - low-level and hard for simple use-cases
+- [(re)Tire](https://github.com/karmi/retire)
+- [RubberBand](https://github.com/grantr/rubberband) - EOL, no Mongoid
+- [Mebla](https://github.com/cousine/mebla) - long dead
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mongoid-elasticsearch'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mongoid-elasticsearch
+
+## Usage
+
+### Basic:
+
+    class Post
+      include Mongoid::Document
+      include Mongoid::Elasticsearch
+      elasticsearch!
+    end
+
+    Post.es.search 'test text' # shortcut for Post.es.search({q: 'test text'})
+    result = Post.es.search body: {query: {...}, facets: {...}} etc
+    result.raw_response
+    result.results # by default returns an Enumerable with Post instances exactly
+                   # like they were loaded from MongoDB
+    Post.es.index.create # create index (done automatically on app boot)
+    Post.es.index.delete # drop index
+    Post.es.index.reset # recreate index
+    Post.es.index.refresh # force index update (useful for specs)
+    Post.es.client # Elasticsearch::Client instance
+
+### Completion: 
+
+    include Mongoid::Elasticsearch
+    elasticsearch! index_mappings: {
+      name: {
+        type: 'multi_field',
+        fields: {
+          name: {type: 'string', boost: 10},
+          suggest: {type: 'completion'}
+        }
+      },
+      desc: {type: 'string'},
+    }
+
+    Post.es.completion('te', 'name.suggest') # requires ES 0.90.3
+
+### Search multiple models:
+
+    # By default only searches in indexes managed by Mongoid::Elasticsearch
+    # to ignore other apps indexes in same ES instance
+    response = Mongoid::Elasticsearch.search 'test'
+
+
+search syntax docs: http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions#search-instance_method
+
+ES Actions docs: http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions
+
+ES Indices docs: http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Indices/Actions
+
+ES docs: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/index.html
+
+### Advanced:
+
+prefix all app's index names:
+
+    Mongoid::Elasticsearch.prefix = 'my_app'
+
+default options for Elasticsearch::Client.new (url etc)
+ 
+    Mongoid::Elasticsearch.client_options = {hosts: ['localhost']}
+
+index definition options and custom model serialization:
+
+    include Mongoid::Elasticsearch
+    elasticsearch!({
+      # index name (prefix is added)
+      index_name: 'mongoid_es_news',
+      
+      # don't use global name prefix
+      prefix_name: false,
+      
+      # elasticsearch index definition
+      index_options: {},
+
+      # or only mappings with empty options
+      index_mappings: {
+        name: {
+          type: 'multi_field',
+          fields: {
+            name:     {type: 'string', analyzer: 'snowball'},
+            raw:      {type: 'string', index: :not_analyzed},
+            suggest:  {type: 'completion'} 
+          }
+        },
+        tags: {type: 'string', include_in_all: false}
+      },
+      wrapper: :load
+    })
+    
+    # customize what gets sent to elasticsearch:
+    def as_indexed_json
+      # id field is properly added automatically
+      {
+        name: name,
+        excerpt: excerpt
+      }
+      # mongoid_slug note: add _slugs to as_indexed_json, NOT slug
+    end
+    
+Example mapping with boost field:
+
+    elasticsearch!({
+      index_name: Rails.env.test? ? 'vv_test_articles' : 'vv_articles',
+      index_options: {
+        settings: {
+          index: {
+            analysis: {
+              analyzer: {
+                my_analyzer: {
+                  type: "snowball",
+                  language: "Russian"
+                }
+              }
+            }
+          }
+        },
+        mappings: {
+          "articles/article" => {
+            _boost: {name: '_boost', null_value: 1},
+            properties: {
+              name: {type: 'string', boost: 10, analyzer: 'my_analyzer'},
+              tags: {type: 'string', analyzer: 'my_analyzer'}
+            }
+          }
+        }
+      },
+      wrapper: :load
+    })
+    
+    
+[Mapping definition docs](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html)    
+
+### Pagination
+
+```= paginate @posts``` should work as normal with Kaminari after you do:
+
+    @posts = Post.es.search(params[:search], page: params[:page])
+    # or
+    @posts = Post.es.search({
+      body: {
+        query: {
+          query_string: {
+            query: params[:search]
+          }
+        },
+        filter: {
+          term: {community_id: @community.id.to_s}
+        }
+      }},
+      page: params[:page], wrapper: :load
+    )
+
+### Reindexing
+
+#### All models
+
+```rake es:reindex``` will reindex all indices managed by Mongoid::Elasticsearch
+
+#### One model - Simple bulk
+
+This is the preferred (fastest) method to reindex everything
+    
+    Music::Video.es.index_all
+
+#### One model - Simple
+
+    Communities::Thread.es.index.reset
+    Communities::Thread.enabled.each do |ingr|
+      ingr.es_update
+    end
+
+### Possible wrappers for results:
+
+- :hash - raw hash from ES
+- :mash - [Hashie::Mash](https://github.com/intridea/hashie#mash) (gem '[hashie](https://github.com/intridea/hashie)' must be added to gemfile)
+- :load - load each found model by ID from database
+- :model - create a model instance from data stored in elasticsearch
+
+See more examples in specs.
+
+### Index creation
+
+This gem by default automatically creates indexes for all configured models on application startup.
+
+Set ```Mongoid::Elasticsearch.autocreate_indexes = false``` in an initalizer to prevent automatic creation for all indexes.
+
+You can always use ```rake es:create``` to create all indexes or call ```Mongoid::Elasticsearch.create_all_indexes!```.
+
+Indexes defined with option ```skip_create: true``` are not created with all other indexes and must be created manually with ```Model.es.index.create```
+
+
+#### Util
+
+    # Escape string so it can be safely passed to ES (removes all special characters)
+    Mongoid::Elasticsearch::Utils.clean(s)
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+desc 'Default: run specs.'
+task :default => :spec
+
+desc "Run specs"
+RSpec::Core::RakeTask.new

data/lib/mongoid/elasticsearch/callbacks.rb

@@ -0,0 +1,40 @@
+module Mongoid
+  module Elasticsearch
+    module Callbacks
+      extend ActiveSupport::Concern
+
+      included do
+        after_save :update_es_index
+        after_destroy :update_es_index
+        
+        ## THIS FUNCTION HAS BEEN UPDATED TO DO THE UPDATE ONLY IF THE OP_SUCCESS IS TRUE OR NIL, IN CASE THE RECORD RESPONDS TO OP_SUCCESS.
+        def update_es_index
+          if self.respond_to? :op_success
+            if self.op_success.nil?
+              es_update
+            else
+              es_update if self.op_success == true
+            end
+          else
+            es_update
+          end
+        end
+      end
+
+      module ClassMethods
+        def without_es_update!( &block )
+          skip_callback( :save, :after, :update_es_index )
+          skip_callback( :destroy, :after, :update_es_index )
+          
+          result = yield
+
+          set_callback( :save, :after, :update_es_index )
+          set_callback( :destroy, :after, :update_es_index )
+          
+          result
+        end
+      end
+    end
+  end
+end
+

data/lib/mongoid/elasticsearch/es.rb

@@ -0,0 +1,111 @@
+module Mongoid
+  module Elasticsearch
+    class Es
+      INDEX_STEP = 100
+      attr_reader :klass, :version
+
+      def initialize(klass)
+        @klass = klass
+        @version = Gem::Version.new(client.info['version']['number'])
+      end
+
+      def client
+        # dup is needed because Elasticsearch::Client.new changes options hash inplace
+        @client ||= ::Elasticsearch::Client.new klass.es_client_options.dup
+      end
+
+      def index
+        @index ||= Index.new(self)
+      end
+
+      def index_all(step_size = INDEX_STEP)
+        index.reset
+        q = klass.asc(:id)
+        steps = (q.count / step_size) + 1
+        last_id = nil
+        steps.times do |step|
+          if last_id
+            docs = q.gt(id: last_id).limit(step_size).to_a
+          else
+            docs = q.limit(step_size).to_a
+          end
+          last_id = docs.last.try(:id)
+          docs = docs.map do |obj|
+            if obj.es_index?
+              { index: {data: obj.as_indexed_json}.merge(_id: obj.id.to_s) }
+            else
+              nil
+            end
+          end.reject { |obj| obj.nil? }
+          next if docs.empty?
+          client.bulk({body: docs}.merge(type_options))
+          if block_given?
+            yield steps, step
+          end
+        end       
+      end
+
+      def search(query, options = {})
+        if query.is_a?(String)
+          query = {q: Utils.clean(query)}
+        end
+
+        page = options[:page]
+        per_page = options[:per_page].nil? ? options[:per] : options[:per_page]
+
+        query[:size] = ( per_page.to_i ) if per_page
+        query[:from] = ( page.to_i <= 1 ? 0 : (per_page.to_i * (page.to_i-1)) ) if page && per_page
+
+        options[:wrapper] ||= klass.es_wrapper
+
+        Response.new(client, query.merge(custom_type_options(options)), false, klass, options)
+      end
+
+      def all(options = {})
+        search({body: {query: {match_all: {}}}}, options)
+      end
+
+      def options_for(obj)
+        {id: obj.id.to_s}.merge type_options
+      end
+
+      def custom_type_options(options)
+        if !options[:include_type].nil? && options[:include_type] == false
+          {index: index.name}
+        else
+          type_options
+        end
+      end
+
+      def type_options
+        {index: index.name, type: index.type}
+      end
+
+      def index_item(obj)
+        client.index({body: obj.as_indexed_json}.merge(options_for(obj)))
+      end
+
+      def remove_item(obj)
+        client.delete(options_for(obj).merge(ignore: 404))
+      end
+
+      def completion_supported?
+        @version > Gem::Version.new('0.90.2')
+      end
+
+      def completion(text, field = "suggest")
+        raise "Completion not supported in ES #{@version}" unless completion_supported?
+        body = {
+          q: {
+            text: Utils.clean(text),
+            completion: {
+              field: field
+            }
+          }
+        }
+        results = client.suggest(index: index.name, body: body)
+        results['q'][0]['options']
+      end
+    end
+  end
+end