chewy 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84273a240a5c4dac56535e864d042f647b655bef
-  data.tar.gz: a66632f1c947d9c8f446ef581139de5175c55763
+  metadata.gz: 8eb12f365d07168ff5e5f52511029a07a5eccda1
+  data.tar.gz: aae1608aa3438ea0bc28ed43bf65ccde19e2bc8b
 SHA512:
-  metadata.gz: 08810417cade81d8cc563438139c1558dde2efc7321eeb1a988399b406c1339ca3bdacb56f083d6d4fc0ebdfbf3c8b7bf232c40ac034abe69f2e868d296821bd
-  data.tar.gz: 7c0cf989f04a92fea804ff67490f623037119ed58ab54ba3e4bfdce272f2d546403313e0afb11fd522794473e9b04719d0b24a9753299538c574f8d0c326efaf
+  metadata.gz: 2db0c6c51c6fc9061c5e751e2759213723b5a2524c4443e7688bd792b81615104d6f5ba698b5de33a0c91030045666d20957255a13e3d706c8f76b68bf7d4174
+  data.tar.gz: bab3cb005e8f0264022c39ecaa4d04b81b9dbccfc4de7fd28c637250746eb18ffea1d6cef89c2dcc7c09f4a4dccb0120a89e7bf15be1e1bf769dc85c10b5b4f1

data/.gitignore

@@ -17,3 +17,5 @@ test/tmp
 test/version_tmp
 tmp
 .rvmrc
+_site
+.sass-cache

data/.rspec

@@ -1,3 +1,2 @@
 --color
 --backtrace
---order random

data/.travis.yml

@@ -37,6 +37,6 @@ matrix:
     - rvm: 2.2.0
       gemfile: gemfiles/rails.3.2.activerecord.will_paginate.gemfile
 before_install:
-  - curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.4.tar.gz | tar xz -C /tmp
+  - curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.5.0.tar.gz | tar xz -C /tmp
 before_script:
-  - TEST_CLUSTER_COMMAND="/tmp/elasticsearch-1.4.4/bin/elasticsearch" rake elasticsearch:start
+  - TEST_CLUSTER_COMMAND="/tmp/elasticsearch-1.5.0/bin/elasticsearch" rake elasticsearch:start

data/Appraisals

@@ -2,12 +2,14 @@
   appraise "rails.#{version}.activerecord" do
     gem 'activerecord', "~> #{version}.0"
     gem 'activesupport', "~> #{version}.0"
+    gem 'resque', require: false
+    gem 'sidekiq', require: false
   end
 
   appraise "rails.#{version}.activerecord.kaminari" do
     gem 'activerecord', "~> #{version}.0"
     gem 'activesupport', "~> #{version}.0"
-    gem 'kaminari', require: false
+    gem 'kaminari', '0.16.3', require: false
   end
 
   appraise "rails.#{version}.activerecord.will_paginate" do
@@ -21,12 +23,14 @@ end
   appraise "rails.#{version}.mongoid" do
     gem 'mongoid', '~> 4.0.0'
     gem 'activesupport', "~> #{version}.0"
+    gem 'resque', require: false
+    gem 'sidekiq', require: false
   end
 
   appraise "rails.#{version}.mongoid.kaminari" do
     gem 'mongoid', '~> 4.0.0'
     gem 'activesupport', "~> #{version}.0"
-    gem 'kaminari', require: false
+    gem 'kaminari', '0.16.3', require: false
   end
 
   appraise "rails.#{version}.mongoid.will_paginate" do

data/CHANGELOG.md

@@ -1,9 +1,37 @@
 # master
 
+# Version 0.8.0
+
+## Incompatible changes:
+
+  * `:atomic` and `:urgent` strategies are using `import!` method raising exceptions
+
+## Changes
+
+  * Crutches™ technology
+
+  * Added `.script_fields` chainable method to query (@ka8725)
+
+  * `update_index` mocha support (@lardawge)
+
+  * `:resque` async strategy
+
+  * `:sidekiq` async strategy (inspired by @sharkzp)
+
+  * Added `Query#search_type` for `search_type` request option setup (@marshall-lee)
+
+## Bugfixes
+
+  * Rails 4.2 migrations are not raising UndefinedUpdateStrategy anymore on data updates
+
+  * Mongoid random failing specs fixes (@marshall-lee)
+
+# Version 0.7.0
+
 ## Incompatible changes:
 
   * `Chewy.use_after_commit_callbacks = false` returns previous RDBMS behavior
-  in tests.
+  in tests
 
   * ActiveRecord import is now called after_commit instead of after_save and after_destroy
 

data/Gemfile

@@ -4,9 +4,13 @@ gemspec
 
 gem 'activerecord'
 # gem 'mongoid'
+
 # gem 'kaminari', require: false
 # gem 'will_pagnate', require: false
 
+gem 'resque', require: false
+gem 'sidekiq', require: false
+
 group :test do
   gem 'guard'
   gem 'guard-rspec'

data/README.md

@@ -1,3 +1,4 @@
+[![Gem Version](https://badge.fury.io/rb/chewy.svg)](http://badge.fury.io/rb/chewy)
 [![Build Status](https://travis-ci.org/toptal/chewy.png)](https://travis-ci.org/toptal/chewy)
 [![Code Climate](https://codeclimate.com/github/toptal/chewy.png)](https://codeclimate.com/github/toptal/chewy)
 [![Inline docs](http://inch-ci.org/github/toptal/chewy.svg?branch=master)](http://inch-ci.org/github/toptal/chewy)
@@ -195,6 +196,86 @@ See [config.rb](lib/chewy/config.rb) for more details.
   update_index('users#user', :users)
   ```
 
+  In case of belongs_to association you may need to update both associated objects, previous and current:
+
+  ```ruby
+  class City < ActiveRecord::Base
+    belongs_to :country
+
+    update_index('cities#city') { self }
+    update_index 'countries#country' do
+      # For the latest active_record changed values are
+      # already in `previous_changes` hash,
+      # but for mongoid you have to use `changes` hash
+      previous_changes['country_id'] || country
+    end
+  end
+  ```
+
+### Crutches™ technology
+
+Assume you are defining index like this (product has_many categories through product_categories):
+
+```ruby
+class ProductsIndex < Chewy::Index
+  define_type Product.includes(:categories) do
+    field :name
+    field :category_names, value: ->(product) { product.categories.map(&:name) } # or shorter just -> { categories.map(&:name) }
+  end
+end
+```
+
+Then chewy reindexing flow would be look like following pseudo-code (even in mongoid):
+
+```ruby
+Product.includes(:categories).find_in_batches(1000) do |batch|
+  bulk_body = batch.map do |object|
+    {name: object.name, category_names: object.categories.map(&:name)}.to_json
+  end
+  # here we are sending every batch of data to ES
+  Chewy.client.bulk bulk_body
+end
+```
+
+But in rails 4.1 and 4.2 you may face with slow associations problem (take a look on https://github.com/rails/rails/pull/19423) also, there might be really complicated cases when associations are not applicable.
+
+Then you are able to replace rails associations with Chewy Crutches™ technology:
+
+```ruby
+class ProductsIndex < Chewy::Index
+  define_type Product.includes(:categories) do
+    crutch :categories do |collection| # collection here is a current batch of products
+      # data is fetched with a lightweight query without objects initialization
+      data = ProductCategory.joins(:category).where(product_id: collection.map(&:id)).pluck(:product_id, 'categories.name')
+      # then we have to convert fetched data to appropriate format
+      # this will return our data in structure like:
+      # {123 => ['seweets', 'juices'], 456 => ['meat']}
+      data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+    end
+
+    field :name
+    # simply use crutch-fetched data as a value:
+    field :category_names, value: ->(product, crutches) { crutches.categories[product.id] }
+  end
+end
+```
+
+And example flow would be look like this:
+
+```ruby
+Product.includes(:categories).find_in_batches(1000) do |batch|
+  crutches[:categories] = ProductCategory.joins(:category).where(product_id: batch.map(&:id)).pluck(:product_id, 'categories.name')
+    .each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
+
+  bulk_body = batch.map do |object|
+    {name: object.name, category_names: crutches[:categories][object.id]}.to_json
+  end
+  Chewy.client.bulk bulk_body
+end
+```
+
+So Chewy Crutches™ technology is able to increase your indexing performance in some cases up to 100 times or even more depending on your associations complexity.
+
 ### Types access
 
 You are able to access index-defined types with the following API:
@@ -280,6 +361,29 @@ Using this strategy delays index update request until the end of
 block. Updated records are aggregated and index update happens with
 bulk API. So this strategy is highly optimized.
 
+#### `:resque`
+
+Does the same thing as `:atomic`, but in async way using resque.
+Default queue name is `chewy`.
+Patch `Chewy::Strategy::Resque::Worker` for index updates improving.
+
+```ruby
+Chewy.strategy(:resque) do
+  City.popular.map(&:do_some_update_action!)
+end
+```
+
+#### `:sidekiq`
+
+Does the same thing as `:atomic`, but in async way using sidekiq.
+Patch `Chewy::Strategy::Sidekiq::Worker` for index updates improving.
+
+```ruby
+Chewy.strategy(:sidekiq) do
+  City.popular.map(&:do_some_update_action!)
+end
+```
+
 #### `:urgent`
 
 Next strategy is convenient if you are going to update documents in
@@ -339,22 +443,12 @@ city3.do_update! # index updated again
 
 #### Designing own strategies
 
-Async strategy is not implemented yet, but it is planned. So
-it would be a good idea to implements own async strategy for
-particular delayed jobs library or simply threads.
-
 See [strategy/base.rb](lib/chewy/strategy/base.rb) for more details.
 See [strategy/atomic.rb](lib/chewy/strategy/atomic.rb) for example.
 
-### Async reindexing
-
-Chewy is not support async index update, but it's planned. Until you can use third-party solutions, such as [https://github.com/averell23/chewy_kiqqer](https://github.com/averell23/chewy_kiqqer)
-
-Not sure it works currently.
-
 ### Rails application strategies integration
 
-There is a couple of pre-defined strategies for your rails application. At first, rails console uses `:urgent` strategy by default, except the sandbox case. Whan you are running sandbox it switches to `bypass` strategy to avoid index polluting.
+There is a couple of pre-defined strategies for your rails application. At first, rails console uses `:urgent` strategy by default, except the sandbox case. When you are running sandbox it switches to `bypass` strategy to avoid index polluting.
 
 Also migrations are wrapped with `:bypass` strategy. Because the main behavor implies that indexes are resetted after migration, so there is no need for extra index updates.
 Also indexing might be broken during migrations because of the outdated schema.
@@ -688,7 +782,7 @@ Compliance cheatsheet for filters and DSL expressions:
 
   ```json
   {"has_child": {"type": "blog_tag", "query": {"term": {"tag": "something"}}}
-  {"has_child": {"type": "comment", "term": {"term": {"user": "john"}}}
+  {"has_child": {"type": "comment", "filter": {"term": {"user": "john"}}}
   ```
 
   ```ruby
@@ -728,6 +822,23 @@ When the response comes back, it will have the ```:facets``` sidechannel include
 < { ... ,"facets":{"countries":{"_type":"terms","missing":?,"total":?,"other":?,"terms":[{"term":"USA","count":?},{"term":"Brazil","count":?}, ...}}
 ```
 
+### Script fields
+
+Script fields allow to execute elasticsearch's scripting language such as groovy, javascript and etc. More about supported languages and what is scripting [here](https://www.elastic.co/guide/en/elasticsearch/reference/0.90/modules-scripting.html). This feature allows to calculate distance between geo points, for example. This is how to use the DSL:
+
+```ruby
+UsersIndex.script_fields(
+  distance: {
+    params: {
+      lat: 37.569976,
+      lon: -122.351591
+    },
+    script: "doc['coordinates'].distanceInMiles(lat, lon)"
+  }
+)
+```
+`coordinates` here is a field with `geo_point` type. There will be `distance` field for the index's model in the search result.
+
 ### Script scoring
 
 Script scoring is used to score the search results. All scores are added to the search request and combined according to boost mode and score mode. This can be useful if, for example, a score function is computationally expensive and it is sufficient to compute the score on a filtered set of documents. For example, you might want to multiply the score by another numeric field in the doc:
@@ -833,27 +944,34 @@ end
 Inside Rails application some index mantaining rake tasks are defined.
 
 ```bash
-rake chewy:reset:all # resets all the existing indexes, declared in app/chewy
-rake chewy:reset # alias for chewy:reset:all
-rake chewy:reset[users] # resets UsersIndex
+rake chewy:reset # resets all the existing indexes, declared in app/chewy
+rake chewy:reset[users] # resets UsersIndex only
 
-rake chewy:update:all # updates all the existing indexes, declared in app/chewy
-rake chewy:update # alias for chewy:update:all
-rake chewy:update[users] # updates UsersIndex
+rake chewy:update # updates all the existing indexes, declared in app/chewy
+rake chewy:update[users] # updates UsersIndex only
 ```
 
+Also `rake chewy:reset` performs zero-downtime reindexing as described here: https://www.elastic.co/blog/changing-mapping-with-zero-downtime. So basically rake task creates new index with uniq suffix and then simply aliases it to the common index name. Previous index is deleted afterwards (see `Chewy::Index.reset!` for more details).
+
+
 ### Rspec integration
 
 Just add `require 'chewy/rspec'` to your spec_helper.rb and you will get additional features:
 See [update_index.rb](lib/chewy/rspec/update_index.rb) for more details.
 
+If you use `DatabaseCleaner` in your tests with `transaction` (strategy)[https://github.com/DatabaseCleaner/database_cleaner#how-to-use] you may run into the problem that `ActiveRecord`'s models are not indexed automatically on save them despite of the fact that you set the callbacks to do this with the `update_index` method. The issue arises because `chewy` indexes data on `after_commit` run as default but all `after_commit` callbacks are not run with the `DatabaseCleaner`'s' `transaction` strategy. You can solve the issue by changing the `Chewy.use_after_commit_callbacks` option. Just add the following initializer in your Rails application:
+
+```ruby
+#config/initializers/chewy.rb
+Chewy.use_after_commit_callbacks = !Rails.env.test?
+```
+
 ## TODO a.k.a coming soon:
 
 * Typecasting support
 * Advanced (simplyfied) query DSL: `UsersIndex.query { email == 'my@gmail.com' }` will produce term query
 * update_all support
 * Maybe, closer ORM/ODM integration, creating index classes implicitly
-* Async indexes updating
 
 ## Contributing
 

data/chewy.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'sqlite3'
   spec.add_development_dependency 'database_cleaner', '~> 1.3.0'
   spec.add_development_dependency 'elasticsearch-extensions'
+  spec.add_development_dependency 'resque_spec'
   spec.add_development_dependency 'rubysl', '~> 2.0' if RUBY_ENGINE == 'rbx'
 
   spec.add_dependency 'activesupport', '>= 3.2'

data/gemfiles/rails.3.2.activerecord.gemfile

@@ -4,6 +4,8 @@ source "https://rubygems.org"
 
 gem "activerecord", "~> 3.2.0"
 gem "activesupport", "~> 3.2.0"
+gem "resque", :require => false
+gem "sidekiq", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.3.2.activerecord.kaminari.gemfile

@@ -4,7 +4,7 @@ source "https://rubygems.org"
 
 gem "activerecord", "~> 3.2.0"
 gem "activesupport", "~> 3.2.0"
-gem "kaminari", :require => false
+gem "kaminari", "0.16.3", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.4.0.activerecord.gemfile

@@ -4,6 +4,8 @@ source "https://rubygems.org"
 
 gem "activerecord", "~> 4.0.0"
 gem "activesupport", "~> 4.0.0"
+gem "resque", :require => false
+gem "sidekiq", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.4.0.activerecord.kaminari.gemfile

@@ -4,7 +4,7 @@ source "https://rubygems.org"
 
 gem "activerecord", "~> 4.0.0"
 gem "activesupport", "~> 4.0.0"
-gem "kaminari", :require => false
+gem "kaminari", "0.16.3", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.4.0.mongoid.gemfile

@@ -4,6 +4,8 @@ source "https://rubygems.org"
 
 gem "mongoid", "~> 4.0.0"
 gem "activesupport", "~> 4.0.0"
+gem "resque", :require => false
+gem "sidekiq", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.4.0.mongoid.kaminari.gemfile

@@ -4,7 +4,7 @@ source "https://rubygems.org"
 
 gem "mongoid", "~> 4.0.0"
 gem "activesupport", "~> 4.0.0"
-gem "kaminari", :require => false
+gem "kaminari", "0.16.3", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.4.1.activerecord.gemfile

@@ -4,6 +4,8 @@ source "https://rubygems.org"
 
 gem "activerecord", "~> 4.1.0"
 gem "activesupport", "~> 4.1.0"
+gem "resque", :require => false
+gem "sidekiq", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.4.1.activerecord.kaminari.gemfile

@@ -4,7 +4,7 @@ source "https://rubygems.org"
 
 gem "activerecord", "~> 4.1.0"
 gem "activesupport", "~> 4.1.0"
-gem "kaminari", :require => false
+gem "kaminari", "0.16.3", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.4.1.mongoid.gemfile

@@ -4,6 +4,8 @@ source "https://rubygems.org"
 
 gem "mongoid", "~> 4.0.0"
 gem "activesupport", "~> 4.1.0"
+gem "resque", :require => false
+gem "sidekiq", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.4.1.mongoid.kaminari.gemfile

@@ -4,7 +4,7 @@ source "https://rubygems.org"
 
 gem "mongoid", "~> 4.0.0"
 gem "activesupport", "~> 4.1.0"
-gem "kaminari", :require => false
+gem "kaminari", "0.16.3", :require => false
 
 group :test do
   gem "guard"

data/gemfiles/rails.4.2.activerecord.gemfile

@@ -4,6 +4,8 @@ source "https://rubygems.org"
 
 gem "activerecord", "~> 4.2.0"
 gem "activesupport", "~> 4.2.0"
+gem "resque", :require => false
+gem "sidekiq", :require => false
 
 group :test do
   gem "guard"