chewy 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12e0c49636421098824baf242c46477ec44a1253
-  data.tar.gz: d8861ffbacc81ec75d4c9a40b36ec19b345ef3aa
+  metadata.gz: 84273a240a5c4dac56535e864d042f647b655bef
+  data.tar.gz: a66632f1c947d9c8f446ef581139de5175c55763
 SHA512:
-  metadata.gz: a29788d87989658ea5a7017554217b21bfdf6a52c7486a740c977261f5a095b591d5f4f352bb1ccbe5f8a26c74f8e355c0f4c2ce4b896a3a71cdb6106974434f
-  data.tar.gz: cffc378f5fe51f0a9507c43315491f005b6f1ad9d993946786583a54df95b44bcba56483d186c6356d2fef2168dc692979146a900b8c33870782eb84cc4c6cb6
+  metadata.gz: 08810417cade81d8cc563438139c1558dde2efc7321eeb1a988399b406c1339ca3bdacb56f083d6d4fc0ebdfbf3c8b7bf232c40ac034abe69f2e868d296821bd
+  data.tar.gz: 7c0cf989f04a92fea804ff67490f623037119ed58ab54ba3e4bfdce272f2d546403313e0afb11fd522794473e9b04719d0b24a9753299538c574f8d0c326efaf

data/.gitignore

@@ -3,7 +3,8 @@
 .bundle
 .config
 .yardoc
-Gemfile*.lock
+Gemfile.lock
+gemfiles/*.lock
 InstalledFiles
 _yardoc
 coverage

data/.travis.yml

@@ -1,36 +1,42 @@
 language: ruby
+services:
+  - mongodb
 rvm:
   - 2.0.0
-  - 2.1.3
+  - 2.1.5
+  - 2.2.0
   # - rbx
 gemfile:
-  - Gemfile
-  - gemfiles/Gemfile.rails-3.2.active_record
-  - gemfiles/Gemfile.rails-3.2.active_record.kaminari
-  - gemfiles/Gemfile.rails-3.2.active_record.will_paginate
-  - gemfiles/Gemfile.rails-4.0.active_record
-  - gemfiles/Gemfile.rails-4.0.active_record.kaminari
-  - gemfiles/Gemfile.rails-4.0.active_record.will_paginate
-  - gemfiles/Gemfile.rails-4.0.mongoid
-  - gemfiles/Gemfile.rails-4.0.mongoid.kaminari
-  - gemfiles/Gemfile.rails-4.0.mongoid.will_paginate
-  - gemfiles/Gemfile.rails-4.1.active_record
-  - gemfiles/Gemfile.rails-4.1.active_record.kaminari
-  - gemfiles/Gemfile.rails-4.1.active_record.will_paginate
-  - gemfiles/Gemfile.rails-4.1.mongoid
-  - gemfiles/Gemfile.rails-4.1.mongoid.kaminari
-  - gemfiles/Gemfile.rails-4.1.mongoid.will_paginate
-  # - gemfiles/Gemfile.rails-4.2.active_record
-  # - gemfiles/Gemfile.rails-4.2.active_record.kaminari
-  # - gemfiles/Gemfile.rails-4.2.active_record.will_paginate
-  # - gemfiles/Gemfile.rails-4.2.mongoid
-  # - gemfiles/Gemfile.rails-4.2.mongoid.kaminari
-  # - gemfiles/Gemfile.rails-4.2.mongoid.will_paginate
-
+  - gemfiles/rails.3.2.activerecord.gemfile
+  - gemfiles/rails.3.2.activerecord.kaminari.gemfile
+  - gemfiles/rails.3.2.activerecord.will_paginate.gemfile
+  - gemfiles/rails.4.0.activerecord.gemfile
+  - gemfiles/rails.4.0.activerecord.kaminari.gemfile
+  - gemfiles/rails.4.0.activerecord.will_paginate.gemfile
+  - gemfiles/rails.4.1.activerecord.gemfile
+  - gemfiles/rails.4.1.activerecord.kaminari.gemfile
+  - gemfiles/rails.4.1.activerecord.will_paginate.gemfile
+  - gemfiles/rails.4.2.activerecord.gemfile
+  - gemfiles/rails.4.2.activerecord.kaminari.gemfile
+  - gemfiles/rails.4.2.activerecord.will_paginate.gemfile
+  - gemfiles/rails.4.0.mongoid.gemfile
+  - gemfiles/rails.4.0.mongoid.kaminari.gemfile
+  - gemfiles/rails.4.0.mongoid.will_paginate.gemfile
+  - gemfiles/rails.4.1.mongoid.gemfile
+  - gemfiles/rails.4.1.mongoid.kaminari.gemfile
+  - gemfiles/rails.4.1.mongoid.will_paginate.gemfile
+  - gemfiles/rails.4.2.mongoid.gemfile
+  - gemfiles/rails.4.2.mongoid.kaminari.gemfile
+  - gemfiles/rails.4.2.mongoid.will_paginate.gemfile
+matrix:
+  exclude:
+    - rvm: 2.2.0
+      gemfile: gemfiles/rails.3.2.activerecord.gemfile
+    - rvm: 2.2.0
+      gemfile: gemfiles/rails.3.2.activerecord.kaminari.gemfile
+    - rvm: 2.2.0
+      gemfile: gemfiles/rails.3.2.activerecord.will_paginate.gemfile
 before_install:
-  - curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.3.0.tar.gz | tar xz -C /tmp
+  - curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.4.tar.gz | tar xz -C /tmp
 before_script:
-  - TEST_CLUSTER_COMMAND="/tmp/elasticsearch-1.3.0/bin/elasticsearch" rake elasticsearch:start
-services:
-  - mongodb
-
+  - TEST_CLUSTER_COMMAND="/tmp/elasticsearch-1.4.4/bin/elasticsearch" rake elasticsearch:start

data/Appraisals

@@ -0,0 +1,37 @@
+%w(3.2 4.0 4.1 4.2).each do |version|
+  appraise "rails.#{version}.activerecord" do
+    gem 'activerecord', "~> #{version}.0"
+    gem 'activesupport', "~> #{version}.0"
+  end
+
+  appraise "rails.#{version}.activerecord.kaminari" do
+    gem 'activerecord', "~> #{version}.0"
+    gem 'activesupport', "~> #{version}.0"
+    gem 'kaminari', require: false
+  end
+
+  appraise "rails.#{version}.activerecord.will_paginate" do
+    gem 'activerecord', "~> #{version}.0"
+    gem 'activesupport', "~> #{version}.0"
+    gem 'will_paginate', require: false
+  end
+end
+
+%w(4.0 4.1 4.2).each do |version|
+  appraise "rails.#{version}.mongoid" do
+    gem 'mongoid', '~> 4.0.0'
+    gem 'activesupport', "~> #{version}.0"
+  end
+
+  appraise "rails.#{version}.mongoid.kaminari" do
+    gem 'mongoid', '~> 4.0.0'
+    gem 'activesupport', "~> #{version}.0"
+    gem 'kaminari', require: false
+  end
+
+  appraise "rails.#{version}.mongoid.will_paginate" do
+    gem 'mongoid', '~> 4.0.0'
+    gem 'activesupport', "~> #{version}.0"
+    gem 'will_paginate', require: false
+  end
+end

data/CHANGELOG.md

@@ -1,5 +1,116 @@
 # master
 
+## Incompatible changes:
+
+  * `Chewy.use_after_commit_callbacks = false` returns previous RDBMS behavior
+  in tests.
+
+  * ActiveRecord import is now called after_commit instead of after_save and after_destroy
+
+  * Import now respects default scope and removes unmatched documents
+
+  * `delete_from_index?` method is deprecated, use
+
+    ```ruby
+      define_type User, delete_if: ->{ removed? } do
+        ...
+      end
+    ```
+
+  * `Chewy.request_strategy` to configure action controller's request wrapping strategy
+
+  * `Chewy.root_strategy` to configure the first strategy in stack
+
+  * Default strategy for controller actions is `:atomic`
+
+  * Default strategy for activerecord migrations is `:bypass`
+
+  * Default strategy for sandbox console is `:bypass`
+
+  * Default strategy for rails console is `:urgent`
+
+  * `Chewy.configuration` was renamed to `Chewy.settings`
+
+  * Reworked index update strategies implementation. `Chewy.atomic`
+  and `Chewy.urgent_update` are now deprecated in favour of the new
+  `Chewy.strategy` API.
+
+  * Loading objects for object-sourced types using `wrap` method is
+  deprecated, `load_one` method should be used instead. Or method name
+  might be passed to `define_type`:
+
+    ```ruby
+      class GeoData
+        def self.get_data(elasticsearch_document)
+          REDIS.get("geo_data_#{elasticsearch_document.id}")
+        end
+      end
+
+      ...
+        define_type GeoData, load_one_method: :get_data do
+          ...
+        end
+    ```
+
+## Changes
+
+  * Multiple enhancements by @DNNX
+
+  * Added `script_fields` to search crteria (@ka8725)
+
+  * ORM adapters now completely relies on the default scope. This means every scope or objects passed to import are merged with default scope so basically there is no need to define `delete_if` block. Default scope strongly restricts objects which may land in the current index.
+
+    ```ruby
+      define_type Country.where("rating > 30") do
+
+      end
+
+      # this code would import only countries with rating between 30 and 50
+      CountriesIndex::Country.import(Country.where("rating < 50"))
+
+      # the same is true for arrays of objects or ids
+      CountriesIndex::Country.import(Country.where("rating < 50").to_a)
+      CountriesIndex::Country.import(Country.where("rating < 50").pluck(:id))
+    ```
+
+  * Object adapter supports custom initial import and load methods, so it
+  could be configured to be used with procs or any class responding to `call`
+  method.
+
+    ```ruby
+      class GeoData
+        def self.call
+          REDIS.get_all
+        end
+      end
+
+      ...
+        define_type GeoData do
+          ...
+        end
+    ```
+
+  * Nested fields value procs additional arguments: parent objects.
+
+    ```ruby
+      define_type Country do
+        field :name
+        field :cities do
+          field :district, value: ->(city, country) { city.districts if country.main? }
+        end
+      end
+    ```
+
+  * Implemented basic named scopes
+
+## Bugfixes
+
+  * `script_score` allow options (@joeljunstrom)
+
+  * Chewy indexes eaged loading fixes (@leemhenson)
+
+  * `Chewy::Index.import nil` imports nothing instead of initial data
+
 # Version 0.6.2
 
 ## Changes
@@ -38,14 +149,14 @@
 
 # Version 0.5.2
 
-## Changes
-
-  * `Chewy.massacre` aliased to `Chewy.delete_all` method deletes all the indexes with current prefix
-
 ## Incompatible changes:
 
   * `Chewy::Type::Base` removed in favour of using `Chewy::Type` as a base class for all types
 
+## Changes
+
+  * `Chewy.massacre` aliased to `Chewy.delete_all` method deletes all the indexes with current prefix
+
 ## Bugfixes:
 
   * Advanced type classes resolving (@inbeom)

data/Gemfile

@@ -1,12 +1,11 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in chewy.gemspec
 gemspec
 
 gem 'activerecord'
 # gem 'mongoid'
-gem 'kaminari', require: false
-# gem 'will_paginate', require: false
+# gem 'kaminari', require: false
+# gem 'will_pagnate', require: false
 
 group :test do
   gem 'guard'

data/README.md

@@ -2,6 +2,9 @@
 [![Code Climate](https://codeclimate.com/github/toptal/chewy.png)](https://codeclimate.com/github/toptal/chewy)
 [![Inline docs](http://inch-ci.org/github/toptal/chewy.svg?branch=master)](http://inch-ci.org/github/toptal/chewy)
 
+<p align="right">Sponsored by</p>
+<p align="right"><a href="http://www.toptal.com/"><img src="http://www.toptal.com/assets/public/blocks/logo/big.png" alt="Toptal" width="105" height="34"></a></p>
+
 # Chewy
 
 Chewy is ODM and wrapper for official elasticsearch client (https://github.com/elasticsearch/elasticsearch-ruby)
@@ -18,7 +21,7 @@ Chewy is ODM and wrapper for official elasticsearch client (https://github.com/e
 
 * Bulk import everywhere.
 
-  Chewy utilizes bulk ES API for full reindexing or index updates. Also it uses atomic updates concept. All the changed objects are collected inside the atomic block and index is updated once at the end of it with all the collected object. See `Chewy.atomic` for more details.
+  Chewy utilizes bulk ES API for full reindexing or index updates. Also it uses atomic updates concept. All the changed objects are collected inside the atomic block and index is updated once at the end of it with all the collected object. See `Chewy.strategy(:atomic)` for more details.
 
 * Powerful querying DSL.
 
@@ -40,15 +43,15 @@ Or install it yourself as:
 
 ## Usage
 
-### Client configuration
+### Client settings
 
-There are 2 ways to configure Chewy client: `Chewy.configuration` hash and `chewy.yml`
+There are 2 ways to configure Chewy client: `Chewy.settings` hash and `chewy.yml`
 
 You can create this file manually or run `rails g chewy:install` to do that with yaml way
 
 ```ruby
 # config/initializers/chewy.rb
-Chewy.configuration = {host: 'localhost:9250'} # do not use environments
+Chewy.settings = {host: 'localhost:9250'} # do not use environments
 ```
 
 ```yaml
@@ -64,7 +67,7 @@ development:
 The result config merges both hashes. Client options are passed as is to Elasticsearch::Transport::Client except the `:prefix` - it is used internally by chewy to create prefixed index names:
 
 ```ruby
-  Chewy.configuration = {prefix: 'test'}
+  Chewy.settings = {prefix: 'test'}
   UsersIndex.index_name # => 'test_users'
 ```
 
@@ -108,6 +111,8 @@ See [config.rb](lib/chewy/config.rb) for more details.
       field :projects do # the same block syntax for multi_field, if `:type` is specified
         field :title
         field :description # default data type is `string`
+        # additional top-level objects passed to value proc:
+        field :categories, value: ->(project, user) { project.categories.map(&:name) if user.active? }
       end
       field :rating, type: 'integer' # custom data type
       field :created, type: 'date', include_in_all: false,
@@ -196,7 +201,7 @@ You are able to access index-defined types with the following API:
 
 ```ruby
 UsersIndex::User # => UsersIndex::User
-UsersIndex.types_hash['user'] # => UsersIndex::User
+UsersIndex.type_hash['user'] # => UsersIndex::User
 UsersIndex.user # => UsersIndex::User
 UsersIndex.types # => [UsersIndex::User]
 UsersIndex.type_names # => ['user']
@@ -226,57 +231,147 @@ UsersIndex.import user: User.where('rating > 100') # import only active users to
 UsersIndex.reset! # purges index and imports default data for all types
 ```
 
-Also if passed user is `#destroyed?` or `#delete_from_index?` or specified id does not exists in the database, import will perform delete from index action for this object.
+Also if passed user is `#destroyed?`, or satisfy `delete_if` type option, or specified id does not exists in the database, import will perform delete from index action for this object.
+
+```ruby
+define_type User, delete_if: :deleted_at
+define_type User, delete_if: -> { deleted_at }
+define_type User, delete_if: ->(user) { user.deleted_at }
+```
 
 See [actions.rb](lib/chewy/index/actions.rb) for more details.
 
-### Observing strategies
+### Index update strategies
+
+Assume you've got the following code:
+
+```ruby
+class City < ActiveRecord::Base
+  update_index 'cities#city', :self
+end
+
+class CitiesIndex < Chewy::Index
+  define_type City do
+    field :name
+  end
+end
+```
+
+If you'll perform something like `City.first.save!` you'll get
+UndefinedUpdateStrategy exception instead of normal object saving
+and index update. This exception forces you to choose appropriate
+update strategy for current context.
 
-There are 3 strategies for index updating: do not update index at all, update right after save and cumulative update. The first is by default.
+If you want to return behavior was before 0.7.0 - just set
+`Chewy.root_strategy = :bypass`.
 
-**WARN: It is preferred to use `Chewy.atomic` block in most cases due to performance restrictions of the urgent updates!**
+#### `:atomic`
 
-#### Updating index on-demand
+The main strategy here is `:atomic`. Assume you have to update a
+lot of records in db.
 
-By default Chewy indexes are not updated when the observed model is saved or destroyed.
-This depends on the `Chewy.urgent_update` (false by default) or on the per-model update config.
-If you will perform `Chewy.urgent_update = true`, all the models will start to update elasticsearch
-index right after save.
+```ruby
+Chewy.strategy(:atomic) do
+  City.popular.map(&:do_some_update_action!)
+end
+```
 
-Note than urgent update options affects only outside-atomic-block behavour. Inside
-the `Chewy.atomic { }` block indexes updates as described below.
+Using this strategy delays index update request until the end of
+block. Updated records are aggregated and index update happens with
+bulk API. So this strategy is highly optimized.
 
-#### Using atomic updates
+#### `:urgent`
 
-To perform atomic cummulative updates, use `Chewy.atomic`:
+Next strategy is convenient if you are going to update documents in
+index one-by-one.
 
 ```ruby
-Chewy.atomic do
-  user.each { |user| user.update_attributes(name: user.name.strip) }
+Chewy.strategy(:urgent) do
+  City.popular.map(&:do_some_update_action!)
 end
 ```
 
-Index update will be performed once per Chewy.atomic block for every affected type.
-This strategy is highly usable for rails actions:
+This code would perform `City.popular.count` requests for ES
+documents update.
+
+Seems to be convenient for usage in e.g. rails console with
+non-block notation:
 
 ```ruby
-class ApplicationController < ActionController::Base
-  around_action :chewy_atomic
+> Chewy.strategy(:urgent)
+> City.popular.map(&:do_some_update_action!)
+```
 
-  def chewy_atomic
-    Chewy.atomic do
-      yield
-    end
+#### `:bypass`
+
+Bypass strategy simply silences index updates.
+
+#### Nesting
+
+Strategies are designed to allow nesting, so it is possible
+to redefine it for nested contexts.
+
+```ruby
+Chewy.strategy(:atomic) do
+  city1.do_update!
+  Chewy.strategy(:urgent) do
+    city2.do_update!
+    city3.do_update!
+    # there will be 2 update index requests for city2 and city3
   end
+  city4..do_update!
+  # city1 and city4 will be grouped in one index update request
 end
 ```
 
-Also atomic blocks might be nested and don't affect each other.
+#### Non-block notation
+
+It is possible to nest strategies without blocks:
+
+```ruby
+Chewy.strategy(:urgent)
+city1.do_update! # index updated
+Chewy.strategy(:bypass)
+city2.do_update! # update bypassed
+Chewy.strategy.pop
+city3.do_update! # index updated again
+```
+
+#### Designing own strategies
+
+Async strategy is not implemented yet, but it is planned. So
+it would be a good idea to implements own async strategy for
+particular delayed jobs library or simply threads.
+
+See [strategy/base.rb](lib/chewy/strategy/base.rb) for more details.
+See [strategy/atomic.rb](lib/chewy/strategy/atomic.rb) for example.
 
 ### Async reindexing
 
 Chewy is not support async index update, but it's planned. Until you can use third-party solutions, such as [https://github.com/averell23/chewy_kiqqer](https://github.com/averell23/chewy_kiqqer)
 
+Not sure it works currently.
+
+### Rails application strategies integration
+
+There is a couple of pre-defined strategies for your rails application. At first, rails console uses `:urgent` strategy by default, except the sandbox case. Whan you are running sandbox it switches to `bypass` strategy to avoid index polluting.
+
+Also migrations are wrapped with `:bypass` strategy. Because the main behavor implies that indexes are resetted after migration, so there is no need for extra index updates.
+Also indexing might be broken during migrations because of the outdated schema.
+
+Controller actions are wrapped with `:atomic` strategy with middleware just to reduce amount of index update requests inside actions.
+
+It is also a good idea to set up `:bypass` strategy inside your test suite and import objects manually only when needed, plus use `Chewy.massacre` when needed to flush test ES indexes before every example. This will allow to minimize unnecessary ES requests and reduce overhead.
+
+
+```ruby
+RSpec.configure do |config|
+  config.before(:suite) do
+    Chewy.strategy(:bypass)
+  end
+end
+```
+
 ### Index querying
 
 ```ruby
@@ -692,7 +787,7 @@ Chewy has notifing the following events:
     {index: 30, delete: 5}
     ```
 
-  * `payload[:erorrs]`: might not exists. Contains grouped errors with objects ids list:
+  * `payload[:errors]`: might not exists. Contains grouped errors with objects ids list:
 
     ```ruby
     {index: {
@@ -709,26 +804,26 @@ To integrate with NewRelic you may use the following example source (config/init
 
 ```ruby
 ActiveSupport::Notifications.subscribe('import_objects.chewy') do |name, start, finish, id, payload|
-  metrics = "Database/ElasticSearch/import"
+  metric_name = "Database/ElasticSearch/import"
   duration = (finish - start).to_f
   logged = "#{payload[:type]} #{payload[:import].to_a.map{ |i| i.join(':') }.join(', ')}"
 
-  self.class.trace_execution_scoped([metrics]) do
+  self.class.trace_execution_scoped([metric_name]) do
     NewRelic::Agent.instance.transaction_sampler.notice_sql(logged, nil, duration)
-    NewRelic::Agent.instance.sql_sampler.notice_sql(logged, metrics, nil, duration)
-    NewRelic::Agent.instance.stats_engine.record_metrics(metrics, duration)
+    NewRelic::Agent.instance.sql_sampler.notice_sql(logged, metric_name, nil, duration)
+    NewRelic::Agent.record_metric(metric_name, duration)
   end
 end
 
 ActiveSupport::Notifications.subscribe('search_query.chewy') do |name, start, finish, id, payload|
-  metrics = "Database/ElasticSearch/search"
+  metric_name = "Database/ElasticSearch/search"
   duration = (finish - start).to_f
-  logged = "#{payload[:index]} #{payload[:request]}"
+  logged = "#{payload[:type].presence || payload[:index]} #{payload[:request]}"
 
-  self.class.trace_execution_scoped([metrics]) do
+  self.class.trace_execution_scoped([metric_name]) do
     NewRelic::Agent.instance.transaction_sampler.notice_sql(logged, nil, duration)
-    NewRelic::Agent.instance.sql_sampler.notice_sql(logged, metrics, nil, duration)
-    NewRelic::Agent.instance.stats_engine.record_metrics(metrics, duration)
+    NewRelic::Agent.instance.sql_sampler.notice_sql(logged, metric_name, nil, duration)
+    NewRelic::Agent.record_metric(metric_name, duration)
   end
 end
 ```