chewy 0.1.0 → 0.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MzNiZmUxMGUxNzE1ODdkYzQzYTAyMzAwN2VhNDhmZDkzYzUyMDZhNQ==
-  data.tar.gz: !binary |-
-    MWIwNDA3MzMyNzRlZGM2ODllODA4ZjAxMzI0NzM5OTc3MGU4NjViMA==
+SHA1:
+  metadata.gz: 37ef7870c9b4ea62e334143eddf486f55e21627e
+  data.tar.gz: 57cd914528a78448c58f89d687f84a666b8a2ffc
 SHA512:
-  metadata.gz: !binary |-
-    MDk1NWIzZmYzYmZhNjkzOTI0OTY0MWVkYjYzNDE2N2FiZGUzOGQ1YjI0NzQ4
-    OTgzMTgyNDI2M2RlMjJhMWQzMjk1MzZjOGFkODQ3NzFjZmEwOTQ2M2UzNDMy
-    MDJjY2Y0ZmFiZjU5NjY3MTE5MzQ3NDk0ZDg4MzdjZDJkOTZhNGI=
-  data.tar.gz: !binary |-
-    NWEyODY1M2JhZjEwMDI1MTUyNGQ1NzM4ZTU0NTgyYWRkOWM3NWI0ZmQ2ZmY1
-    ODkyMjc5OTQwNThlODA2YjY3MGM5MTQyZDA5NzMzZDBjOWYyYmI2NmMyNGQx
-    OTU3ZGM3OTEyOGJjMWIyMmEwYWY2MjkzOGJmM2Q3MzQ2MzgzMGE=
+  metadata.gz: 9e923803456e4d75abe810db4c2cceb6bd4cdd758de480f937131570830dcb7e5e1053a4db06ec1aec62f74cf0abe43173f02d461a56e83e9705f2f819d0b7df
+  data.tar.gz: 53e95e0795081fda77fbb8858207081eded90ab8d5ce5b01b4ae2f140641f71e1db11c3af470f321c4ece7f881916045286a4e22b0ab81694ca5bdb01b271655

data/.travis.yml

@@ -2,8 +2,9 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
+  - 2.1.0
   # - rbx
 before_install:
-  - curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.9.tar.gz | tar xz -C /tmp
+  - curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.10.tar.gz | tar xz -C /tmp
 before_script:
-  - TEST_CLUSTER_COMMAND="/tmp/elasticsearch-0.90.9/bin/elasticsearch" rake elasticsearch:start
+  - TEST_CLUSTER_COMMAND="/tmp/elasticsearch-0.90.10/bin/elasticsearch" rake elasticsearch:start

data/CHANGELOG.md

@@ -1,10 +1,36 @@
 # master
 
+  * Reworked import error handling. Now all the import errors from ElasticSearch are handled properly, also import method returns true of false depending on the import process success.
+
+  * `Chewy::Index.import` now takes types hash as argument within options hash:
+
+    `PlacesIndex.import city: City.enabled, country: Country.enabled, refresh: false`
+
+  * Old indexes cleanup after reset.
+
+  * Added index prefixes.
+
+  * `define_type` now takes options for adapter.
+
+  * `chewy:reset` and `chewy:reset:all` rake tasks are now trying to reset index with zero downtime if it is possible.
+
+  * Added `chewy:update:all` rake task.
+
+  * Methods `.create`, `.create!`, `.delete`, `.delete`, `reset!` are now supports index name suffix passing as the first argument. See [actions.rb](lib/chewy/index/actions.rb) for more details.
+
+  * Method `reset` renamed to `reset!`.
+
+  * Added common loading scope for AR adapter. Also removed scope proc argument, now it executes just in main load scope context.
+
+    `CitiesIndex.all.load(scope: {city: City.include(:country)})`
+    `CitiesIndex.all.load(scope: {city: -> { include(:country) }})`
+    `CitiesIndex.all.load(scope: ->{ include(:country) })`
+
 # Version 0.1.0
 
-  * Added filters simplified DSL. See [context.rb](lib/chewy/query/context.rb) for more info.
+  * Added filters simplified DSL. See [filters.rb](lib/chewy/query/filters.rb) for more details.
 
-  * Queries and filters join system reworked. See [query.rb](lib/chewy/query.rb) for more info.
+  * Queries and filters join system reworked. See [query.rb](lib/chewy/query.rb) for more details.
 
   * Added query `merge` method
 

data/README.md

@@ -5,6 +5,24 @@
 
 Chewy is ODM and wrapper for official elasticsearch client (https://github.com/elasticsearch/elasticsearch-ruby)
 
+## Why chewy?
+
+* Index classes are independant from ORM/ODM models.
+
+  Now implementing, e.g. cross-model autocomplete is much easier. You can just define index and work with it in object-oriented style. You can define several types for index - one per indexed model.
+
+* Every index is observable by all the related models.
+
+  Most of the indexed models a related to other and somtimes it is nessesary to denormalize this related data and put at the same object. Like you need to index array of tags with article together. Chewy allows you to specify updatable index for every model separately. So, corresponding articles will be reindexed on the any tag update.
+
+* Bulk import everywhere.
+
+  Chewy utilizes bulk ES API for full reindexing or index updates. Also it uses atomic updates concept. All the changed objects are collected inside the atomic block and index is updated once at the end of it with all the collected object. See `Chewy.atomic` for more details.
+
+* Powerful querying DSL.
+
+  Chewy has AR-style query DSL. It is chainable, mergable and lazy. So you can produce queries in the most efficient way. Also it has object-oriented query and filter builders.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -21,7 +39,39 @@ Or install it yourself as:
 
 ## Usage
 
-### Index file
+### Client configuration
+
+There are 2 ways to configure Chewy client: `Chewy.client_options` hash and `chewy.yml`
+
+```ruby
+# config/initializers/chewy.rb
+Chewy.client_options = {host: 'localhost:9250'} # do not use environments
+```
+
+```yaml
+# config/chewy.yml
+# separate environment configs
+test:
+  host: 'localhost:9250'
+  prefix: 'test'
+development:
+  host: 'localhost:9250'
+```
+
+The result config merges both hashes. Client options are passed as is to Elasticsearch::Transport::Client except the `:prefix` - it is used internally by chewy to create prefixed index names:
+
+```ruby
+  Chewy.client_options = {prefix: 'testing'}
+  UsersIndex.index_name # => 'testing_users'
+```
+
+Also logger might be set explicitly:
+
+```ruby
+Chewy.logger = Logger.new
+```
+
+### Index definition
 
 1. Create `/app/chewy/users_index.rb`
 
@@ -162,9 +212,13 @@ UsersIndex::User.import User.where('rating > 100').to_a # or import specified us
 UsersIndex::User.import [1, 2, 42] # pass even ids for import, it will be handled in the most effective way
 
 UsersIndex.import # import every defined type
-UsersIndex.reset # purges index and imports default data for all types
+UsersIndex.import user: User.where('rating > 100') # import only active users to `user` type.
+  # Other index types, if exists, will be imported with default scope from the type definition.
+UsersIndex.reset! # purges index and imports default data for all types
 ```
 
+See [actions.rb](lib/chewy/index/actions.rb) for more details.
+
 Also if passed user is #destroyed? or specified id is not existing in the database, import will perform `delete` index for this it
 
 ### Observing strategies
@@ -253,7 +307,7 @@ UsersIndex::User.filter{ name == 'Fred' }.filter{ age < 42 }.filter_mode(:should
 UsersIndex::User.filter{ name == 'Fred' }.filter{ age < 42 }.filter_mode('75%') # will be wrapped with bool `should` filter with `minimum_should_match: '75%'`
 ```
 
-See [query.rb](lib/chewy/query.rb) for more info.
+See [query.rb](lib/chewy/query.rb) for more details.
 
 ### Filters query DSL.
 
@@ -500,7 +554,41 @@ Compliance cheatsheet for filters and DSL expressions:
   UsersIndex.filter{ !(age < 42) }
   ```
 
-See [context.rb](lib/chewy/query/context.rb) for more info.
+* Match all filter
+
+  ```json
+  {"match_all": {}}
+  ```
+
+  ```ruby
+  UsersIndex.filter{ match_all }
+  ```
+
+* Has child filter
+
+  ```json
+  {"has_child": {"type": "blog_tag", "query": {"term": {"tag": "something"}}}
+  {"has_child": {"type": "comment", "term": {"term": {"user": "john"}}}
+  ```
+
+  ```ruby
+  UsersIndex.filter{ has_child(:blog_tag).query(term: {tag: 'something'}) }
+  UsersIndex.filter{ has_child(:comment).filter{ user == 'john' } }
+  ```
+
+* Has parent filter
+
+  ```json
+  {"has_parent": {"type": "blog", "query": {"term": {"tag": "something"}}}}
+  {"has_parent": {"type": "blog", "filter": {"term": {"text": "bonsai three"}}}}
+  ```
+
+  ```ruby
+  UsersIndex.filter{ has_parent(:blog).query(term: {tag: 'something'}) }
+  UsersIndex.filter{ has_parent(:blog).filter{ text == 'bonsai three' } }
+  ```
+
+See [filters.rb](lib/chewy/query/filters.rb) for more details.
 
 ### Objects loading
 
@@ -510,12 +598,73 @@ It is possible to load source objects from database for every search result:
 scope = UsersIndex.filter(range: {rating: {gte: 100}})
 
 scope.load # => will return User instances array (not a scope because )
-scope.load(user: { scope: ->(_) { includes(:country) }}) # you can also pass loading scopes for each
-                                                         # possibly returned type
-scope.load(user: { scope: User.includes(:country) }) # the second scope passing way
+scope.load(user: { scope: ->{ includes(:country) }}) # you can also pass loading scopes for each
+                                                     # possibly returned type
+scope.load(user: { scope: User.includes(:country) }) # the second scope passing way.
+scope.load(scope: ->{ includes(:country) }) # and more common scope applied to every loaded object type.
+
 scope.only(:id).load # it is optimal to request ids only if you are not planning to use type objects
 ```
 
+### `ActiveSupport::Notifications` support
+
+Chewy has notifing the following events:
+
+#### `search_query.chewy` payload
+
+  * `payload[:index]`: requested index class
+  * `payload[:request]`: request hash
+
+#### `import_objects.chewy` payload
+
+  * `payload[:type]`: currently imported type
+  * `payload[:import]`: imports stast, total imported and deleted objects count:
+
+    ```ruby
+    {index: 30, delete: 5}
+    ```
+
+  * `payload[:erorrs]`: might not exists. Contains grouped errors with objects ids list:
+
+    ```ruby
+    {index: {
+      'error 1 text' => ['1', '2', '3'],
+      'error 2 text' => ['4']
+    }, delete: {
+      'delete error text' => ['10', '12']
+    }}
+    ```
+
+#### NewRelic integration
+
+To integrate with NewRelic you may use the following example source (config/initializers/chewy.rb):
+
+```ruby
+ActiveSupport::Notifications.subscribe('import_objects.chewy') do |name, start, finish, id, payload|
+  metrics = "Database/ElasticSearch/import"
+  duration = (finish - start).to_f
+  logged = "#{payload[:type]} #{payload[:import].to_a.map{ |i| i.join(':') }.join(', ')}"
+
+  self.class.trace_execution_scoped([metrics]) do
+    NewRelic::Agent.instance.transaction_sampler.notice_sql(logged, nil, duration)
+    NewRelic::Agent.instance.sql_sampler.notice_sql(logged, metrics, nil, duration)
+    NewRelic::Agent.instance.stats_engine.record_metrics(metrics, duration)
+  end
+end
+
+ActiveSupport::Notifications.subscribe('search_query.chewy') do |name, start, finish, id, payload|
+  metrics = "Database/ElasticSearch/search"
+  duration = (finish - start).to_f
+  logged = "#{payload[:index]} #{payload[:request]}"
+
+  self.class.trace_execution_scoped([metrics]) do
+    NewRelic::Agent.instance.transaction_sampler.notice_sql(logged, nil, duration)
+    NewRelic::Agent.instance.sql_sampler.notice_sql(logged, metrics, nil, duration)
+    NewRelic::Agent.instance.stats_engine.record_metrics(metrics, duration)
+  end
+end
+```
+
 ### Rake tasks
 
 Inside Rails application some index mantaining rake tasks are defined.

data/filters

@@ -49,7 +49,8 @@ not
   email != 'email'
 
 script s()
-query q()
+match all
+  match_all
 
 has child
   has_child('type').query()
@@ -59,8 +60,10 @@ has parent
   has_parent('type').filter()
 nested
   name.nested()
-match all
-  match_all
+
+query q()
+type
+  .types()
 
 geo bounding box
 geo distance
@@ -70,7 +73,6 @@ geoshape
 geohash cell
 
 indices
-type
 ids
 
 limit

data/lib/chewy/index.rb

@@ -1,9 +1,11 @@
 require 'chewy/index/actions'
+require 'chewy/index/aliases'
 require 'chewy/index/search'
 
 module Chewy
   class Index
     include Actions
+    include Aliases
     include Search
 
     singleton_class.delegate :client, to: 'Chewy'
@@ -14,8 +16,67 @@ module Chewy
     class_attribute :_settings
     self._settings = {}
 
-    def self.define_type(name_or_scope, &block)
-      type_class = Chewy::Type.new(self, name_or_scope, &block)
+    # Setups or returns ElasticSearch index name
+    #
+    #   class UsersIndex < Chewy::Index
+    #   end
+    #   UsersIndex.index_name # => 'users'
+    #
+    #   class UsersIndex < Chewy::Index
+    #     index_name 'dudes'
+    #   end
+    #   UsersIndex.index_name # => 'dudes'
+    #
+    def self.index_name(suggest = nil)
+      if suggest
+        @index_name = build_index_name(suggest, prefix: Chewy.client_options[:prefix])
+      else
+        @index_name ||= begin
+          build_index_name(
+            name.gsub(/Index\Z/, '').demodulize.underscore,
+            prefix: Chewy.client_options[:prefix]
+          ) if name
+        end
+      end
+      @index_name or raise UndefinedIndex
+    end
+
+    # Defines type for the index. Arguments depends on adapter used. For
+    # ActiveRecord you can pass model or scope and options
+    #
+    #   class CarsIndex < Chewy::Index
+    #     define_type Car do
+    #       ...
+    #     end # defines VehiclesIndex::Car type
+    #   end
+    #
+    # Type name might be passed in complicated cases:
+    #
+    #   class VehiclesIndex < Chewy::Index
+    #     define_type Vehicle.cars.includes(:manufacturer), name: 'cars' do
+    #        ...
+    #     end # defines VehiclesIndex::Cars type
+    #
+    #     define_type Vehicle.motocycles.includes(:manufacturer), name: 'motocycles' do
+    #        ...
+    #     end # defines VehiclesIndex::Motocycles type
+    #   end
+    #
+    # For plain objects:
+    #
+    #   class PlanesIndex < Chewy::Index
+    #     define_type :plane do
+    #       ...
+    #     end # defines PlanesIndex::Plane type
+    #   end
+    #
+    # The main difference between using plain objects or ActiveRecord models for indexing
+    # is import. If you will call `CarsIndex::Car.import` - it will import all the cars
+    # automatically, while `PlanesIndex::Plane.import(my_planes)` requires import data to be
+    # passed.
+    #
+    def self.define_type(target, options = {}, &block)
+      type_class = Chewy::Type.new(self, target, options, &block)
       self.type_hash = type_hash.merge(type_class.type_name => type_class)
 
       class_eval <<-RUBY, __FILE__, __LINE__ + 1
@@ -25,6 +86,17 @@ module Chewy
       RUBY
     end
 
+    # Types method has double usage.
+    # If no arguments are passed - it returns array of defined types:
+    #
+    #   UsersIndex.types # => [UsersIndex::Admin, UsersIndex::Manager, UsersIndex::User]
+    #
+    # If arguments are passed it treats like a part of chainable query dsl and
+    # adds types array for index to select.
+    #
+    #   UsersIndex.filters { name =~ 'ro' }.types(:admin, :manager)
+    #   UsersIndex.types(:admin, :manager).filters { name =~ 'ro' } # the same as the first example
+    #
     def self.types *args
       if args.any?
         all.types *args
@@ -33,21 +105,54 @@ module Chewy
       end
     end
 
+    # Returns defined types names:
+    #
+    #   UsersIndex.type_names # => ['admin', 'manager', 'user']
+    #
     def self.type_names
       type_hash.keys
     end
 
+    # Used as a part of index definition DSL. Defines settings:
+    #
+    #   class UsersIndex < Chewy::Index
+    #     settings analysis: {
+    #       analyzer: {
+    #         name: {
+    #           tokenizer: 'standard',
+    #           filter: ['lowercase', 'icu_folding', 'names_nysiis']
+    #         }
+    #       }
+    #     }
+    #   end
+    #
+    # See http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-update-settings.html
+    # for more details
+    #
     def self.settings(params)
       self._settings = params
     end
 
-    def self.index_name(suggest = nil)
-      if suggest
-        @index_name = suggest.to_s
-      else
-        @index_name ||= (name.gsub(/Index\Z/, '').demodulize.underscore if name)
-      end
-      @index_name or raise UndefinedIndex
+    # Perform import operation for every defined type
+    #
+    #   UsersIndex.import
+    #   UsersIndex.import refresh: false # to disable index refreshing after import
+    #   UsersIndex.import suffix: Time.now.to_i # imports data to index with specified suffix if such is exists
+    #   UsersIndex.import batch_size: 300 # import batch size
+    #
+    def self.import options = {}
+      objects = options.extract!(*type_names.map(&:to_sym))
+      types.map do |type|
+        args = [objects[type.type_name.to_sym], options.dup].reject(&:blank?)
+        type.import *args
+      end.all?
+    end
+
+  private
+
+    def self.build_index_name *args
+      options = args.extract_options!
+      [options[:prefix], args.first || index_name, options[:suffix]].reject(&:blank?).join(?_)
     end
 
     def self.settings_hash
@@ -70,14 +175,5 @@ module Chewy
     def self.search_type
       type_names
     end
-
-    def self.import
-      types.all? { |t| t.import }
-    end
-
-    def self.reset
-      purge!
-      import
-    end
   end
 end