chewy 0.3.0 → 0.4.0

data/lib/chewy/rspec/update_index.rb

@@ -1,14 +1,83 @@
+require 'i18n/core_ext/hash'
+
+# Rspec matcher `update_index`
+# To use it - add `require 'chewy/rspec'` to the `spec_helper.rb`
+# Simple usage - just pass type as argument.
+#
+#   specify { expect { user.save! }.to update_index(UsersIndex::User) }
+#   specify { expect { user.save! }.to update_index('users#user') }
+#   specify { expect { user.save! }.not_to update_index('users#user') }
+#
+# This example will pass as well because user1 was reindexed
+# and nothing was said about user2:
+#
+#   specify { expect { [user1, user2].map(&:save!) }
+#     .to update_index(UsersIndex.user).and_reindex(user1) }
+#
+# If you need to specify reindexed records strictly - use `only` chain.
+# Combined matcher chain methods:
+#
+#   specify { expect { user1.destroy!; user2.save! } }
+#     .to update_index(UsersIndex:User).and_reindex(user2).and_delete(user1)
+#
 RSpec::Matchers.define :update_index do |type_name, options = {}|
+
+  # Specify indexed records by passing record itself or id.
+  #
+  #   specify { expect { user.save! }.to update_index(UsersIndex::User).and_reindex(user)
+  #   specify { expect { user.save! }.to update_index(UsersIndex::User).and_reindex(42)
+  #   specify { expect { [user1, user2].map(&:save!) }
+  #     .to update_index(UsersIndex::User).and_reindex(user1, user2) }
+  #   specify { expect { [user1, user2].map(&:save!) }
+  #     .to update_index(UsersIndex::User).and_reindex(user1).and_reindex(user2) }
+  #
+  # Specify indexing count for every particular record. Useful in case
+  # urgent index updates.
+  #
+  #   specify { expect { 2.times { user.save! } }
+  #     .to update_index(UsersIndex::User).and_reindex(user, times: 2) }
+  #
+  # Specify reindexed attributes. Note that arrays are
+  # compared position-independantly.
+  #
+  #   specify { expect { user.update_attributes!(name: 'Duke') }
+  #     .to update_index(UsersIndex.user).and_reindex(user, with: {name: 'Duke'}) }
+  #
+  # You can combine all the options and chain `and_reindex` method to
+  # specify options for every indexed record:
+  #
+  #   specify { expect { 2.times { [user1, user2].map { |u| u.update_attributes!(name: "Duke#{u.id}") } } }
+  #     .to update_index(UsersIndex.user)
+  #     .and_reindex(user1, with: {name: 'Duke42'}) }
+  #     .and_reindex(user2, times: 1, with: {name: 'Duke43'}) }
+  #
   chain(:and_reindex) do |*args|
     @reindex ||= {}
     @reindex.merge!(extract_documents(*args))
   end
 
+  # Specify deleted records with record itself or id passed.
+  #
+  #   specify { expect { user.destroy! }.to update_index(UsersIndex::User).and_delete(user) }
+  #   specify { expect { user.destroy! }.to update_index(UsersIndex::User).and_delete(user.id) }
+  #
   chain(:and_delete) do |*args|
     @delete ||= {}
     @delete.merge!(extract_documents(*args))
   end
 
+  # Used for specifying than no other records would be indexed or deleted:
+  #
+  #   specify { expect { [user1, user2].map(&:save!) }
+  #     .to update_index(UsersIndex.user).and_reindex(user1, user2).only }
+  #   specify { expect { [user1, user2].map(&:destroy!) }
+  #     .to update_index(UsersIndex.user).and_delete(user1, user2).only }
+  #
+  # This example will fail:
+  #
+  #   specify { expect { [user1, user2].map(&:save!) }
+  #     .to update_index(UsersIndex.user).and_reindex(user1).only }
+  #
   chain(:only) do |*args|
     @only = true
   end
@@ -23,9 +92,8 @@ RSpec::Matchers.define :update_index do |type_name, options = {}|
     updated = []
     type.stub(:bulk) do |options|
       updated += options[:body].map do |updated_document|
-        updated_document = updated_document.symbolize_keys
+        updated_document = updated_document.deep_symbolize_keys
         body = updated_document[:index] || updated_document[:delete]
-        body[:data] = body[:data].symbolize_keys if body[:data]
         updated_document
       end
       {}
@@ -59,7 +127,7 @@ RSpec::Matchers.define :update_index do |type_name, options = {}|
       document[:match_count] = (!document[:expected_count] && document[:real_count] > 0) ||
         (document[:expected_count] && document[:expected_count] == document[:real_count])
       document[:match_attributes] = document[:expected_attributes].blank? ||
-        document[:real_attributes].slice(*document[:expected_attributes].keys) == document[:expected_attributes]
+        compare_attributes(document[:expected_attributes], document[:real_attributes])
     end
     @delete.each do |_, document|
       document[:match_count] = (!document[:expected_count] && document[:real_count] > 0) ||
@@ -123,7 +191,7 @@ RSpec::Matchers.define :update_index do |type_name, options = {}|
     options = args.extract_options!
 
     expected_count = options[:times] || options[:count]
-    expected_attributes = (options[:with] || options[:attributes] || {}).symbolize_keys!
+    expected_attributes = (options[:with] || options[:attributes] || {}).deep_symbolize_keys
 
     Hash[args.flatten.map do |document|
       id = document.respond_to?(:id) ? document.id.to_s : document.to_s
@@ -136,4 +204,27 @@ RSpec::Matchers.define :update_index do |type_name, options = {}|
       }]
     end]
   end
+
+  def compare_attributes expected, real
+    expected.inject(true) do |result, (key, value)|
+      equal = if value.is_a?(Array) && real[key].is_a?(Array)
+        array_difference(value, real[key]) && array_difference(real[key], value)
+      elsif value.is_a?(Hash) && real[key].is_a?(Hash)
+        compare_attributes(value, real[key])
+      else
+        real[key] == value
+      end
+      result && equal
+    end
+  end
+
+  def array_difference first, second
+    difference = first.to_ary.dup
+    second.to_ary.each do |element|
+      if index = difference.index(element)
+        difference.delete_at(index)
+      end
+    end
+    difference.none?
+  end
 end

data/lib/chewy/type/adapter/active_record.rb

@@ -36,6 +36,16 @@ module Chewy
         #
         #   <tt>:batch_size</tt> - import batch size, 1000 objects by default
         #
+        # Method handles destroyed objects as well. In case of objects AcriveRecord::Relation
+        # or array passed, objects, responding with true to `destroyed?` method will be deleted
+        # from index. In case of ids array passed - documents with missing records ids will be
+        # deleted from index:
+        #
+        #   users = User.all
+        #   users.each { |user| user.destroy if user.incative? }
+        #   UsersIndex::User.import users # inactive users will be deleted from index
+        #   UsersIndex::User.import users.map(&:id) # deleted user ids will be deleted from index
+        #
         def import *args, &block
           import_options = args.extract_options!
           import_options[:batch_size] ||= BATCH_SIZE
@@ -65,7 +75,7 @@ module Chewy
 
           additional_scope = load_options[load_options[:_type].type_name.to_sym].try(:[], :scope) || load_options[:scope]
 
-          scope = model.where(id: objects.map(&:id))
+          scope = scoped_model(objects.map(&:id))
           loaded_objects = if additional_scope.is_a?(Proc)
             scope.instance_exec(&additional_scope)
           elsif additional_scope.is_a?(::ActiveRecord::Relation)
@@ -85,7 +95,7 @@ module Chewy
           ids = ids.map(&:to_i).uniq
 
           indexed = true
-          merged_scope(model.where(id: ids)).find_in_batches(import_options.slice(:batch_size)) do |objects|
+          merged_scope(scoped_model(ids)).find_in_batches(import_options.slice(:batch_size)) do |objects|
             ids -= objects.map(&:id)
             indexed &= block.call index: objects
           end
@@ -107,6 +117,10 @@ module Chewy
           scope ? scope.clone.merge(target) : target
         end
 
+        def scoped_model(ids)
+          model.where(Hash[model.primary_key.to_sym || :id, ids])
+        end
+
         def model_all
           ::ActiveRecord::VERSION::MAJOR < 4 ? model.scoped : model.all
         end

data/lib/chewy/type/import.rb

@@ -17,7 +17,9 @@ module Chewy
         #
         def import *args
           import_options = args.extract_options!
-          bulk_options = import_options.extract!(:refresh, :suffix).reverse_merge!(refresh: true)
+          bulk_options = import_options.reject { |k, v| ![:refresh, :suffix].include?(k) }.reverse_merge!(refresh: true)
+
+          index.create!(bulk_options.slice(:suffix)) unless index.exists?
 
           ActiveSupport::Notifications.instrument 'import_objects.chewy', type: self do |payload|
             adapter.import(*args, import_options) do |action_objects|
@@ -57,6 +59,9 @@ module Chewy
         # Adds `:suffix` option to bulk import to index with specified suffix.
         def bulk options = {}
           suffix = options.delete(:suffix)
+
+          Chewy.wait_for_status
+
           result = client.bulk options.merge(index: index.build_index_name(suffix: suffix), type: type_name)
 
           extract_errors result

data/lib/chewy/type/mapping.rb

@@ -5,17 +5,111 @@ module Chewy
 
       included do
         class_attribute :root_object, instance_reader: false, instance_writer: false
+        class_attribute :_templates
       end
 
       module ClassMethods
-        def root(options = {}, &block)
-          raise "Root is already defined" if self.root_object
+        # Defines root object for mapping and is optional for type
+        # definition. Use it only if you need to pass options for root
+        # object mapping, such as `date_detection` or `dynamic_date_formats`
+        #
+        #   class UsersIndex < Chewy::Index
+        #     define_type User do
+        #       # root object defined implicitly and optionless for current type
+        #       field :full_name, type: 'string'
+        #     end
+        #   end
+        #
+        #   class CarsIndex < Chewy::Index
+        #     define_type Car do
+        #       # explicit root definition with additional options
+        #       root dynamic_date_formats: ['yyyy-MM-dd'] do
+        #         field :model_name, type: 'string'
+        #       end
+        #     end
+        #   end
+        #
+        def root options = {}, &block
+          raise "Root is already defined" if root_object
           build_root(options, &block)
         end
 
-        def field(*args, &block)
+        # Defines mapping field for current type
+        #
+        #   class UsersIndex < Chewy::Index
+        #     define_type User do
+        #       # passing all the options to field definition:
+        #       field :full_name, type: 'string', analyzer: 'special'
+        #     end
+        #   end
+        #
+        # The `type` is optional and defaults to `string` if not defined:
+        #
+        #   field :full_name
+        #
+        # Also, multiple fields might be defined with one call and
+        # with the same options:
+        #
+        #   field :first_name, :last_name, analyzer: 'special'
+        #
+        # The only special option in the field definition
+        # is `:value`. If no `:value` specified then just corresponding
+        # method will be called for the indexed object. Also
+        # `:value` might be a proc or indexed object method name:
+        #
+        #   class User < ActiveRecord::Base
+        #     def user_full_name
+        #       [first_name, last_name].join(' ')
+        #     end
+        #   end
+        #
+        #   field :full_name, type: 'string', value: :user_full_name
+        #
+        # The proc evaluates inside the indexed object context if
+        # its arity is 0 and in present contexts if there is an argument:
+        #
+        #   field :full_name, type: 'string', value: -> { [first_name, last_name].join(' ') }
+        #
+        #   separator = ' '
+        #   field :full_name, type: 'string', value: ->(user) { [user.first_name, user.last_name].join(separator) }
+        #
+        # If array was returned as value - it will be put in index as well.
+        #
+        #   field :tags, type: 'string', value: -> { tags.map(&:name) }
+        #
+        # Fields supports nesting in case of `object` field type. If
+        # `user.quiz` will return an array of objects, then result index content
+        # will be an array of hashes, if `user.quiz` is not a collection association
+        # then just values hash will be put in the index.
+        #
+        #   field :quiz, type: 'object' do
+        #     field :question, :answer
+        #     field :score, type: 'integer'
+        #   end
+        #
+        # Nested fields are composed from nested objects:
+        #
+        #   field :name, type: 'object', value: -> { name_translations } do
+        #     field :ru, value: ->(name) { name['ru'] }
+        #     field :en, value: ->(name) { name['en'] }
+        #   end
+        #
+        # Off course it is possible to define object fields contents dynamically
+        # but make sure evaluation proc returns hash:
+        #
+        #   field :name, type: 'object', value: -> { name_translations }
+        #
+        # The special case is `multi_field`. In that case field composition
+        # changes satisfy elasticsearch rules:
+        #
+        #   field :full_name, type: 'multi_field', value: ->{ full_name.try(:strip) } do
+        #     field :full_name, index: 'analyzed', analyzer: 'name'
+        #     field :sorted, index: 'analyzed', analyzer: 'sorted'
+        #   end
+        #
+        def field *args, &block
           options = args.extract_options!
-          build_root unless self.root_object
+          build_root unless root_object
 
           if args.size > 1
             args.map { |name| field(name, options) }
@@ -24,13 +118,42 @@ module Chewy
           end
         end
 
+        # Defines dynamic template in mapping root objests
+        #
+        #   class CarsIndex < Chewy::Index
+        #     define_type Car do
+        #       template 'model.*', type: 'string', analyzer: 'special'
+        #       field 'model', type: 'object' # here we can put { ru: 'Мерседес', en: 'Mercedes' }
+        #                                     # and template will be applyed to this field
+        #     end
+        #   end
+        #
+        # Name for each template is generated with the following
+        # rule: "template_#{dynamic_templates.size + 1}".
+        #
+        #   template 'tit*', mapping_hash
+        #   template 'title.*', mapping_hash # dot in template causes "path_match" using
+        #   template /tit.+/, mapping_hash # using "match_pattern": "regexp"
+        #   template /title\..+/, mapping_hash # "\." - escaped dot causes "path_match" using
+        #   template /tit.+/, 'string' mapping_hash # "match_mapping_type" as the optionsl second argument
+        #   template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
+        #
+        def template *args
+          build_root unless root_object
+
+          root_object.dynamic_template *args
+        end
+        alias_method :dynamic_template, :template
+
+        # Returns compiled mappings hash for current type
+        #
         def mappings_hash
           root_object ? root_object.mappings_hash : {}
         end
 
       private
 
-        def expand_nested(field, &block)
+        def expand_nested field, &block
           @_current_field.nested(field) if @_current_field
           if block
             previous_field, @_current_field = @_current_field, field
@@ -39,7 +162,7 @@ module Chewy
           end
         end
 
-        def build_root(options = {}, &block)
+        def build_root options = {}, &block
           self.root_object = Chewy::Fields::Root.new(type_name, options)
           expand_nested(self.root_object, &block)
           @_current_field = self.root_object

data/lib/chewy/type/observe.rb

@@ -9,6 +9,9 @@ module Chewy
           method = args.first
 
           update = Proc.new do
+            update_options = options.reverse_merge(urgent: Chewy.urgent_update)
+            clear_association_cache if update_options[:urgent]
+
             backreference = if method && method.to_s == 'self'
               self
             elsif method
@@ -17,8 +20,7 @@ module Chewy
               instance_eval(&block)
             end
 
-            Chewy.derive_type(type_name).update_index(backreference,
-              options.reverse_merge(urgent: Chewy.urgent_update))
+            Chewy.derive_type(type_name).update_index(backreference, update_options)
           end
 
           after_save &update

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/spec/chewy/fields/root_spec.rb

@@ -3,4 +3,42 @@ require 'spec_helper'
 describe Chewy::Fields::Root do
   specify { described_class.new('name').value.should be_a(Proc) }
   # TODO: add 'should_behave_like base_field'
+
+  subject(:field) { described_class.new('product') }
+
+  describe '#dynamic_template' do
+    specify do
+      field.dynamic_template 'hello', type: 'string'
+      field.dynamic_template 'hello*', :integer
+      field.dynamic_template 'hello.*'
+      field.dynamic_template /hello/
+      field.dynamic_template /hello.*/
+      field.dynamic_template template_42: {mapping: {}, match: ''}
+      field.dynamic_template /hello\..*/
+
+      field.mappings_hash.should == {product: {dynamic_templates: [
+        {template_1: {mapping: {type: 'string'}, match: 'hello'}},
+        {template_2: {mapping: {}, match_mapping_type: 'integer', match: 'hello*'}},
+        {template_3: {mapping: {}, path_match: 'hello.*'}},
+        {template_4: {mapping: {}, match: 'hello', match_pattern: 'regexp'}},
+        {template_5: {mapping: {}, match: 'hello.*', match_pattern: 'regexp'}},
+        {template_42: {mapping: {}, match: ''}},
+        {template_7: {mapping: {}, path_match: 'hello\..*', match_pattern: 'regexp'}}
+      ]}}
+    end
+
+    context do
+      subject(:field) { described_class.new('product', dynamic_templates: [
+        {template_42: {mapping: {}, match: ''}}
+      ]) }
+
+      specify do
+        field.dynamic_template 'hello', type: 'string'
+        field.mappings_hash.should == {product: {dynamic_templates: [
+          {template_42: {mapping: {}, match: ''}},
+          {template_1: {mapping: {type: 'string'}, match: 'hello'}}
+        ]}}
+      end
+    end
+  end
 end

data/spec/chewy/index/actions_spec.rb

@@ -284,7 +284,7 @@ describe Chewy::Index::Actions do
           define_type City do
             field :name, type: 'object'
           end
-        end.tap(&:create!)
+        end
       end
 
       specify { CitiesIndex.import(city: dummy_cities).should == false }
@@ -308,7 +308,7 @@ describe Chewy::Index::Actions do
           define_type City do
             field :name, type: 'object'
           end
-        end.tap(&:create!)
+        end
       end
 
       specify { expect { CitiesIndex.import!(city: dummy_cities) }.to raise_error Chewy::FailedImport }