chewy 0.8.4 → 5.1.0

data/lib/chewy/type.rb

@@ -1,19 +1,20 @@
 require 'chewy/search'
+require 'chewy/type/adapter/object'
+require 'chewy/type/adapter/active_record'
+require 'chewy/type/adapter/mongoid'
+require 'chewy/type/adapter/sequel'
 require 'chewy/type/mapping'
 require 'chewy/type/wrapper'
 require 'chewy/type/observe'
 require 'chewy/type/actions'
+require 'chewy/type/syncer'
 require 'chewy/type/crutch'
 require 'chewy/type/import'
 require 'chewy/type/witchcraft'
-require 'chewy/type/adapter/object'
-require 'chewy/type/adapter/active_record'
-require 'chewy/type/adapter/mongoid'
-require 'chewy/type/adapter/sequel'
 
 module Chewy
   class Type
-    IMPORT_OPTIONS_KEYS = [:batch_size, :bulk_size, :refresh, :consistency, :replication]
+    IMPORT_OPTIONS_KEYS = %i[batch_size bulk_size refresh consistency replication raw_import journal pipeline].freeze
 
     include Search
     include Mapping
@@ -24,66 +25,93 @@ module Chewy
     include Witchcraft
     include Import
 
-    singleton_class.delegate :index_name, :client, to: :index
+    singleton_class.delegate :index_name, :derivable_index_name, :client, to: :index
 
     class_attribute :_default_import_options
     self._default_import_options = {}
 
-    # Chewy index current type belongs to. Defined inside `Chewy.create_type`
-    #
-    def self.index
-      raise NotImplementedError
-    end
+    class << self
+      # Chewy index current type belongs to. Defined inside `Chewy.create_type`
+      #
+      def index
+        raise NotImplementedError, 'Looks like this type was defined outside the index scope and `.index` method is undefined for it'
+      end
 
-    # Current type adapter. Defined inside `Chewy.create_type`, derived from
-    # `Chewy::Index.define_type` arguments.
-    #
-    def self.adapter
-      raise NotImplementedError
-    end
+      # Current type adapter. Defined inside `Chewy.create_type`, derived from
+      # `Chewy::Index.define_type` arguments.
+      #
+      def adapter
+        raise NotImplementedError
+      end
 
-    # Returns type name string
-    #
-    def self.type_name
-      adapter.type_name
-    end
+      # Returns type name string
+      #
+      def type_name
+        adapter.type_name
+      end
 
-    # Returns list of public class methods defined in current type
-    #
-    def self.scopes
-      public_methods - Chewy::Type.public_methods
-    end
+      # Appends type name to {Chewy::Index.derivable_name}
+      #
+      # @example
+      #   class Namespace::UsersIndex < Chewy::Index
+      #     define_type User
+      #   end
+      #   UsersIndex::User.derivable_name # => 'namespace/users#user'
+      #
+      # @see Chewy::Index.derivable_name
+      # @return [String, nil] derivable name or nil when it is impossible to calculate
+      def derivable_name
+        @derivable_name ||= [index.derivable_name, type_name].join('#') if index && index.derivable_name
+      end
 
-    def self.default_import_options(params)
-      params.assert_valid_keys(IMPORT_OPTIONS_KEYS)
-      self._default_import_options = _default_import_options.merge(params)
-    end
+      # This method is an API shared with {Chewy::Index}, added for convenience.
+      #
+      # @return [Chewy::Type] array containing itself
+      def types
+        [self]
+      end
 
-    def self.method_missing(method, *args, &block)
-      if index.scopes.include?(method)
-        define_singleton_method method do |*args, &block|
-          all.scoping { index.public_send(method, *args, &block) }
+      # Returns list of public class methods defined in current type
+      #
+      def scopes
+        public_methods - Chewy::Type.public_methods
+      end
+
+      def default_import_options(params)
+        params.assert_valid_keys(IMPORT_OPTIONS_KEYS)
+        self._default_import_options = _default_import_options.merge(params)
+      end
+
+      def method_missing(method, *args, &block)
+        if index.scopes.include?(method)
+          define_singleton_method method do |*method_args, &method_block|
+            all.scoping { index.public_send(method, *method_args, &method_block) }
+          end
+          send(method, *args, &block)
+        else
+          super
         end
-        send(method, *args, &block)
-      else
-        super
       end
-    end
 
-    def self.const_missing(name)
-      to_resolve = "#{self.to_s}::#{name}"
-      to_resolve[index.to_s] = ''
+      def respond_to_missing?(method, _)
+        index.scopes.include?(method) || super
+      end
+
+      def const_missing(name)
+        to_resolve = "#{self}::#{name}"
+        to_resolve[index.to_s] = ''
 
-      @__resolved_constants ||= {}
+        @__resolved_constants ||= {}
 
-      if to_resolve.empty? || @__resolved_constants[to_resolve]
+        if to_resolve.empty? || @__resolved_constants[to_resolve]
+          super
+        else
+          @__resolved_constants[to_resolve] = true
+          to_resolve.constantize
+        end
+      rescue NotImplementedError
         super
-      else
-        @__resolved_constants[to_resolve] = true
-        to_resolve.constantize
       end
-    rescue NotImplementedError
-      super
     end
   end
 end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '0.8.4'
+  VERSION = '5.1.0'.freeze
 end

data/lib/chewy.rb

@@ -1,3 +1,4 @@
+require 'active_support/version'
 require 'active_support/concern'
 require 'active_support/deprecation'
 require 'active_support/json'
@@ -16,9 +17,32 @@ require 'active_support/core_ext/string/inflections'
 require 'i18n/core_ext/hash'
 require 'chewy/backports/deep_dup' unless Object.respond_to?(:deep_dup)
 require 'singleton'
+require 'base64'
 
 require 'elasticsearch'
 
+def try_require(path)
+  require path
+rescue LoadError
+  nil
+end
+
+try_require 'kaminari'
+try_require 'kaminari/core'
+try_require 'will_paginate'
+try_require 'will_paginate/collection'
+try_require 'parallel'
+
+ActiveSupport.on_load(:active_record) do
+  try_require 'will_paginate/active_record'
+  try_require 'kaminari/activerecord'
+end
+
+ActiveSupport.on_load(:mongoid) do
+  try_require 'will_paginate/mongoid'
+  try_require 'kaminari/mongoid'
+end
+
 require 'chewy/version'
 require 'chewy/errors'
 require 'chewy/config'
@@ -31,44 +55,24 @@ require 'chewy/index'
 require 'chewy/type'
 require 'chewy/fields/base'
 require 'chewy/fields/root'
-require 'chewy/railtie' if defined?(::Rails)
-
-begin
-  require 'kaminari'
-  require 'chewy/query/pagination/kaminari'
-rescue LoadError
-end
-
-begin
-  require 'will_paginate'
-  require 'will_paginate/collection'
-  require 'chewy/query/pagination/will_paginate'
-rescue LoadError
-end
+require 'chewy/journal'
+require 'chewy/railtie' if defined?(::Rails::Railtie)
 
 ActiveSupport.on_load(:active_record) do
   extend Chewy::Type::Observe::ActiveRecordMethods
-
-  begin
-    require 'will_paginate/active_record'
-  rescue LoadError
-  end
 end
 
 ActiveSupport.on_load(:mongoid) do
-  module Mongoid::Document::ClassMethods
-    include Chewy::Type::Observe::MongoidMethods
-  end
-
-  begin
-    require 'will_paginate/mongoid'
-    require 'chewy/query/pagination/will_paginate'
-  rescue LoadError
+  module Mongoid
+    module Document
+      module ClassMethods
+        include Chewy::Type::Observe::MongoidMethods
+      end
+    end
   end
 end
 
 module Chewy
-
   @adapters = [
     Chewy::Type::Adapter::ActiveRecord,
     Chewy::Type::Adapter::Mongoid,
@@ -79,44 +83,62 @@ module Chewy
   class << self
     attr_accessor :adapters
 
-    # Derives type from string `index#type` representation:
-    #
-    #   Chewy.derive_type('users#user') # => UsersIndex::User
-    #
-    # If index has only one type - it is possible to derive it without specification:
+    # Derives a single type for the passed string identifier if possible.
     #
-    #   Chewy.derive_type('users') # => UsersIndex::User
+    # @example
+    #   Chewy.derive_types(UsersIndex::User) # => UsersIndex::User
+    #   Chewy.derive_types('namespace/users') # => Namespace::UsersIndex::User
+    #   Chewy.derive_types('places') # => raises Chewy::UnderivableType
+    #   Chewy.derive_types('places#city') # => PlacesIndex::City
     #
-    # If index has more then one type - it raises Chewy::UnderivableType.
-    #
-    def derive_type name
+    # @param name [String, Chewy::Type] string type identifier
+    # @raise [Chewy::UnderivableType] in cases when it is impossble to find index or type or more than one type found
+    # @return [Chewy::Type] an array of derived types
+    def derive_type(name)
       return name if name.is_a?(Class) && name < Chewy::Type
 
-      index_name, type_name = name.split('#', 2)
-      class_name = "#{index_name.camelize}Index"
+      types = derive_types(name)
+      raise Chewy::UnderivableType, "Index `#{types.first.index}` has more than one type, please specify type via `#{types.first.index.derivable_name}#type_name`" unless types.one?
+      types.first
+    end
+
+    # Derives all the types for the passed string identifier if possible.
+    #
+    # @example
+    #   Chewy.derive_types('namespace/users') # => [Namespace::UsersIndex::User]
+    #   Chewy.derive_types('places') # => [PlacesIndex::City, PlacesIndex::Country]
+    #   Chewy.derive_types('places#city') # => [PlacesIndex::City]
+    #
+    # @param from [String] string type identifier
+    # @raise [Chewy::UnderivableType] in cases when it is impossible to find index or type
+    # @return [Array<Chewy::Type>] an array of derived types
+    def derive_types(from)
+      return from.types if from.is_a?(Class) && (from < Chewy::Index || from < Chewy::Type)
+
+      index_name, type_name = from.split('#', 2)
+      class_name = "#{index_name.camelize.gsub(/Index\z/, '')}Index"
       index = class_name.safe_constantize
-      raise Chewy::UnderivableType.new("Can not find index named `#{class_name}`") unless index && index < Chewy::Index
-      type = if type_name.present?
-        index.type_hash[type_name] or raise Chewy::UnderivableType.new("Index `#{class_name}` doesn`t have type named `#{type_name}`")
-      elsif index.types.one?
-        index.types.first
+      raise Chewy::UnderivableType, "Can not find index named `#{class_name}`" unless index && index < Chewy::Index
+      if type_name.present?
+        type = index.type_hash[type_name] or raise Chewy::UnderivableType, "Index `#{class_name}` doesn`t have type named `#{type_name}`"
+        [type]
       else
-        raise Chewy::UnderivableType.new("Index `#{class_name}` has more than one type, please specify type via `#{index_name}#type_name`")
+        index.types
       end
     end
 
     # Creates Chewy::Type ancestor defining index and adapter methods.
     #
-    def create_type index, target, options = {}, &block
+    def create_type(index, target, options = {}, &block)
       type = Class.new(Chewy::Type)
 
-      adapter = adapters.find { |adapter| adapter.accepts?(target) }.new(target, options)
+      adapter = adapters.find { |klass| klass.accepts?(target) }.new(target, options)
 
       index.const_set(adapter.name, type)
       type.send(:define_singleton_method, :index) { index }
       type.send(:define_singleton_method, :adapter) { adapter }
 
-      type.class_eval &block if block
+      type.class_eval(&block) if block
       type
     end
 
@@ -144,7 +166,7 @@ module Chewy
     # Be careful, if current prefix is blank, this will destroy all the indexes.
     #
     def massacre
-      Chewy.client.indices.delete(index: [Chewy.configuration[:prefix], '*'].delete_if(&:blank?).join(?_))
+      Chewy.client.indices.delete(index: [Chewy.configuration[:prefix], '*'].reject(&:blank?).join('_'))
       Chewy.wait_for_status
     end
     alias_method :delete_all, :massacre
@@ -172,7 +194,7 @@ module Chewy
     #   Chewy.strategy.pop
     #   city3.do_update! # index updated again
     #
-    def strategy name = nil, &block
+    def strategy(name = nil, &block)
       Thread.current[:chewy_strategy] ||= Chewy::Strategy.new
       if name
         if block
@@ -188,11 +210,32 @@ module Chewy
     def config
       Chewy::Config.instance
     end
-    delegate *Chewy::Config.delegated, to: :config
+    delegate(*Chewy::Config.delegated, to: :config)
 
     def repository
       Chewy::Repository.instance
     end
-    delegate *Chewy::Repository.delegated, to: :repository
+    delegate(*Chewy::Repository.delegated, to: :repository)
+
+    def create_indices
+      Chewy::Index.descendants.each(&:create)
+    end
+
+    def create_indices!
+      Chewy::Index.descendants.each(&:create!)
+    end
+
+    def eager_load!
+      return unless defined?(Chewy::Railtie)
+      dirs = Chewy::Railtie.all_engines.map { |engine| engine.paths[Chewy.configuration[:indices_path]] }.compact.map(&:existent).flatten.uniq
+
+      dirs.each do |dir|
+        Dir.glob(File.join(dir, '**/*.rb')).each do |file|
+          require_dependency file
+        end
+      end
+    end
   end
 end
+
+require 'chewy/stash'

data/lib/generators/chewy/install_generator.rb

@@ -1,11 +1,11 @@
 module Chewy
   module Generators
     class InstallGenerator < Rails::Generators::Base
-      source_root File.expand_path("../../templates", __FILE__)
+      source_root File.expand_path('../../templates', __FILE__)
 
       def copy_configuration
-        template "chewy.yml", "config/chewy.yml"
+        template 'chewy.yml', 'config/chewy.yml'
       end
     end
   end
-end
+end

data/lib/sequel/plugins/chewy_observe.rb

@@ -36,40 +36,25 @@ module Sequel
           callback_options = ChewyObserve.extract_callback_options!(args)
           update_proc = ChewyObserve.update_proc(type_name, *args, &block)
 
-          if Chewy.use_after_commit_callbacks
-            set_callback(:commit, callback_options, &update_proc)
-            set_callback(:destroy_commit, callback_options, &update_proc)
-          else
-            set_callback(:save, callback_options, &update_proc)
-            set_callback(:destroy, callback_options, &update_proc)
-          end
+          set_callback(:save, callback_options, &update_proc)
+          set_callback(:destroy, callback_options, &update_proc)
         end
       end
 
       # Instance level methods for Sequel::Model
       #
       module InstanceMethods
-        def after_commit
-          run_callbacks(:commit) do
-            super
-          end
-        end
-
-        def after_destroy_commit
-          run_callbacks(:destroy_commit) do
-            super
-          end
-        end
-
         def after_save
           run_callbacks(:save) do
             super
+            db.after_commit {} if Chewy.use_after_commit_callbacks
           end
         end
 
         def after_destroy
           run_callbacks(:destroy) do
             super
+            db.after_commit {} if Chewy.use_after_commit_callbacks
           end
         end
       end

data/lib/tasks/chewy.rake

@@ -1,49 +1,112 @@
 require 'chewy/rake_helper'
 
+def parse_classes(args)
+  if args.present? && args.first.tr!('-', '')
+    {except: args}
+  else
+    {only: args}
+  end
+end
+
+def parse_parallel_args(args)
+  options = {}
+  options[:parallel] = args.first =~ /\A\d+\z/ ? Integer(args.shift) : true
+  options.merge!(parse_classes(args))
+end
+
+def parse_journal_args(args)
+  options = {}
+  options[:time] = Time.parse(args.shift) if args.first =~ /\A\d+/
+  options.merge!(parse_classes(args))
+end
+
 namespace :chewy do
-  desc 'Destroy, recreate and import data to specified index'
-  task :reset, [:index] => :environment do |task, args|
-    Chewy::RakeHelper.subscribed_task_stats do
+  desc 'Destroys, recreates and imports data for the specified indexes or all of them'
+  task reset: :environment do |_task, args|
+    Chewy::RakeHelper.reset(parse_classes(args.extras))
+  end
 
-      if args[:index].present?
-        Chewy::RakeHelper.reset_index(args[:index])
-      else
-        Chewy::RakeHelper.reset_all
-      end
+  desc 'Resets data for the specified indexes or all of them only if the index specification is changed'
+  task upgrade: :environment do |_task, args|
+    Chewy::RakeHelper.upgrade(parse_classes(args.extras))
+  end
+
+  desc 'Updates data for the specified indexes/types or all of them'
+  task update: :environment do |_task, args|
+    Chewy::RakeHelper.update(parse_classes(args.extras))
+  end
+
+  desc 'Synchronizes data for the specified indexes/types or all of them'
+  task sync: :environment do |_task, args|
+    Chewy::RakeHelper.sync(parse_classes(args.extras))
+  end
+
+  desc 'Resets all the indexes with the specification changed and synchronizes the rest of them'
+  task deploy: :environment do
+    processed = Chewy::RakeHelper.upgrade
+    Chewy::RakeHelper.sync(except: processed)
+  end
+
+  namespace :parallel do
+    desc 'Parallel version of `rake chewy:reset`'
+    task reset: :environment do |_task, args|
+      Chewy::RakeHelper.reset(parse_parallel_args(args.extras))
+    end
+
+    desc 'Parallel version of `rake chewy:upgrade`'
+    task upgrade: :environment do |_task, args|
+      Chewy::RakeHelper.upgrade(parse_parallel_args(args.extras))
+    end
+
+    desc 'Parallel version of `rake chewy:update`'
+    task update: :environment do |_task, args|
+      Chewy::RakeHelper.update(parse_parallel_args(args.extras))
+    end
+
+    desc 'Parallel version of `rake chewy:sync`'
+    task sync: :environment do |_task, args|
+      Chewy::RakeHelper.sync(parse_parallel_args(args.extras))
+    end
+
+    desc 'Parallel version of `rake chewy:deploy`'
+    task deploy: :environment do |_task, args|
+      parallel = args.extras.first =~ /\A\d+\z/ ? Integer(args.extras.first) : true
+      processed = Chewy::RakeHelper.upgrade(parallel: parallel)
+      Chewy::RakeHelper.sync(except: processed, parallel: parallel)
     end
   end
 
-  namespace :reset do
-    desc 'Destroy, recreate and import data for all found indexes'
-    task all: :environment do
-      ActiveSupport::Deprecation.warn('`rake chewy:reset:all` is deprecated and will be removed soon. Use `rake chewy:reset` instead')
+  namespace :journal do
+    desc 'Applies changes that were done after the specified time for the specified indexes/types or all of them'
+    task apply: :environment do |_task, args|
+      Chewy::RakeHelper.journal_apply(parse_journal_args(args.extras))
+    end
 
-      Chewy::RakeHelper.subscribed_task_stats do
-        Chewy::RakeHelper.reset_all
-      end
+    desc 'Removes journal records created before the specified timestamp for the specified indexes/types or all of them'
+    task clean: :environment do |_task, args|
+      Chewy::RakeHelper.journal_clean(parse_journal_args(args.extras))
     end
   end
 
-  desc 'Updates data specified index'
-  task :update, [:index] => :environment do |task, args|
+  task apply_changes_from: :environment do |_task, args|
+    ActiveSupport::Deprecation.warn '`rake chewy:apply_changes_from` is deprecated and will be removed soon, use `rake chewy:journal:apply` instead'
+
     Chewy::RakeHelper.subscribed_task_stats do
+      params = args.extras
 
-      if args[:index].present?
-        Chewy::RakeHelper.update_index(args[:index])
+      if params.empty?
+        puts 'Please specify a timestamp like chewy:apply_changes_from[1469528705]'
       else
-        Chewy::RakeHelper.update_all
+        timestamp, retries = params
+        time = Time.at(timestamp.to_i)
+        Chewy::Journal.new.apply(time, retries: (retries.to_i if retries))
       end
     end
   end
 
-  namespace :update do
-    desc 'Updates data for all found indexes'
-    task all: :environment do
-      ActiveSupport::Deprecation.warn('`rake chewy:update:all` is deprecated and will be removed soon. Use `rake chewy:update` instead')
+  task clean_journal: :environment do |_task, args|
+    ActiveSupport::Deprecation.warn '`rake chewy:clean_journal` is deprecated and will be removed soon, use `rake chewy:journal:clean` instead'
 
-      Chewy::RakeHelper.subscribed_task_stats do
-        Chewy::RakeHelper.update_all
-      end
-    end
+    Chewy::Journal.new.clean(args.extras.first)
   end
 end