chewy 0.6.2 → 0.7.0

data/lib/chewy/railtie.rb

@@ -1,27 +1,65 @@
 module Chewy
   class Railtie < Rails::Railtie
+    def self.all_engines
+      Rails::Engine.subclasses.map(&:instance) + [Rails.application]
+    end
+
+    class RequestStrategy
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if env['PATH_INFO'] =~ /^\/assets\//
+          @app.call(env)
+        else
+          Chewy.strategy(Chewy.request_strategy) { @app.call(env) }
+        end
+      end
+    end
+
+    module MigrationStrategy
+      extend ActiveSupport::Concern
+      included do
+        alias_method_chain :migrate, :chewy
+      end
+
+      def migrate_with_chewy(*args)
+        Chewy.strategy(:bypass) { migrate_without_chewy(*args) }
+      end
+    end
+
     rake_tasks do
       load 'tasks/chewy.rake'
     end
 
-    initializer 'chewy.add_app_chewy_path' do |app|
-      app.config.paths.add 'app/chewy'
-    end
+    console do |app|
+      Chewy.logger = ActiveRecord::Base.logger if defined?(ActiveRecord)
 
-    initializer 'chewy.add_requests_logging' do |app|
-      ActiveSupport::Notifications.subscribe('import_objects.chewy') do |name, start, finish, id, payload|
-        duration = ((finish - start).to_f * 10000).round / 10.0
-        Rails.logger.debug("  \e[1m\e[32m#{payload[:type]} Import (#{duration}ms)\e[0m #{payload[:import]}")
+      if app.sandbox?
+        Chewy.strategy(:bypass)
+      else
+        Chewy.strategy(:urgent)
       end
+    end
+
+    initializer 'chewy.logger', after: 'active_record.logger' do
+      ActiveSupport.on_load(:active_record)  { Chewy.logger ||= ActiveRecord::Base.logger }
+    end
 
-      ActiveSupport::Notifications.subscribe('search_query.chewy') do |name, start, finish, id, payload|
-        duration = ((finish - start).to_f * 10000).round / 10.0
-        Rails.logger.debug("  \e[1m\e[32m#{payload[:index]} Search (#{duration}ms)\e[0m #{payload[:request]}")
+    initializer 'chewy.migration_strategy' do
+      ActiveSupport.on_load(:active_record) do
+        ActiveRecord::Migration.send(:include, MigrationStrategy)
       end
+    end
 
-      ActiveSupport::Notifications.subscribe('delete_query.chewy') do |name, start, finish, id, payload|
-        duration = ((finish - start).to_f * 10000).round / 10.0
-        Rails.logger.debug("  \e[1m\e[32m#{payload[:index]} Delete by Query (#{duration}ms)\e[0m #{payload[:request]}")
+    initializer 'chewy.request_strategy' do |app|
+      app.config.middleware.insert_after(Rack::Runtime, RequestStrategy)
+    end
+
+    initializer 'chewy.add_app_chewy_path' do |app|
+      Chewy::Railtie.all_engines.each do |engine|
+        engine.paths.add 'app/chewy'
       end
     end
   end

data/lib/chewy/repository.rb

@@ -0,0 +1,61 @@
+module Chewy
+  class Repository
+    include Singleton
+
+    attr_reader :analyzers, :tokenizers, :filters, :char_filters
+
+    def self.delegated
+      public_instance_methods - self.superclass.public_instance_methods - Singleton.public_instance_methods
+    end
+
+    def self.repository name
+      plural_name = name.to_s.pluralize
+
+      class_eval <<-METHOD, __FILE__, __LINE__ + 1
+        def #{name}(name, options = nil)
+          options ? #{plural_name}[name.to_sym] = options : #{plural_name}[name.to_sym]
+        end
+      METHOD
+    end
+
+    # Analysers repository:
+    #
+    #   Chewy.analyzer :my_analyzer2, {
+    #     type: custom,
+    #     tokenizer: 'my_tokenizer1',
+    #     filter : ['my_token_filter1', 'my_token_filter2']
+    #     char_filter : ['my_html']
+    #   }
+    #   Chewy.analyzer(:my_analyzer2) # => {type: 'custom', tokenizer: ...}
+    #
+    repository :analyzer
+
+    # Tokenizers repository:
+    #
+    #   Chewy.tokenizer :my_tokenizer1, {type: standard, max_token_length: 900}
+    #   Chewy.tokenizer(:my_tokenizer1) # => {type: standard, max_token_length: 900}
+    #
+    repository :tokenizer
+
+    # Token filters repository:
+    #
+    #   Chewy.filter :my_token_filter1, {type: stop, stopwords: [stop1, stop2, stop3, stop4]}
+    #   Chewy.filter(:my_token_filter1) # => {type: stop, stopwords: [stop1, stop2, stop3, stop4]}
+    #
+    repository :filter
+
+    # Char filters repository:
+    #
+    #   Chewy.char_filter :my_html, {type: html_strip, escaped_tags: [xxx, yyy], read_ahead: 1024}
+    #   Chewy.char_filter(:my_html) # => {type: html_strip, escaped_tags: [xxx, yyy], read_ahead: 1024}
+    #
+    repository :char_filter
+
+    def initialize
+      @analyzers = {}
+      @tokenizers = {}
+      @filters = {}
+      @char_filters = {}
+    end
+  end
+end

data/lib/chewy/rspec/update_index.rb

@@ -18,7 +18,7 @@ require 'i18n/core_ext/hash'
 # Combined matcher chain methods:
 #
 #   specify { expect { user1.destroy!; user2.save! } }
-#     .to update_index(UsersIndex:User).and_reindex(user2).and_delete(user1)
+#     .to update_index(UsersIndex:User).and_reindex(user2).and_delete(user1) }
 #
 RSpec::Matchers.define :update_index do |type_name, options = {}|
 
@@ -107,11 +107,8 @@ RSpec::Matchers.define :update_index do |type_name, options = {}|
       {}
     end
 
-    if options[:atomic] == false
-      block.call
-    else
-      Chewy.atomic { block.call }
-    end
+    ActiveSupport::Deprecation.warn('`atomic: false` option is removed and not effective anymore, use `strategy: :atomic` option instead') if options.key?(:atomic)
+    Chewy.strategy(options[:strategy] || :atomic) { block.call }
 
     @updated.each do |updated_document|
       if body = updated_document[:index]

data/lib/chewy/search.rb

@@ -15,20 +15,41 @@ module Chewy
 
     module ClassMethods
       def all
-        Chewy::Query.new(search_index, types: search_type)
+        query_class.scopes.last || query_class.new(self)
       end
 
       def search_string query, options = {}
-        options = options.merge(index: search_index.index_name, type: search_type, q: query)
-        client.search(options)
+        options = options.merge(
+          index: all._indexes.map(&:index_name),
+          type: all._types.map(&:type_name),
+          q: query)
+        Chewy.client.search(options)
       end
 
-      def search_index
-        raise NotImplementedError
+    private
+
+      def query_class
+        @query_class ||= begin
+          query_class = Class.new(Chewy::Query)
+          if self < Chewy::Type
+            index_scopes = index.scopes - scopes
+
+            delegate_scoped index, query_class, index_scopes
+            delegate_scoped index, self, index_scopes
+          end
+          delegate_scoped self, query_class, scopes
+          const_set('Query', query_class)
+        end
       end
 
-      def search_type
-        raise NotImplementedError
+      def delegate_scoped source, destination, methods
+        methods.each do |method|
+          destination.class_eval do
+            define_method method do |*args, &block|
+              scoping { source.public_send(method, *args, &block) }
+            end
+          end
+        end
       end
     end
   end

data/lib/chewy/strategy.rb

@@ -0,0 +1,60 @@
+require 'chewy/strategy/base'
+require 'chewy/strategy/bypass'
+require 'chewy/strategy/urgent'
+require 'chewy/strategy/atomic'
+
+module Chewy
+  # This class represents strategies stack with `:base`
+  # Strategy on top of it. This causes raising exceptions
+  # on every index update attempt, so other strategy must
+  # be choosen.
+  #
+  #   User.first.save # Raises UndefinedUpdateStrategy exception
+  #
+  #   Chewy.strategy(:atomic) do
+  #     User.last.save # Save user according to the `:atomic` strategy rules
+  #   end
+  #
+  class Strategy
+    def initialize
+      @stack = [resolve(Chewy.root_strategy).new]
+    end
+
+    def current
+      @stack.last
+    end
+
+    def push name
+      result = @stack.push resolve(name).new
+      debug "[#{@stack.size}] <- #{current.name}"
+      result
+    end
+
+    def pop
+      raise "Can't pop root strategy" if @stack.one?
+      debug "[#{@stack.size}] -> #{current.name}"
+      result = @stack.pop.tap(&:leave)
+      result
+    end
+
+    def wrap name
+      push name
+      yield
+    ensure
+      pop
+    end
+
+  private
+
+    def debug string
+      if Chewy.logger && Chewy.logger.debug?
+        line = caller.detect { |line| line !~ %r{lib/chewy/strategy.rb:|lib/chewy.rb:} }
+        Chewy.logger.debug(["Chewy strategies stack: #{string}", line.sub(/:in\s.+$/, '')].join(' @ '))
+      end
+    end
+
+    def resolve name
+      "Chewy::Strategy::#{name.to_s.camelize}".constantize or raise "Can't find update strategy `#{name}`"
+    end
+  end
+end

data/lib/chewy/strategy/atomic.rb

@@ -0,0 +1,31 @@
+module Chewy
+  class Strategy
+    # This strategy accomulates all the objects prepared for
+    # indexing and fires index process when strategy is popped
+    # from the strategies stack.
+    #
+    #   Chewy.strategy(:atomic) do
+    #     User.all.map(&:save) # Does nothing here
+    #     Post.all.map(&:save) # And here
+    #     # It imports all the changed users and posts right here
+    #     # before block leaving with bulk ES API, kinda optimization
+    #   end
+    #
+    class Atomic < Base
+      def initialize
+        @stash = {}
+      end
+
+      def update type, objects, options = {}
+        ActiveSupport::Deprecation.warn("`urgent: true` option is deprecated and is not effective inside `:atomic` strategy, use `Chewy.strategy(:urgent)` strategy instead") if options.key?(:urgent)
+
+        @stash[type] ||= []
+        @stash[type] |= type.adapter.identify(objects)
+      end
+
+      def leave
+        @stash.all? { |type, ids| type.import(ids) }
+      end
+    end
+  end
+end

data/lib/chewy/strategy/base.rb

@@ -0,0 +1,27 @@
+module Chewy
+  class Strategy
+    # This strategy raises exception on every index update
+    # asking to choose some other strategy.
+    #
+    #   Chewy.strategy(:base) do
+    #     User.all.map(&:save) # Raises UndefinedUpdateStrategy exception
+    #   end
+    #
+    class Base
+      def name
+        self.class.name.demodulize.underscore.to_sym
+      end
+      # This method called when some model tries to update index
+      #
+      def update type, objects, options = {}
+        raise UndefinedUpdateStrategy.new(type)
+      end
+
+      # This method called when strategy pops from the
+      # strategies stack
+      #
+      def leave
+      end
+    end
+  end
+end

data/lib/chewy/strategy/bypass.rb

@@ -0,0 +1,15 @@
+module Chewy
+  class Strategy
+    # This strategy basically does nothing.
+    #
+    #   Chewy.strategy(:bypass) do
+    #     User.all.map(&:save) # Does nothing here
+    #     # Does not update index all over the block.
+    #   end
+    #
+    class Bypass < Base
+      def update type, objects, options = {}
+      end
+    end
+  end
+end

data/lib/chewy/strategy/urgent.rb

@@ -0,0 +1,17 @@
+module Chewy
+  class Strategy
+    # This strategy updates index on demand. Not the best
+    # strategy in case of optimization. If you need to update
+    # indexes with bulk API calls - use :atomic instead.
+    #
+    #   Chewy.strategy(:urgent) do
+    #     User.all.map(&:save) # Updates index on every `save` call
+    #   end
+    #
+    class Urgent < Base
+      def update type, objects, options = {}
+        type.import(Array.wrap(objects))
+      end
+    end
+  end
+end

data/lib/chewy/type.rb

@@ -17,26 +17,40 @@ module Chewy
     include Actions
     include Import
 
-    singleton_class.delegate :client, to: :index
+    singleton_class.delegate :index_name, :client, to: :index
 
+    # Chewy index current type blongs to. Defined inside `Chewy.create_type`
+    #
     def self.index
       raise NotImplementedError
     end
 
+    # Current type adapter. Defined inside `Chewy.create_type`, derived from
+    # `Chewy::Index.define_type` arguments.
+    #
     def self.adapter
       raise NotImplementedError
     end
 
+    # Returns type name string
+    #
     def self.type_name
       adapter.type_name
     end
 
-    def self.search_index
-      index
+    # Returns list of public class methods defined in current type
+    #
+    def self.scopes
+      public_methods - Chewy::Type.public_methods
     end
 
-    def self.search_type
-      type_name
+    def self.method_missing(method, *args, &block)
+      if index.scopes.include?(method)
+        define_singleton_method method do |*args, &block|
+          all.scoping { index.public_send(method, *args, &block) }
+        end
+        send(method, *args, &block)
+      end
     end
 
     def self.const_missing(name)

data/lib/chewy/type/adapter/active_record.rb

@@ -1,142 +1,53 @@
-require 'chewy/type/adapter/base'
+require 'chewy/type/adapter/orm'
 
 module Chewy
   class Type
     module Adapter
-      class ActiveRecord < Base
-        def initialize *args
-          @options = args.extract_options!
-          subject = args.first
-          if subject.is_a?(::ActiveRecord::Relation)
-            @model = subject.klass
-            @scope = subject
-          else
-            @model = subject
-          end
-        end
-
-        def name
-          @name ||= (options[:name].present? ? options[:name].to_s.camelize : model.model_name.to_s).demodulize
-        end
-
-        # Import method fo ActiveRecord takes import data and import options
-        #
-        # Import data types:
-        #
-        #   * Nothing passed - imports all the model data
-        #   * ActiveRecord scope
-        #   * Objects collection
-        #   * Ids collection
-        #
-        # Import options:
-        #
-        #   <tt>:batch_size</tt> - import batch size, 1000 objects by default
-        #
-        # Method handles destroyed objects as well. In case of objects AcriveRecord::Relation
-        # or array passed, objects, responding with true to `destroyed?` method will be deleted
-        # from index. In case of ids array passed - documents with missing records ids will be
-        # deleted from index:
-        #
-        #   users = User.all
-        #   users.each { |user| user.destroy if user.incative? }
-        #   UsersIndex::User.import users # inactive users will be deleted from index
-        #   # or
-        #   UsersIndex::User.import users.map(&:id) # deleted user ids will be deleted from index
-        #
-        # Also there is custom API method `delete_from_index?`. It it returns `true`
-        # object will be deleted from index. Note that if this method is defined and
-        # return `false` Chewy will still check `destroyed?` method. This is useful
-        # for paranoid objects sdeleting implementation.
-        #
-        #   class User
-        #     alias_method :delete_from_index?, :deleted_at?
-        #   end
-        #
-        #   users = User.all
-        #   users.each { |user| user.deleted_at = Time.now }
-        #   UsersIndex::User.import users # paranoid deleted users will be deleted from index
-        #   # or
-        #   UsersIndex::User.import users.map(&:id) # user ids will be deleted from index
-        #
-        def import *args, &block
-          import_options = args.extract_options!
-          import_options[:batch_size] ||= BATCH_SIZE
-          collection = args.none? ? model_all :
-            (args.one? && args.first.is_a?(::ActiveRecord::Relation) ? args.first : args.flatten.compact)
+      class ActiveRecord < Orm
+      private
 
-          if collection.is_a?(::ActiveRecord::Relation)
-            result = true
-            merged_scope(collection).find_in_batches(import_options.slice(:batch_size)) do |group|
-              result &= block.call grouped_objects(group)
-            end
-            result
-          else
-            if collection.all? { |object| object.respond_to?(:id) }
-              collection.each_slice(import_options[:batch_size]).map do |group|
-                block.call grouped_objects(group)
-              end.all?
-            else
-              import_ids(collection, import_options, &block)
-            end
+        def cleanup_default_scope!
+          if Chewy.logger && (@default_scope.arel.orders.present? ||
+             @default_scope.arel.limit.present? || @default_scope.arel.offset.present?)
+            Chewy.logger.warn('Default type scope order, limit and offest are ignored and will be nullified')
           end
-        end
 
-        def load *args
-          load_options = args.extract_options!
-          objects = args.flatten
-
-          additional_scope = load_options[load_options[:_type].type_name.to_sym].try(:[], :scope) || load_options[:scope]
-
-          scope = scoped_model(objects.map(&:id))
-          loaded_objects = if additional_scope.is_a?(Proc)
-            scope.instance_exec(&additional_scope)
-          elsif additional_scope.is_a?(::ActiveRecord::Relation)
-            scope.merge(additional_scope)
-          else
-            scope
-          end.index_by { |object| object.id.to_s }
-
-          objects.map { |object| loaded_objects[object.id.to_s] }
+          @default_scope = @default_scope.reorder(nil).limit(nil).offset(nil)
         end
 
-      private
+        def import_scope(scope, batch_size)
+          scope = scope.reorder(target.primary_key.to_sym).limit(batch_size)
 
-        attr_reader :model, :scope, :options
+          ids = pluck_ids(scope)
+          result = true
 
-        def import_ids(ids, import_options = {}, &block)
-          ids.uniq!
-
-          indexed = true
-          merged_scope(scoped_model(ids)).find_in_batches(import_options.slice(:batch_size)) do |objects|
-            ids -= objects.map(&:id)
-            indexed &= block.call(grouped_objects(objects))
+          while ids.any?
+            result &= yield grouped_objects(default_scope_where_ids_in(ids))
+            break if ids.size < batch_size
+            ids = pluck_ids(scope.where(scope.table[target.primary_key].gt(ids.last)))
           end
 
-          deleted = ids.each_slice(import_options[:batch_size]).map do |group|
-            block.call(delete: group)
-          end.all?
+          result
+        end
 
-          indexed && deleted
+        def pluck_ids(scope)
+          scope.pluck(target.primary_key.to_sym)
         end
 
-        def grouped_objects(objects)
-          objects.group_by do |object|
-            delete = object.delete_from_index? if object.respond_to?(:delete_from_index?)
-            delete ||= object.destroyed?
-            delete ? :delete : :index
-          end
+        def scope_where_ids_in(scope, ids)
+          scope.where(target.primary_key => ids)
         end
 
-        def merged_scope(target)
-          scope ? scope.clone.merge(target) : target
+        def all_scope
+          target.where(nil)
         end
 
-        def scoped_model(ids)
-          model.where(Hash[model.primary_key.to_sym || :id, ids])
+        def relation_class
+          ::ActiveRecord::Relation
         end
 
-        def model_all
-          ::ActiveRecord::VERSION::MAJOR < 4 ? model.scoped : model.all
+        def object_class
+          ::ActiveRecord::Base
         end
       end
     end