chewy 0.8.1 → 0.8.2

data/lib/chewy/type/adapter/base.rb

@@ -7,6 +7,12 @@ module Chewy
 
         attr_reader :target, :options
 
+        # Returns `true` if this adapter is applicable for the given target.
+        #
+        def self.accepts? target
+          true
+        end
+
         # Camelcased name, used as type class constant name.
         # For returned value 'Product' will be generated class name `ProductsIndex::Product`
         #

data/lib/chewy/type/adapter/mongoid.rb

@@ -5,6 +5,12 @@ module Chewy
     module Adapter
       class Mongoid < Orm
 
+        def self.accepts?(target)
+          defined?(::Mongoid::Document) && (
+            target.is_a?(Class) && target.ancestors.include?(::Mongoid::Document) ||
+            target.is_a?(::Mongoid::Criteria))
+        end
+
         def identify collection
           super(collection).map { |id| id.is_a?(BSON::ObjectId) ? id.to_s : id }
         end
@@ -13,7 +19,7 @@ module Chewy
 
         def cleanup_default_scope!
           if Chewy.logger && @default_scope.options.values_at(:sort, :limit, :skip).compact.present?
-            Chewy.logger.warn('Default type scope order, limit and offest are ignored and will be nullified')
+            Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified')
           end
 
           @default_scope = @default_scope.reorder(nil)

data/lib/chewy/type/adapter/orm.rb

@@ -53,7 +53,7 @@ module Chewy
         # deleted from index:
         #
         #   users = User.all
-        #   users.each { |user| user.destroy if user.incative? }
+        #   users.each { |user| user.destroy if user.inactive? }
         #   UsersIndex::User.import users # inactive users will be deleted from index
         #   # or
         #   UsersIndex::User.import users.map(&:id) # deleted user ids will be deleted from index

data/lib/chewy/type/adapter/sequel.rb

@@ -0,0 +1,125 @@
+require 'chewy/type/adapter/base'
+
+module Chewy
+  class Type
+    module Adapter
+      class Sequel < Base
+
+        attr_reader :default_dataset
+
+        def self.accepts?(target)
+          defined?(::Sequel::Model) && (
+            target.is_a?(Class) && target < ::Sequel::Model || target.is_a?(::Sequel::Dataset))
+        end
+
+        def initialize(*args)
+          @options = args.extract_options!
+
+          if dataset? args.first
+            dataset = args.first
+            @target = dataset.model
+            @default_dataset = dataset.unordered.unlimited
+          else
+            model = args.first
+            @target = model
+            @default_dataset = model.where(nil)
+          end
+        end
+
+        def name
+          @name ||= (options[:name].presence || target.name).camelize.demodulize
+        end
+
+        def identify(obj)
+          if dataset? obj
+            obj.select_map(target_pk)
+          else
+            Array.wrap(obj).map do |item|
+              model?(item) ? item.pk : item
+            end
+          end
+        end
+
+        def import(*args, &block)
+          import_options = args.extract_options!
+          batch_size = import_options[:batch_size] || BATCH_SIZE
+
+          if args.empty?
+            import_dataset(default_dataset, batch_size, &block)
+          elsif args.one? && dataset?(args.first)
+            import_dataset(args.first, batch_size, &block)
+          else
+            import_models(args.flatten.compact, batch_size, &block)
+          end
+        end
+
+        def load(*args)
+          load_options = args.extract_options!
+          index_ids = args.flatten.map(&:id)  # args contains index instances
+
+          type_name = load_options[:_type].type_name.to_sym
+          additional_scope = load_options[type_name].try(:[], :scope) || load_options[:scope]
+
+          dataset = select_by_ids(target, index_ids)
+
+          if additional_scope.is_a?(Proc)
+            index_ids.map!(&:to_s)
+            dataset.instance_exec(&additional_scope).to_a.select do |model|
+              index_ids.include? model.pk.to_s
+            end
+          else
+            dataset.to_a
+          end
+        end
+
+        private
+
+        def import_dataset(dataset, batch_size)
+          dataset = dataset.limit(batch_size)
+
+          dataset.db.transaction(isolation: :committed) do
+            0.step(Float::INFINITY, batch_size).lazy
+              .map { |offset| dataset.offset(offset).to_a }
+              .take_while(&:any?)
+              .map { |items| yield grouped_objects(items) }
+              .reduce(:&)
+          end
+        end
+
+        def import_models(objects, batch_size)
+          objects_by_id = objects.index_by do |item|
+            model?(item) ? item.pk : item
+          end
+
+          indexed = objects_by_id.keys.each_slice(batch_size).map do |ids|
+            models = select_by_ids(default_dataset, ids).to_a
+            models.each { |model| objects_by_id.delete(model.pk) }
+            models.empty? || yield(grouped_objects(models))
+          end
+
+          deleted = objects_by_id.keys.each_slice(batch_size).map do |ids|
+            yield delete: objects_by_id.values_at(*ids)
+          end
+
+          indexed.all? && deleted.all?
+        end
+
+        def select_by_ids(dataset, ids)
+          dataset.where(target_pk => Array.wrap(ids))
+        end
+
+        def target_pk
+          target.primary_key
+        end
+
+        def dataset?(obj)
+          obj.is_a? ::Sequel::Dataset
+        end
+
+        def model?(obj)
+          obj.is_a? ::Sequel::Model
+        end
+      end
+    end
+  end
+end

data/lib/chewy/type/mapping.rb

@@ -6,6 +6,8 @@ module Chewy
       included do
         class_attribute :root_object, instance_reader: false, instance_writer: false
         class_attribute :_templates
+        class_attribute :_agg_defs
+        self._agg_defs = {}
       end
 
       module ClassMethods
@@ -118,7 +120,30 @@ module Chewy
           end
         end
 
-        # Defines dynamic template in mapping root objests
+        # Defines an aggregation that can be bound to a query or filter
+        #
+        #   Suppose that a user has posts and each post has ratings
+        #   avg_post_rating is the mean of all ratings
+        #
+        #   class UsersIndex < Chewy::Index
+        #     define_type User do
+        #       field :posts do
+        #         field :rating
+        #       end
+        #
+        #       agg :avg_rating do
+        #         { avg: { field: 'posts.rating' } }
+        #       end
+        #     end
+        #   end
+        def agg *args, &block
+          options = args.extract_options!
+          build_root unless root_object
+          self._agg_defs = _agg_defs.merge(args.first => block)
+        end
+        alias_method :aggregation, :agg
+
+        # Defines dynamic template in mapping root objects
         #
         #   class CarsIndex < Chewy::Index
         #     define_type Car do

data/lib/chewy/type/observe.rb

@@ -3,41 +3,64 @@ module Chewy
     module Observe
       extend ActiveSupport::Concern
 
-      def self.update_proc(type_name, *args, &block)
-        options = args.extract_options!
-        method = args.first
-
-        Proc.new do
-          backreference = if method && method.to_s == 'self'
-            self
-          elsif method
-            send(method)
-          else
-            instance_eval(&block)
+      module Helpers
+        def update_proc(type_name, *args, &block)
+          options = args.extract_options!
+          method = args.first
+
+          Proc.new do
+            backreference = if method && method.to_s == 'self'
+              self
+            elsif method
+              send(method)
+            else
+              instance_eval(&block)
+            end
+
+            reference = if type_name.is_a?(Proc)
+              type_name.arity == 0 ?
+                instance_exec(&type_name) :
+                type_name.call(self)
+            else
+              type_name
+            end
+
+            Chewy.derive_type(reference).update_index(backreference, options)
           end
+        end
 
-          Chewy.derive_type(type_name).update_index(backreference, options)
+        def extract_callback_options!(args)
+          options = args.extract_options!
+          options.each_key.with_object({}) { |key, hash|
+            hash[key] = options.delete(key) if [:if, :unless].include?(key)
+          }.tap {
+            args.push(options) unless options.empty?
+          }
         end
       end
 
+      extend Helpers
+
       module MongoidMethods
         def update_index(type_name, *args, &block)
+          callback_options = Observe.extract_callback_options!(args)
           update_proc = Observe.update_proc(type_name, *args, &block)
 
-          after_save &update_proc
-          after_destroy &update_proc
+          after_save(callback_options, &update_proc)
+          after_destroy(callback_options, &update_proc)
         end
       end
 
       module ActiveRecordMethods
         def update_index(type_name, *args, &block)
+          callback_options = Observe.extract_callback_options!(args)
           update_proc = Observe.update_proc(type_name, *args, &block)
 
           if Chewy.use_after_commit_callbacks
-            after_commit &update_proc
+            after_commit(callback_options, &update_proc)
           else
-            after_save &update_proc
-            after_destroy &update_proc
+            after_save(callback_options, &update_proc)
+            after_destroy(callback_options, &update_proc)
           end
         end
       end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '0.8.1'
+  VERSION = '0.8.2'
 end

data/lib/sequel/plugins/chewy_observe.rb

@@ -0,0 +1,71 @@
+require 'active_support/callbacks'
+
+module Sequel
+  module Plugins
+    # This Sequel plugin adds support for chewy's model-observing hook for
+    # updating indexes after model save or destroy.
+    #
+    # Usage:
+    #
+    #   # Make all model subclasses support the `update_index` hook (called
+    #   # before loading subclasses).
+    #   Sequel::Model.plugin :chewy_observe
+    #
+    #   # Make the Album class support the `update_index` hooks.
+    #   Album.plugin :chewy_observe
+    #
+    #   # Declare one or more `update_index` observers in model.
+    #   class Album < Sequel::Model
+    #     update_index('albums#album') { self }
+    #   end
+    #
+    module ChewyObserve
+      extend ::Chewy::Type::Observe::Helpers
+
+      def self.apply(model)
+        model.instance_eval do
+          include ActiveSupport::Callbacks
+          define_callbacks :commit, :save, :destroy
+        end
+      end
+
+      # Class level methods for Sequel::Model
+      #
+      module ClassMethods
+        def update_index(type_name, *args, &block)
+          callback_options = ChewyObserve.extract_callback_options!(args)
+          update_proc = ChewyObserve.update_proc(type_name, *args, &block)
+
+          if Chewy.use_after_commit_callbacks
+            set_callback(:commit, callback_options, &update_proc)
+          else
+            set_callback(:save, callback_options, &update_proc)
+            set_callback(:destroy, callback_options, &update_proc)
+          end
+        end
+      end
+
+      # Instance level methods for Sequel::Model
+      #
+      module InstanceMethods
+        def after_commit
+          run_callbacks(:commit) do
+            super
+          end
+        end
+
+        def after_save
+          run_callbacks(:save) do
+            super
+          end
+        end
+
+        def after_destroy
+          run_callbacks(:destroy) do
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tasks/chewy.rake

@@ -1,83 +1,41 @@
-def subscribe_task_stats!
-  ActiveSupport::Notifications.subscribe('import_objects.chewy') do |name, start, finish, id, payload|
-    duration = (finish - start).round(2)
-    puts "  Imported #{payload[:type]} for #{duration}s, documents total: #{payload[:import].try(:[], :index).to_i}"
-    payload[:errors].each do |action, errors|
-      puts "    #{action.to_s.humanize} errors:"
-      errors.each do |error, documents|
-        puts "      `#{error}`"
-        puts "        on #{documents.count} documents: #{documents}"
-      end
-    end if payload[:errors]
-  end
-end
-
-def eager_load_chewy!
-  dirs = Chewy::Railtie.all_engines.map { |engine| engine.paths['app/chewy'].existent }.flatten.uniq
-
-  dirs.each do |dir|
-    Dir.glob(File.join(dir, '**/*.rb')).each do |file|
-      require_dependency file
-    end
-  end
-end
-
-def normalize_index index
-  "#{index.to_s.gsub(/index$/i, '').camelize}Index".constantize
-end
-
-def reset_index index
-  index = normalize_index(index)
-  puts "Resetting #{index}"
-  index.reset! (Time.now.to_f * 1000).round
-end
-
-def reset_all
-  eager_load_chewy!
-  Chewy::Index.descendants.each { |index| reset_index index }
-end
-
-def update_index index
-  index = normalize_index(index)
-  puts "Updating #{index}"
-  if index.exists?
-    index.import
-  else
-    puts "Index `#{index.index_name}` does not exists. Use rake chewy:reset[#{index.index_name}] to create and update it."
-  end
-end
-
-def update_all
-  eager_load_chewy!
-  Chewy::Index.descendants.each { |index| update_index index }
-end
+require 'chewy/rake_helper'
 
 namespace :chewy do
   desc 'Destroy, recreate and import data to specified index'
   task :reset, [:index] => :environment do |task, args|
-    subscribe_task_stats!
-    args[:index].present? ? reset_index(args[:index]) : reset_all
+    Chewy::RakeHelper.subscribe_task_stats!
+
+    if args[:index].present?
+      Chewy::RakeHelper.reset_index(args[:index])
+    else
+      Chewy::RakeHelper.reset_all
+    end
   end
 
   namespace :reset do
     desc 'Destroy, recreate and import data for all found indexes'
     task all: :environment do
-      subscribe_task_stats!
-      reset_all
+      Chewy::RakeHelper.subscribe_task_stats!
+      Chewy::RakeHelper.reset_all
     end
   end
 
   desc 'Updates data specified index'
   task :update, [:index] => :environment do |task, args|
-    subscribe_task_stats!
-    args[:index].present? ? update_index(args[:index]) : update_all
+    Chewy::RakeHelper.subscribe_task_stats!
+
+    if args[:index].present?
+      Chewy::RakeHelper.update_index(args[:index])
+    else
+      Chewy::RakeHelper.update_all
+    end
   end
 
   namespace :update do
     desc 'Updates data for all found indexes'
     task all: :environment do
-      subscribe_task_stats!
-      update_all
+      Chewy::RakeHelper.subscribe_task_stats!
+      Chewy::RakeHelper.update_all
     end
   end
 end