activerecord_reindex 0.1.2 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6508847b036e9191ba24d150c3ca6fd3aff0d147
-  data.tar.gz: 1a73c90b46cb0b0f76ddad37e2451737b1f53112
+  metadata.gz: 3935a55283f2824e669f9f5b669b82ff4242779e
+  data.tar.gz: 848a4b8e5ceb4379846ce2119ff38292fbe56494
 SHA512:
-  metadata.gz: 583b1b18878781670b7180cdb15453e7e0306b46dbc30f3f3655e638f2e7108e367818ed6b3b8d6b0f6c4f17db28214a1512607b6d8e39101cf89d894d27ebe2
-  data.tar.gz: 6d5367d14ca8223e851c8d6a98b26102db0cf48db5562c8351c4f5c2a673679d08fcebec36e307db52a2e7cdf56be78ac615d7045b519e945846ce6e92dd40f8
+  metadata.gz: 6a35107ab674882f57ad17d89ada77a3fcd34ead94553633774abae201308b6cd112860275920bfeafbb0b2546199b34940164b9007348f12c824c31fa95d06b
+  data.tar.gz: 86da00f80b987fef72f7662104661b3ad43598d80b2fa7c4c93417a3ddd29b42a559940dff20424a46b38b75494eb91f20f1767c3d64baf0e0a32ac1de6498b2

data/lib/activerecord_reindex/adapter.rb

@@ -1,21 +1,43 @@
 # frozen_string_literal: true
 # author: Vadim Shaveiko <@vshaveyko>
 # Abstract adapter class
-# New adapters should implement :call method that will perform reindexing of the given record
-# Example:
-#   def call(record)
-#     reindex_it(record)
-#   end
+# New adapters should implement :_single_reindex and :_mass_reindex methods that will perform reindexing of the given records
+#
 module ActiverecordReindex
   class Adapter
 
-    # check if record of this class can be reindexed
+    #
+    # check if record of this class can be reindexed ==
     # check if klass inherits from elasticsearch-model base class
-    # and have method required for reindexing
+    #
     def self._check_elasticsearch_connection(klass)
       klass < Elasticsearch::Model
     end
 
+    #
+    # updates index directly in elasticsearch through
+    # Elasticsearch::Model instance method
+    # if class not inherited from Elasticsearch::Model it skips since it cannot be reindexed
+    #
+    # ***nasty-stuff***
+    #   hooking into update_document has sudden side-effect
+    #   if associations defined two-way they will trigger reindex recursively and result in StackLevelTooDeep
+    #   hence to prevent this we're passing request_record to adapter
+    #   request record is record that initted reindex for current record as association
+    #   we will skip it in associations reindex to prevent recursive reindex and StackLevelTooDeep error
+    #
+    def self.call(request_record, record: nil, records: nil)
+      if record
+        return unless _check_elasticsearch_connection(record.class)
+
+        _single_reindex(request_record, record)
+      elsif records && record = records.first
+        return unless _check_elasticsearch_connection(record.class)
+
+        _mass_reindex(request_record, record.class.name, records)
+      end
+    end
+
   end
 
 end

data/lib/activerecord_reindex/association.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 # frozen_string_literal: true
 # author: Vadim Shaveiko <@vshaveyko>
-
 # Adds reindex option to associations
 # values accepted are true, :async. Default false.
 # If true it will add syncronous elasticsearch reindex callbacks on:
@@ -9,6 +8,8 @@
 # 2. record destroyed
 # 3. record index updated
 # if :async it will add async callbacks in same cases
+require_relative 'reindex_hook'
+
 module ActiveRecord
   module Associations
     module Builder
@@ -60,7 +61,11 @@ module ActiveRecord
           def add_destroy_reindex_callback(model, reflection, async:)
             return if [:destroy, :delete_all].include? reflection.options[:dependent]
 
-            model.after_commit on: :destroy, &callback(async, reflection)
+            destroy_callback = callback(async, reflection)
+
+            model.after_commit(on: :destroy) do
+              destroy_callback.call(self)
+            end
           end
 
           # add callback to reindex associations on update
@@ -70,12 +75,20 @@ module ActiveRecord
           def add_update_reindex_callback(model, reflection, async:)
             return if model < Elasticsearch::Model
 
-            model.after_commit on: :update, &callback(async, reflection)
+            # for why it is needed see reindex_hook.rb
+            model.include ActiverecordReindex::ReindexHook
+
+            update_callback = callback(async, reflection)
+
+            model.after_commit(on: :update) do
+              next unless changed_index_relevant_attributes?
+              update_callback.call(self)
+            end
           end
 
           # callback methods defined in ActiveRecord::Base monkeypatch
           def callback(async, reflection)
-            async ? -> { reindex_async(reflection) } : -> { reindex_sync(reflection) }
+            async ? ->(record) { record.reindex_async(reflection) } : ->(record) { record.reindex_sync(reflection) }
           end
 
         end

data/lib/activerecord_reindex/async_adapter.rb

@@ -1,40 +1,78 @@
 # frozen_string_literal: true
 # author: Vadim Shaveiko <@vshaveyko>
-
 require_relative 'adapter'
 module ActiverecordReindex
+  #
   # Asyncronouse reindex adapter
   # uses Jobs for reindexing records asyncronously
   # Using ActiveJob as dependency bcs activerecord is required for this so
   # in most cases it would be used with rails hence with ActiveJob
   # later can think about adding support for differnt job adapters
+  #
   class AsyncAdapter < Adapter
 
+    #
     # Job wrapper. Queues elastic_index queue for each reindex
+    #
     class UpdateJob < ::ActiveJob::Base
 
-      # TODO: make queue name configurable
-      queue_as :elastic_index
+      queue_as ActiverecordReindex.config.index_queue
 
       def perform(klass, id, request_record_klass, request_record_id)
         klass = klass.constantize
+
         request_record = request_record_klass.constantize.find(request_record_id)
+
         klass.find(id).__elasticsearch__.update_document(request_record: request_record)
       end
 
     end
 
+    class MassUpdateJob < ::ActiveJob::Base
+
+      queue_as ActiverecordReindex.config.index_queue
+
+      def perform(klass, ids, request_record_klass, request_record_id)
+        klass = klass.constantize
+
+        request_record = request_record_klass.constantize.find(request_record_id)
+
+        klass.find(ids).each do |record|
+          record.__elasticsearch__.update_document(request_record: request_record)
+        end
+      end
+
+    end
+
     class << self
 
-      # ***nasty-stuff***
-      #   hooking into update_document has sudden side-effect
-      #   if associations defined two-way they will trigger reindex recursively and result in StackLevelTooDeep
-      #   hence to prevent this we're passing request_record to adapter
-      #   request record is record that initted reindex for current record as association
-      #   we will skip it in associations reindex to prevent recursive reindex and StackLevelTooDeep error
-      def call(record, request_record)
-        return unless _check_elasticsearch_connection(record.class)
-        UpdateJob.perform_later(record.class.to_s, record.id, request_record.class.to_s, request_record.id)
+      private
+
+      #
+      # UpdateJob is default for this
+      # uses configured job class otherwise
+      #
+      def _single_reindex(request_record, record)
+        ActiverecordReindex.config.index_class
+          .perform_later(record.class.name,
+                         record.id,
+                         request_record.class.name,
+                         request_record.id)
+      end
+
+      #
+      # MassUpdateJob is default for this
+      # uses configured job class otherwise
+      #
+      # used for saving time on creating jobs in realtime
+      # create one job that will reindex all records internally
+      #
+      def _mass_reindex(request_record, class_name, records)
+        ActiverecordReindex.config.mass_index_class
+          .perform_later(class_name,
+                         records.ids,
+                         request_record.class.name,
+                         request_record.id)
       end
 
     end

data/lib/activerecord_reindex/base.rb

@@ -14,7 +14,8 @@ module ActiveRecord
       super
       class << child
 
-        attr_accessor :reindexer, :async_adapter, :sync_adapter, :sync_reindexable_reflections, :async_reindexable_reflections
+        attr_accessor :reindexer, :async_adapter, :sync_adapter, :sync_reindexable_reflections,
+                      :async_reindexable_reflections, :reindex_attr_blacklist, :reindex_attr_whitelist
 
       end
 
@@ -25,7 +26,12 @@ module ActiveRecord
       child.reindexer = ActiverecordReindex::Reindexer.new
       # TODO: provide config for changing adapters
       # For now can set adapter through writers inside class
-      child.async_adapter = ActiverecordReindex::AsyncAdapter
+      if ActiverecordReindex.config.async_reindex_only_in_production? && !Rails.env.production?
+        child.async_adapter = ActiverecordReindex::SyncAdapter
+      else
+        child.async_adapter = ActiverecordReindex::AsyncAdapter
+      end
+
       child.sync_adapter = ActiverecordReindex::SyncAdapter
     end
 
@@ -45,5 +51,26 @@ module ActiveRecord
           .call(self, reflection: reflection, skip_record: skip_record)
     end
 
+    def changed_index_relevant_attributes?
+      return true unless self.class.reindex_attr_blacklist || self.class.reindex_attr_whitelist
+      changed = changed_attributes.keys
+      wl = self.class.reindex_attr_whitelist&.map(&:to_sym)
+      bl = self.class.reindex_attr_blacklist&.map(&:to_sym)
+
+      if wl
+        whitelisted = wl & changed
+      else
+        whitelisted = changed
+      end
+
+      if bl
+        blacklisted = changed - bl
+      else
+        blacklisted = []
+      end
+
+      !(whitelisted - blacklisted).empty?
+    end
+
   end
 end

data/lib/activerecord_reindex/reflection_reindex.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+# author: Vadim Shaveiko <@vshaveyko>
+# helpers for reindexing all required reflections on record's class
+module ActiverecordReindex
+  module ReflectionReindex
+
+    def update_document_hook(request_record)
+      return unless _active_record_model?(self.class)
+      _reindex_reflections(self.class, request_record)
+    end
+
+    private
+
+    def _active_record_model?(klass)
+      klass < ActiveRecord::Base
+    end
+
+    def _reindex_reflections(klass, request_record)
+      klass.sync_reindexable_reflections.each do |reflection|
+        target.reindex_sync(reflection, skip_record: request_record)
+      end
+
+      klass.async_reindexable_reflections.each do |reflection|
+        target.reindex_async(reflection, skip_record: request_record)
+      end
+    end
+
+  end
+end

data/lib/activerecord_reindex/reindex_hook.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+# author: Vadim Shaveiko <@vshaveyko>
+module ActiverecordReindex
+  # Since recursive reindexing can go through models that does not
+  # exist in elastic and not inherited from Elsaticsearch::Model
+  # they will not have __elasticsearch__.update_document method
+  # those will stop recursive reindexing chain in its way
+  # To prevent this we emulate __elasticsearch__.update_document on them
+  # and continue reindex chain as is, without reindexing this records
+  module ReindexHook
+
+    def __elasticsearch__
+      @__elasticsearch__ ||= ActiverecordReindex::ElasticsearchProxy.new(self.class)
+    end
+
+  end
+
+  # Proxy to imitate missing __elasticsearch__ on models without
+  # Elasticsearch::Model included
+  require 'activerecord_reindex/reflection_reindex'
+  class ElasticsearchProxy
+
+    include ActiverecordReindex::ReflectionReindex
+
+    def initialize(klass)
+      @klass = klass
+    end
+
+    def update_document(*, request_record: nil)
+      # defined in ActiverecordReindex::ReflectionReindex
+      update_document_hook(request_record)
+    end
+
+  end
+
+end

data/lib/activerecord_reindex/reindexer.rb

@@ -3,61 +3,69 @@
 module ActiverecordReindex
   class Reindexer
 
+    #
     # chain strategy before actual executing
     # strategy can be either sync or async
     # corresponding to type of reindexing
     # additional strategies can be defined and specified by user
+    #
     def with_strategy(strategy)
       @strategy = strategy
       self
     end
 
+    #
     # reindex records associated with given record on given association
     # if association is collection(has_many, has_many_through, has_and_belongs_to_many)
     #   get all associated recrods and reindex them
     # else
     #   reindex given record associted one
+    #
     def call(record, reflection:, skip_record:)
       if reflection.collection?
         _reindex_collection(reflection, record, skip_record)
       else
-        associated_record = record.public_send(reflection.name)
-        return if associated_record == skip_record
-        _update_index(associated_record, record)
+        _reindex_single(reflection, record, skip_record)
       end
     end
 
     private
 
-    # TODO: add bulk reindex if need performance
+    #
     # raise if strategy was not specified or doesn't respond to call which is required for strategy
-    # pass record to strategy and execute reindex
-    # clear strategy to not mess up future reindexing
-    def _update_index(associated_record, record)
-      _check_strategy
-
-      @strategy.call(associated_record, record)
-
-      _clear_strategy
-    end
-
+    #
     def _check_strategy
       raise ArgumentError, 'No strategy specified.' unless @strategy
       raise ArgumentError, "Strategy specified incorrect. Check if #{@strategy} responds to :call." unless @strategy.respond_to? :call
     end
 
+    # clear strategy to not mess up future reindexing
     def _clear_strategy
       @strategy = nil
     end
 
+    #
+    # TODO: got clearing collection bug, write test for it
+    #
+    def _reindex_single(reflection, record, skip_record)
+      _check_strategy
+
+      associated_record = record.public_send(reflection.name)
+      return if associated_record == skip_record
+      @strategy.call(record, record: associated_record)
+
+      _clear_strategy
+    end
+
     def _reindex_collection(reflection, record, skip_record)
-      collection = record.public_send(reflection.name)
+      _check_strategy
 
+      collection = record.public_send(reflection.name)
       collection -= [skip_record] if reflection.klass == skip_record.class
 
-      collection.each do |associated_record|
-        _update_index(associated_record, record)
-      end
+      @strategy.call(record, records: collection)
+
+      _clear_strategy
     end
 
   end

data/lib/activerecord_reindex/sync_adapter.rb

@@ -7,19 +7,22 @@ module ActiverecordReindex
 
     class << self
 
-      # updates index directly in elasticsearch through
-      # Elasticsearch::Model instance method
-      # if class not inherited from Elasticsearch::Model it skips since it cannot be reindexing
-      # TODO: show error\warning about trying to reindex record that is not connection to elastic
-      # ***nasty-stuff***
-      #   hooking into update_document has sudden side-effect
-      #   if associations defined two-way they will trigger reindex recursively and result in StackLevelTooDeep
-      #   hence to prevent this we're passing request_record to adapter
-      #   request record is record that initted reindex for current record as association
-      #   we will skip it in associations reindex to prevent recursive reindex and StackLevelTooDeep error
-      def call(record, request_record)
-        return unless _check_elasticsearch_connection(record.class)
+      private
+
+      def _single_reindex(request_record, record)
+        _update_index_on_record(record, request_record)
+      end
+
+      def _mass_reindex(request_record, _class_name, records)
+        records.each do |record|
+          _update_index_on_record(record, request_record)
+        end
+      end
+
+      def _update_index_on_record(record, request_record)
         record.__elasticsearch__.update_document(request_record: request_record)
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        record.__elasticsearch__.index_document
       end
 
     end

data/lib/activerecord_reindex/update_document_monkey_patch.rb

@@ -1,36 +1,27 @@
 # frozen_string_literal: true
 # author: Vadim Shaveiko <@vshaveyko>
+require 'activerecord_reindex/reflection_reindex'
+
 module Elasticsearch
   module Model
     module Indexing
       module InstanceMethods
 
+        include ActiverecordReindex::ReflectionReindex
+
         alias original_update_document update_document
 
+        #
         # monkey patch update_document method from elasticsearch gem
         # use +super+ and hook on reindex to reindex associations
+        #
         # for why request_record needed here and what it is see sync_adapter.rb
+        #
         def update_document(*args, request_record: nil)
-          if _active_record_model?(self.class)
-            _reindex_reflections(self.class, request_record)
-          end
-          original_update_document(*args)
-        end
-
-        private
+          # defined in ActiverecordReindex::ReflectionReindex
+          update_document_hook(request_record)
 
-        def _active_record_model?(klass)
-          klass < ActiveRecord::Base
-        end
-
-        def _reindex_reflections(klass, request_record)
-          klass.sync_reindexable_reflections.each do |reflection|
-            target.reindex_sync(reflection, skip_record: request_record)
-          end
-
-          klass.async_reindexable_reflections.each do |reflection|
-            target.reindex_async(reflection, skip_record: request_record)
-          end
+          original_update_document(*args)
         end
 
       end

data/lib/activerecord_reindex/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # author: Vadim Shaveiko <@vshaveyko>
 module ActiverecordReindex
-  VERSION = '0.1.2'
+  VERSION = '0.2.1'
 end

data/lib/activerecord_reindex.rb

@@ -2,6 +2,49 @@
 # author: Vadim Shaveiko <@vshaveyko>
 require 'activerecord_reindex/version'
 
+module ActiverecordReindex
+
+  class Config
+
+    attr_accessor :index_queue, :index_class, :mass_index_class, :async_reindex_only_in_production
+
+    def initialize
+      @index_queue = :elastic_index
+      @async_reindex_only_in_production = false
+    end
+
+    def async_reindex_only_in_production?
+      @async_reindex_only_in_production
+    end
+
+    def index_class
+      @index_class || ActiverecordReindex::AsyncAdapter::UpdateJob
+    end
+
+    def mass_index_class
+      @mass_index_class || ActiverecordReindex::AsyncAdapter::MassUpdateJob
+    end
+
+  end
+
+  class << self
+
+    def configure
+      yield config
+    end
+
+    def config
+      @_configuration ||= Config.new
+    end
+
+    def reset_configuration
+      @_configuration = nil
+    end
+
+  end
+
+end
+
 # monkey patch active record associations
 require 'active_record'
 require 'active_job'
@@ -13,6 +56,3 @@ require 'activerecord_reindex/association_reflection'
 # monkey patch elasticsearch/model
 require 'elasticsearch/model'
 require 'activerecord_reindex/update_document_monkey_patch'
-
-module ActiverecordReindex
-end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: activerecord_reindex
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.1
 platform: ruby
 authors:
 - vs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-20 00:00:00.000000000 Z
+date: 2017-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -24,20 +24,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: activejob
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: elasticsearch-model
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +52,8 @@ files:
 - lib/activerecord_reindex/association_reflection.rb
 - lib/activerecord_reindex/async_adapter.rb
 - lib/activerecord_reindex/base.rb
+- lib/activerecord_reindex/reflection_reindex.rb
+- lib/activerecord_reindex/reindex_hook.rb
 - lib/activerecord_reindex/reindexer.rb
 - lib/activerecord_reindex/sync_adapter.rb
 - lib/activerecord_reindex/update_document_monkey_patch.rb
@@ -90,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Add Elasticsearch reindex option to ActiveRecord associations