elasticsearch-model-transactional_callbacks 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: dd0a0e00471ca007bf1fe2da314b50cabc442a6caade0732ae83132d880d69f2
+  data.tar.gz: 9a259c08c9a516118694b8a9b38433051b4d6ca4f7f08aabe1a2a243a18641a3
+SHA512:
+  metadata.gz: 9df1626b1e0b86eb567177fb8a19d2475e4ae18038a1050e15b4e85023f86a68ef73cf2738e15d42958de739a52186783a6f0ae1778514d32dca65d090435905
+  data.tar.gz: 7c2cd5646a7c8b636702bd7484e702a46cf9f086effb4bb0fc7b4d77416c5c8feec07d3fff7e450a48baa05eac5ef93ba482b2e3fa4363b1104efab4fbfde973

data/README.md

@@ -0,0 +1,104 @@
+# Elasticsearch::Model::TransactionalCallbacks
+
+The [`elasticsearch-model`](https://github.com/elastic/elasticsearch-rails/tree/master/elasticsearch-model)
+works great in simplifying the integration of Ruby classes ("models") with the
+[Elasticsearch](http://www.elasticsearch.org/) search and analytics engine.
+But, it come short with support for updating the indexed documents asynchronously.
+
+Built-in support for updating the indexed documents comes in the form of `Elasticsearch::Model::Callbacks`
+which will updates each related documents individually inside the same thread where the changes were made.
+Depending on the size of your application, and the size of the changes itself, triggering N number of
+indexing request to Elasticsearch could amount to nothing, or it could slow down the request-response
+cycle considerably and render it unusable.
+
+This gem aim to solve this by providing a way to update the index asynchronously via `ActiveJob`.
+
+## Usage
+
+The minimum is to include `Elasticsearch::Model::TransactionalCallbacks` into any model
+which could benefit from asynchronous indexing, e.g.
+
+```ruby
+class User < ApplicationRecord
+  include Elasticsearch::Model
+  include Elasticsearch::Model::TransactionalCallbacks
+
+  index_name 'users'
+  document_type 'user'
+
+  mappings do
+    # indexes for users
+  end
+end
+```
+
+But, this will end up trading n+1 on updating index with n+1 on database queries in case your `#as_indexed_json`
+pulls data from associated models, e.g.
+
+```ruby
+class Post < ApplicationRecord
+  include Elasticsearch::Model
+  include Elasticsearch::Model::TransactionalCallbacks
+
+  has_many :taggings, as: :taggable
+  has_many :tags, through: :taggings
+
+  index_name 'posts'
+  document_type 'post'
+
+  mappings dynamic: false do
+    indexes :subject, type: 'text', analyzer: 'english'
+    indexes :tags, type: 'keyword'
+  end
+
+  def as_indexed_json(_options = {})
+    {
+      subject: subject,
+      tags: tags.map(&:key) # FIXME: this triggers n+1 queries
+    }
+  end
+end
+```
+
+to get around this, you can define a `scope` called `preload_for_index` like so:
+
+```ruby
+class Post < ApplicationRecord
+  # ...snip...
+  scope :preload_for_import, -> { preload(:tags) }
+  # ...snip...
+end
+```
+
+and it will be automatically called by the library.
+
+## Compatibility
+This library is compatible and tested with Elasticsearch 5. Some works might be needed to make it works with Elasticsearch 6.
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'elasticsearch-model-transactional_callbacks'
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+```bash
+$ gem install elasticsearch-model-transactional_callbacks
+```
+
+## Contributing
+Any and all kind of help are welcomed! Especially interested in:
+
+- sample use cases which are not yet supported,
+- compatibility with elasticsearch 6.0
+
+feel free to file an issue/PR with sample mapping!
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Elasticsearch::Model::TransactionalCallbacks'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test

data/lib/elasticsearch/model/transactional_callbacks.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'elasticsearch/model'
+require_relative './transactional_callbacks/railtie'
+require_relative './transactional_callbacks/manager'
+
+module Elasticsearch
+  module Model
+    # Extend ElasticSearch::Model with transactional callbacks for asynchronous indexing
+    module TransactionalCallbacks
+      extend ActiveSupport::Concern
+
+      included do
+        after_commit :batch_index_document, on: :create
+        after_commit :batch_update_document, on: :update
+        after_commit :batch_delete_document, on: :destroy
+      end
+
+      def batch_index_document
+        Manager.queue.push(:index, self)
+      end
+
+      def batch_update_document
+        Manager.queue.push(:update, self)
+      end
+
+      def batch_delete_document
+        Manager.queue.push(:delete, self)
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/transactional_callbacks/bulk_indexing_job.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Elasticsearch
+  module Model
+    module TransactionalCallbacks
+      ##
+      # Background job which handles the request to index/update/delete documents asynchronously
+      #
+      #   Elasticsearch::Model::TransactionalCallbacks::BulkIndexingJob.perform_later(
+      #     document_type: {
+      #       index: [{ _id: document.id }],
+      #       update: [{ _id: document.id }],
+      #       delete: [{ _id: document.id }],
+      #     }
+      #   )
+      #
+      class BulkIndexingJob < ::ActiveJob::Base
+        queue_as :default
+
+        def perform(indexables)
+          indexables.each do |document_type, action_map|
+            klass = document_type.to_s.camelcase.constantize
+            body = transform_batches(klass, action_map)
+
+            response = bulk_index klass, body
+
+            ::Rails.logger.error "[ELASTICSEARCH] Bulk request failed: #{response['items']}" if response&.dig('errors')
+          end
+        end
+
+        private
+
+          def transform_batches(klass, action_map)
+            reverse_map = build_reverse_map(action_map)
+            resources = klass.where id: reverse_map.keys
+
+            preload(resources).find_each.map { |resource|
+              action, option = reverse_map[resource.id]
+
+              send "transform_#{action}", resource, option
+            } + action_map.fetch(:delete, []).map { |option|
+              transform_delete(option)
+            }
+          end
+
+          def build_reverse_map(action_map)
+            action_map.each_with_object({}) { |map, memo|
+              action, options = map
+
+              next if action == :delete
+
+              options.each do |option|
+                memo[option[:_id]] = [action, option]
+              end
+            }
+          end
+
+          def preload(resources)
+            resources.respond_to?(:preload_for_import) ? resources.preload_for_import : resources
+          end
+
+          def transform_index(resource, option)
+            { index: option.merge(data: to_indexed_json(resource)) }
+          end
+          # elasticsearch do support update operation in their bulk API,
+          # but it will fail in case the update is done to missing documents,
+          # while index work for both new and existing document
+          #
+          # because of this, we choose to use index for update to avoid issue with race condition
+          # where a document is updated immediately after it is created,
+          # on which elasticsearch might not be aware of the document yet
+          alias transform_update transform_index
+
+          def transform_delete(option)
+            { delete: option }
+          end
+
+          def to_indexed_json(resource)
+            return resource.as_indexed_json if resource.respond_to?(:as_indexed_json)
+
+            resource.__elasticsearch__.as_indexed_json
+          end
+
+          def bulk_index(klass, body)
+            return if body.blank?
+
+            klass.__elasticsearch__.client.bulk(
+              index: klass.index_name,
+              type: klass.document_type,
+              body: body
+            )
+          end
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/transactional_callbacks/manager.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative './bulk_indexing_job'
+require_relative './queue'
+
+module Elasticsearch
+  module Model
+    module TransactionalCallbacks
+      module Manager # :nodoc:
+        class << self
+          def capture
+            counter_stack.push(:lol)
+
+            yield.tap do
+              register_job if counter_stack.length == 1
+            end
+          ensure
+            counter_stack.pop
+          end
+
+          def queue
+            Thread.current[:elasticsearch_transactional_queue] ||= Queue.new
+          end
+
+          private
+
+            def counter_stack
+              Thread.current[:elasticsearch_transactional_counter] ||= []
+            end
+
+            def register_job
+              return if queue.empty?
+
+              BulkIndexingJob.perform_later(queue.to_h)
+
+              queue.reset!
+            end
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/transactional_callbacks/queue.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Elasticsearch
+  module Model
+    module TransactionalCallbacks
+      ##
+      #  Responsible for storing a queue of resources to be indexed/updated/deleted from elasticsearch
+      #
+      class Queue
+        attr_reader :state
+
+        delegate :empty?, to: :state
+
+        def initialize
+          reset!
+        end
+
+        def reset!
+          @state = {}
+        end
+
+        def push(action, resource)
+          do_push action, resource, _id: resource.id, _parent: parent_id(resource)
+        end
+
+        def push_all(action, relation)
+          return unless ::Elasticsearch::Model::TransactionalCallbacks.in?(relation.included_modules)
+
+          pluck_ids(relation) do |ids|
+            do_push action, relation, ids
+          end
+        end
+
+        def to_h
+          state
+        end
+
+        private
+
+          def do_push(action, resource_or_relation, options)
+            type = document_type(resource_or_relation)
+
+            prepare_state_for(type)
+
+            state[type][action] << options.compact
+            state[type][action].uniq!
+          end
+
+          def prepare_state_for(type)
+            state[type] ||= {
+              index: [],
+              update: [],
+              delete: []
+            }
+          end
+
+          def resource_class(resource_or_relation)
+            resource_or_relation.respond_to?(:document_type) ? resource_or_relation : resource_or_relation.class
+          end
+
+          def document_type(resource_or_relation)
+            resource_class(resource_or_relation).document_type.to_sym
+          end
+
+          def child?(resource_or_relation)
+            parent_type(resource_or_relation).present?
+          end
+
+          def parent_type(resource_or_relation)
+            resource_class(resource_or_relation).mapping.options.dig(:_parent, :type)
+          end
+
+          def parent_id(resource)
+            return unless child?(resource)
+
+            resource.public_send "#{parent_type resource}_id"
+          end
+
+          def pluck_ids(relation)
+            if child?(relation)
+              relation.pluck(:id, "#{parent_type relation}_id").map do |ids|
+                yield _id: ids[0], _parent: ids[1]
+              end
+            else
+              relation.pluck(:id).map do |id|
+                yield _id: id
+              end
+            end
+          end
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/transactional_callbacks/railtie.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative './relation'
+require_relative './transaction'
+
+module Elasticsearch
+  module Model
+    module TransactionalCallbacks
+      class Railtie < ::Rails::Railtie # :nodoc:
+        initializer 'elasticsearch.model.transactional_callbacks.initialize' do
+          ActiveSupport.on_load(:active_record) do
+            ActiveRecord::Relation.prepend Elasticsearch::Model::TransactionalCallbacks::Relation
+
+            extend Elasticsearch::Model::TransactionalCallbacks::Transaction
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/transactional_callbacks/relation.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative './manager'
+
+module Elasticsearch
+  module Model
+    module TransactionalCallbacks
+      ##
+      # Override .update_all and .delete_all of ActiveRecord::Relation to batch update/delete
+      # the index if the resources in question have a coresponding elasticsearch index
+      #
+      # This module are automatically included into ActiveRecord::Relation inside of railtie
+      #
+      module Relation
+        def update_all(*arguments)
+          Manager.capture do
+            Manager.queue.push_all(:update, self)
+
+            super(*arguments)
+          end
+        end
+
+        def delete_all
+          Manager.capture do
+            Manager.queue.push_all(:delete, self)
+
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/transactional_callbacks/transaction.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative './manager'
+
+module Elasticsearch
+  module Model
+    module TransactionalCallbacks
+      ##
+      # Override ActiveRecord::Base.transaction to allow Manager to listen for
+      # any indexing request from active record after_commit callback
+      #
+      # This module are automatically included into ActiveRecord::Base inside of railtie
+      #
+      module Transaction
+        def transaction(*args, &block)
+          Manager.capture do
+            super(*args, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/transactional_callbacks/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Elasticsearch
+  module Model
+    module TransactionalCallbacks
+      VERSION = '0.1.0'
+    end
+  end
+end

data/lib/tasks/elasticsearch/model/transactional_callbacks_tasks.rake

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# desc "Explaining what the task does"
+# task :elasticsearch_model_transactional_callbacks do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: elasticsearch-model-transactional_callbacks
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ignatius Reza
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-02-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: elasticsearch-model
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: minitest-ci
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-stub_any_instance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.6
+description: Reduce load from your application server by offloading indexing into
+  background jobs
+email:
+- lyoneil.de.sire@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/elasticsearch/model/transactional_callbacks.rb
+- lib/elasticsearch/model/transactional_callbacks/bulk_indexing_job.rb
+- lib/elasticsearch/model/transactional_callbacks/manager.rb
+- lib/elasticsearch/model/transactional_callbacks/queue.rb
+- lib/elasticsearch/model/transactional_callbacks/railtie.rb
+- lib/elasticsearch/model/transactional_callbacks/relation.rb
+- lib/elasticsearch/model/transactional_callbacks/transaction.rb
+- lib/elasticsearch/model/transactional_callbacks/version.rb
+- lib/tasks/elasticsearch/model/transactional_callbacks_tasks.rake
+homepage: https://github.com/ignatiusreza/elasticsearch-model-transactional_callbacks
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.2
+signing_key: 
+specification_version: 4
+summary: Extend ElasticSearch::Model with transactional callbacks for asynchronous
+  indexing
+test_files: []