esse-async_indexing 0.0.2

data/lib/esse/async_indexing/actions/import_batch_id.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing::Actions
+  class ImportBatchId
+    def self.call(index_class_name, repo_name, batch_id, options = {})
+      _index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)
+      queue = Esse::RedisStorage::Queue.for(repo: repo_class)
+
+      kwargs = Esse::HashUtils.deep_transform_keys(options, &:to_sym)
+      kwargs[:context] ||= {}
+      result = 0
+      ids_from_batch = []
+      queue.fetch(batch_id) do |ids|
+        ids_from_batch = ids
+        kwargs[:context][:id] = ids
+        result = repo_class.import(**kwargs)
+      end
+      [result, ids_from_batch]
+    end
+  end
+end

data/lib/esse/async_indexing/actions/index_document.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing::Actions
+  class IndexDocument
+    DOC_ARGS = %i[lazy_attributes]
+
+    def self.call(index_class_name, repo_name, document_id, options = {})
+      index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)
+      bulk_opts = Esse::HashUtils.deep_transform_keys(options, &:to_sym)
+      bulk_opts.delete_if { |k, _| DOC_ARGS.include?(k) }
+      find_opts = options.slice(*DOC_ARGS)
+
+      doc = repo_class.documents(**find_opts, id: document_id).first
+      return :not_found unless doc
+
+      index_class.index(doc, **bulk_opts)
+      :indexed
+    end
+  end
+end

data/lib/esse/async_indexing/actions/update_document.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing::Actions
+  class UpdateDocument
+    DOC_ARGS = %i[lazy_attributes]
+
+    def self.call(index_class_name, repo_name, document_id, options = {})
+      index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)
+      bulk_opts = Esse::HashUtils.deep_transform_keys(options, &:to_sym)
+      bulk_opts.delete_if { |k, _| DOC_ARGS.include?(k) }
+      find_opts = options.slice(*DOC_ARGS)
+
+      doc = repo_class.documents(**find_opts, id: document_id).first
+      return :not_found unless doc
+
+      index_class.update(doc, **bulk_opts)
+      :indexed
+    end
+  end
+end

data/lib/esse/async_indexing/actions/update_lazy_document_attribute.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing::Actions
+  class UpdateLazyDocumentAttribute
+    def self.call(index_class_name, repo_name, attr_name, ids, options = {})
+      _index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)
+      kwargs = Esse::HashUtils.deep_transform_keys(options, &:to_sym)
+
+      attr_name = repo_class.lazy_document_attributes.keys.find { |key| key.to_s == attr_name.to_s }
+      repo_class.update_documents_attribute(attr_name, ids, **kwargs)
+      ids
+    end
+  end
+end

data/lib/esse/async_indexing/actions/upsert_document.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing::Actions
+  class UpsertDocument
+    DOC_ARGS = %i[lazy_attributes]
+    # OPERATIONS = %w[index update delete]
+
+    def self.call(index_class_name, repo_name, document_id, operation = "index", options = {})
+      case operation
+      when "delete"
+        DeleteDocument.call(index_class_name, repo_name, document_id, options)
+      when "update"
+        result = UpdateDocument.call(index_class_name, repo_name, document_id, options)
+        return result if result != :not_found
+        DeleteDocument.call(index_class_name, repo_name, document_id, options)
+      when "index"
+        result = IndexDocument.call(index_class_name, repo_name, document_id, options)
+        return result if result != :not_found
+        DeleteDocument.call(index_class_name, repo_name, document_id, options)
+      else
+        raise ArgumentError, "operation must be one of index, update, delete"
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/actions.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Esse
+  module AsyncIndexing
+    module Actions
+    end
+  end
+end
+
+require_relative "actions/coerce_index_repository"
+require_relative "actions/batch_import_all"
+require_relative "actions/batch_import"
+require_relative "actions/batch_delete"
+require_relative "actions/batch_update"
+require_relative "actions/bulk_update_lazy_document_attribute"
+require_relative "actions/delete_document"
+require_relative "actions/import_batch_id"
+require_relative "actions/index_document"
+require_relative "actions/update_document"
+require_relative "actions/upsert_document"
+require_relative "actions/update_lazy_document_attribute"

data/lib/esse/async_indexing/active_record.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Esse
+  module AsyncIndexing
+    ESSE_ACTIVE_RECORD_MINIMAL_VERSION = ::Gem::Version.new("0.3.5")
+
+    def self.__validate_active_record_version!
+      unless defined?(Esse::ActiveRecord::Callbacks)
+        raise <<~MSG
+          To use async indexing ActiveRecord callbacks you need to install and require the `esse-active_record` gem.
+
+          Add this line to your application's Gemfile:
+          gem 'esse-active_record', '~> 0.3.5'
+        MSG
+      end
+      require "esse/active_record/version"
+      if ::Gem::Version.new(Esse::ActiveRecord::VERSION) < ESSE_ACTIVE_RECORD_MINIMAL_VERSION
+        raise <<~MSG
+          The esse-active_record gem version #{ESSE_ACTIVE_RECORD_MINIMAL_VERSION} or higher is required. Please update the gem to the latest version.
+        MSG
+      end
+    end
+
+    def self.__register_active_record_callbacks!
+      callbacks = Esse::ActiveRecord::Callbacks
+      unless callbacks.registered?(:async_indexing, :create)
+        callbacks.register_callback(:async_indexing, :create, Esse::AsyncIndexing::ActiveRecordCallbacks::OnCreate)
+      end
+      unless callbacks.registered?(:async_indexing, :update)
+        callbacks.register_callback(:async_indexing, :update, Esse::AsyncIndexing::ActiveRecordCallbacks::OnUpdate)
+      end
+      unless callbacks.registered?(:async_indexing, :destroy)
+        callbacks.register_callback(:async_indexing, :destroy, Esse::AsyncIndexing::ActiveRecordCallbacks::OnDestroy)
+      end
+      unless callbacks.registered?(:async_update_lazy_attribute, :create)
+        callbacks.register_callback(:async_update_lazy_attribute, :create, Esse::AsyncIndexing::ActiveRecordCallbacks::LazyUpdateAttribute)
+      end
+      unless callbacks.registered?(:async_update_lazy_attribute, :update)
+        callbacks.register_callback(:async_update_lazy_attribute, :update, Esse::AsyncIndexing::ActiveRecordCallbacks::LazyUpdateAttribute)
+      end
+      unless callbacks.registered?(:async_update_lazy_attribute, :destroy)
+        callbacks.register_callback(:async_update_lazy_attribute, :destroy, Esse::AsyncIndexing::ActiveRecordCallbacks::LazyUpdateAttribute)
+      end
+    end
+
+    # Usage:
+    # class User < ApplicationRecord
+    #  include Esse::AsyncIndexing::ActiveRecord::Model
+    module ActiveRecord
+      module Model
+        def self.included(base)
+          unless base.respond_to?(:esse_index)
+            base.include(Esse::ActiveRecord::Model)
+          end
+          base.extend(ClassMethods)
+        end
+
+        module ClassMethods
+          # Define callback on create/update/delete to push a job to the async indexing the document.
+          #
+          # @param [String] index_repo_name The path of index and repository name.
+          #   For example a index with a single repository named `users` is `users`. And a index with
+          #   multiple repositories named `animals` and `dog` as the repository name is `animals/dog`.
+          #   For namespace, use `/` as the separator.
+          # @raise [ArgumentError] when the repo and events are already registered
+          # @raise [ArgumentError] when the specified index have multiple repos
+          def async_index_callback(index_repo_name, on: %i[create update destroy], with: nil, **options, &block)
+            options[:service_name] = ::Esse::AsyncIndexing.service_name(options[:service_name])
+            Array(on).each do |event|
+              esse_callback(index_repo_name, :async_indexing, on: event, with: with, **options, &block)
+            end
+          end
+
+          # Define callback on create/update/delete to push a job to the async update a lazy attribute.
+          #
+          # @param [String] index_repo_name The path of index and repository name.
+          #   For example a index with a single repository named `users` is `users`. And a index with
+          #   multiple repositories named `animals` and `dog` as the repository name is `animals/dog`.
+          #   For namespace, use `/` as the separator.
+          # @param [String, Symbol] attribute_name The name of the lazy attribute to update.
+          # @raise [ArgumentError] when the repo and events are already registered
+          # @raise [ArgumentError] when the specified index have multiple repos
+          def async_update_lazy_attribute_callback(index_repo_name, attribute_name, on: %i[create update destroy], **options, &block)
+            options[:attribute_name] = attribute_name
+            options[:service_name] = ::Esse::AsyncIndexing.service_name(options[:service_name])
+            esse_callback(index_repo_name, :async_update_lazy_attribute, identifier_suffix: attribute_name.to_sym, on: on, **options, &block)
+          end
+        end
+      end
+    end
+  end
+end
+
+Esse::AsyncIndexing.__validate_active_record_version!
+
+require_relative "active_record_callbacks/callback"
+require_relative "active_record_callbacks/on_create"
+require_relative "active_record_callbacks/on_update"
+require_relative "active_record_callbacks/on_destroy"
+require_relative "active_record_callbacks/lazy_update_attribute"
+
+Esse::AsyncIndexing.__register_active_record_callbacks!

data/lib/esse/async_indexing/active_record_callbacks/callback.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  module ActiveRecordCallbacks
+    class Callback < ::Esse::ActiveRecord::Callback
+      attr_reader :service_name
+
+      def initialize(service_name:, with: nil, **kwargs)
+        @service_name = service_name
+        @with = with
+        super(**kwargs)
+      end
+
+      protected
+
+      def resolve_document_id(model)
+        resolve_document_ids(model).first
+      end
+
+      def resolve_document_ids(model)
+        ::Esse::ArrayUtils.wrap(block_result || model).map do |record|
+          record.is_a?(::ActiveRecord::Base) ? record.id : record
+        end.compact
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/active_record_callbacks/lazy_update_attribute.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  module ActiveRecordCallbacks
+    class LazyUpdateAttribute < Callback
+      LAZY_ATTR_WORKER = "Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob"
+
+      attr_reader :attribute_name
+
+      def initialize(service_name:, attribute_name:, with: nil, **kwargs)
+        @attribute_name = attribute_name
+        super(service_name: service_name, **kwargs)
+      end
+
+      def call(model)
+        if (doc_ids = resolve_document_ids(model))
+          Esse::AsyncIndexing.worker(LAZY_ATTR_WORKER, service: service_name)
+            .with_args(repo.index.name, repo.repo_name, attribute_name.to_s, doc_ids, Esse::HashUtils.deep_transform_keys(options, &:to_s))
+            .push
+        end
+
+        true
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/active_record_callbacks/on_create.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  module ActiveRecordCallbacks
+    class OnCreate < Callback
+      def call(model)
+        if (doc_id = resolve_document_id(model))
+          repo.async_indexing_job_for(:index).call(**options, service: service_name, repo: repo, operation: :index, id: doc_id)
+        end
+
+        true
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/active_record_callbacks/on_destroy.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  module ActiveRecordCallbacks
+    class OnDestroy < Callback
+      def call(model)
+        if (doc_id = resolve_document_id(model))
+          repo.async_indexing_job_for(:delete).call(**options, service: service_name, repo: repo, operation: :delete, id: doc_id)
+        end
+
+        true
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/active_record_callbacks/on_update.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  module ActiveRecordCallbacks
+    class OnUpdate < Callback
+      def call(model)
+        doc_id = resolve_document_id(model)
+        return true unless doc_id
+
+        kwargs = {service: service_name, repo: repo, id: doc_id}
+        if with == :update
+          repo.async_indexing_job_for(:update).call(**options, **kwargs, operation: :update)
+        else
+          repo.async_indexing_job_for(:index).call(**options, **kwargs, operation: :index)
+        end
+
+        true
+      end
+
+      protected
+
+      def with
+        @with || :index
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/adapters/adapter.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  module Adapters
+    class Adapter
+      # Push the worker job to the service
+      # @param _worker [Esse::AsyncIndexing::Worker] An instance of background worker
+      # @abstract Child classes should override this method
+      def self.push(_worker)
+        raise NotImplemented
+      end
+
+      # Coerces the raw payload into an instance of Worker
+      # @param payload [Object] the object that should be coerced to a Worker
+      # @options options [Hash] list of options that will be passed along to the Worker instance
+      # @return [Esse::AsyncIndexing::Worker] and instance of Esse::AsyncIndexing::Worker
+      # @abstract Child classes should override this method
+      def self.coerce_to_worker(payload, **options)
+        raise NotImplemented
+      end
+
+      protected
+
+      def normalize_before_push
+        # noop
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/adapters/faktory.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  module Adapters
+    # This is a Faktory adapter that converts Esse::AsyncIndexing::Worker object into a faktory readable format
+    # and then push the jobs into the service.
+    class Faktory < Adapter
+      attr_reader :worker, :queue
+
+      def initialize(worker)
+        @worker = worker
+        @queue = worker.options.fetch(:queue, "default")
+
+        @payload = worker.payload.merge(
+          "jobtype" => worker.worker_class,
+          "queue" => @queue,
+          "retry" => parse_retry(worker.options[:retry])
+        )
+        @payload["created_at"] ||= Time.now.to_f
+      end
+
+      # Coerces the raw payload into an instance of Worker
+      # @param payload [Hash] The job as json from redis
+      # @options options [Hash] list of options that will be passed along to the Worker instance
+      # @return [Esse::AsyncIndexing::Worker] and instance of Esse::AsyncIndexing::Worker
+      def self.coerce_to_worker(payload, **options)
+        raise(Error, "invalid payload") unless payload.is_a?(Hash)
+        raise(Error, "invalid payload") unless payload["jobtype"].is_a?(String)
+
+        options[:retry] ||= payload["retry"] if payload.key?("retry")
+        options[:queue] ||= payload["queue"] if payload.key?("queue")
+
+        Esse::AsyncIndexing.worker(payload["jobtype"], **options, service: :faktory).tap do |worker|
+          worker.with_args(*Array(payload["args"])) if payload.key?("args")
+          worker.with_job_jid(payload["jid"]) if payload.key?("jid")
+          worker.created_at(payload["created_at"]) if payload.key?("created_at")
+          worker.enqueued_at(payload["enqueued_at"]) if payload.key?("enqueued_at")
+          worker.at(payload["at"]) if payload.key?("at")
+        end
+      end
+
+      # Initializes adapter and push job into the faktory service
+      #
+      # @param worker [Esse::AsyncIndexing::Worker] An instance of Esse::AsyncIndexing::Worker
+      # @return [Hash] Job payload
+      # @see push method for more details
+      def self.push(worker)
+        new(worker).push
+      end
+
+      # Push job to Faktory
+      #   * If job has the 'at' key. Then schedule it
+      #   * Otherwise enqueue for immediate execution
+      #
+      # @raise [Esse::AsyncIndexing::Error] raise and error when faktory dependency is not loaded
+      # @return [Hash] Payload that was sent to server
+      def push
+        unless Object.const_defined?(:Faktory)
+          raise Esse::AsyncIndexing::Error, <<~ERR
+            Faktory client for ruby is not loaded. You must install and require https://github.com/contribsys/faktory_worker_ruby.
+          ERR
+        end
+        normalize_before_push
+
+        pool = Thread.current[:faktory_via_pool] || ::Faktory.server_pool
+        ::Faktory.client_middleware.invoke(@payload, pool) do
+          pool.with do |c|
+            c.push(@payload)
+          end
+        end
+        @payload
+      end
+
+      protected
+
+      # Convert worker retry value acording to the Go struct datatype.
+      #
+      # * 25 is the default.
+      # * 0 means the job is completely ephemeral. No matter if it fails or succeeds, it will be discarded.
+      # * -1 means the job will go straight to the Dead set if it fails, no retries.
+      def parse_retry(value)
+        case value
+        when Numeric then value.to_i
+        when false then -1
+        else
+          25
+        end
+      end
+
+      def parse_time(value)
+        case value
+        when Numeric then Time.at(value).to_datetime.rfc3339(9)
+        when Time then value.to_datetime.rfc3339(9)
+        when DateTime then value.rfc3339(9)
+        end
+      end
+
+      def to_json(value)
+        MultiJson.dump(value, mode: :compat)
+      end
+
+      def normalize_before_push
+        @payload["enqueued_at"] ||= Time.now.to_f
+        {"created_at" => false, "enqueued_at" => false, "at" => true}.each do |field, past_remove|
+          # Optimization to enqueue something now that is scheduled to go out now or in the past
+          if (time = @payload.delete(field)) &&
+              (!past_remove || (past_remove && time > Time.now.to_f))
+            @payload[field] = parse_time(time)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/adapters/sidekiq.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  module Adapters
+    # This is a Sidekiq adapter that converts Esse::AsyncIndexing::Worker object into a sidekiq readable format
+    # and then push the jobs into the service.
+    class Sidekiq < Adapter
+      attr_reader :worker, :queue
+
+      def initialize(worker)
+        @worker = worker
+        @queue = worker.options.fetch(:queue, "default")
+
+        @payload = worker.payload.merge(
+          "class" => worker.worker_class,
+          "retry" => worker.options.fetch(:retry, true),
+          "queue" => @queue
+        )
+        @payload["created_at"] ||= Time.now.to_f
+      end
+
+      # Coerces the raw payload into an instance of Worker
+      # @param payload [Hash] The job as json from redis
+      # @options options [Hash] list of options that will be passed along to the Worker instance
+      # @return [Esse::AsyncIndexing::Worker] and instance of Esse::AsyncIndexing::Worker
+      def self.coerce_to_worker(payload, **options)
+        raise(Error, "invalid payload") unless payload.is_a?(Hash)
+        raise(Error, "invalid payload") unless payload["class"].is_a?(String)
+
+        options[:retry] ||= payload["retry"] if payload.key?("retry")
+        options[:queue] ||= payload["queue"] if payload.key?("queue")
+
+        Esse::AsyncIndexing.worker(payload["class"], **options, service: :sidekiq).tap do |worker|
+          worker.with_args(*Array(payload["args"])) if payload.key?("args")
+          worker.with_job_jid(payload["jid"]) if payload.key?("jid")
+          worker.created_at(payload["created_at"]) if payload.key?("created_at")
+          worker.enqueued_at(payload["enqueued_at"]) if payload.key?("enqueued_at")
+          worker.at(payload["at"]) if payload.key?("at")
+        end
+      end
+
+      # Initializes adapter and push job into the sidekiq service
+      #
+      # @param worker [Esse::AsyncIndexing::Worker] An instance of Esse::AsyncIndexing::Worker
+      # @return [Hash] Job payload
+      # @see push method for more details
+      def self.push(worker)
+        new(worker).push
+      end
+
+      # Push sidekiq to the Sidekiq(Redis actually).
+      #   * If job has the 'at' key. Then schedule it
+      #   * Otherwise enqueue for immediate execution
+      #
+      # @return [Hash] Payload that was sent to redis
+      def push
+        normalize_before_push
+        # Optimization to enqueue something now that is scheduled to go out now or in the past
+        if (timestamp = @payload.delete("at")) && (timestamp > Time.now.to_f)
+          Esse.config.async_indexing.sidekiq.redis_pool.with do |redis|
+            redis.zadd(scheduled_queue_name, timestamp.to_f.to_s, to_json(@payload))
+          end
+        else
+          Esse.config.async_indexing.sidekiq.redis_pool.with do |redis|
+            redis.lpush(immediate_queue_name, to_json(@payload))
+          end
+        end
+        @payload
+      end
+
+      protected
+
+      def namespace
+        Esse.config.async_indexing.sidekiq.namespace
+      end
+
+      def scheduled_queue_name
+        "#{namespace}:schedule"
+      end
+
+      def immediate_queue_name
+        "#{namespace}:queue:#{queue}"
+      end
+
+      def to_json(value)
+        MultiJson.dump(value, mode: :compat)
+      end
+
+      def normalize_before_push
+        @payload["enqueued_at"] = Time.now.to_f
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/adapters.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Esse
+  module AsyncIndexing
+    module Adapters
+    end
+  end
+end
+
+require_relative "adapters/adapter"
+require_relative "adapters/sidekiq"
+require_relative "adapters/faktory"

data/lib/esse/async_indexing/cli/async_import.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "esse/cli/index/base_operation"
+
+class Esse::AsyncIndexing::CLI::AsyncImport < Esse::CLI::Index::BaseOperation
+  WORKER_NAME = "Esse::AsyncIndexing::Jobs::ImportBatchIdJob"
+
+  def run
+    validate_options!
+    indices.each do |index|
+      repos = if (repo = @options[:repo])
+        [index.repo(repo)]
+      else
+        index.repo_hash.values
+      end
+
+      repos.each do |repo|
+        unless Esse::AsyncIndexing.async_indexing_repo?(repo)
+          raise Esse::CLI::InvalidOption, <<~MSG
+            The #{repo} repository does not support async indexing. Make sure you have the `plugin :async_indexing` in your `#{index}` class and the :#{repo.repo_name} collection implements the `#each_batch_ids` method.
+          MSG
+        end
+
+        enqueuer = if (caller = repo.async_indexing_jobs[:import])
+          ->(ids) { caller.call(service_name, repo, :import, ids, **bulk_options) }
+        else
+          queue = Esse::RedisStorage::Queue.for(repo: repo)
+          ->(ids) do
+            batch_id = queue.enqueue(values: ids)
+            Esse::AsyncIndexing.worker(WORKER_NAME, service: service_name)
+              .with_args(repo.index.name, repo.repo_name, batch_id, Esse::HashUtils.deep_transform_keys(bulk_options, &:to_s))
+              .push
+          end
+        end
+
+        repo.batch_ids.each(&enqueuer)
+      end
+    end
+  end
+
+  private
+
+  def bulk_options
+    @bulk_options ||= begin
+      hash = @options.slice(*@options.keys - Esse::CLI_IGNORE_OPTS - [:repo, :service])
+      hash.delete(:context) if hash[:context].nil? || hash[:context].empty?
+      hash
+    end
+  end
+
+  def validate_options!
+    validate_indices_option!
+  end
+
+  def service_name
+    (@options[:service] || Esse.config.async_indexing.services.first)&.to_sym
+  end
+end