esse-async_indexing 0.0.2

data/lib/esse/async_indexing/cli.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "esse/cli"
+
+module Esse::AsyncIndexing
+  module CLI
+  end
+end
+
+Esse::CLI::Index.class_eval do
+  desc "async_import *INDEX_CLASSES", "Async Import documents from the given classes into the index"
+  option :repo, type: :string, default: nil, alias: "-r", desc: "Repository to use for import"
+  option :suffix, type: :string, default: nil, aliases: "-s", desc: "Suffix to append to index name"
+  option :context, type: :hash, default: {}, required: true, desc: "List of options to pass to the index class"
+  option :service, type: :string, default: nil, alias: "-s", desc: "Service to use for async import: sidekiq, faktory"
+  option :eager_include_document_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to include to the bulk index request. Or pass `true` to include all lazy attributes"
+  option :lazy_update_document_attributes, type: :string, default: nil, desc: "Comma separated list of lazy document attributes to bulk update after the bulk index request Or pass `true` to include all lazy attributes"
+  def async_import(*index_classes)
+    opts = Esse::HashUtils.deep_transform_keys(options.to_h, &:to_sym)
+    opts[:service] ||= Esse.config.async_indexing.services.first
+    opts.delete(:lazy_update_document_attributes) if opts[:lazy_update_document_attributes] == "false"
+    opts.delete(:eager_include_document_attributes) if opts[:eager_include_document_attributes] == "false"
+    if (val = opts[:eager_include_document_attributes])
+      opts[:eager_include_document_attributes] = (val == "true") ? true : val.split(",")
+    end
+    if (val = opts[:lazy_update_document_attributes])
+      opts[:lazy_update_document_attributes] = (val == "true") ? true : val.split(",")
+    end
+    require "esse/async_indexing/cli/async_import"
+    Esse::AsyncIndexing::CLI::AsyncImport.new(indices: index_classes, **opts).run
+  end
+end

data/lib/esse/async_indexing/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require_relative "configuration"
+require_relative "configuration/base"
+require_relative "configuration/faktory"
+require_relative "configuration/sidekiq"
+
+module Esse
+  module AsyncIndexing
+    module Config
+      def self.included(base)
+        base.__send__(:include, InstanceMethods)
+      end
+
+      module InstanceMethods
+        def async_indexing
+          @async_indexing ||= AsyncIndexing::Configuration.new
+          if block_given?
+            yield @async_indexing
+          else
+            @async_indexing
+          end
+        end
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/configuration/base.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class Esse::AsyncIndexing::Configuration::Base
+  class << self
+    private
+
+    def attribute_accessor(field, validator: nil, normalizer: nil, default: nil)
+      normalizer ||= :"normalize_#{field}"
+      validator ||= :"validate_#{field}"
+
+      define_method(field) do
+        unless instance_variable_defined?(:"@#{field}")
+          return if default.nil?
+
+          send(:"#{field}=", default.respond_to?(:call) ? default.call : default)
+        end
+        instance_variable_get(:"@#{field}")
+      end
+
+      define_method(:"#{field}=") do |value|
+        value = send(normalizer, field, value) if respond_to?(normalizer, true)
+        send(validator, field, value) if respond_to?(validator, true)
+
+        instance_variable_set(:"@#{field}", value)
+      end
+    end
+  end
+
+  # A Hash with all workers definitions. The worker class name must be the main hash key
+  # Example:
+  #   "FaktoryIndexWorker":
+  #     retry: false
+  #     queue: "indexing"
+  #     adapter: "faktory"
+  #   "FaktoryBatchIndexWorker":
+  #     retry: 5
+  #     queue: "batch_index"
+  #     adapter: "faktory"
+  attribute_accessor :workers, default: {}
+
+  def worker_options(class_name)
+    class_name = class_name.to_s
+    if strict? && !workers.key?(class_name)
+      raise Esse::AsyncIndexing::NotDefinedWorkerError.new(class_name)
+    end
+
+    workers.fetch(class_name, {})
+  end
+
+  def strict?
+    true
+  end
+
+  protected
+
+  def normalize_workers(_, value)
+    return unless value.is_a?(Hash)
+
+    hash = Esse::AsyncIndexing::Workers::DEFAULT.values.map { |v| [v, {}] }.to_h
+    value.each do |class_name, opts|
+      hash[class_name.to_s] = Esse::HashUtils.deep_transform_keys(opts.to_h, &:to_sym)
+    end
+    hash
+  end
+end

data/lib/esse/async_indexing/configuration/faktory.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  class Configuration::Faktory < Configuration::Base
+  end
+end

data/lib/esse/async_indexing/configuration/sidekiq.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  class Configuration::Sidekiq < Configuration::Base
+    attribute_accessor :redis
+    attribute_accessor :namespace, default: "sidekiq"
+
+    def redis_pool
+      @redis_pool ||= Esse::RedisStorage::Pool.new(redis)
+    end
+  end
+end

data/lib/esse/async_indexing/configuration.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Esse
+  module AsyncIndexing
+    class ConfigService < Set
+      def sidekiq?
+        include?(:sidekiq)
+      end
+
+      def faktory?
+        include?(:faktory)
+      end
+    end
+
+    class Configuration
+      def services
+        @services ||= ConfigService.new
+      end
+
+      def faktory
+        @faktory ||= begin
+          services.add(:faktory)
+          Configuration::Faktory.new
+        end
+        if block_given?
+          yield @faktory
+        else
+          @faktory
+        end
+      end
+
+      def sidekiq
+        @sidekiq ||= begin
+          services.add(:sidekiq)
+          Configuration::Sidekiq.new
+        end
+        if block_given?
+          yield @sidekiq
+        else
+          @sidekiq
+        end
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/errors.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  class Error < StandardError
+  end
+
+  class NotDefinedWorkerError < Error
+    def initialize(worker_name)
+      super("Worker `#{worker_name}` is not defined.")
+    end
+  end
+end

data/lib/esse/async_indexing/jobs/bulk_update_lazy_document_attribute_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob
+  def perform(index_class_name, repo_name, attribute_name, batch_id, options = {})
+    Esse::AsyncIndexing::Actions::BulkUpdateLazyDocumentAttribute.call(index_class_name, repo_name, attribute_name, batch_id, options)
+  end
+end

data/lib/esse/async_indexing/jobs/document_delete_by_id_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob
+  def perform(index_class_name, repo_name, document_id, options = {})
+    Esse::AsyncIndexing::Actions::DeleteDocument.call(index_class_name, repo_name, document_id, options)
+  end
+end

data/lib/esse/async_indexing/jobs/document_index_by_id_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob
+  def perform(index_class_name, repo_name, document_id, options = {})
+    Esse::AsyncIndexing::Actions::IndexDocument.call(index_class_name, repo_name, document_id, options)
+  end
+end

data/lib/esse/async_indexing/jobs/document_update_by_id_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob
+  def perform(index_class_name, repo_name, document_id, options = {})
+    Esse::AsyncIndexing::Actions::UpdateDocument.call(index_class_name, repo_name, document_id, options)
+  end
+end

data/lib/esse/async_indexing/jobs/document_upsert_by_id_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob
+  def perform(index_class_name, repo_name, document_id, operation = "index", options = {})
+    Esse::AsyncIndexing::Actions::UpsertDocument.call(index_class_name, repo_name, document_id, operation, options)
+  end
+end

data/lib/esse/async_indexing/jobs/import_all_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Esse::AsyncIndexing::Jobs::ImportAllJob
+  def perform(index_class_name, repo_name, options = {})
+    Esse::AsyncIndexing::Actions::BatchImportAll.call(index_class_name, repo_name, options)
+  end
+end

data/lib/esse/async_indexing/jobs/import_batch_id_job.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class Esse::AsyncIndexing::Jobs::ImportBatchIdJob
+  LAZY_ATTR_WORKER = "Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob"
+
+  def perform(index_class_name, repo_name, batch_id, options = {})
+    total, ids = Esse::AsyncIndexing::Actions::ImportBatchId.call(index_class_name, repo_name, batch_id, options)
+
+    options = Esse::HashUtils.deep_transform_keys(options, &:to_s)
+    return total if total.zero?
+    return total if lazy_already_imported?(options)
+    return total unless self.class.respond_to?(:bg_worker_options)
+
+    _index_class, repo_class = Esse::AsyncIndexing::Actions::CoerceIndexRepository.call(index_class_name, repo_name)
+
+    repo_class.lazy_document_attributes.each_key do |attr_name|
+      queue = Esse::RedisStorage::Queue.for(repo: repo_class, attribute_name: attr_name)
+      queue.enqueue(id: batch_id, values: ids)
+      Esse::AsyncIndexing.worker(LAZY_ATTR_WORKER, service: self.class.bg_worker_options[:service])
+        .with_args(index_class_name, repo_name, attr_name.to_s, batch_id, options)
+        .push
+    end
+    total
+  end
+
+  protected
+
+  # The `import` action already eager or lazy load the document attributes when some of these options are set.
+  def lazy_already_imported?(options)
+    eager = options.delete("eager_include_document_attributes") || false
+    lazy = options.delete("lazy_update_document_attributes") || false
+    eager || lazy
+  end
+end

data/lib/esse/async_indexing/jobs/update_lazy_document_attribute_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob
+  def perform(index_class_name, repo_name, attribute_name, ids, options = {})
+    Esse::AsyncIndexing::Actions::UpdateLazyDocumentAttribute.call(index_class_name, repo_name, attribute_name, ids, options)
+  end
+end

data/lib/esse/async_indexing/testing.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Esse
+  module AsyncIndexing
+    class Testing
+      class << self
+        def enable!
+          Thread.current[:esse_async_indexing_testing] = true
+        end
+
+        def disable!
+          Thread.current[:esse_async_indexing_testing] = false
+        end
+
+        def enabled?
+          Thread.current[:esse_async_indexing_testing] == true
+        end
+
+        def disabled?
+          !enabled?
+        end
+      end
+    end
+  end
+end
+
+Esse::AsyncIndexing::Testing.disable!
+
+module Esse::AsyncIndexing::Jobs
+  class << self
+    def jobs
+      @jobs ||= []
+    end
+
+    def push(job)
+      jobs.push(job)
+    end
+
+    def clear
+      jobs.clear
+    end
+
+    def size
+      jobs.size
+    end
+
+    def jobs_for(service: nil, class_name: nil)
+      filtered = jobs
+      if service
+        filtered = filtered.select { |job| job["service"] == service.to_s }
+      end
+      if class_name
+        filtered = filtered.select { |job| job["__class_name__"] == class_name.to_s }
+      end
+      filtered
+    end
+  end
+end
+
+module Esse::AsyncIndexing::JobsInterceptorAdapter
+  def push
+    return super unless Esse::AsyncIndexing::Testing.enabled?
+
+    normalize_before_push
+    test_payload = @payload.dup
+    if @payload["jobtype"]
+      test_payload["service"] = "faktory"
+      test_payload["__class_name__"] = @payload["jobtype"]
+    else
+      test_payload["service"] = "sidekiq"
+      test_payload["__class_name__"] = @payload["class"]
+    end
+    Esse::AsyncIndexing::Jobs.push(test_payload)
+  end
+end
+
+Esse::AsyncIndexing::SERVICES.each_value do |adapter|
+  adapter.prepend(Esse::AsyncIndexing::JobsInterceptorAdapter)
+end

data/lib/esse/async_indexing/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Esse
+  module AsyncIndexing
+    VERSION = "0.0.2"
+  end
+end

data/lib/esse/async_indexing/worker.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Esse::AsyncIndexing
+  class Worker
+    attr_reader :options, :payload, :worker_class, :service
+
+    attr_reader :arguments
+
+    def initialize(worker_class, service: nil, **options)
+      @worker_class = worker_class
+      @service = service
+      @options = options
+      @payload = {}
+    end
+
+    def self.coerce(service:, payload:, **opts)
+      Esse::AsyncIndexing::SERVICES.fetch(service).coerce_to_worker(payload, **opts)
+    end
+
+    %i[created_at enqueued_at].each do |method_name|
+      define_method method_name do |value|
+        @payload[method_name.to_s] =
+          case value
+          when Numeric then value.to_f
+          when String then Time.parse(value).to_f
+          when Time, DateTime then value.to_f
+          else
+            raise ArgumentError, format("The %<v>p is not a valid value for %<m>s.", v: value, m: method_name)
+          end
+
+        self
+      end
+    end
+
+    # Adds arguments to the job
+    # @return self
+    def with_args(*args)
+      @payload["args"] = args
+
+      self
+    end
+
+    # Schedule the time when a job will be executed. Jobs which are scheduled in the past are enqueued for immediate execution.
+    # @param timestamp [Numeric] timestamp, numeric or something that acts numeric.
+    # @return self
+    def in(timestamp)
+      now = Time.now.to_f
+      timestamp = Time.parse(timestamp) if timestamp.is_a?(String)
+      int = timestamp.respond_to?(:strftime) ? timestamp.to_f : now + timestamp.to_f
+      return self if int <= now
+
+      @payload["at"] = int
+      @payload["created_at"] = now
+
+      self
+    end
+    alias_method :at, :in
+
+    def with_job_jid(jid = nil)
+      @payload["jid"] ||= jid || Esse::AsyncIndexing.jid
+
+      self
+    end
+
+    # @return Response of service
+    # @see Esse::AsyncIndexing::Adapters::** for more details
+    def push
+      unless Esse::AsyncIndexing::SERVICES.key?(service)
+        raise Esse::AsyncIndexing::Error, format("Service %<service>p is not implemented. Please use one of #{Esse::AsyncIndexing::SERVICES.keys.map(&:inspect).join(" or ")}.", service: service)
+      end
+      @payload["created_at"] ||= Time.now.to_f
+      worker_to_push = with_job_jid
+      Esse::AsyncIndexing::SERVICES[service].push(worker_to_push)
+    end
+
+    def eql?(other)
+      return false unless other.is_a?(self.class)
+
+      worker_class == other.worker_class &&
+        payload == other.payload &&
+        options == other.options
+    end
+    alias_method :==, :eql?
+  end
+end

data/lib/esse/async_indexing/workers/faktory.rb

@@ -0,0 +1,28 @@
+# frizen_string_literal: true
+
+require_relative "shared_class_methods"
+
+module Esse::AsyncIndexing
+  module Workers
+    module Faktory
+      def self.extended(base)
+        base.include(::Faktory::Job) if defined?(::Faktory)
+        base.extend SharedClassMethods
+        base.extend ClassMethods
+      end
+
+      module ClassMethods
+        def service_worker_options
+          default_queue = Esse.config.async_indexing.faktory.workers.dig(name, :queue)
+          default_retry = Esse.config.async_indexing.faktory.workers.dig(name, :retry)
+          default_queue ||= ::Faktory.default_job_options["queue"] if defined?(::Faktory)
+          default_retry ||= ::Faktory.default_job_options["retry"] if defined?(::Faktory)
+          {
+            queue: default_queue || "default",
+            retry: default_retry || 25
+          }
+        end
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/workers/shared_class_methods.rb

@@ -0,0 +1,26 @@
+# frizen_string_literal: true
+
+module Esse::AsyncIndexing
+  module Workers
+    module SharedClassMethods
+      def perform_async(*args)
+        build_worker.with_args(*args).push
+      end
+
+      def perform_in(interval, *args)
+        build_worker.with_args(*args).at(interval).push
+      end
+      alias_method :perform_at, :perform_in
+
+      protected
+
+      def service_worker_options
+        {}
+      end
+
+      def build_worker
+        Esse::AsyncIndexing.worker(name, **service_worker_options.merge(bg_worker_options))
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/workers/sidekiq.rb

@@ -0,0 +1,28 @@
+# frizen_string_literal: true
+
+require_relative "shared_class_methods"
+
+module Esse::AsyncIndexing
+  module Workers
+    module Sidekiq
+      def self.extended(base)
+        base.include(::Sidekiq::Worker) if defined?(::Sidekiq)
+        base.extend SharedClassMethods
+        base.extend ClassMethods
+      end
+
+      module ClassMethods
+        def service_worker_options
+          default_queue = Esse.config.async_indexing.sidekiq.workers.dig(name, :queue)
+          default_retry = Esse.config.async_indexing.sidekiq.workers.dig(name, :retry)
+          default_queue ||= ::Sidekiq.default_worker_options["queue"] if defined?(::Sidekiq)
+          default_retry ||= ::Sidekiq.default_worker_options["retry"] if defined?(::Sidekiq)
+          {
+            queue: default_queue || "default",
+            retry: default_retry || 15
+          }
+        end
+      end
+    end
+  end
+end

data/lib/esse/async_indexing/workers.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require_relative "worker"
+
+module Esse
+  module AsyncIndexing
+    module Workers
+      DEFAULT = {
+        "esse/async_indexing/jobs/bulk_update_lazy_document_attribute_job" => "Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob",
+        "esse/async_indexing/jobs/document_delete_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentDeleteByIdJob",
+        "esse/async_indexing/jobs/document_index_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentIndexByIdJob",
+        "esse/async_indexing/jobs/document_update_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentUpdateByIdJob",
+        "esse/async_indexing/jobs/document_upsert_by_id_job" => "Esse::AsyncIndexing::Jobs::DocumentUpsertByIdJob",
+        "esse/async_indexing/jobs/import_all_job" => "Esse::AsyncIndexing::Jobs::ImportAllJob",
+        "esse/async_indexing/jobs/import_batch_id_job" => "Esse::AsyncIndexing::Jobs::ImportBatchIdJob",
+        "esse/async_indexing/jobs/update_lazy_document_attribute_job" => "Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob"
+      }
+
+      # The backend service may live in a different application, so they are not required by default.
+      # That's why we have a separate structure to let enqueue jobs even without having the explicit worker class loaded.
+      # This method will require all the internal jobs and configure them according to the defined options.
+      def self.install!(service, **options)
+        return if @installed_services&.include?(service.to_sym)
+
+        DEFAULT.each do |job, worker_name|
+          Kernel.require(job)
+          worker = Esse::AsyncIndexing::Jobs.const_get(worker_name.split("::").last)
+          worker.extend(self.for(service, **options))
+        end
+        @installed_services = Array(@installed_services) << service.to_sym
+      end
+
+      def self.for(service, **options)
+        require_relative "workers/#{service}"
+        service = service.to_sym
+        worker_options = options.merge(service: service)
+        module_name = service.to_s.split(/_/i).collect! { |w| w.capitalize }.join
+        mod = Esse::AsyncIndexing::Workers.const_get(module_name)
+        mod.module_eval do
+          define_method(:bg_worker_options) do
+            worker_options
+          end
+        end
+        mod
+      end
+    end
+  end
+end

data/lib/esse/async_indexing.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "esse"
+require "esse-redis_storage"
+require "forwardable"
+require "securerandom"
+require "time"
+require "multi_json"
+
+module Esse
+  module AsyncIndexing
+    module Actions
+    end
+
+    module Jobs
+    end
+
+    module Workers
+    end
+  end
+end
+
+require_relative "async_indexing/version"
+require_relative "async_indexing/actions"
+require_relative "async_indexing/adapters"
+require_relative "async_indexing/config"
+require_relative "async_indexing/errors"
+require_relative "async_indexing/workers"
+require_relative "plugins/async_indexing"
+
+module Esse::AsyncIndexing
+  SERVICES = {
+    sidekiq: Adapters::Sidekiq,
+    faktory: Adapters::Faktory
+  }
+
+  # @param worker_class [String] The worker class name
+  # @param options [Hash] Options that will be passed along to the worker instance
+  # @return [Esse::AsyncIndexing::Worker] An instance of worker
+  def self.worker(worker_class, service: nil, **options)
+    serv_name = service_name(service)
+    Worker.new(worker_class, **Esse.config.async_indexing.send(service).worker_options(worker_class).merge(options), service: serv_name)
+  end
+
+  def self.service_name(identifier = nil)
+    identifier ||= Esse.config.async_indexing.services.first
+    if identifier.nil?
+      raise ArgumentError, "There are no async indexing services configured. Please configure at least one service or pass the service name as an argument."
+    end
+
+    if SERVICES[identifier.to_sym].nil?
+      raise ArgumentError, "Invalid service: #{identifier.inspect}, valid services are: #{SERVICES.keys.join(", ")}"
+    end
+
+    identifier.to_sym
+  end
+
+  def self.jid
+    SecureRandom.hex(12)
+  end
+
+  def self.async_indexing_repo?(repo)
+    return false unless repo.is_a?(Class) && repo < Esse::Repository
+
+    repo.respond_to?(:implement_batch_ids?) && repo.implement_batch_ids?
+  end
+end
+
+Esse::Config.__send__ :include, Esse::AsyncIndexing::Config
+if defined?(Esse::CLI)
+  require_relative "async_indexing/cli"
+end