esse-async_indexing 0.0.2 → 0.1.0.rc1

data/lib/esse/async_indexing/actions/bulk_update_lazy_document_attribute.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Esse::AsyncIndexing::Actions
-  class BulkUpdateLazyDocumentAttribute
-    def self.call(index_class_name, repo_name, attr_name, batch_id, options = {})
-      _index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)
-      queue = Esse::RedisStorage::Queue.for(repo: repo_class, attribute_name: attr_name)
-
-      kwargs = Esse::HashUtils.deep_transform_keys(options, &:to_sym)
-
-      attr_name = repo_class.lazy_document_attributes.keys.find { |key| key.to_s == attr_name.to_s }
-      updated_ids = []
-      queue.fetch(batch_id) do |ids|
-        updated_ids = ids
-        repo_class.update_documents_attribute(attr_name, ids, **kwargs)
-      end
-      updated_ids
-    end
-  end
-end

data/lib/esse/async_indexing/actions/import_batch_id.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Esse::AsyncIndexing::Actions
-  class ImportBatchId
-    def self.call(index_class_name, repo_name, batch_id, options = {})
-      _index_class, repo_class = CoerceIndexRepository.call(index_class_name, repo_name)
-      queue = Esse::RedisStorage::Queue.for(repo: repo_class)
-
-      kwargs = Esse::HashUtils.deep_transform_keys(options, &:to_sym)
-      kwargs[:context] ||= {}
-      result = 0
-      ids_from_batch = []
-      queue.fetch(batch_id) do |ids|
-        ids_from_batch = ids
-        kwargs[:context][:id] = ids
-        result = repo_class.import(**kwargs)
-      end
-      [result, ids_from_batch]
-    end
-  end
-end

data/lib/esse/async_indexing/adapters/adapter.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Esse::AsyncIndexing
-  module Adapters
-    class Adapter
-      # Push the worker job to the service
-      # @param _worker [Esse::AsyncIndexing::Worker] An instance of background worker
-      # @abstract Child classes should override this method
-      def self.push(_worker)
-        raise NotImplemented
-      end
-
-      # Coerces the raw payload into an instance of Worker
-      # @param payload [Object] the object that should be coerced to a Worker
-      # @options options [Hash] list of options that will be passed along to the Worker instance
-      # @return [Esse::AsyncIndexing::Worker] and instance of Esse::AsyncIndexing::Worker
-      # @abstract Child classes should override this method
-      def self.coerce_to_worker(payload, **options)
-        raise NotImplemented
-      end
-
-      protected
-
-      def normalize_before_push
-        # noop
-      end
-    end
-  end
-end

data/lib/esse/async_indexing/adapters/faktory.rb

@@ -1,114 +0,0 @@
-# frozen_string_literal: true
-
-module Esse::AsyncIndexing
-  module Adapters
-    # This is a Faktory adapter that converts Esse::AsyncIndexing::Worker object into a faktory readable format
-    # and then push the jobs into the service.
-    class Faktory < Adapter
-      attr_reader :worker, :queue
-
-      def initialize(worker)
-        @worker = worker
-        @queue = worker.options.fetch(:queue, "default")
-
-        @payload = worker.payload.merge(
-          "jobtype" => worker.worker_class,
-          "queue" => @queue,
-          "retry" => parse_retry(worker.options[:retry])
-        )
-        @payload["created_at"] ||= Time.now.to_f
-      end
-
-      # Coerces the raw payload into an instance of Worker
-      # @param payload [Hash] The job as json from redis
-      # @options options [Hash] list of options that will be passed along to the Worker instance
-      # @return [Esse::AsyncIndexing::Worker] and instance of Esse::AsyncIndexing::Worker
-      def self.coerce_to_worker(payload, **options)
-        raise(Error, "invalid payload") unless payload.is_a?(Hash)
-        raise(Error, "invalid payload") unless payload["jobtype"].is_a?(String)
-
-        options[:retry] ||= payload["retry"] if payload.key?("retry")
-        options[:queue] ||= payload["queue"] if payload.key?("queue")
-
-        Esse::AsyncIndexing.worker(payload["jobtype"], **options, service: :faktory).tap do |worker|
-          worker.with_args(*Array(payload["args"])) if payload.key?("args")
-          worker.with_job_jid(payload["jid"]) if payload.key?("jid")
-          worker.created_at(payload["created_at"]) if payload.key?("created_at")
-          worker.enqueued_at(payload["enqueued_at"]) if payload.key?("enqueued_at")
-          worker.at(payload["at"]) if payload.key?("at")
-        end
-      end
-
-      # Initializes adapter and push job into the faktory service
-      #
-      # @param worker [Esse::AsyncIndexing::Worker] An instance of Esse::AsyncIndexing::Worker
-      # @return [Hash] Job payload
-      # @see push method for more details
-      def self.push(worker)
-        new(worker).push
-      end
-
-      # Push job to Faktory
-      #   * If job has the 'at' key. Then schedule it
-      #   * Otherwise enqueue for immediate execution
-      #
-      # @raise [Esse::AsyncIndexing::Error] raise and error when faktory dependency is not loaded
-      # @return [Hash] Payload that was sent to server
-      def push
-        unless Object.const_defined?(:Faktory)
-          raise Esse::AsyncIndexing::Error, <<~ERR
-            Faktory client for ruby is not loaded. You must install and require https://github.com/contribsys/faktory_worker_ruby.
-          ERR
-        end
-        normalize_before_push
-
-        pool = Thread.current[:faktory_via_pool] || ::Faktory.server_pool
-        ::Faktory.client_middleware.invoke(@payload, pool) do
-          pool.with do |c|
-            c.push(@payload)
-          end
-        end
-        @payload
-      end
-
-      protected
-
-      # Convert worker retry value acording to the Go struct datatype.
-      #
-      # * 25 is the default.
-      # * 0 means the job is completely ephemeral. No matter if it fails or succeeds, it will be discarded.
-      # * -1 means the job will go straight to the Dead set if it fails, no retries.
-      def parse_retry(value)
-        case value
-        when Numeric then value.to_i
-        when false then -1
-        else
-          25
-        end
-      end
-
-      def parse_time(value)
-        case value
-        when Numeric then Time.at(value).to_datetime.rfc3339(9)
-        when Time then value.to_datetime.rfc3339(9)
-        when DateTime then value.rfc3339(9)
-        end
-      end
-
-      def to_json(value)
-        MultiJson.dump(value, mode: :compat)
-      end
-
-      def normalize_before_push
-        @payload["enqueued_at"] ||= Time.now.to_f
-        {"created_at" => false, "enqueued_at" => false, "at" => true}.each do |field, past_remove|
-          # Optimization to enqueue something now that is scheduled to go out now or in the past
-          if (time = @payload.delete(field)) &&
-              (!past_remove || (past_remove && time > Time.now.to_f))
-            @payload[field] = parse_time(time)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/esse/async_indexing/adapters/sidekiq.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-module Esse::AsyncIndexing
-  module Adapters
-    # This is a Sidekiq adapter that converts Esse::AsyncIndexing::Worker object into a sidekiq readable format
-    # and then push the jobs into the service.
-    class Sidekiq < Adapter
-      attr_reader :worker, :queue
-
-      def initialize(worker)
-        @worker = worker
-        @queue = worker.options.fetch(:queue, "default")
-
-        @payload = worker.payload.merge(
-          "class" => worker.worker_class,
-          "retry" => worker.options.fetch(:retry, true),
-          "queue" => @queue
-        )
-        @payload["created_at"] ||= Time.now.to_f
-      end
-
-      # Coerces the raw payload into an instance of Worker
-      # @param payload [Hash] The job as json from redis
-      # @options options [Hash] list of options that will be passed along to the Worker instance
-      # @return [Esse::AsyncIndexing::Worker] and instance of Esse::AsyncIndexing::Worker
-      def self.coerce_to_worker(payload, **options)
-        raise(Error, "invalid payload") unless payload.is_a?(Hash)
-        raise(Error, "invalid payload") unless payload["class"].is_a?(String)
-
-        options[:retry] ||= payload["retry"] if payload.key?("retry")
-        options[:queue] ||= payload["queue"] if payload.key?("queue")
-
-        Esse::AsyncIndexing.worker(payload["class"], **options, service: :sidekiq).tap do |worker|
-          worker.with_args(*Array(payload["args"])) if payload.key?("args")
-          worker.with_job_jid(payload["jid"]) if payload.key?("jid")
-          worker.created_at(payload["created_at"]) if payload.key?("created_at")
-          worker.enqueued_at(payload["enqueued_at"]) if payload.key?("enqueued_at")
-          worker.at(payload["at"]) if payload.key?("at")
-        end
-      end
-
-      # Initializes adapter and push job into the sidekiq service
-      #
-      # @param worker [Esse::AsyncIndexing::Worker] An instance of Esse::AsyncIndexing::Worker
-      # @return [Hash] Job payload
-      # @see push method for more details
-      def self.push(worker)
-        new(worker).push
-      end
-
-      # Push sidekiq to the Sidekiq(Redis actually).
-      #   * If job has the 'at' key. Then schedule it
-      #   * Otherwise enqueue for immediate execution
-      #
-      # @return [Hash] Payload that was sent to redis
-      def push
-        normalize_before_push
-        # Optimization to enqueue something now that is scheduled to go out now or in the past
-        if (timestamp = @payload.delete("at")) && (timestamp > Time.now.to_f)
-          Esse.config.async_indexing.sidekiq.redis_pool.with do |redis|
-            redis.zadd(scheduled_queue_name, timestamp.to_f.to_s, to_json(@payload))
-          end
-        else
-          Esse.config.async_indexing.sidekiq.redis_pool.with do |redis|
-            redis.lpush(immediate_queue_name, to_json(@payload))
-          end
-        end
-        @payload
-      end
-
-      protected
-
-      def namespace
-        Esse.config.async_indexing.sidekiq.namespace
-      end
-
-      def scheduled_queue_name
-        "#{namespace}:schedule"
-      end
-
-      def immediate_queue_name
-        "#{namespace}:queue:#{queue}"
-      end
-
-      def to_json(value)
-        MultiJson.dump(value, mode: :compat)
-      end
-
-      def normalize_before_push
-        @payload["enqueued_at"] = Time.now.to_f
-      end
-    end
-  end
-end

data/lib/esse/async_indexing/adapters.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-module Esse
-  module AsyncIndexing
-    module Adapters
-    end
-  end
-end
-
-require_relative "adapters/adapter"
-require_relative "adapters/sidekiq"
-require_relative "adapters/faktory"

data/lib/esse/async_indexing/configuration/base.rb

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-class Esse::AsyncIndexing::Configuration::Base
-  class << self
-    private
-
-    def attribute_accessor(field, validator: nil, normalizer: nil, default: nil)
-      normalizer ||= :"normalize_#{field}"
-      validator ||= :"validate_#{field}"
-
-      define_method(field) do
-        unless instance_variable_defined?(:"@#{field}")
-          return if default.nil?
-
-          send(:"#{field}=", default.respond_to?(:call) ? default.call : default)
-        end
-        instance_variable_get(:"@#{field}")
-      end
-
-      define_method(:"#{field}=") do |value|
-        value = send(normalizer, field, value) if respond_to?(normalizer, true)
-        send(validator, field, value) if respond_to?(validator, true)
-
-        instance_variable_set(:"@#{field}", value)
-      end
-    end
-  end
-
-  # A Hash with all workers definitions. The worker class name must be the main hash key
-  # Example:
-  #   "FaktoryIndexWorker":
-  #     retry: false
-  #     queue: "indexing"
-  #     adapter: "faktory"
-  #   "FaktoryBatchIndexWorker":
-  #     retry: 5
-  #     queue: "batch_index"
-  #     adapter: "faktory"
-  attribute_accessor :workers, default: {}
-
-  def worker_options(class_name)
-    class_name = class_name.to_s
-    if strict? && !workers.key?(class_name)
-      raise Esse::AsyncIndexing::NotDefinedWorkerError.new(class_name)
-    end
-
-    workers.fetch(class_name, {})
-  end
-
-  def strict?
-    true
-  end
-
-  protected
-
-  def normalize_workers(_, value)
-    return unless value.is_a?(Hash)
-
-    hash = Esse::AsyncIndexing::Workers::DEFAULT.values.map { |v| [v, {}] }.to_h
-    value.each do |class_name, opts|
-      hash[class_name.to_s] = Esse::HashUtils.deep_transform_keys(opts.to_h, &:to_sym)
-    end
-    hash
-  end
-end

data/lib/esse/async_indexing/configuration/faktory.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-module Esse::AsyncIndexing
-  class Configuration::Faktory < Configuration::Base
-  end
-end

data/lib/esse/async_indexing/configuration/sidekiq.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-module Esse::AsyncIndexing
-  class Configuration::Sidekiq < Configuration::Base
-    attribute_accessor :redis
-    attribute_accessor :namespace, default: "sidekiq"
-
-    def redis_pool
-      @redis_pool ||= Esse::RedisStorage::Pool.new(redis)
-    end
-  end
-end

data/lib/esse/async_indexing/jobs/bulk_update_lazy_document_attribute_job.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-class Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob
-  def perform(index_class_name, repo_name, attribute_name, batch_id, options = {})
-    Esse::AsyncIndexing::Actions::BulkUpdateLazyDocumentAttribute.call(index_class_name, repo_name, attribute_name, batch_id, options)
-  end
-end

data/lib/esse/async_indexing/jobs/import_batch_id_job.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-class Esse::AsyncIndexing::Jobs::ImportBatchIdJob
-  LAZY_ATTR_WORKER = "Esse::AsyncIndexing::Jobs::BulkUpdateLazyDocumentAttributeJob"
-
-  def perform(index_class_name, repo_name, batch_id, options = {})
-    total, ids = Esse::AsyncIndexing::Actions::ImportBatchId.call(index_class_name, repo_name, batch_id, options)
-
-    options = Esse::HashUtils.deep_transform_keys(options, &:to_s)
-    return total if total.zero?
-    return total if lazy_already_imported?(options)
-    return total unless self.class.respond_to?(:bg_worker_options)
-
-    _index_class, repo_class = Esse::AsyncIndexing::Actions::CoerceIndexRepository.call(index_class_name, repo_name)
-
-    repo_class.lazy_document_attributes.each_key do |attr_name|
-      queue = Esse::RedisStorage::Queue.for(repo: repo_class, attribute_name: attr_name)
-      queue.enqueue(id: batch_id, values: ids)
-      Esse::AsyncIndexing.worker(LAZY_ATTR_WORKER, service: self.class.bg_worker_options[:service])
-        .with_args(index_class_name, repo_name, attr_name.to_s, batch_id, options)
-        .push
-    end
-    total
-  end
-
-  protected
-
-  # The `import` action already eager or lazy load the document attributes when some of these options are set.
-  def lazy_already_imported?(options)
-    eager = options.delete("eager_include_document_attributes") || false
-    lazy = options.delete("lazy_update_document_attributes") || false
-    eager || lazy
-  end
-end

data/lib/esse/async_indexing/jobs/update_lazy_document_attribute_job.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-class Esse::AsyncIndexing::Jobs::UpdateLazyDocumentAttributeJob
-  def perform(index_class_name, repo_name, attribute_name, ids, options = {})
-    Esse::AsyncIndexing::Actions::UpdateLazyDocumentAttribute.call(index_class_name, repo_name, attribute_name, ids, options)
-  end
-end

data/lib/esse/async_indexing/testing.rb

@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-module Esse
-  module AsyncIndexing
-    class Testing
-      class << self
-        def enable!
-          Thread.current[:esse_async_indexing_testing] = true
-        end
-
-        def disable!
-          Thread.current[:esse_async_indexing_testing] = false
-        end
-
-        def enabled?
-          Thread.current[:esse_async_indexing_testing] == true
-        end
-
-        def disabled?
-          !enabled?
-        end
-      end
-    end
-  end
-end
-
-Esse::AsyncIndexing::Testing.disable!
-
-module Esse::AsyncIndexing::Jobs
-  class << self
-    def jobs
-      @jobs ||= []
-    end
-
-    def push(job)
-      jobs.push(job)
-    end
-
-    def clear
-      jobs.clear
-    end
-
-    def size
-      jobs.size
-    end
-
-    def jobs_for(service: nil, class_name: nil)
-      filtered = jobs
-      if service
-        filtered = filtered.select { |job| job["service"] == service.to_s }
-      end
-      if class_name
-        filtered = filtered.select { |job| job["__class_name__"] == class_name.to_s }
-      end
-      filtered
-    end
-  end
-end
-
-module Esse::AsyncIndexing::JobsInterceptorAdapter
-  def push
-    return super unless Esse::AsyncIndexing::Testing.enabled?
-
-    normalize_before_push
-    test_payload = @payload.dup
-    if @payload["jobtype"]
-      test_payload["service"] = "faktory"
-      test_payload["__class_name__"] = @payload["jobtype"]
-    else
-      test_payload["service"] = "sidekiq"
-      test_payload["__class_name__"] = @payload["class"]
-    end
-    Esse::AsyncIndexing::Jobs.push(test_payload)
-  end
-end
-
-Esse::AsyncIndexing::SERVICES.each_value do |adapter|
-  adapter.prepend(Esse::AsyncIndexing::JobsInterceptorAdapter)
-end

data/lib/esse/async_indexing/worker.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-module Esse::AsyncIndexing
-  class Worker
-    attr_reader :options, :payload, :worker_class, :service
-
-    attr_reader :arguments
-
-    def initialize(worker_class, service: nil, **options)
-      @worker_class = worker_class
-      @service = service
-      @options = options
-      @payload = {}
-    end
-
-    def self.coerce(service:, payload:, **opts)
-      Esse::AsyncIndexing::SERVICES.fetch(service).coerce_to_worker(payload, **opts)
-    end
-
-    %i[created_at enqueued_at].each do |method_name|
-      define_method method_name do |value|
-        @payload[method_name.to_s] =
-          case value
-          when Numeric then value.to_f
-          when String then Time.parse(value).to_f
-          when Time, DateTime then value.to_f
-          else
-            raise ArgumentError, format("The %<v>p is not a valid value for %<m>s.", v: value, m: method_name)
-          end
-
-        self
-      end
-    end
-
-    # Adds arguments to the job
-    # @return self
-    def with_args(*args)
-      @payload["args"] = args
-
-      self
-    end
-
-    # Schedule the time when a job will be executed. Jobs which are scheduled in the past are enqueued for immediate execution.
-    # @param timestamp [Numeric] timestamp, numeric or something that acts numeric.
-    # @return self
-    def in(timestamp)
-      now = Time.now.to_f
-      timestamp = Time.parse(timestamp) if timestamp.is_a?(String)
-      int = timestamp.respond_to?(:strftime) ? timestamp.to_f : now + timestamp.to_f
-      return self if int <= now
-
-      @payload["at"] = int
-      @payload["created_at"] = now
-
-      self
-    end
-    alias_method :at, :in
-
-    def with_job_jid(jid = nil)
-      @payload["jid"] ||= jid || Esse::AsyncIndexing.jid
-
-      self
-    end
-
-    # @return Response of service
-    # @see Esse::AsyncIndexing::Adapters::** for more details
-    def push
-      unless Esse::AsyncIndexing::SERVICES.key?(service)
-        raise Esse::AsyncIndexing::Error, format("Service %<service>p is not implemented. Please use one of #{Esse::AsyncIndexing::SERVICES.keys.map(&:inspect).join(" or ")}.", service: service)
-      end
-      @payload["created_at"] ||= Time.now.to_f
-      worker_to_push = with_job_jid
-      Esse::AsyncIndexing::SERVICES[service].push(worker_to_push)
-    end
-
-    def eql?(other)
-      return false unless other.is_a?(self.class)
-
-      worker_class == other.worker_class &&
-        payload == other.payload &&
-        options == other.options
-    end
-    alias_method :==, :eql?
-  end
-end

data/lib/esse/async_indexing/workers/faktory.rb

@@ -1,28 +0,0 @@
-# frizen_string_literal: true
-
-require_relative "shared_class_methods"
-
-module Esse::AsyncIndexing
-  module Workers
-    module Faktory
-      def self.extended(base)
-        base.include(::Faktory::Job) if defined?(::Faktory)
-        base.extend SharedClassMethods
-        base.extend ClassMethods
-      end
-
-      module ClassMethods
-        def service_worker_options
-          default_queue = Esse.config.async_indexing.faktory.workers.dig(name, :queue)
-          default_retry = Esse.config.async_indexing.faktory.workers.dig(name, :retry)
-          default_queue ||= ::Faktory.default_job_options["queue"] if defined?(::Faktory)
-          default_retry ||= ::Faktory.default_job_options["retry"] if defined?(::Faktory)
-          {
-            queue: default_queue || "default",
-            retry: default_retry || 25
-          }
-        end
-      end
-    end
-  end
-end

data/lib/esse/async_indexing/workers/shared_class_methods.rb

@@ -1,26 +0,0 @@
-# frizen_string_literal: true
-
-module Esse::AsyncIndexing
-  module Workers
-    module SharedClassMethods
-      def perform_async(*args)
-        build_worker.with_args(*args).push
-      end
-
-      def perform_in(interval, *args)
-        build_worker.with_args(*args).at(interval).push
-      end
-      alias_method :perform_at, :perform_in
-
-      protected
-
-      def service_worker_options
-        {}
-      end
-
-      def build_worker
-        Esse::AsyncIndexing.worker(name, **service_worker_options.merge(bg_worker_options))
-      end
-    end
-  end
-end

data/lib/esse/async_indexing/workers/sidekiq.rb

@@ -1,28 +0,0 @@
-# frizen_string_literal: true
-
-require_relative "shared_class_methods"
-
-module Esse::AsyncIndexing
-  module Workers
-    module Sidekiq
-      def self.extended(base)
-        base.include(::Sidekiq::Worker) if defined?(::Sidekiq)
-        base.extend SharedClassMethods
-        base.extend ClassMethods
-      end
-
-      module ClassMethods
-        def service_worker_options
-          default_queue = Esse.config.async_indexing.sidekiq.workers.dig(name, :queue)
-          default_retry = Esse.config.async_indexing.sidekiq.workers.dig(name, :retry)
-          default_queue ||= ::Sidekiq.default_worker_options["queue"] if defined?(::Sidekiq)
-          default_retry ||= ::Sidekiq.default_worker_options["retry"] if defined?(::Sidekiq)
-          {
-            queue: default_queue || "default",
-            retry: default_retry || 15
-          }
-        end
-      end
-    end
-  end
-end